1 /* ACLE support for AArch64 SVE
2 Copyright (C) 2018-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
11 GCC is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
20 #define IN_TARGET_CODE 1
24 #include "coretypes.h"
30 #include "insn-codes.h"
33 #include "diagnostic.h"
35 #include "basic-block.h"
37 #include "fold-const.h"
39 #include "gimple-iterator.h"
43 #include "tree-vector-builder.h"
44 #include "stor-layout.h"
47 #include "gimple-fold.h"
48 #include "langhooks.h"
49 #include "stringpool.h"
51 #include "aarch64-sve-builtins.h"
52 #include "aarch64-sve-builtins-base.h"
53 #include "aarch64-sve-builtins-sve2.h"
54 #include "aarch64-sve-builtins-sme.h"
55 #include "aarch64-sve-builtins-shapes.h"
56 #include "aarch64-builtins.h"
58 namespace aarch64_sve
{
60 /* Static information about each single-predicate or single-vector
62 struct vector_type_info
64 /* The name of the type as declared by arm_sve.h. */
65 const char *acle_name
;
67 /* The name of the type specified in AAPCS64. The type is always
68 available under this name, even when arm_sve.h isn't included. */
71 /* The C++ mangling of ABI_NAME. */
72 const char *mangled_name
;
75 /* Describes a function decl. */
76 class GTY(()) registered_function
79 /* The ACLE function that the decl represents. */
80 function_instance instance
GTY ((skip
));
82 /* The decl itself. */
85 /* The architecture extensions that the function requires, as a set of
86 AARCH64_FL_* flags. */
87 aarch64_feature_flags required_extensions
;
89 /* True if the decl represents an overloaded function that needs to be
90 resolved by function_resolver. */
94 /* Hash traits for registered_function. */
95 struct registered_function_hasher
: nofree_ptr_hash
<registered_function
>
97 typedef function_instance compare_type
;
99 static hashval_t
hash (value_type
);
100 static bool equal (value_type
, const compare_type
&);
103 /* Information about each single-predicate or single-vector type. */
104 static CONSTEXPR
const vector_type_info vector_types
[] = {
105 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
106 { #ACLE_NAME, #ABI_NAME, "u" #NCHARS #ABI_NAME },
107 #include "aarch64-sve-builtins.def"
110 /* The function name suffix associated with each predication type. */
111 static const char *const pred_suffixes
[NUM_PREDS
+ 1] = {
121 /* Static information about each mode_suffix_index. */
122 CONSTEXPR
const mode_suffix_info mode_suffixes
[] = {
123 #define VECTOR_TYPE_none NUM_VECTOR_TYPES
124 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) \
125 { "_" #NAME, VECTOR_TYPE_##BASE, VECTOR_TYPE_##DISPLACEMENT, UNITS_##UNITS },
126 #include "aarch64-sve-builtins.def"
127 #undef VECTOR_TYPE_none
128 { "", NUM_VECTOR_TYPES
, NUM_VECTOR_TYPES
, UNITS_none
}
131 /* Static information about each type_suffix_index. */
132 CONSTEXPR
const type_suffix_info type_suffixes
[NUM_TYPE_SUFFIXES
+ 1] = {
133 #define DEF_SVE_NEON_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE, \
136 VECTOR_TYPE_##ACLE_TYPE, \
139 BITS / BITS_PER_UNIT, \
140 TYPE_##CLASS == TYPE_signed || TYPE_##CLASS == TYPE_unsigned, \
141 TYPE_##CLASS == TYPE_unsigned, \
142 TYPE_##CLASS == TYPE_float, \
143 TYPE_##CLASS != TYPE_bool, \
144 TYPE_##CLASS == TYPE_bool, \
150 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
151 DEF_SVE_NEON_TYPE_SUFFIX (NAME, ACLE_TYPE, CLASS, BITS, MODE, \
152 ARM_NEON_H_TYPES_LAST, ARM_NEON_H_TYPES_LAST)
153 #define DEF_SME_ZA_SUFFIX(NAME, BITS, MODE) \
158 BITS / BITS_PER_UNIT, \
167 ARM_NEON_H_TYPES_LAST, \
168 ARM_NEON_H_TYPES_LAST },
169 #include "aarch64-sve-builtins.def"
170 { "", NUM_VECTOR_TYPES
, TYPE_bool
, 0, 0, false, false, false, false,
171 false, false, 0, VOIDmode
, ARM_NEON_H_TYPES_LAST
, ARM_NEON_H_TYPES_LAST
}
174 CONSTEXPR
const group_suffix_info group_suffixes
[] = {
175 #define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) \
176 { "_" #NAME, VG, VECTORS_PER_TUPLE },
177 #include "aarch64-sve-builtins.def"
181 /* Define a TYPES_<combination> macro for each combination of type
182 suffixes that an ACLE function can have, where <combination> is the
183 name used in DEF_SVE_FUNCTION entries.
185 Use S (T) for single type suffix T and D (T1, T2) for a pair of type
186 suffixes T1 and T2. Use commas to separate the suffixes.
188 Although the order shouldn't matter, the convention is to sort the
189 suffixes lexicographically after dividing suffixes into a type
190 class ("b", "f", etc.) and a numerical bit count. */
192 /* _b8 _b16 _b32 _b64. */
193 #define TYPES_all_pred(S, D) \
194 S (b8), S (b16), S (b32), S (b64)
196 /* _c8 _c16 _c32 _c64. */
197 #define TYPES_all_count(S, D) \
198 S (c8), S (c16), S (c32), S (c64)
200 /* _b8 _b16 _b32 _b64
201 _c8 _c16 _c32 _c64. */
202 #define TYPES_all_pred_count(S, D) \
203 TYPES_all_pred (S, D), \
204 TYPES_all_count (S, D)
206 /* _f16 _f32 _f64. */
207 #define TYPES_all_float(S, D) \
208 S (f16), S (f32), S (f64)
210 /* _s8 _s16 _s32 _s64. */
211 #define TYPES_all_signed(S, D) \
212 S (s8), S (s16), S (s32), S (s64)
215 _s8 _s16 _s32 _s64. */
216 #define TYPES_all_float_and_signed(S, D) \
217 TYPES_all_float (S, D), TYPES_all_signed (S, D)
219 /* _u8 _u16 _u32 _u64. */
220 #define TYPES_all_unsigned(S, D) \
221 S (u8), S (u16), S (u32), S (u64)
223 /* _s8 _s16 _s32 _s64
224 _u8 _u16 _u32 _u64. */
225 #define TYPES_all_integer(S, D) \
226 TYPES_all_signed (S, D), TYPES_all_unsigned (S, D)
230 _u8 _u16 _u32 _u64. */
231 #define TYPES_all_arith(S, D) \
232 TYPES_all_float (S, D), TYPES_all_integer (S, D)
237 _u8 _u16 _u32 _u64. */
238 #define TYPES_all_data(S, D) \
239 S (bf16), TYPES_all_arith (S, D)
242 #define TYPES_b(S, D) \
246 #define TYPES_c(S, D) \
250 #define TYPES_b_unsigned(S, D) \
255 #define TYPES_b_integer(S, D) \
256 S (s8), TYPES_b_unsigned (S, D)
260 #define TYPES_bh_integer(S, D) \
261 S (s8), S (s16), S (u8), S (u16)
264 #define TYPES_bs_unsigned(S, D) \
268 #define TYPES_bhs_signed(S, D) \
269 S (s8), S (s16), S (s32)
272 #define TYPES_bhs_unsigned(S, D) \
273 S (u8), S (u16), S (u32)
277 #define TYPES_bhs_integer(S, D) \
278 TYPES_bhs_signed (S, D), TYPES_bhs_unsigned (S, D)
284 #define TYPES_bhs_data(S, D) \
285 S (bf16), S (f16), S (f32), TYPES_bhs_integer (S, D)
287 /* _s16_s8 _s32_s16 _s64_s32
288 _u16_u8 _u32_u16 _u64_u32. */
289 #define TYPES_bhs_widen(S, D) \
290 D (s16, s8), D (s32, s16), D (s64, s32), \
291 D (u16, u8), D (u32, u16), D (u64, u32)
295 #define TYPES_h_integer(S, D) \
299 #define TYPES_hs_signed(S, D) \
304 #define TYPES_hs_integer(S, D) \
305 TYPES_hs_signed (S, D), S (u16), S (u32)
308 #define TYPES_hs_float(S, D) \
315 #define TYPES_hs_data(S, D) \
316 S (bf16), S (f16), S (f32), TYPES_hs_integer (S, D)
319 #define TYPES_hd_unsigned(S, D) \
322 /* _s16 _s32 _s64. */
323 #define TYPES_hsd_signed(S, D) \
324 S (s16), S (s32), S (s64)
328 #define TYPES_hsd_integer(S, D) \
329 TYPES_hsd_signed (S, D), S (u16), S (u32), S (u64)
332 #define TYPES_s_float(S, D) \
338 #define TYPES_s_float_hsd_integer(S, D) \
339 TYPES_s_float (S, D), TYPES_hsd_integer (S, D)
344 #define TYPES_s_float_sd_integer(S, D) \
345 TYPES_s_float (S, D), TYPES_sd_integer (S, D)
348 #define TYPES_s_signed(S, D) \
352 #define TYPES_s_unsigned(S, D) \
356 #define TYPES_s_integer(S, D) \
357 TYPES_s_signed (S, D), TYPES_s_unsigned (S, D)
360 #define TYPES_sd_signed(S, D) \
364 #define TYPES_sd_unsigned(S, D) \
369 #define TYPES_sd_integer(S, D) \
370 TYPES_sd_signed (S, D), TYPES_sd_unsigned (S, D)
375 #define TYPES_sd_data(S, D) \
376 S (f32), S (f64), TYPES_sd_integer (S, D)
381 #define TYPES_all_float_and_sd_integer(S, D) \
382 TYPES_all_float (S, D), TYPES_sd_integer (S, D)
385 #define TYPES_d_float(S, D) \
389 #define TYPES_d_unsigned(S, D) \
394 #define TYPES_d_integer(S, D) \
395 S (s64), TYPES_d_unsigned (S, D)
400 #define TYPES_d_data(S, D) \
401 TYPES_d_float (S, D), TYPES_d_integer (S, D)
403 /* All the type combinations allowed by svcvt. */
404 #define TYPES_cvt(S, D) \
405 D (f16, f32), D (f16, f64), \
406 D (f16, s16), D (f16, s32), D (f16, s64), \
407 D (f16, u16), D (f16, u32), D (f16, u64), \
409 D (f32, f16), D (f32, f64), \
410 D (f32, s32), D (f32, s64), \
411 D (f32, u32), D (f32, u64), \
413 D (f64, f16), D (f64, f32), \
414 D (f64, s32), D (f64, s64), \
415 D (f64, u32), D (f64, u64), \
418 D (s32, f16), D (s32, f32), D (s32, f64), \
419 D (s64, f16), D (s64, f32), D (s64, f64), \
422 D (u32, f16), D (u32, f32), D (u32, f64), \
423 D (u64, f16), D (u64, f32), D (u64, f64)
426 #define TYPES_cvt_bfloat(S, D) \
429 /* { _bf16 _f16 } x _f32. */
430 #define TYPES_cvt_h_s_float(S, D) \
431 D (bf16, f32), D (f16, f32)
435 #define TYPES_cvt_long(S, D) \
436 D (f32, f16), D (f64, f32)
439 #define TYPES_cvt_narrow_s(S, D) \
444 #define TYPES_cvt_narrow(S, D) \
445 D (f16, f32), TYPES_cvt_narrow_s (S, D)
447 /* { _s32 _u32 } x _f32
449 _f32 x { _s32 _u32 }. */
450 #define TYPES_cvt_s_s(S, D) \
456 /* { _s32 _s64 } x { _b8 _b16 _b32 _b64 }
458 #define TYPES_inc_dec_n1(D, A) \
459 D (A, b8), D (A, b16), D (A, b32), D (A, b64)
460 #define TYPES_inc_dec_n(S, D) \
461 TYPES_inc_dec_n1 (D, s32), \
462 TYPES_inc_dec_n1 (D, s64), \
463 TYPES_inc_dec_n1 (D, u32), \
464 TYPES_inc_dec_n1 (D, u64)
466 /* { _s16 _u16 } x _s32
469 #define TYPES_qcvt_x2(S, D) \
474 /* { _s8 _u8 } x _s32
481 #define TYPES_qcvt_x4(S, D) \
491 #define TYPES_qrshr_x2(S, D) \
496 #define TYPES_qrshru_x2(S, D) \
503 #define TYPES_qrshr_x4(S, D) \
511 #define TYPES_qrshru_x4(S, D) \
515 /* { _bf16 } { _bf16 }
516 { _f16 _f32 _f64 } { _f16 _f32 _f64 }
517 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 }
518 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 }. */
519 #define TYPES_reinterpret1(D, A) \
521 D (A, f16), D (A, f32), D (A, f64), \
522 D (A, s8), D (A, s16), D (A, s32), D (A, s64), \
523 D (A, u8), D (A, u16), D (A, u32), D (A, u64)
524 #define TYPES_reinterpret(S, D) \
525 TYPES_reinterpret1 (D, bf16), \
526 TYPES_reinterpret1 (D, f16), \
527 TYPES_reinterpret1 (D, f32), \
528 TYPES_reinterpret1 (D, f64), \
529 TYPES_reinterpret1 (D, s8), \
530 TYPES_reinterpret1 (D, s16), \
531 TYPES_reinterpret1 (D, s32), \
532 TYPES_reinterpret1 (D, s64), \
533 TYPES_reinterpret1 (D, u8), \
534 TYPES_reinterpret1 (D, u16), \
535 TYPES_reinterpret1 (D, u32), \
536 TYPES_reinterpret1 (D, u64)
540 #define TYPES_reinterpret_b(S, D) \
544 /* { _b8 _b16 _b32 _b64 } x { _s32 _s64 }
546 #define TYPES_while1(D, bn) \
547 D (bn, s32), D (bn, s64), D (bn, u32), D (bn, u64)
548 #define TYPES_while(S, D) \
549 TYPES_while1 (D, b8), \
550 TYPES_while1 (D, b16), \
551 TYPES_while1 (D, b32), \
552 TYPES_while1 (D, b64)
554 /* { _b8 _b16 _b32 _b64 } x { _s64 }
556 #define TYPES_while_x(S, D) \
557 D (b8, s64), D (b8, u64), \
558 D (b16, s64), D (b16, u64), \
559 D (b32, s64), D (b32, u64), \
560 D (b64, s64), D (b64, u64)
562 /* { _c8 _c16 _c32 _c64 } x { _s64 }
564 #define TYPES_while_x_c(S, D) \
565 D (c8, s64), D (c8, u64), \
566 D (c16, s64), D (c16, u64), \
567 D (c32, s64), D (c32, u64), \
568 D (c64, s64), D (c64, u64)
573 #define TYPES_s_narrow_fsu(S, D) \
574 D (f32, f16), D (s32, s16), D (u32, u16)
576 /* _za8 _za16 _za32 _za64 _za128. */
577 #define TYPES_all_za(S, D) \
578 S (za8), S (za16), S (za32), S (za64), S (za128)
581 #define TYPES_d_za(S, D) \
584 /* { _za8 } x { _s8 _u8 }
586 { _za16 } x { _bf16 _f16 _s16 _u16 }
588 { _za32 } x { _f32 _s32 _u32 }
590 { _za64 } x { _f64 _s64 _u64 }. */
591 #define TYPES_za_bhsd_data(S, D) \
592 D (za8, s8), D (za8, u8), \
593 D (za16, bf16), D (za16, f16), D (za16, s16), D (za16, u16), \
594 D (za32, f32), D (za32, s32), D (za32, u32), \
595 D (za64, f64), D (za64, s64), D (za64, u64)
599 { _za128 } x { _bf16 }
601 { _s8 _s16 _s32 _s64 }
602 { _u8 _u16 _u32 _u64 }. */
604 #define TYPES_za_all_data(S, D) \
605 TYPES_za_bhsd_data (S, D), \
606 TYPES_reinterpret1 (D, za128)
609 #define TYPES_za_s_b_signed(S, D) \
613 #define TYPES_za_s_b_unsigned(S, D) \
616 /* _za32 x { _s8 _u8 }. */
617 #define TYPES_za_s_b_integer(S, D) \
618 D (za32, s8), D (za32, u8)
620 /* _za32 x { _s16 _u16 }. */
621 #define TYPES_za_s_h_integer(S, D) \
622 D (za32, s16), D (za32, u16)
624 /* _za32 x { _bf16 _f16 _s16 _u16 }. */
625 #define TYPES_za_s_h_data(S, D) \
626 D (za32, bf16), D (za32, f16), D (za32, s16), D (za32, u16)
629 #define TYPES_za_s_unsigned(S, D) \
632 /* _za32 x { _s32 _u32 }. */
633 #define TYPES_za_s_integer(S, D) \
634 D (za32, s32), D (za32, u32)
637 #define TYPES_za_s_float(S, D) \
640 /* _za32 x { _f32 _s32 _u32 }. */
641 #define TYPES_za_s_data(S, D) \
642 D (za32, f32), D (za32, s32), D (za32, u32)
644 /* _za64 x { _s16 _u16 }. */
645 #define TYPES_za_d_h_integer(S, D) \
646 D (za64, s16), D (za64, u16)
649 #define TYPES_za_d_float(S, D) \
652 /* _za64 x { _s64 _u64 }. */
653 #define TYPES_za_d_integer(S, D) \
654 D (za64, s64), D (za64, u64)
656 /* _za32 x { _s8 _u8 _bf16 _f16 _f32 }. */
657 #define TYPES_mop_base(S, D) \
658 D (za32, s8), D (za32, u8), D (za32, bf16), D (za32, f16), D (za32, f32)
661 #define TYPES_mop_base_signed(S, D) \
665 #define TYPES_mop_base_unsigned(S, D) \
668 /* _za64 x { _s16 _u16 }. */
669 #define TYPES_mop_i16i64(S, D) \
670 D (za64, s16), D (za64, u16)
673 #define TYPES_mop_i16i64_signed(S, D) \
677 #define TYPES_mop_i16i64_unsigned(S, D) \
681 #define TYPES_za(S, D) \
684 /* Describe a pair of type suffixes in which only the first is used. */
685 #define DEF_VECTOR_TYPE(X) { TYPE_SUFFIX_ ## X, NUM_TYPE_SUFFIXES }
687 /* Describe a pair of type suffixes in which both are used. */
688 #define DEF_DOUBLE_TYPE(X, Y) { TYPE_SUFFIX_ ## X, TYPE_SUFFIX_ ## Y }
690 /* Create an array that can be used in aarch64-sve-builtins.def to
691 select the type suffixes in TYPES_<NAME>. */
692 #define DEF_SVE_TYPES_ARRAY(NAME) \
693 static const type_suffix_pair types_##NAME[] = { \
694 TYPES_##NAME (DEF_VECTOR_TYPE, DEF_DOUBLE_TYPE), \
695 { NUM_TYPE_SUFFIXES, NUM_TYPE_SUFFIXES } \
698 /* For functions that don't take any type suffixes. */
699 static const type_suffix_pair types_none
[] = {
700 { NUM_TYPE_SUFFIXES
, NUM_TYPE_SUFFIXES
},
701 { NUM_TYPE_SUFFIXES
, NUM_TYPE_SUFFIXES
}
704 /* Create an array for each TYPES_<combination> macro above. */
705 DEF_SVE_TYPES_ARRAY (all_pred
);
706 DEF_SVE_TYPES_ARRAY (all_count
);
707 DEF_SVE_TYPES_ARRAY (all_pred_count
);
708 DEF_SVE_TYPES_ARRAY (all_float
);
709 DEF_SVE_TYPES_ARRAY (all_signed
);
710 DEF_SVE_TYPES_ARRAY (all_float_and_signed
);
711 DEF_SVE_TYPES_ARRAY (all_unsigned
);
712 DEF_SVE_TYPES_ARRAY (all_integer
);
713 DEF_SVE_TYPES_ARRAY (all_arith
);
714 DEF_SVE_TYPES_ARRAY (all_data
);
715 DEF_SVE_TYPES_ARRAY (b
);
716 DEF_SVE_TYPES_ARRAY (b_unsigned
);
717 DEF_SVE_TYPES_ARRAY (b_integer
);
718 DEF_SVE_TYPES_ARRAY (bh_integer
);
719 DEF_SVE_TYPES_ARRAY (bs_unsigned
);
720 DEF_SVE_TYPES_ARRAY (bhs_signed
);
721 DEF_SVE_TYPES_ARRAY (bhs_unsigned
);
722 DEF_SVE_TYPES_ARRAY (bhs_integer
);
723 DEF_SVE_TYPES_ARRAY (bhs_data
);
724 DEF_SVE_TYPES_ARRAY (bhs_widen
);
725 DEF_SVE_TYPES_ARRAY (c
);
726 DEF_SVE_TYPES_ARRAY (h_integer
);
727 DEF_SVE_TYPES_ARRAY (hs_signed
);
728 DEF_SVE_TYPES_ARRAY (hs_integer
);
729 DEF_SVE_TYPES_ARRAY (hs_float
);
730 DEF_SVE_TYPES_ARRAY (hs_data
);
731 DEF_SVE_TYPES_ARRAY (hd_unsigned
);
732 DEF_SVE_TYPES_ARRAY (hsd_signed
);
733 DEF_SVE_TYPES_ARRAY (hsd_integer
);
734 DEF_SVE_TYPES_ARRAY (s_float
);
735 DEF_SVE_TYPES_ARRAY (s_float_hsd_integer
);
736 DEF_SVE_TYPES_ARRAY (s_float_sd_integer
);
737 DEF_SVE_TYPES_ARRAY (s_signed
);
738 DEF_SVE_TYPES_ARRAY (s_unsigned
);
739 DEF_SVE_TYPES_ARRAY (s_integer
);
740 DEF_SVE_TYPES_ARRAY (sd_signed
);
741 DEF_SVE_TYPES_ARRAY (sd_unsigned
);
742 DEF_SVE_TYPES_ARRAY (sd_integer
);
743 DEF_SVE_TYPES_ARRAY (sd_data
);
744 DEF_SVE_TYPES_ARRAY (all_float_and_sd_integer
);
745 DEF_SVE_TYPES_ARRAY (d_float
);
746 DEF_SVE_TYPES_ARRAY (d_unsigned
);
747 DEF_SVE_TYPES_ARRAY (d_integer
);
748 DEF_SVE_TYPES_ARRAY (d_data
);
749 DEF_SVE_TYPES_ARRAY (cvt
);
750 DEF_SVE_TYPES_ARRAY (cvt_bfloat
);
751 DEF_SVE_TYPES_ARRAY (cvt_h_s_float
);
752 DEF_SVE_TYPES_ARRAY (cvt_long
);
753 DEF_SVE_TYPES_ARRAY (cvt_narrow_s
);
754 DEF_SVE_TYPES_ARRAY (cvt_narrow
);
755 DEF_SVE_TYPES_ARRAY (cvt_s_s
);
756 DEF_SVE_TYPES_ARRAY (inc_dec_n
);
757 DEF_SVE_TYPES_ARRAY (qcvt_x2
);
758 DEF_SVE_TYPES_ARRAY (qcvt_x4
);
759 DEF_SVE_TYPES_ARRAY (qrshr_x2
);
760 DEF_SVE_TYPES_ARRAY (qrshr_x4
);
761 DEF_SVE_TYPES_ARRAY (qrshru_x2
);
762 DEF_SVE_TYPES_ARRAY (qrshru_x4
);
763 DEF_SVE_TYPES_ARRAY (reinterpret
);
764 DEF_SVE_TYPES_ARRAY (reinterpret_b
);
765 DEF_SVE_TYPES_ARRAY (while);
766 DEF_SVE_TYPES_ARRAY (while_x
);
767 DEF_SVE_TYPES_ARRAY (while_x_c
);
768 DEF_SVE_TYPES_ARRAY (s_narrow_fsu
);
769 DEF_SVE_TYPES_ARRAY (all_za
);
770 DEF_SVE_TYPES_ARRAY (d_za
);
771 DEF_SVE_TYPES_ARRAY (za_bhsd_data
);
772 DEF_SVE_TYPES_ARRAY (za_all_data
);
773 DEF_SVE_TYPES_ARRAY (za_s_b_signed
);
774 DEF_SVE_TYPES_ARRAY (za_s_b_unsigned
);
775 DEF_SVE_TYPES_ARRAY (za_s_b_integer
);
776 DEF_SVE_TYPES_ARRAY (za_s_h_integer
);
777 DEF_SVE_TYPES_ARRAY (za_s_h_data
);
778 DEF_SVE_TYPES_ARRAY (za_s_unsigned
);
779 DEF_SVE_TYPES_ARRAY (za_s_integer
);
780 DEF_SVE_TYPES_ARRAY (za_s_float
);
781 DEF_SVE_TYPES_ARRAY (za_s_data
);
782 DEF_SVE_TYPES_ARRAY (za_d_h_integer
);
783 DEF_SVE_TYPES_ARRAY (za_d_float
);
784 DEF_SVE_TYPES_ARRAY (za_d_integer
);
785 DEF_SVE_TYPES_ARRAY (mop_base
);
786 DEF_SVE_TYPES_ARRAY (mop_base_signed
);
787 DEF_SVE_TYPES_ARRAY (mop_base_unsigned
);
788 DEF_SVE_TYPES_ARRAY (mop_i16i64
);
789 DEF_SVE_TYPES_ARRAY (mop_i16i64_signed
);
790 DEF_SVE_TYPES_ARRAY (mop_i16i64_unsigned
);
791 DEF_SVE_TYPES_ARRAY (za
);
793 static const group_suffix_index groups_none
[] = {
794 GROUP_none
, NUM_GROUP_SUFFIXES
797 static const group_suffix_index groups_x2
[] = { GROUP_x2
, NUM_GROUP_SUFFIXES
};
799 static const group_suffix_index groups_x12
[] = {
800 GROUP_none
, GROUP_x2
, NUM_GROUP_SUFFIXES
803 static const group_suffix_index groups_x4
[] = { GROUP_x4
, NUM_GROUP_SUFFIXES
};
805 static const group_suffix_index groups_x24
[] = {
806 GROUP_x2
, GROUP_x4
, NUM_GROUP_SUFFIXES
809 static const group_suffix_index groups_x124
[] = {
810 GROUP_none
, GROUP_x2
, GROUP_x4
, NUM_GROUP_SUFFIXES
813 static const group_suffix_index groups_x1234
[] = {
814 GROUP_none
, GROUP_x2
, GROUP_x3
, GROUP_x4
, NUM_GROUP_SUFFIXES
817 static const group_suffix_index groups_vg1x2
[] = {
818 GROUP_vg1x2
, NUM_GROUP_SUFFIXES
821 static const group_suffix_index groups_vg1x4
[] = {
822 GROUP_vg1x4
, NUM_GROUP_SUFFIXES
825 static const group_suffix_index groups_vg1x24
[] = {
826 GROUP_vg1x2
, GROUP_vg1x4
, NUM_GROUP_SUFFIXES
829 static const group_suffix_index groups_vg2
[] = {
830 GROUP_vg2x1
, GROUP_vg2x2
, GROUP_vg2x4
, NUM_GROUP_SUFFIXES
833 static const group_suffix_index groups_vg4
[] = {
834 GROUP_vg4x1
, GROUP_vg4x2
, GROUP_vg4x4
, NUM_GROUP_SUFFIXES
837 static const group_suffix_index groups_vg24
[] = {
838 GROUP_vg2
, GROUP_vg4
, NUM_GROUP_SUFFIXES
841 /* Used by functions that have no governing predicate. */
842 static const predication_index preds_none
[] = { PRED_none
, NUM_PREDS
};
844 /* Used by functions that have a governing predicate but do not have an
846 static const predication_index preds_implicit
[] = { PRED_implicit
, NUM_PREDS
};
848 /* Used by functions that only support "_m" predication. */
849 static const predication_index preds_m
[] = { PRED_m
, NUM_PREDS
};
851 /* Used by functions that allow merging and "don't care" predication,
852 but are not suitable for predicated MOVPRFX. */
853 static const predication_index preds_mx
[] = {
854 PRED_m
, PRED_x
, NUM_PREDS
857 /* Used by functions that allow merging, zeroing and "don't care"
859 static const predication_index preds_mxz
[] = {
860 PRED_m
, PRED_x
, PRED_z
, NUM_PREDS
863 /* Used by functions that have the mxz predicated forms above, and in addition
864 have an unpredicated form. */
865 static const predication_index preds_mxz_or_none
[] = {
866 PRED_m
, PRED_x
, PRED_z
, PRED_none
, NUM_PREDS
869 /* Used by functions that allow merging and zeroing predication but have
871 static const predication_index preds_mz
[] = { PRED_m
, PRED_z
, NUM_PREDS
};
873 /* Used by functions that have an unpredicated form and a _z predicated
875 static const predication_index preds_z_or_none
[] = {
876 PRED_z
, PRED_none
, NUM_PREDS
879 /* Used by (mostly predicate) functions that only support "_z" predication. */
880 static const predication_index preds_z
[] = { PRED_z
, NUM_PREDS
};
882 /* Used by SME instructions that always merge into ZA. */
883 static const predication_index preds_za_m
[] = { PRED_za_m
, NUM_PREDS
};
885 /* A list of all arm_sve.h functions. */
886 static CONSTEXPR
const function_group_info function_groups
[] = {
887 #define DEF_SVE_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
888 { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_##GROUPS, \
889 preds_##PREDS, REQUIRED_EXTENSIONS },
890 #include "aarch64-sve-builtins.def"
893 /* A list of all arm_neon_sve_bridge.h ACLE functions. */
894 static CONSTEXPR
const function_group_info neon_sve_function_groups
[] = {
895 #define DEF_NEON_SVE_FUNCTION(NAME, SHAPE, TYPES, GROUPS, PREDS) \
896 { #NAME, &neon_sve_bridge_functions::NAME, &shapes::SHAPE, types_##TYPES, \
897 groups_##GROUPS, preds_##PREDS, 0 },
898 #include "aarch64-neon-sve-bridge-builtins.def"
901 /* A list of all arm_sme.h functions. */
902 static CONSTEXPR
const function_group_info sme_function_groups
[] = {
903 #define DEF_SME_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
904 { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_##GROUPS, \
905 preds_##PREDS, REQUIRED_EXTENSIONS },
906 #define DEF_SME_ZA_FUNCTION_GS(NAME, SHAPE, TYPES, GROUPS, PREDS) \
907 { #NAME, &functions::NAME##_za, &shapes::SHAPE, types_##TYPES, \
908 groups_##GROUPS, preds_##PREDS, (REQUIRED_EXTENSIONS | AARCH64_FL_ZA_ON) },
909 #include "aarch64-sve-builtins-sme.def"
912 /* The scalar type associated with each vector type. */
913 extern GTY(()) tree scalar_types
[NUM_VECTOR_TYPES
+ 1];
914 tree scalar_types
[NUM_VECTOR_TYPES
+ 1];
916 /* The single-predicate and single-vector types, with their built-in
917 "__SV..._t" name. Allow an index of NUM_VECTOR_TYPES, which always
918 yields a null tree. */
919 static GTY(()) tree abi_vector_types
[NUM_VECTOR_TYPES
+ 1];
921 /* Same, but with the arm_sve.h "sv..._t" name. */
922 extern GTY(()) tree acle_vector_types
[MAX_TUPLE_SIZE
][NUM_VECTOR_TYPES
+ 1];
923 tree acle_vector_types
[MAX_TUPLE_SIZE
][NUM_VECTOR_TYPES
+ 1];
925 /* The svpattern enum type. */
926 extern GTY(()) tree acle_svpattern
;
929 /* The svprfop enum type. */
930 extern GTY(()) tree acle_svprfop
;
933 /* The list of all registered function decls, indexed by code. */
934 static GTY(()) vec
<registered_function
*, va_gc
> *registered_functions
;
936 /* Stores the starting function index for each pragma handler. */
937 static unsigned int initial_indexes
[NUM_PRAGMA_HANDLERS
];
939 /* All registered function decls, hashed on the function_instance
940 that they implement. This is used for looking up implementations of
941 overloaded functions. */
942 static hash_table
<registered_function_hasher
> *function_table
;
944 /* Index 0 maps all overloaded function names that we've registered so far to
945 their associated function_instances. Index 1 does the same for functions
946 that we've skipped over without registering. In both cases, the map keys
947 are IDENTIFIER_NODEs. */
948 static GTY(()) hash_map
<tree
, registered_function
*> *overload_names
[2];
950 /* Record that TYPE is an ABI-defined SVE type that contains NUM_ZR SVE vectors
951 and NUM_PR SVE predicates. MANGLED_NAME, if nonnull, is the ABI-defined
952 mangling of the type. ACLE_NAME is the <arm_sve.h> name of the type. */
954 add_sve_type_attribute (tree type
, unsigned int num_zr
, unsigned int num_pr
,
955 const char *mangled_name
, const char *acle_name
)
957 tree mangled_name_tree
958 = (mangled_name
? get_identifier (mangled_name
) : NULL_TREE
);
960 tree value
= tree_cons (NULL_TREE
, get_identifier (acle_name
), NULL_TREE
);
961 value
= tree_cons (NULL_TREE
, mangled_name_tree
, value
);
962 value
= tree_cons (NULL_TREE
, size_int (num_pr
), value
);
963 value
= tree_cons (NULL_TREE
, size_int (num_zr
), value
);
964 TYPE_ATTRIBUTES (type
) = tree_cons (get_identifier ("SVE type"), value
,
965 TYPE_ATTRIBUTES (type
));
968 /* If TYPE is an ABI-defined SVE type, return its attribute descriptor,
969 otherwise return null. */
971 lookup_sve_type_attribute (const_tree type
)
973 if (type
== error_mark_node
)
975 return lookup_attribute ("SVE type", TYPE_ATTRIBUTES (type
));
978 /* Force TYPE to be a sizeless type. */
980 make_type_sizeless (tree type
)
982 TYPE_ATTRIBUTES (type
) = tree_cons (get_identifier ("SVE sizeless type"),
983 NULL_TREE
, TYPE_ATTRIBUTES (type
));
986 /* Return true if TYPE is a sizeless type. */
988 sizeless_type_p (const_tree type
)
990 if (type
== error_mark_node
)
992 return lookup_attribute ("SVE sizeless type", TYPE_ATTRIBUTES (type
));
995 /* Return true if CANDIDATE is equivalent to MODEL_TYPE for overloading
998 matches_type_p (const_tree model_type
, const_tree candidate
)
1000 if (VECTOR_TYPE_P (model_type
))
1002 if (!VECTOR_TYPE_P (candidate
)
1003 || maybe_ne (TYPE_VECTOR_SUBPARTS (model_type
),
1004 TYPE_VECTOR_SUBPARTS (candidate
))
1005 || TYPE_MODE (model_type
) != TYPE_MODE (candidate
))
1008 model_type
= TREE_TYPE (model_type
);
1009 candidate
= TREE_TYPE (candidate
);
1011 return (candidate
!= error_mark_node
1012 && TYPE_MAIN_VARIANT (model_type
) == TYPE_MAIN_VARIANT (candidate
));
1015 /* If TYPE is a valid SVE element type, return the corresponding type
1016 suffix, otherwise return NUM_TYPE_SUFFIXES. */
1017 static type_suffix_index
1018 find_type_suffix_for_scalar_type (const_tree type
)
1020 /* A linear search should be OK here, since the code isn't hot and
1021 the number of types is only small. */
1022 for (unsigned int suffix_i
= 0; suffix_i
< NUM_TYPE_SUFFIXES
; ++suffix_i
)
1023 if (type_suffixes
[suffix_i
].vector_p
)
1025 vector_type_index vector_i
= type_suffixes
[suffix_i
].vector_type
;
1026 if (matches_type_p (scalar_types
[vector_i
], type
))
1027 return type_suffix_index (suffix_i
);
1029 return NUM_TYPE_SUFFIXES
;
1032 /* Return the implicit group suffix for intrinsics that operate on NVECTORS
1034 static group_suffix_index
1035 num_vectors_to_group (unsigned int nvectors
)
1039 case 1: return GROUP_none
;
1040 case 2: return GROUP_x2
;
1041 case 3: return GROUP_x3
;
1042 case 4: return GROUP_x4
;
1047 /* Return the vector type associated with TYPE. */
1049 get_vector_type (sve_type type
)
1051 auto vector_type
= type_suffixes
[type
.type
].vector_type
;
1052 return acle_vector_types
[type
.num_vectors
- 1][vector_type
];
1055 /* If FNDECL is an SVE builtin, return its function instance, otherwise
1057 const function_instance
*
1058 lookup_fndecl (tree fndecl
)
1060 if (!fndecl_built_in_p (fndecl
, BUILT_IN_MD
))
1063 unsigned int code
= DECL_MD_FUNCTION_CODE (fndecl
);
1064 if ((code
& AARCH64_BUILTIN_CLASS
) != AARCH64_BUILTIN_SVE
)
1067 unsigned int subcode
= code
>> AARCH64_BUILTIN_SHIFT
;
1068 return &(*registered_functions
)[subcode
]->instance
;
1072 /* Report that LOCATION has a call to FNDECL in which argument ARGNO
1073 was not an integer constant expression. ARGNO counts from zero. */
1075 report_non_ice (location_t location
, tree fndecl
, unsigned int argno
)
1077 error_at (location
, "argument %d of %qE must be an integer constant"
1078 " expression", argno
+ 1, fndecl
);
1081 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1082 the value ACTUAL, whereas the function requires a value in the range
1083 [MIN, MAX]. ARGNO counts from zero. */
1085 report_out_of_range (location_t location
, tree fndecl
, unsigned int argno
,
1086 HOST_WIDE_INT actual
, HOST_WIDE_INT min
,
1090 error_at (location
, "passing %wd to argument %d of %qE, which expects"
1091 " the value %wd", actual
, argno
+ 1, fndecl
, min
);
1093 error_at (location
, "passing %wd to argument %d of %qE, which expects"
1094 " a value in the range [%wd, %wd]", actual
, argno
+ 1, fndecl
,
1098 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1099 the value ACTUAL, whereas the function requires either VALUE0 or
1100 VALUE1. ARGNO counts from zero. */
1102 report_neither_nor (location_t location
, tree fndecl
, unsigned int argno
,
1103 HOST_WIDE_INT actual
, HOST_WIDE_INT value0
,
1104 HOST_WIDE_INT value1
)
1106 error_at (location
, "passing %wd to argument %d of %qE, which expects"
1107 " either %wd or %wd", actual
, argno
+ 1, fndecl
, value0
, value1
);
1110 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1111 the value ACTUAL, whereas the function requires one of VALUE0..3.
1112 ARGNO counts from zero. */
1114 report_not_one_of (location_t location
, tree fndecl
, unsigned int argno
,
1115 HOST_WIDE_INT actual
, HOST_WIDE_INT value0
,
1116 HOST_WIDE_INT value1
, HOST_WIDE_INT value2
,
1117 HOST_WIDE_INT value3
)
1119 error_at (location
, "passing %wd to argument %d of %qE, which expects"
1120 " %wd, %wd, %wd or %wd", actual
, argno
+ 1, fndecl
, value0
, value1
,
1124 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
1125 the value ACTUAL, whereas the function requires a valid value of
1126 enum type ENUMTYPE. ARGNO counts from zero. */
1128 report_not_enum (location_t location
, tree fndecl
, unsigned int argno
,
1129 HOST_WIDE_INT actual
, tree enumtype
)
1131 error_at (location
, "passing %wd to argument %d of %qE, which expects"
1132 " a valid %qT value", actual
, argno
+ 1, fndecl
, enumtype
);
1135 /* Return a hash code for a function_instance. */
1137 function_instance::hash () const
1140 /* BASE uniquely determines BASE_NAME, so we don't need to hash both. */
1143 h
.add_int (mode_suffix_id
);
1144 h
.add_int (type_suffix_ids
[0]);
1145 h
.add_int (type_suffix_ids
[1]);
1146 h
.add_int (group_suffix_id
);
1151 /* Return a set of CP_* flags that describe what the function could do,
1152 taking the command-line flags into account. */
1154 function_instance::call_properties () const
1156 unsigned int flags
= base
->call_properties (*this);
1158 /* -fno-trapping-math means that we can assume any FP exceptions
1159 are not user-visible. */
1160 if (!flag_trapping_math
)
1161 flags
&= ~CP_RAISE_FP_EXCEPTIONS
;
1166 /* Return true if calls to the function could read some form of
1169 function_instance::reads_global_state_p () const
1171 unsigned int flags
= call_properties ();
1173 /* Preserve any dependence on rounding mode, flush to zero mode, etc.
1174 There is currently no way of turning this off; in particular,
1175 -fno-rounding-math (which is the default) means that we should make
1176 the usual assumptions about rounding mode, which for intrinsics means
1177 acting as the instructions do. */
1178 if (flags
& CP_READ_FPCR
)
1181 /* Handle direct reads of global state. */
1182 return flags
& (CP_READ_MEMORY
| CP_READ_FFR
| CP_READ_ZA
| CP_READ_ZT0
);
1185 /* Return true if calls to the function could modify some form of
1188 function_instance::modifies_global_state_p () const
1190 unsigned int flags
= call_properties ();
1192 /* Preserve any exception state written back to the FPCR,
1193 unless -fno-trapping-math says this is unnecessary. */
1194 if (flags
& CP_RAISE_FP_EXCEPTIONS
)
1197 /* Treat prefetches as modifying global state, since that's the
1198 only means we have of keeping them in their correct position. */
1199 if (flags
& CP_PREFETCH_MEMORY
)
1202 /* Handle direct modifications of global state. */
1203 return flags
& (CP_WRITE_MEMORY
| CP_WRITE_FFR
| CP_WRITE_ZA
| CP_WRITE_ZT0
);
1206 /* Return true if calls to the function could raise a signal. */
1208 function_instance::could_trap_p () const
1210 unsigned int flags
= call_properties ();
1212 /* Handle functions that could raise SIGFPE. */
1213 if (flags
& CP_RAISE_FP_EXCEPTIONS
)
1216 /* Handle functions that could raise SIGBUS or SIGSEGV. */
1217 if (flags
& (CP_READ_MEMORY
| CP_WRITE_MEMORY
))
1224 registered_function_hasher::hash (value_type value
)
1226 return value
->instance
.hash ();
1230 registered_function_hasher::equal (value_type value
, const compare_type
&key
)
1232 return value
->instance
== key
;
1235 sve_switcher::sve_switcher (aarch64_feature_flags flags
)
1236 : aarch64_simd_switcher (AARCH64_FL_F16
| AARCH64_FL_SVE
| flags
)
1238 /* Changing the ISA flags and have_regs_of_mode should be enough here.
1239 We shouldn't need to pay the compile-time cost of a full target
1241 m_old_maximum_field_alignment
= maximum_field_alignment
;
1242 maximum_field_alignment
= 0;
1244 memcpy (m_old_have_regs_of_mode
, have_regs_of_mode
,
1245 sizeof (have_regs_of_mode
));
1246 for (int i
= 0; i
< NUM_MACHINE_MODES
; ++i
)
1247 if (aarch64_sve_mode_p ((machine_mode
) i
))
1248 have_regs_of_mode
[i
] = true;
1251 sve_switcher::~sve_switcher ()
1253 memcpy (have_regs_of_mode
, m_old_have_regs_of_mode
,
1254 sizeof (have_regs_of_mode
));
1255 maximum_field_alignment
= m_old_maximum_field_alignment
;
1258 function_builder::function_builder (handle_pragma_index pragma_index
,
1259 bool function_nulls
)
1261 m_overload_type
= build_function_type (void_type_node
, void_list_node
);
1262 m_direct_overloads
= lang_GNU_CXX ();
1264 if (initial_indexes
[pragma_index
] == 0)
1266 unsigned int index
= vec_safe_length (registered_functions
);
1267 initial_indexes
[pragma_index
] = index
;
1270 m_function_index
= initial_indexes
[pragma_index
];
1271 m_function_nulls
= function_nulls
;
1273 gcc_obstack_init (&m_string_obstack
);
1276 function_builder::~function_builder ()
1278 obstack_free (&m_string_obstack
, NULL
);
1281 /* Add NAME to the end of the function name being built. */
1283 function_builder::append_name (const char *name
)
1285 obstack_grow (&m_string_obstack
, name
, strlen (name
));
1288 /* Zero-terminate and complete the function name being built. */
1290 function_builder::finish_name ()
1292 obstack_1grow (&m_string_obstack
, 0);
1293 return (char *) obstack_finish (&m_string_obstack
);
1296 /* Return the overloaded or full function name for INSTANCE; OVERLOADED_P
1297 selects which. Allocate the string on m_string_obstack; the caller
1298 must use obstack_free to free it after use. */
1300 function_builder::get_name (const function_instance
&instance
,
1303 /* __arm_* functions are listed as arm_*, so that the associated GCC
1304 code is not in the implementation namespace. */
1305 if (strncmp (instance
.base_name
, "arm_", 4) == 0)
1307 append_name (instance
.base_name
);
1309 switch (instance
.displacement_units ())
1315 append_name ("_offset");
1318 case UNITS_elements
:
1319 append_name ("_index");
1323 append_name ("_vnum");
1327 append_name (instance
.mode_suffix ().string
);
1328 for (unsigned int i
= 0; i
< 2; ++i
)
1329 if (!overloaded_p
|| instance
.shape
->explicit_type_suffix_p (i
))
1330 append_name (instance
.type_suffix (i
).string
);
1331 if (!overloaded_p
|| instance
.shape
->explicit_group_suffix_p ())
1332 append_name (instance
.group_suffix ().string
);
1333 append_name (pred_suffixes
[instance
.pred
]);
1334 return finish_name ();
1337 /* Add attribute NAME to ATTRS. */
1339 add_attribute (const char *name
, tree attrs
)
1341 return tree_cons (get_identifier (name
), NULL_TREE
, attrs
);
1344 /* Add attribute NS::NAME to ATTRS. */
1346 add_attribute (const char *ns
, const char *name
, tree value
, tree attrs
)
1348 return tree_cons (build_tree_list (get_identifier (ns
),
1349 get_identifier (name
)),
1353 /* Attribute arm::NAME describes shared state that is an input if IS_IN
1354 and an output if IS_OUT. Check whether a call with call properties
1355 CALL_FLAGS needs such an attribute. Add it to in-progress attribute
1356 list ATTRS if so. Return the new attribute list. */
1358 add_shared_state_attribute (const char *name
, bool is_in
, bool is_out
,
1359 unsigned int call_flags
, tree attrs
)
1361 struct state_flag_info
1364 unsigned int read_flag
;
1365 unsigned int write_flag
;
1367 static state_flag_info state_flags
[] =
1369 { "za", CP_READ_ZA
, CP_WRITE_ZA
},
1370 { "zt0", CP_READ_ZT0
, CP_WRITE_ZT0
}
1373 tree args
= NULL_TREE
;
1374 for (const auto &state_flag
: state_flags
)
1376 auto all_flags
= state_flag
.read_flag
| state_flag
.write_flag
;
1377 auto these_flags
= ((is_in
? state_flag
.read_flag
: 0)
1378 | (is_out
? state_flag
.write_flag
: 0));
1379 if ((call_flags
& all_flags
) == these_flags
)
1381 tree value
= build_string (strlen (state_flag
.name
) + 1,
1383 args
= tree_cons (NULL_TREE
, value
, args
);
1387 attrs
= add_attribute ("arm", name
, args
, attrs
);
1391 /* Return the appropriate function attributes for INSTANCE, which requires
1392 the feature flags in REQUIRED_EXTENSIONS. */
1394 function_builder::get_attributes (const function_instance
&instance
,
1395 aarch64_feature_flags required_extensions
)
1397 tree attrs
= NULL_TREE
;
1399 if (required_extensions
& AARCH64_FL_SM_ON
)
1400 attrs
= add_attribute ("arm", "streaming", NULL_TREE
, attrs
);
1401 else if (!(required_extensions
& AARCH64_FL_SM_OFF
))
1402 attrs
= add_attribute ("arm", "streaming_compatible", NULL_TREE
, attrs
);
1404 attrs
= add_shared_state_attribute ("in", true, false,
1405 instance
.call_properties (), attrs
);
1406 attrs
= add_shared_state_attribute ("out", false, true,
1407 instance
.call_properties (), attrs
);
1408 attrs
= add_shared_state_attribute ("inout", true, true,
1409 instance
.call_properties (), attrs
);
1411 if (!instance
.modifies_global_state_p ())
1413 if (instance
.reads_global_state_p ())
1414 attrs
= add_attribute ("pure", attrs
);
1416 attrs
= add_attribute ("const", attrs
);
1419 if (!flag_non_call_exceptions
|| !instance
.could_trap_p ())
1420 attrs
= add_attribute ("nothrow", attrs
);
1422 return add_attribute ("leaf", attrs
);
1425 /* Add a function called NAME with type FNTYPE and attributes ATTRS.
1426 INSTANCE describes what the function does and OVERLOADED_P indicates
1427 whether it is overloaded. REQUIRED_EXTENSIONS are the set of
1428 architecture extensions that the function requires. */
1429 registered_function
&
1430 function_builder::add_function (const function_instance
&instance
,
1431 const char *name
, tree fntype
, tree attrs
,
1432 aarch64_feature_flags required_extensions
,
1436 unsigned int length
= vec_safe_length (registered_functions
);
1437 unsigned int code
= (m_function_index
<< AARCH64_BUILTIN_SHIFT
) | AARCH64_BUILTIN_SVE
;
1438 /* We need to be able to generate placeholders to enusre that we have a
1439 consistent numbering scheme for function codes between the C and C++
1440 frontends, so that everything ties up in LTO.
1442 Currently, tree-streamer-in.cc:unpack_ts_function_decl_value_fields
1443 validates that tree nodes returned by TARGET_BUILTIN_DECL are non-NULL and
1444 some node other than error_mark_node. This is a holdover from when builtin
1445 decls were streamed by code rather than by value.
1447 Ultimately, we should be able to remove this validation of BUILT_IN_MD
1448 nodes and remove the target hook. For now, however, we need to appease the
1449 validation and return a non-NULL, non-error_mark_node node, so we
1450 arbitrarily choose integer_zero_node. */
1451 tree decl
= placeholder_p
|| m_function_nulls
1453 : simulate_builtin_function_decl (input_location
, name
, fntype
,
1456 registered_function
&rfn
= *ggc_alloc
<registered_function
> ();
1457 rfn
.instance
= instance
;
1459 rfn
.required_extensions
= required_extensions
;
1460 rfn
.overloaded_p
= overloaded_p
;
1461 if (m_function_index
>= length
)
1462 vec_safe_push (registered_functions
, &rfn
);
1464 (*registered_functions
)[m_function_index
] = &rfn
;
1470 /* Add a built-in function for INSTANCE, with the argument types given
1471 by ARGUMENT_TYPES and the return type given by RETURN_TYPE.
1472 REQUIRED_EXTENSIONS are the set of architecture extensions that the
1473 function requires. FORCE_DIRECT_OVERLOADS is true if there is a
1474 one-to-one mapping between "short" and "full" names, and if standard
1475 overload resolution therefore isn't necessary. */
1478 add_unique_function (const function_instance
&instance
,
1480 vec
<tree
> &argument_types
,
1481 aarch64_feature_flags required_extensions
,
1482 bool force_direct_overloads
)
1484 /* Add the function under its full (unique) name. */
1485 char *name
= get_name (instance
, false);
1486 tree fntype
= build_function_type_array (return_type
,
1487 argument_types
.length (),
1488 argument_types
.address ());
1489 tree attrs
= get_attributes (instance
, required_extensions
);
1490 registered_function
&rfn
= add_function (instance
, name
, fntype
, attrs
,
1491 required_extensions
, false, false);
1493 /* Enter the function into the hash table. */
1494 if (!m_function_nulls
)
1496 hashval_t hash
= instance
.hash ();
1497 registered_function
**rfn_slot
1498 = function_table
->find_slot_with_hash (instance
, hash
, INSERT
);
1499 gcc_assert (!*rfn_slot
);
1503 /* Also add the function under its overloaded alias, if we want
1504 a separate decl for each instance of an overloaded function. */
1505 char *overload_name
= get_name (instance
, true);
1506 if (strcmp (name
, overload_name
) != 0)
1508 /* Attribute lists shouldn't be shared. */
1509 tree attrs
= get_attributes (instance
, required_extensions
);
1510 bool placeholder_p
= !(m_direct_overloads
|| force_direct_overloads
);
1511 add_function (instance
, overload_name
, fntype
, attrs
,
1512 required_extensions
, false, placeholder_p
);
1515 obstack_free (&m_string_obstack
, name
);
1518 /* Add one function decl for INSTANCE, to be used with manual overload
1519 resolution. REQUIRED_EXTENSIONS are the set of architecture extensions
1520 that the function requires.
1522 For simplicity, deal with duplicate attempts to add the same function,
1523 including cases in which the new function requires more features than
1524 the original one did. In that case we'll check whether the required
1525 features are available as part of resolving the function to the
1526 relevant unique function. */
1529 add_overloaded_function (const function_instance
&instance
,
1530 aarch64_feature_flags required_extensions
)
1532 auto &name_map
= overload_names
[m_function_nulls
];
1534 name_map
= hash_map
<tree
, registered_function
*>::create_ggc ();
1536 char *name
= get_name (instance
, true);
1537 tree id
= get_identifier (name
);
1538 if (registered_function
**map_value
= name_map
->get (id
))
1539 gcc_assert ((*map_value
)->instance
== instance
1540 && ((*map_value
)->required_extensions
1541 & ~required_extensions
) == 0);
1544 registered_function
&rfn
1545 = add_function (instance
, name
, m_overload_type
, NULL_TREE
,
1546 required_extensions
, true, m_direct_overloads
);
1547 name_map
->put (id
, &rfn
);
1549 obstack_free (&m_string_obstack
, name
);
1552 /* If we are using manual overload resolution, add one function decl
1553 for each overloaded function in GROUP. Take the function base name
1554 from GROUP and the mode from MODE. */
1556 function_builder::add_overloaded_functions (const function_group_info
&group
,
1557 mode_suffix_index mode
)
1559 bool explicit_type0
= (*group
.shape
)->explicit_type_suffix_p (0);
1560 bool explicit_type1
= (*group
.shape
)->explicit_type_suffix_p (1);
1561 bool explicit_group
= (*group
.shape
)->explicit_group_suffix_p ();
1562 auto add_function
= [&](const type_suffix_pair
&types
,
1563 group_suffix_index group_suffix_id
,
1566 function_instance
instance (group
.base_name
, *group
.base
,
1567 *group
.shape
, mode
, types
,
1568 group_suffix_id
, group
.preds
[pi
]);
1569 add_overloaded_function (instance
, group
.required_extensions
);
1572 auto add_group_suffix
= [&](group_suffix_index group_suffix_id
,
1575 if (mode
== MODE_single
1576 && group_suffixes
[group_suffix_id
].vectors_per_tuple
== 1)
1579 if (!explicit_type0
&& !explicit_type1
)
1580 /* Deal with the common case in which there is one overloaded
1581 function for all type combinations. */
1582 add_function (types_none
[0], group_suffix_id
, pi
);
1584 for (unsigned int ti
= 0; group
.types
[ti
][0] != NUM_TYPE_SUFFIXES
;
1587 /* Stub out the types that are determined by overload
1589 type_suffix_pair types
= {
1590 explicit_type0
? group
.types
[ti
][0] : NUM_TYPE_SUFFIXES
,
1591 explicit_type1
? group
.types
[ti
][1] : NUM_TYPE_SUFFIXES
1593 add_function (types
, group_suffix_id
, pi
);
1597 for (unsigned int pi
= 0; group
.preds
[pi
] != NUM_PREDS
; ++pi
)
1599 for (unsigned int gi
= 0; group
.groups
[gi
] != NUM_GROUP_SUFFIXES
; ++gi
)
1600 add_group_suffix (group
.groups
[gi
], pi
);
1602 add_group_suffix (GROUP_none
, pi
);
1605 /* Register all the functions in GROUP. */
1607 function_builder::register_function_group (const function_group_info
&group
)
1609 (*group
.shape
)->build (*this, group
);
1612 function_call_info::function_call_info (location_t location_in
,
1613 const function_instance
&instance_in
,
1615 : function_instance (instance_in
), location (location_in
), fndecl (fndecl_in
)
1619 function_resolver::function_resolver (location_t location
,
1620 const function_instance
&instance
,
1621 tree fndecl
, vec
<tree
, va_gc
> &arglist
)
1622 : function_call_info (location
, instance
, fndecl
), m_arglist (arglist
)
1626 /* Return the <stdint.h> name associated with TYPE. Using the <stdint.h>
1627 name should be more user-friendly than the underlying canonical type,
1628 since it makes the signedness and bitwidth explicit. */
1630 function_resolver::get_scalar_type_name (type_suffix_index type
)
1632 return vector_types
[type_suffixes
[type
].vector_type
].acle_name
+ 2;
1635 /* Return the type of argument I, or error_mark_node if it isn't
1638 function_resolver::get_argument_type (unsigned int i
)
1640 tree arg
= m_arglist
[i
];
1641 return arg
== error_mark_node
? arg
: TREE_TYPE (arg
);
1644 /* Return true if argument I is some form of scalar value. */
1646 function_resolver::scalar_argument_p (unsigned int i
)
1648 tree type
= get_argument_type (i
);
1649 return (INTEGRAL_TYPE_P (type
)
1650 /* Allow pointer types, leaving the frontend to warn where
1652 || POINTER_TYPE_P (type
)
1653 || SCALAR_FLOAT_TYPE_P (type
));
1656 /* Report that argument ARGNO was expected to have NUM_VECTORS vectors.
1657 TYPE is the type that ARGNO actually has. */
1659 function_resolver::report_incorrect_num_vectors (unsigned int argno
,
1661 unsigned int num_vectors
)
1663 if (num_vectors
== 1)
1664 error_at (location
, "passing %qT to argument %d of %qE, which"
1665 " expects a single SVE vector rather than a tuple",
1666 get_vector_type (type
), argno
+ 1, fndecl
);
1667 else if (type
.num_vectors
== 1
1668 && type
.type
!= TYPE_SUFFIX_b
)
1669 /* num_vectors is always != 1, so the singular isn't needed. */
1670 error_n (location
, num_vectors
, "%qT%d%qE%d",
1671 "passing single vector %qT to argument %d"
1672 " of %qE, which expects a tuple of %d vectors",
1673 get_vector_type (type
), argno
+ 1, fndecl
, num_vectors
);
1675 /* num_vectors is always != 1, so the singular isn't needed. */
1676 error_n (location
, num_vectors
, "%qT%d%qE%d",
1677 "passing %qT to argument %d of %qE, which"
1678 " expects a tuple of %d vectors", get_vector_type (type
),
1679 argno
+ 1, fndecl
, num_vectors
);
1682 /* Report that arguments FIRST_ARGNO and ARGNO have different numbers
1683 of vectors, but are required to have the same number of vectors.
1684 FIRST_TYPE and TYPE are the types that arguments FIRST_ARGNO and
1685 ARGNO actually have. */
1687 function_resolver::report_mismatched_num_vectors (unsigned int first_argno
,
1688 sve_type first_type
,
1692 /* If the tuple size is implied by the group suffix, and if the first
1693 type had the right number of vectors, treat argument ARGNO as being
1694 individually wrong, rather than wrong in relation to FIRST_ARGNO. */
1695 if (group_suffix_id
!= GROUP_none
1696 && first_type
.num_vectors
== vectors_per_tuple ())
1698 report_incorrect_num_vectors (argno
, type
, first_type
.num_vectors
);
1702 /* Make sure that FIRST_TYPE itself is sensible before using it
1703 as a basis for an error message. */
1704 if (resolve_to (mode_suffix_id
, first_type
) == error_mark_node
)
1707 if (type
.num_vectors
!= 1 && first_type
.num_vectors
== 1)
1708 error_at (location
, "passing tuple %qT to argument %d of %qE after"
1709 " passing single vector %qT to argument %d",
1710 get_vector_type (type
), argno
+ 1, fndecl
,
1711 get_vector_type (first_type
), first_argno
+ 1);
1712 else if (type
.num_vectors
== 1 && first_type
.num_vectors
!= 1)
1713 error_at (location
, "passing single vector %qT to argument %d"
1714 " of %qE after passing tuple %qT to argument %d",
1715 get_vector_type (type
), argno
+ 1, fndecl
,
1716 get_vector_type (first_type
), first_argno
+ 1);
1718 error_at (location
, "passing mismatched tuple types %qT and %qT"
1719 " to arguments %d and %d of %qE",
1720 get_vector_type (first_type
), get_vector_type (type
),
1721 first_argno
+ 1, argno
+ 1, fndecl
);
1724 /* Report that the function has no form that takes type TYPE.
1725 Return error_mark_node. */
1727 function_resolver::report_no_such_form (sve_type type
)
1729 error_at (location
, "%qE has no form that takes %qT arguments",
1730 fndecl
, get_vector_type (type
));
1731 return error_mark_node
;
1734 /* Silently check whether there is an instance of the function with the
1735 mode suffix given by MODE, the type suffixes given by TYPE0 and TYPE1,
1736 and the group suffix given by GROUP. Return its function decl if so,
1737 otherwise return null. */
1739 function_resolver::lookup_form (mode_suffix_index mode
,
1740 type_suffix_index type0
,
1741 type_suffix_index type1
,
1742 group_suffix_index group
)
1744 type_suffix_pair types
= { type0
, type1
};
1745 function_instance
instance (base_name
, base
, shape
, mode
, types
,
1747 registered_function
*rfn
1748 = function_table
->find_with_hash (instance
, instance
.hash ());
1749 return rfn
? rfn
->decl
: NULL_TREE
;
1752 /* Silently check whether there is an instance of the function that has the
1753 mode suffix given by MODE and the type and group suffixes implied by TYPE.
1754 If the overloaded function has an explicit first type suffix (like
1755 conversions do), TYPE describes the implicit second type suffix.
1756 Otherwise, TYPE describes the only type suffix.
1758 Return the decl of the function if it exists, otherwise return null. */
1760 function_resolver::lookup_form (mode_suffix_index mode
, sve_type type
)
1762 type_suffix_index type0
= type_suffix_ids
[0];
1763 type_suffix_index type1
= type_suffix_ids
[1];
1764 (type0
== NUM_TYPE_SUFFIXES
? type0
: type1
) = type
.type
;
1766 group_suffix_index group
= group_suffix_id
;
1767 if (group
== GROUP_none
&& type
.num_vectors
!= vectors_per_tuple ())
1768 group
= num_vectors_to_group (type
.num_vectors
);
1770 return lookup_form (mode
, type0
, type1
, group
);
1773 /* Resolve the function to one with the mode suffix given by MODE, the
1774 type suffixes given by TYPE0 and TYPE1, and group suffix given by
1775 GROUP. Return its function decl on success, otherwise report an
1776 error and return error_mark_node. */
1778 function_resolver::resolve_to (mode_suffix_index mode
,
1779 type_suffix_index type0
,
1780 type_suffix_index type1
,
1781 group_suffix_index group
)
1783 tree res
= lookup_form (mode
, type0
, type1
, group
);
1786 if (type1
== NUM_TYPE_SUFFIXES
)
1787 return report_no_such_form (type0
);
1788 if (type0
== type_suffix_ids
[0])
1789 return report_no_such_form (type1
);
1790 error_at (location
, "%qE has no form that takes %qT and %qT arguments",
1791 fndecl
, get_vector_type (type0
), get_vector_type (type1
));
1792 return error_mark_node
;
1797 /* Resolve the function to one that has the suffixes associated with MODE
1798 and TYPE; see lookup_form for how TYPE is interpreted. Return the
1799 function decl on success, otherwise report an error and return
1802 function_resolver::resolve_to (mode_suffix_index mode
, sve_type type
)
1804 if (tree res
= lookup_form (mode
, type
))
1807 return report_no_such_form (type
);
1810 /* Like resolve_to, but used for a conversion function with the following
1813 - The function has an explicit first type suffix.
1814 - The elements of the argument (which has type TYPE) might be narrower
1815 or wider than the elements of the return type.
1816 - The return type has enough vectors to represent the converted value
1818 - The group suffix describes the wider of the argument type and the
1821 function_resolver::resolve_conversion (mode_suffix_index mode
, sve_type type
)
1823 auto ret_type
= type_suffix_ids
[0];
1824 unsigned int num_ret_vectors
= (type
.num_vectors
1825 * type_suffixes
[ret_type
].element_bits
1826 / type_suffixes
[type
.type
].element_bits
);
1827 if (num_ret_vectors
== 1
1828 || num_ret_vectors
== 2
1829 || num_ret_vectors
== 4)
1831 unsigned int num_vectors
= MAX (num_ret_vectors
, type
.num_vectors
);
1832 if (tree res
= lookup_form (mode
, { type
.type
, num_vectors
}))
1835 return report_no_such_form (type
);
1838 /* Require argument ARGNO to be an svbool_t or svcount_t predicate.
1839 Return its type on success, otherwise report an error and return
1840 NUM_VECTOR_TYPES. */
1842 function_resolver::infer_predicate_type (unsigned int argno
)
1844 tree actual
= get_argument_type (argno
);
1845 if (actual
== error_mark_node
)
1846 return NUM_VECTOR_TYPES
;
1848 for (auto index
: { VECTOR_TYPE_svbool_t
, VECTOR_TYPE_svcount_t
})
1849 if (matches_type_p (acle_vector_types
[0][index
], actual
))
1852 error_at (location
, "passing %qT to argument %d of %qE, which expects"
1853 " an %qs or %qs", actual
, argno
+ 1, fndecl
, "svbool_t",
1855 return NUM_VECTOR_TYPES
;
1858 /* Require argument ARGNO to be a 32-bit or 64-bit scalar integer type.
1859 Return the associated type suffix on success, otherwise report an
1860 error and return NUM_TYPE_SUFFIXES. */
1862 function_resolver::infer_integer_scalar_type (unsigned int argno
)
1864 tree actual
= get_argument_type (argno
);
1865 if (actual
== error_mark_node
)
1866 return NUM_TYPE_SUFFIXES
;
1868 /* Allow enums and booleans to decay to integers, for compatibility
1869 with C++ overloading rules. */
1870 if (INTEGRAL_TYPE_P (actual
))
1872 bool uns_p
= TYPE_UNSIGNED (actual
);
1873 /* Honor the usual integer promotions, so that resolution works
1874 in the same way as for C++. */
1875 if (TYPE_PRECISION (actual
) < 32)
1876 return TYPE_SUFFIX_s32
;
1877 if (TYPE_PRECISION (actual
) == 32)
1878 return uns_p
? TYPE_SUFFIX_u32
: TYPE_SUFFIX_s32
;
1879 if (TYPE_PRECISION (actual
) == 64)
1880 return uns_p
? TYPE_SUFFIX_u64
: TYPE_SUFFIX_s64
;
1883 error_at (location
, "passing %qT to argument %d of %qE, which expects"
1884 " a 32-bit or 64-bit integer type", actual
, argno
+ 1, fndecl
);
1885 return NUM_TYPE_SUFFIXES
;
1888 /* Return arguments ARGNO and ARGNO + 1 to be 64-bit scalar integers
1889 of the same signedness, or be a combination that converts unambiguously
1890 to such a pair. Return the associated type suffix if they are,
1891 otherwise report an error and return NUM_TYPE_SUFFIXES. */
1893 function_resolver::infer_64bit_scalar_integer_pair (unsigned int argno
)
1895 /* Require two scalar integers, with one having 64 bits and the other
1896 one being no bigger. */
1897 tree types
[] = { get_argument_type (argno
), get_argument_type (argno
+ 1) };
1898 if (!INTEGRAL_TYPE_P (types
[0])
1899 || !INTEGRAL_TYPE_P (types
[1])
1900 || MAX (TYPE_PRECISION (types
[0]), TYPE_PRECISION (types
[1])) != 64)
1902 error_at (location
, "passing %qT and %qT to arguments %d and %d of %qE,"
1903 " which expects a pair of 64-bit integers", types
[0], types
[1],
1904 argno
+ 1, argno
+ 2, fndecl
);
1905 return NUM_TYPE_SUFFIXES
;
1908 /* Allow signed integers smaller than int64_t to be paired with an int64_t.
1909 Allow unsigned integers smaller than uint64_t to be paired with any
1911 for (int i
= 0; i
< 2; ++i
)
1913 if (TYPE_PRECISION (types
[i
]) != 64)
1916 if (TYPE_UNSIGNED (types
[1 - i
]) != TYPE_UNSIGNED (types
[i
]))
1918 if (TYPE_PRECISION (types
[1 - i
]) == 64)
1920 if (!TYPE_UNSIGNED (types
[1 - i
]))
1923 return TYPE_UNSIGNED (types
[i
]) ? TYPE_SUFFIX_u64
: TYPE_SUFFIX_s64
;
1926 error_at (location
, "passing mismatched integer types %qT and %qT"
1927 " to arguments %d and %d of %qE", types
[0], types
[1],
1928 argno
+ 1, argno
+ 2, fndecl
);
1929 return NUM_TYPE_SUFFIXES
;
1932 /* Require argument ARGNO to be a pointer to a scalar type that has a
1933 corresponding type suffix. Return that type suffix on success,
1934 otherwise report an error and return NUM_TYPE_SUFFIXES.
1935 GATHER_SCATTER_P is true if the function is a gather/scatter
1936 operation, and so requires a pointer to 32-bit or 64-bit data. */
1938 function_resolver::infer_pointer_type (unsigned int argno
,
1939 bool gather_scatter_p
)
1941 tree actual
= get_argument_type (argno
);
1942 if (actual
== error_mark_node
)
1943 return NUM_TYPE_SUFFIXES
;
1945 if (TREE_CODE (actual
) != POINTER_TYPE
)
1947 error_at (location
, "passing %qT to argument %d of %qE, which"
1948 " expects a pointer type", actual
, argno
+ 1, fndecl
);
1949 if (VECTOR_TYPE_P (actual
) && gather_scatter_p
)
1950 inform (location
, "an explicit type suffix is needed"
1951 " when using a vector of base addresses");
1952 return NUM_TYPE_SUFFIXES
;
1955 tree target
= TREE_TYPE (actual
);
1956 type_suffix_index type
= find_type_suffix_for_scalar_type (target
);
1957 if (type
== NUM_TYPE_SUFFIXES
)
1959 error_at (location
, "passing %qT to argument %d of %qE, but %qT is not"
1960 " a valid SVE element type", actual
, argno
+ 1, fndecl
,
1961 build_qualified_type (target
, 0));
1962 return NUM_TYPE_SUFFIXES
;
1964 unsigned int bits
= type_suffixes
[type
].element_bits
;
1965 if (gather_scatter_p
&& bits
!= 32 && bits
!= 64)
1967 error_at (location
, "passing %qT to argument %d of %qE, which"
1968 " expects a pointer to 32-bit or 64-bit elements",
1969 actual
, argno
+ 1, fndecl
);
1970 return NUM_TYPE_SUFFIXES
;
1976 /* If TYPE is an SVE predicate or vector type, or a tuple of such a type,
1977 return the associated sve_type, otherwise return an invalid sve_type. */
1979 find_sve_type (const_tree type
)
1981 /* A linear search should be OK here, since the code isn't hot and
1982 the number of types is only small. */
1983 for (unsigned int size_i
= 0; size_i
< MAX_TUPLE_SIZE
; ++size_i
)
1984 for (unsigned int suffix_i
= 0; suffix_i
< NUM_TYPE_SUFFIXES
; ++suffix_i
)
1986 vector_type_index type_i
= type_suffixes
[suffix_i
].vector_type
;
1987 tree this_type
= acle_vector_types
[size_i
][type_i
];
1988 if (this_type
&& matches_type_p (this_type
, type
))
1989 return { type_suffix_index (suffix_i
), size_i
+ 1 };
1995 /* Require argument ARGNO to be an SVE type (i.e. something that can be
1996 represented by sve_type). Return the (valid) type if it is, otherwise
1997 report an error and return an invalid type. */
1999 function_resolver::infer_sve_type (unsigned int argno
)
2001 tree actual
= get_argument_type (argno
);
2002 if (actual
== error_mark_node
)
2005 if (sve_type type
= find_sve_type (actual
))
2008 if (scalar_argument_p (argno
))
2009 error_at (location
, "passing %qT to argument %d of %qE, which"
2010 " expects an SVE type rather than a scalar type",
2011 actual
, argno
+ 1, fndecl
);
2013 error_at (location
, "passing %qT to argument %d of %qE, which"
2014 " expects an SVE type",
2015 actual
, argno
+ 1, fndecl
);
2019 /* Require argument ARGNO to be a single vector or a tuple of NUM_VECTORS
2020 vectors; NUM_VECTORS is 1 for the former. Return the associated type
2021 on success. Report an error on failure. */
2023 function_resolver::infer_vector_or_tuple_type (unsigned int argno
,
2024 unsigned int num_vectors
)
2026 auto type
= infer_sve_type (argno
);
2030 if (type
.num_vectors
== num_vectors
)
2033 report_incorrect_num_vectors (argno
, type
, num_vectors
);
2037 /* Require argument ARGNO to have some form of vector type. Return the
2038 associated type suffix on success, using TYPE_SUFFIX_b for predicates.
2039 Report an error and return NUM_TYPE_SUFFIXES on failure. */
2041 function_resolver::infer_vector_type (unsigned int argno
)
2043 if (auto type
= infer_vector_or_tuple_type (argno
, 1))
2045 return NUM_TYPE_SUFFIXES
;
2048 /* Like infer_vector_type, but also require the type to be integral. */
2050 function_resolver::infer_integer_vector_type (unsigned int argno
)
2052 type_suffix_index type
= infer_vector_type (argno
);
2053 if (type
== NUM_TYPE_SUFFIXES
)
2056 if (!type_suffixes
[type
].integer_p
)
2058 error_at (location
, "passing %qT to argument %d of %qE, which"
2059 " expects a vector of integers", get_argument_type (argno
),
2061 return NUM_TYPE_SUFFIXES
;
2067 /* Require argument ARGNO to have some form of NEON128 vector type. Return the
2068 associated type suffix on success.
2069 Report an error and return NUM_TYPE_SUFFIXES on failure. */
2071 function_resolver::infer_neon128_vector_type (unsigned int argno
)
2073 tree actual
= get_argument_type (argno
);
2074 if (actual
== error_mark_node
)
2075 return NUM_TYPE_SUFFIXES
;
2077 for (unsigned int suffix_i
= 0; suffix_i
< NUM_TYPE_SUFFIXES
; ++suffix_i
)
2079 int neon_index
= type_suffixes
[suffix_i
].neon128_type
;
2080 if (neon_index
!= ARM_NEON_H_TYPES_LAST
)
2082 tree type
= aarch64_simd_types
[neon_index
].itype
;
2083 if (type
&& matches_type_p (type
, actual
))
2084 return type_suffix_index (suffix_i
);
2088 error_at (location
, "passing %qT to argument %d of %qE, which"
2089 " expects a 128 bit NEON vector type", actual
, argno
+ 1, fndecl
);
2090 return NUM_TYPE_SUFFIXES
;
2094 /* Like infer_vector_type, but also require the type to be an unsigned
2097 function_resolver::infer_unsigned_vector_type (unsigned int argno
)
2099 type_suffix_index type
= infer_vector_type (argno
);
2100 if (type
== NUM_TYPE_SUFFIXES
)
2103 if (!type_suffixes
[type
].unsigned_p
)
2105 error_at (location
, "passing %qT to argument %d of %qE, which"
2106 " expects a vector of unsigned integers",
2107 get_argument_type (argno
), argno
+ 1, fndecl
);
2108 return NUM_TYPE_SUFFIXES
;
2114 /* Like infer_vector_type, but also require the element size to be
2117 function_resolver::infer_sd_vector_type (unsigned int argno
)
2119 type_suffix_index type
= infer_vector_type (argno
);
2120 if (type
== NUM_TYPE_SUFFIXES
)
2123 unsigned int bits
= type_suffixes
[type
].element_bits
;
2124 if (bits
!= 32 && bits
!= 64)
2126 error_at (location
, "passing %qT to argument %d of %qE, which"
2127 " expects a vector of 32-bit or 64-bit elements",
2128 get_argument_type (argno
), argno
+ 1, fndecl
);
2129 return NUM_TYPE_SUFFIXES
;
2135 /* If the function operates on tuples of vectors, require argument ARGNO to be
2136 a tuple with the appropriate number of vectors, otherwise require it to be
2137 a single vector. Return the associated type on success. Report an error
2140 function_resolver::infer_tuple_type (unsigned int argno
)
2142 return infer_vector_or_tuple_type (argno
, vectors_per_tuple ());
2145 /* PRED_TYPE is the type of a governing predicate argument and DATA_TYPE
2146 is the type of an argument that it predicates. Require the two types
2147 to "agree": svcount_t must be used for multiple vectors and svbool_t
2150 Return true if they do agree, otherwise report an error and
2152 bool function_resolver::
2153 require_matching_predicate_type (vector_type_index pred_type
,
2156 if (pred_type
== VECTOR_TYPE_svbool_t
&& data_type
.num_vectors
== 1)
2159 if (pred_type
== VECTOR_TYPE_svcount_t
&& data_type
.num_vectors
!= 1)
2162 /* Make sure that FIRST_TYPE itself is sensible before using it
2163 as a basis for an error message. */
2164 if (resolve_to (mode_suffix_id
, data_type
) == error_mark_node
)
2167 if (data_type
.num_vectors
> 1)
2168 error_at (location
, "operations on multiple vectors must be predicated"
2169 " by %qs rather than %qs", "svcount_t", "svbool_t");
2171 error_at (location
, "operations on single vectors must be predicated"
2172 " by %qs rather than %qs", "svbool_t", "svcount_t");
2176 /* Require argument ARGNO to be a vector or scalar argument. Return true
2177 if it is, otherwise report an appropriate error. */
2179 function_resolver::require_vector_or_scalar_type (unsigned int argno
)
2181 tree actual
= get_argument_type (argno
);
2182 if (actual
== error_mark_node
)
2185 if (!scalar_argument_p (argno
) && !VECTOR_TYPE_P (actual
))
2187 error_at (location
, "passing %qT to argument %d of %qE, which"
2188 " expects a vector or scalar type", actual
, argno
+ 1, fndecl
);
2195 /* Require argument ARGNO to have vector type TYPE, in cases where this
2196 requirement holds for all uses of the function. Return true if the
2197 argument has the right form, otherwise report an appropriate error. */
2199 function_resolver::require_vector_type (unsigned int argno
,
2200 vector_type_index type
)
2202 tree expected
= acle_vector_types
[0][type
];
2203 tree actual
= get_argument_type (argno
);
2204 if (actual
== error_mark_node
)
2207 if (!matches_type_p (expected
, actual
))
2209 error_at (location
, "passing %qT to argument %d of %qE, which"
2210 " expects %qT", actual
, argno
+ 1, fndecl
, expected
);
2216 /* Like require_vector_type, but TYPE is inferred from argument FIRST_ARGNO
2217 rather than being a fixed part of the function signature. This changes
2218 the nature of the error messages. */
2220 function_resolver::require_matching_vector_type (unsigned int argno
,
2221 unsigned int first_argno
,
2224 sve_type new_type
= infer_sve_type (argno
);
2228 if (type
.num_vectors
!= new_type
.num_vectors
)
2230 report_mismatched_num_vectors (first_argno
, type
, argno
, new_type
);
2234 if (type
!= new_type
)
2236 error_at (location
, "passing %qT to argument %d of %qE, but"
2237 " argument %d had type %qT",
2238 get_vector_type (new_type
), argno
+ 1, fndecl
,
2239 first_argno
+ 1, get_vector_type (type
));
2245 /* Require argument ARGNO to be a vector or tuple type with the following
2248 - the type class must be the same as FIRST_TYPE's if EXPECTED_TCLASS
2249 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
2251 - the element size must be:
2253 - the same as FIRST_TYPE's if EXPECTED_BITS == SAME_SIZE
2254 - half of FIRST_TYPE's if EXPECTED_BITS == HALF_SIZE
2255 - a quarter of FIRST_TYPE's if EXPECTED_BITS == QUARTER_SIZE
2256 - EXPECTED_BITS itself otherwise
2258 - the number of vectors must be the same as FIRST_TYPE's if
2259 EXPECTED_NUM_VECTORS is zero, otherwise it must be EXPECTED_NUM_VECTORS.
2261 Return true if the argument has the required type, otherwise report
2262 an appropriate error.
2264 FIRST_ARGNO is the first argument that is known to have type FIRST_TYPE.
2265 Usually it comes before ARGNO, but sometimes it is more natural to resolve
2266 arguments out of order.
2268 If the required properties depend on FIRST_TYPE then both FIRST_ARGNO and
2269 ARGNO contribute to the resolution process. If the required properties
2270 are fixed, only FIRST_ARGNO contributes to the resolution process.
2272 This function is a bit of a Swiss army knife. The complication comes
2273 from trying to give good error messages when FIRST_ARGNO and ARGNO are
2274 inconsistent, since either of them might be wrong. */
2275 bool function_resolver::
2276 require_derived_vector_type (unsigned int argno
,
2277 unsigned int first_argno
,
2278 sve_type first_type
,
2279 type_class_index expected_tclass
,
2280 unsigned int expected_bits
,
2281 unsigned int expected_num_vectors
)
2283 /* If the type needs to match FIRST_ARGNO exactly, use the preferred
2284 error message for that case. */
2285 if (expected_tclass
== SAME_TYPE_CLASS
2286 && expected_bits
== SAME_SIZE
2287 && expected_num_vectors
== 0)
2289 /* There's no need to resolve this case out of order. */
2290 gcc_assert (argno
> first_argno
);
2291 return require_matching_vector_type (argno
, first_argno
, first_type
);
2294 /* Use FIRST_TYPE to get the expected type class and element size. */
2295 auto &first_type_suffix
= type_suffixes
[first_type
.type
];
2296 type_class_index orig_expected_tclass
= expected_tclass
;
2297 if (expected_tclass
== NUM_TYPE_CLASSES
)
2298 expected_tclass
= first_type_suffix
.tclass
;
2300 unsigned int orig_expected_bits
= expected_bits
;
2301 if (expected_bits
== SAME_SIZE
)
2302 expected_bits
= first_type_suffix
.element_bits
;
2303 else if (expected_bits
== HALF_SIZE
)
2304 expected_bits
= first_type_suffix
.element_bits
/ 2;
2305 else if (expected_bits
== QUARTER_SIZE
)
2306 expected_bits
= first_type_suffix
.element_bits
/ 4;
2308 unsigned int orig_expected_num_vectors
= expected_num_vectors
;
2309 if (expected_num_vectors
== 0)
2310 expected_num_vectors
= first_type
.num_vectors
;
2312 /* If the expected type doesn't depend on FIRST_TYPE at all,
2313 just check for the fixed choice of vector type. */
2314 if (expected_tclass
== orig_expected_tclass
2315 && expected_bits
== orig_expected_bits
2316 && orig_expected_num_vectors
== 1)
2318 const type_suffix_info
&expected_suffix
2319 = type_suffixes
[find_type_suffix (expected_tclass
, expected_bits
)];
2320 return require_vector_type (argno
, expected_suffix
.vector_type
);
2323 /* Require the argument to be some form of SVE vector type,
2324 without being specific about the type of vector we want. */
2325 sve_type actual_type
= infer_sve_type (argno
);
2329 if (actual_type
.num_vectors
!= expected_num_vectors
)
2331 if (orig_expected_num_vectors
== 0)
2332 report_mismatched_num_vectors (first_argno
, first_type
,
2333 argno
, actual_type
);
2335 report_incorrect_num_vectors (argno
, actual_type
,
2336 expected_num_vectors
);
2340 if (orig_expected_tclass
== SAME_TYPE_CLASS
2341 && orig_expected_bits
== SAME_SIZE
)
2343 if (actual_type
.type
== first_type
.type
)
2346 if (first_type
.num_vectors
> 1)
2347 error_at (location
, "passing %qT to argument %d of %qE, but"
2348 " argument %d was a tuple of %qT",
2349 get_vector_type (actual_type
), argno
+ 1, fndecl
,
2350 first_argno
+ 1, get_vector_type (first_type
.type
));
2352 error_at (location
, "passing %qT to argument %d of %qE, but"
2353 " argument %d had type %qT",
2354 get_vector_type (actual_type
), argno
+ 1, fndecl
,
2355 first_argno
+ 1, get_vector_type (first_type
));
2359 /* Exit now if we got the right type. */
2360 auto &actual_type_suffix
= type_suffixes
[actual_type
.type
];
2361 bool tclass_ok_p
= (actual_type_suffix
.tclass
== expected_tclass
);
2362 bool size_ok_p
= (actual_type_suffix
.element_bits
== expected_bits
);
2363 if (tclass_ok_p
&& size_ok_p
)
2366 /* First look for cases in which the actual type contravenes a fixed
2367 size requirement, without having to refer to FIRST_TYPE. */
2368 if (!size_ok_p
&& expected_bits
== orig_expected_bits
)
2370 if (expected_num_vectors
== 1)
2371 error_at (location
, "passing %qT to argument %d of %qE, which"
2372 " expects a vector of %d-bit elements",
2373 get_vector_type (actual_type
), argno
+ 1, fndecl
,
2376 error_at (location
, "passing %qT to argument %d of %qE, which"
2377 " expects vectors of %d-bit elements",
2378 get_vector_type (actual_type
), argno
+ 1, fndecl
,
2383 /* Likewise for a fixed type class requirement. This is only ever
2384 needed for signed and unsigned types, so don't create unnecessary
2385 translation work for other type classes. */
2386 if (!tclass_ok_p
&& orig_expected_tclass
== TYPE_signed
)
2388 if (expected_num_vectors
== 1)
2389 error_at (location
, "passing %qT to argument %d of %qE, which"
2390 " expects a vector of signed integers",
2391 get_vector_type (actual_type
), argno
+ 1, fndecl
);
2393 /* Translation note: could also be written "expects a tuple of
2394 signed integer vectors". */
2395 error_at (location
, "passing %qT to argument %d of %qE, which"
2396 " expects vectors of signed integers",
2397 get_vector_type (actual_type
), argno
+ 1, fndecl
);
2400 if (!tclass_ok_p
&& orig_expected_tclass
== TYPE_unsigned
)
2402 if (expected_num_vectors
== 1)
2403 error_at (location
, "passing %qT to argument %d of %qE, which"
2404 " expects a vector of unsigned integers",
2405 get_vector_type (actual_type
), argno
+ 1, fndecl
);
2407 /* Translation note: could also be written "expects a tuple of
2408 unsigned integer vectors". */
2409 error_at (location
, "passing %qT to argument %d of %qE, which"
2410 " expects vectors of unsigned integers",
2411 get_vector_type (actual_type
), argno
+ 1, fndecl
);
2415 /* Make sure that FIRST_TYPE itself is sensible before using it
2416 as a basis for an error message. */
2417 if (resolve_to (mode_suffix_id
, first_type
) == error_mark_node
)
2420 /* If the arguments have consistent type classes, but a link between
2421 the sizes has been broken, try to describe the error in those terms. */
2422 if (tclass_ok_p
&& orig_expected_bits
== SAME_SIZE
)
2424 if (argno
< first_argno
)
2426 std::swap (argno
, first_argno
);
2427 std::swap (actual_type
, first_type
);
2429 error_at (location
, "arguments %d and %d of %qE must have the"
2430 " same element size, but the values passed here have type"
2431 " %qT and %qT respectively", first_argno
+ 1, argno
+ 1,
2432 fndecl
, get_vector_type (first_type
),
2433 get_vector_type (actual_type
));
2437 /* Likewise in reverse: look for cases in which the sizes are consistent
2438 but a link between the type classes has been broken. */
2440 && orig_expected_tclass
== SAME_TYPE_CLASS
2441 && first_type_suffix
.integer_p
2442 && actual_type_suffix
.integer_p
)
2444 if (argno
< first_argno
)
2446 std::swap (argno
, first_argno
);
2447 std::swap (actual_type
, first_type
);
2449 error_at (location
, "arguments %d and %d of %qE must have the"
2450 " same signedness, but the values passed here have type"
2451 " %qT and %qT respectively", first_argno
+ 1, argno
+ 1,
2452 fndecl
, get_vector_type (first_type
),
2453 get_vector_type (actual_type
));
2457 /* The two arguments are wildly inconsistent. */
2458 type_suffix_index expected_type
2459 = find_type_suffix (expected_tclass
, expected_bits
);
2460 error_at (location
, "passing %qT instead of the expected %qT to argument"
2461 " %d of %qE, after passing %qT to argument %d",
2462 get_vector_type (actual_type
), get_vector_type (expected_type
),
2463 argno
+ 1, fndecl
, get_argument_type (first_argno
),
2468 /* Require argument ARGNO to match argument FIRST_ARGNO, which was inferred
2469 to be a pointer to a scalar element of type TYPE. */
2471 function_resolver::require_matching_pointer_type (unsigned int argno
,
2472 unsigned int first_argno
,
2473 type_suffix_index type
)
2475 type_suffix_index new_type
= infer_pointer_type (argno
);
2476 if (new_type
== NUM_TYPE_SUFFIXES
)
2479 if (type
!= new_type
)
2481 error_at (location
, "passing %qT to argument %d of %qE, but"
2482 " argument %d had type %qT", get_argument_type (argno
),
2483 argno
+ 1, fndecl
, first_argno
+ 1,
2484 get_argument_type (first_argno
));
2490 /* Require argument ARGNO to be a (possibly variable) scalar, using EXPECTED
2491 as the name of its expected type. Return true if the argument has the
2492 right form, otherwise report an appropriate error. */
2494 function_resolver::require_scalar_type (unsigned int argno
,
2495 const char *expected
)
2497 if (!scalar_argument_p (argno
))
2500 error_at (location
, "passing %qT to argument %d of %qE, which"
2501 " expects %qs", get_argument_type (argno
), argno
+ 1,
2508 /* Require argument ARGNO to be a nonscalar type, given that it has already
2509 passed require_vector_or_scalar_type. Return true if it is, otherwise
2510 report an error. This is used when two sets of instructions share the
2511 same overloaded function and one accepts scalars while the other
2514 function_resolver::require_nonscalar_type (unsigned int argno
)
2516 if (scalar_argument_p (argno
))
2518 error_at (location
, "passing %qT to argument %d of %qE, which"
2519 " does not accept scalars for this combination of arguments",
2520 get_argument_type (argno
), argno
+ 1, fndecl
);
2526 /* Require argument ARGNO to be some form of pointer, without being specific
2527 about its target type. Return true if the argument has the right form,
2528 otherwise report an appropriate error. */
2530 function_resolver::require_pointer_type (unsigned int argno
)
2532 if (!scalar_argument_p (argno
))
2534 error_at (location
, "passing %qT to argument %d of %qE, which"
2535 " expects a scalar pointer", get_argument_type (argno
),
2542 /* Argument FIRST_ARGNO is a scalar with type EXPECTED_TYPE, and argument
2543 ARGNO should be consistent with it. Return true if it is, otherwise
2544 report an appropriate error. */
2545 bool function_resolver::
2546 require_matching_integer_scalar_type (unsigned int argno
,
2547 unsigned int first_argno
,
2548 type_suffix_index expected_type
)
2550 type_suffix_index actual_type
= infer_integer_scalar_type (argno
);
2551 if (actual_type
== NUM_TYPE_SUFFIXES
)
2554 if (actual_type
== expected_type
)
2557 error_at (location
, "call to %qE is ambiguous; argument %d has type"
2558 " %qs but argument %d has type %qs", fndecl
,
2559 first_argno
+ 1, get_scalar_type_name (expected_type
),
2560 argno
+ 1, get_scalar_type_name (actual_type
));
2564 /* Require argument ARGNO to be a (possibly variable) scalar, expecting it
2565 to have the following properties:
2567 - the type class must be the same as for type suffix 0 if EXPECTED_TCLASS
2568 is SAME_TYPE_CLASS, otherwise it must be EXPECTED_TCLASS itself.
2570 - the element size must be the same as for type suffix 0 if EXPECTED_BITS
2571 is SAME_TYPE_SIZE, otherwise it must be EXPECTED_BITS itself.
2573 Return true if the argument is valid, otherwise report an appropriate error.
2575 Note that we don't check whether the scalar type actually has the required
2576 properties, since that's subject to implicit promotions and conversions.
2577 Instead we just use the expected properties to tune the error message. */
2578 bool function_resolver::
2579 require_derived_scalar_type (unsigned int argno
,
2580 type_class_index expected_tclass
,
2581 unsigned int expected_bits
)
2583 gcc_assert (expected_tclass
== SAME_TYPE_CLASS
2584 || expected_tclass
== TYPE_signed
2585 || expected_tclass
== TYPE_unsigned
);
2587 /* If the expected type doesn't depend on the type suffix at all,
2588 just check for the fixed choice of scalar type. */
2589 if (expected_tclass
!= SAME_TYPE_CLASS
&& expected_bits
!= SAME_SIZE
)
2591 type_suffix_index expected_type
2592 = find_type_suffix (expected_tclass
, expected_bits
);
2593 return require_scalar_type (argno
, get_scalar_type_name (expected_type
));
2596 if (scalar_argument_p (argno
))
2599 if (expected_tclass
== SAME_TYPE_CLASS
)
2600 /* It doesn't really matter whether the element is expected to be
2601 the same size as type suffix 0. */
2602 error_at (location
, "passing %qT to argument %d of %qE, which"
2603 " expects a scalar element", get_argument_type (argno
),
2606 /* It doesn't seem useful to distinguish between signed and unsigned
2608 error_at (location
, "passing %qT to argument %d of %qE, which"
2609 " expects a scalar integer", get_argument_type (argno
),
2614 /* Require argument ARGNO to be suitable for an integer constant expression.
2615 Return true if it is, otherwise report an appropriate error.
2617 function_checker checks whether the argument is actually constant and
2618 has a suitable range. The reason for distinguishing immediate arguments
2619 here is because it provides more consistent error messages than
2620 require_scalar_type would. */
2622 function_resolver::require_integer_immediate (unsigned int argno
)
2624 if (!scalar_argument_p (argno
))
2626 report_non_ice (location
, fndecl
, argno
);
2632 /* Require argument ARGNO to be a vector base in a gather-style address.
2633 Return its type on success, otherwise return NUM_VECTOR_TYPES. */
2635 function_resolver::infer_vector_base_type (unsigned int argno
)
2637 type_suffix_index type
= infer_vector_type (argno
);
2638 if (type
== NUM_TYPE_SUFFIXES
)
2639 return NUM_VECTOR_TYPES
;
2641 if (type
== TYPE_SUFFIX_u32
|| type
== TYPE_SUFFIX_u64
)
2642 return type_suffixes
[type
].vector_type
;
2644 error_at (location
, "passing %qT to argument %d of %qE, which"
2645 " expects %qs or %qs", get_argument_type (argno
),
2646 argno
+ 1, fndecl
, "svuint32_t", "svuint64_t");
2647 return NUM_VECTOR_TYPES
;
2650 /* Require argument ARGNO to be a vector displacement in a gather-style
2651 address. Return its type on success, otherwise return NUM_VECTOR_TYPES. */
2653 function_resolver::infer_vector_displacement_type (unsigned int argno
)
2655 type_suffix_index type
= infer_integer_vector_type (argno
);
2656 if (type
== NUM_TYPE_SUFFIXES
)
2657 return NUM_VECTOR_TYPES
;
2659 if (type_suffixes
[type
].integer_p
2660 && (type_suffixes
[type
].element_bits
== 32
2661 || type_suffixes
[type
].element_bits
== 64))
2662 return type_suffixes
[type
].vector_type
;
2664 error_at (location
, "passing %qT to argument %d of %qE, which"
2665 " expects a vector of 32-bit or 64-bit integers",
2666 get_argument_type (argno
), argno
+ 1, fndecl
);
2667 return NUM_VECTOR_TYPES
;
2670 /* Require argument ARGNO to be a vector displacement in a gather-style
2671 address. There are three possible uses:
2673 - for loading into elements of type TYPE (when LOAD_P is true)
2674 - for storing from elements of type TYPE (when LOAD_P is false)
2675 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
2677 The overloaded function's mode suffix determines the units of the
2678 displacement (bytes for "_offset", elements for "_index").
2680 Return the associated mode on success, otherwise report an error
2681 and return MODE_none. */
2683 function_resolver::resolve_sv_displacement (unsigned int argno
,
2684 type_suffix_index type
,
2687 if (type
== NUM_TYPE_SUFFIXES
)
2689 /* For prefetches, the base is a void pointer and the displacement
2690 can be any valid offset or index type. */
2691 vector_type_index displacement_vector_type
2692 = infer_vector_displacement_type (argno
);
2693 if (displacement_vector_type
== NUM_VECTOR_TYPES
)
2696 mode_suffix_index mode
= find_mode_suffix (NUM_VECTOR_TYPES
,
2697 displacement_vector_type
,
2698 displacement_units ());
2699 gcc_assert (mode
!= MODE_none
);
2703 unsigned int required_bits
= type_suffixes
[type
].element_bits
;
2704 if (required_bits
== 32
2705 && displacement_units () == UNITS_elements
2706 && !lookup_form (MODE_s32index
, type
)
2707 && !lookup_form (MODE_u32index
, type
))
2709 if (lookup_form (MODE_u32base_index
, type
))
2711 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
2713 gcc_assert (!load_p
);
2714 error_at (location
, "when storing %qT, %qE requires a vector"
2715 " base and a scalar index", get_vector_type (type
),
2719 error_at (location
, "%qE requires a vector base and a scalar"
2723 error_at (location
, "%qE does not support 32-bit vector type %qT",
2724 fndecl
, get_vector_type (type
));
2728 /* Check for some form of vector type, without naming any in particular
2729 as being expected. */
2730 type_suffix_index displacement_type
= infer_vector_type (argno
);
2731 if (displacement_type
== NUM_TYPE_SUFFIXES
)
2734 /* If the displacement type is consistent with the data vector type,
2735 try to find the associated mode suffix. This will fall through
2736 for non-integral displacement types. */
2737 if (type_suffixes
[displacement_type
].element_bits
== required_bits
)
2739 vector_type_index displacement_vector_type
2740 = type_suffixes
[displacement_type
].vector_type
;
2741 mode_suffix_index mode
= find_mode_suffix (NUM_VECTOR_TYPES
,
2742 displacement_vector_type
,
2743 displacement_units ());
2744 if (mode
!= MODE_none
)
2746 if (mode
== MODE_s32offset
2747 && !lookup_form (mode
, type
)
2748 && lookup_form (MODE_u32offset
, type
))
2750 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
2751 error_at (location
, "%qE does not support 32-bit sign-extended"
2752 " offsets", fndecl
);
2754 error_at (location
, "%qE does not support sign-extended"
2755 " offsets", fndecl
);
2762 if (type_suffix_ids
[0] == NUM_TYPE_SUFFIXES
)
2764 /* TYPE has been inferred rather than specified by the user,
2765 so mention it in the error messages. */
2767 error_at (location
, "passing %qT to argument %d of %qE, which when"
2768 " loading %qT expects a vector of %d-bit integers",
2769 get_argument_type (argno
), argno
+ 1, fndecl
,
2770 get_vector_type (type
), required_bits
);
2772 error_at (location
, "passing %qT to argument %d of %qE, which when"
2773 " storing %qT expects a vector of %d-bit integers",
2774 get_argument_type (argno
), argno
+ 1, fndecl
,
2775 get_vector_type (type
), required_bits
);
2778 /* TYPE is part of the function name. */
2779 error_at (location
, "passing %qT to argument %d of %qE, which"
2780 " expects a vector of %d-bit integers",
2781 get_argument_type (argno
), argno
+ 1, fndecl
, required_bits
);
2785 /* Require the arguments starting at ARGNO to form a gather-style address.
2786 There are three possible uses:
2788 - for loading into elements of type TYPE (when LOAD_P is true)
2789 - for storing from elements of type TYPE (when LOAD_P is false)
2790 - for prefetching data (when TYPE is NUM_TYPE_SUFFIXES)
2792 The three possible addresses are:
2794 - a vector base with no displacement
2795 - a vector base and a scalar displacement
2796 - a scalar (pointer) base and a vector displacement
2798 The overloaded function's mode suffix determines whether there is
2799 a displacement, and if so, what units it uses:
2801 - MODE_none: no displacement
2802 - MODE_offset: the displacement is measured in bytes
2803 - MODE_index: the displacement is measured in elements
2805 Return the mode of the non-overloaded function on success, otherwise
2806 report an error and return MODE_none. */
2808 function_resolver::resolve_gather_address (unsigned int argno
,
2809 type_suffix_index type
,
2812 tree actual
= get_argument_type (argno
);
2813 if (actual
== error_mark_node
)
2816 if (displacement_units () != UNITS_none
)
2818 /* Some form of displacement is needed. First handle a scalar
2819 pointer base and a vector displacement. */
2820 if (scalar_argument_p (argno
))
2821 /* Don't check the pointer type here, since there's only one valid
2822 choice. Leave that to the frontend. */
2823 return resolve_sv_displacement (argno
+ 1, type
, load_p
);
2825 if (!VECTOR_TYPE_P (actual
))
2827 error_at (location
, "passing %qT to argument %d of %qE,"
2828 " which expects a vector or pointer base address",
2829 actual
, argno
+ 1, fndecl
);
2834 /* Check for the correct choice of vector base type. */
2835 vector_type_index base_vector_type
;
2836 if (type
== NUM_TYPE_SUFFIXES
)
2838 /* Since prefetches have no type suffix, there is a free choice
2839 between 32-bit and 64-bit base addresses. */
2840 base_vector_type
= infer_vector_base_type (argno
);
2841 if (base_vector_type
== NUM_VECTOR_TYPES
)
2846 /* Check for some form of vector type, without saying which type
2848 type_suffix_index base_type
= infer_vector_type (argno
);
2849 if (base_type
== NUM_TYPE_SUFFIXES
)
2852 /* Check whether the type is the right one. */
2853 unsigned int required_bits
= type_suffixes
[type
].element_bits
;
2854 gcc_assert (required_bits
== 32 || required_bits
== 64);
2855 type_suffix_index required_type
= (required_bits
== 32
2858 if (required_type
!= base_type
)
2860 error_at (location
, "passing %qT to argument %d of %qE,"
2861 " which expects %qT", actual
, argno
+ 1, fndecl
,
2862 get_vector_type (required_type
));
2865 base_vector_type
= type_suffixes
[base_type
].vector_type
;
2868 /* Check the scalar displacement, if any. */
2869 if (displacement_units () != UNITS_none
2870 && !require_scalar_type (argno
+ 1, "int64_t"))
2873 /* Find the appropriate mode suffix. The checks above should have
2874 weeded out all erroneous cases. */
2875 for (unsigned int mode_i
= 0; mode_i
< ARRAY_SIZE (mode_suffixes
); ++mode_i
)
2877 const mode_suffix_info
&mode
= mode_suffixes
[mode_i
];
2878 if (mode
.base_vector_type
== base_vector_type
2879 && mode
.displacement_vector_type
== NUM_VECTOR_TYPES
2880 && mode
.displacement_units
== displacement_units ())
2881 return mode_suffix_index (mode_i
);
2887 /* Require arguments ARGNO and ARGNO + 1 to form an ADR-style address,
2888 i.e. one with a vector of base addresses and a vector of displacements.
2889 The overloaded function's mode suffix determines the units of the
2890 displacement (bytes for "_offset", elements for "_index").
2892 Return the associated mode suffix on success, otherwise report
2893 an error and return MODE_none. */
2895 function_resolver::resolve_adr_address (unsigned int argno
)
2897 vector_type_index base_type
= infer_vector_base_type (argno
);
2898 if (base_type
== NUM_VECTOR_TYPES
)
2901 vector_type_index displacement_type
2902 = infer_vector_displacement_type (argno
+ 1);
2903 if (displacement_type
== NUM_VECTOR_TYPES
)
2906 mode_suffix_index mode
= find_mode_suffix (base_type
, displacement_type
,
2907 displacement_units ());
2908 if (mode
== MODE_none
)
2910 if (mode_suffix_id
== MODE_offset
)
2911 error_at (location
, "cannot combine a base of type %qT with"
2912 " an offset of type %qT",
2913 get_argument_type (argno
), get_argument_type (argno
+ 1));
2915 error_at (location
, "cannot combine a base of type %qT with"
2916 " an index of type %qT",
2917 get_argument_type (argno
), get_argument_type (argno
+ 1));
2922 /* Require the function to have exactly EXPECTED arguments. Return true
2923 if it does, otherwise report an appropriate error. */
2925 function_resolver::check_num_arguments (unsigned int expected
)
2927 if (m_arglist
.length () < expected
)
2928 error_at (location
, "too few arguments to function %qE", fndecl
);
2929 else if (m_arglist
.length () > expected
)
2930 error_at (location
, "too many arguments to function %qE", fndecl
);
2931 return m_arglist
.length () == expected
;
2934 /* If the function is predicated, check that the first argument is a
2935 suitable governing predicate. Also check that there are NOPS further
2936 arguments after any governing predicate, but don't check what they are.
2938 Return true on success, otherwise report a suitable error.
2939 When returning true:
2941 - set I to the number of the first unchecked argument.
2942 - set NARGS to the total number of arguments. */
2944 function_resolver::check_gp_argument (unsigned int nops
,
2945 unsigned int &i
, unsigned int &nargs
)
2947 gcc_assert (pred
!= PRED_za_m
);
2949 if (pred
!= PRED_none
)
2951 /* Unary merge operations should use resolve_unary instead. */
2952 gcc_assert (!shape
->has_merge_argument_p (*this, nops
));
2954 if (!check_num_arguments (nargs
)
2955 || !require_vector_type (i
, gp_type_index ()))
2962 if (!check_num_arguments (nargs
))
2969 /* Finish resolving a function whose final argument can be a vector
2970 or a scalar, with the function having an implicit "_n" suffix
2971 in the latter case. This "_n" form might only exist for certain
2974 ARGNO is the index of the final argument. The inferred type suffix
2975 was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE.
2976 EXPECTED_TCLASS and EXPECTED_BITS describe the expected properties
2977 of the final vector or scalar argument, in the same way as for
2978 require_derived_vector_type. INFERRED_TYPE is the inferred type
2979 suffix itself, or NUM_TYPE_SUFFIXES if it's the same as FIRST_TYPE.
2981 Return the function decl of the resolved function on success,
2982 otherwise report a suitable error and return error_mark_node. */
2983 tree
function_resolver::
2984 finish_opt_n_resolution (unsigned int argno
, unsigned int first_argno
,
2985 type_suffix_index first_type
,
2986 type_class_index expected_tclass
,
2987 unsigned int expected_bits
,
2988 type_suffix_index inferred_type
)
2990 if (inferred_type
== NUM_TYPE_SUFFIXES
)
2991 inferred_type
= first_type
;
2992 tree scalar_form
= lookup_form (MODE_n
, inferred_type
);
2994 /* Allow the final argument to be scalar, if an _n form exists. */
2995 if (scalar_argument_p (argno
))
3000 /* Check the vector form normally. If that succeeds, raise an
3001 error about having no corresponding _n form. */
3002 tree res
= resolve_to (mode_suffix_id
, inferred_type
);
3003 if (res
!= error_mark_node
)
3004 error_at (location
, "passing %qT to argument %d of %qE, but its"
3005 " %qT form does not accept scalars",
3006 get_argument_type (argno
), argno
+ 1, fndecl
,
3007 get_vector_type (first_type
));
3008 return error_mark_node
;
3011 /* If an _n form does exist, provide a more accurate message than
3012 require_derived_vector_type would for arguments that are neither
3013 vectors nor scalars. */
3014 if (scalar_form
&& !require_vector_or_scalar_type (argno
))
3015 return error_mark_node
;
3017 /* Check for the correct vector type. */
3018 if (!require_derived_vector_type (argno
, first_argno
, first_type
,
3019 expected_tclass
, expected_bits
))
3020 return error_mark_node
;
3022 return resolve_to (mode_suffix_id
, inferred_type
);
3025 /* Finish resolving a function whose final argument can be a tuple
3026 or a vector, with the function having an implicit "_single" suffix
3027 in the latter case. This "_single" form might only exist for certain
3030 ARGNO is the index of the final argument. The inferred type suffix
3031 was obtained from argument FIRST_ARGNO, which has type FIRST_TYPE.
3032 EXPECTED_TCLASS gives the expected type class for the final tuple
3035 Return the function decl of the resolved function on success,
3036 otherwise report a suitable error and return error_mark_node. */
3037 tree
function_resolver::
3038 finish_opt_single_resolution (unsigned int argno
, unsigned int first_argno
,
3039 sve_type first_type
,
3040 type_class_index expected_tclass
)
3042 sve_type new_type
= infer_sve_type (argno
);
3044 return error_mark_node
;
3046 /* If the type is a tuple, require it to match the group suffix. */
3047 unsigned int num_vectors
= vectors_per_tuple ();
3048 if (num_vectors
!= 1
3049 && new_type
.num_vectors
!= 1
3050 && new_type
.num_vectors
!= num_vectors
)
3052 report_incorrect_num_vectors (argno
, new_type
, num_vectors
);
3053 return error_mark_node
;
3056 auto expected_num_vectors
= (new_type
.num_vectors
== 1 ? 1 : 0);
3057 if (!require_derived_vector_type (argno
, first_argno
, first_type
,
3058 expected_tclass
, SAME_SIZE
,
3059 expected_num_vectors
))
3060 return error_mark_node
;
3062 if (new_type
.num_vectors
== 1 && first_type
.num_vectors
> 1)
3064 if (tree single_form
= lookup_form (MODE_single
, first_type
))
3067 if (resolve_to (mode_suffix_id
, first_type
) != error_mark_node
)
3068 error_at (location
, "passing %qT to argument %d of %qE, but its"
3069 " %qT form does not accept single vectors",
3070 get_vector_type (new_type
), argno
+ 1, fndecl
,
3071 get_vector_type (first_type
));
3072 return error_mark_node
;
3074 return resolve_to (mode_suffix_id
, first_type
);
3077 /* Resolve a (possibly predicated) unary function. If the function uses
3078 merge predication or if TREAT_AS_MERGE_P is true, there is an extra
3079 vector argument before the governing predicate that specifies the
3080 values of inactive elements. This argument has the following
3083 - the type class must be the same as for active elements if MERGE_TCLASS
3084 is SAME_TYPE_CLASS, otherwise it must be MERGE_TCLASS itself.
3086 - the element size must be the same as for active elements if MERGE_BITS
3087 is SAME_TYPE_SIZE, otherwise it must be MERGE_BITS itself.
3089 Return the function decl of the resolved function on success,
3090 otherwise report a suitable error and return error_mark_node. */
3092 function_resolver::resolve_unary (type_class_index merge_tclass
,
3093 unsigned int merge_bits
,
3094 bool treat_as_merge_p
)
3096 type_suffix_index type
;
3097 if (pred
== PRED_m
|| treat_as_merge_p
)
3099 if (!check_num_arguments (3))
3100 return error_mark_node
;
3101 if (merge_tclass
== SAME_TYPE_CLASS
&& merge_bits
== SAME_SIZE
)
3103 /* The inactive elements are the same as the active elements,
3104 so we can use normal left-to-right resolution. */
3105 if ((type
= infer_vector_type (0)) == NUM_TYPE_SUFFIXES
3106 || !require_vector_type (1, VECTOR_TYPE_svbool_t
)
3107 || !require_matching_vector_type (2, 0, type
))
3108 return error_mark_node
;
3112 /* The inactive element type is a function of the active one,
3113 so resolve the active one first. */
3114 if (!require_vector_type (1, VECTOR_TYPE_svbool_t
)
3115 || (type
= infer_vector_type (2)) == NUM_TYPE_SUFFIXES
3116 || !require_derived_vector_type (0, 2, type
, merge_tclass
,
3118 return error_mark_node
;
3123 /* We just need to check the predicate (if any) and the single
3125 unsigned int i
, nargs
;
3126 if (!check_gp_argument (1, i
, nargs
)
3127 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
3128 return error_mark_node
;
3131 /* Handle convert-like functions in which the first type suffix is
3133 if (type_suffix_ids
[0] != NUM_TYPE_SUFFIXES
)
3134 return resolve_to (mode_suffix_id
, type_suffix_ids
[0], type
);
3136 return resolve_to (mode_suffix_id
, type
);
3139 /* Resolve a (possibly predicated) function that takes NOPS like-typed
3140 vector arguments followed by NIMM integer immediates. Return the
3141 function decl of the resolved function on success, otherwise report
3142 a suitable error and return error_mark_node. */
3144 function_resolver::resolve_uniform (unsigned int nops
, unsigned int nimm
)
3146 unsigned int i
, nargs
;
3147 type_suffix_index type
;
3148 if (!check_gp_argument (nops
+ nimm
, i
, nargs
)
3149 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
3150 return error_mark_node
;
3152 unsigned int first_arg
= i
++;
3153 for (; i
< nargs
- nimm
; ++i
)
3154 if (!require_matching_vector_type (i
, first_arg
, type
))
3155 return error_mark_node
;
3157 for (; i
< nargs
; ++i
)
3158 if (!require_integer_immediate (i
))
3159 return error_mark_node
;
3161 return resolve_to (mode_suffix_id
, type
);
3164 /* Resolve a (possibly predicated) function that offers a choice between
3167 - NOPS like-typed vector arguments or
3168 - NOPS - 1 like-typed vector arguments followed by a scalar argument
3170 Return the function decl of the resolved function on success,
3171 otherwise report a suitable error and return error_mark_node. */
3173 function_resolver::resolve_uniform_opt_n (unsigned int nops
)
3175 unsigned int i
, nargs
;
3176 type_suffix_index type
;
3177 if (!check_gp_argument (nops
, i
, nargs
)
3178 || (type
= infer_vector_type (i
)) == NUM_TYPE_SUFFIXES
)
3179 return error_mark_node
;
3181 unsigned int first_arg
= i
++;
3182 for (; i
< nargs
- 1; ++i
)
3183 if (!require_matching_vector_type (i
, first_arg
, type
))
3184 return error_mark_node
;
3186 return finish_opt_n_resolution (i
, first_arg
, type
);
3189 /* If the call is erroneous, report an appropriate error and return
3190 error_mark_node. Otherwise, if the function is overloaded, return
3191 the decl of the non-overloaded function. Return NULL_TREE otherwise,
3192 indicating that the call should be processed in the normal way. */
3194 function_resolver::resolve ()
3196 return shape
->resolve (*this);
3199 function_checker::function_checker (location_t location
,
3200 const function_instance
&instance
,
3201 tree fndecl
, tree fntype
,
3202 unsigned int nargs
, tree
*args
)
3203 : function_call_info (location
, instance
, fndecl
),
3204 m_fntype (fntype
), m_nargs (nargs
), m_args (args
),
3205 m_base_arg (pred
!= PRED_none
&& pred
!= PRED_za_m
? 1 : 0)
3209 /* Return true if argument ARGNO exists. which it might not for
3210 erroneous calls. It is safe to wave through checks if this
3211 function returns false. */
3213 function_checker::argument_exists_p (unsigned int argno
)
3215 gcc_assert (argno
< (unsigned int) type_num_arguments (m_fntype
));
3216 return argno
< m_nargs
;
3219 /* Check that argument ARGNO is an integer constant expression and
3220 store its value in VALUE_OUT if so. The caller should first
3221 check that argument ARGNO exists. */
3223 function_checker::require_immediate (unsigned int argno
,
3224 HOST_WIDE_INT
&value_out
)
3226 gcc_assert (argno
< m_nargs
);
3227 tree arg
= m_args
[argno
];
3229 /* The type and range are unsigned, so read the argument as an
3230 unsigned rather than signed HWI. */
3231 if (!tree_fits_uhwi_p (arg
))
3233 report_non_ice (location
, fndecl
, argno
);
3237 /* ...but treat VALUE_OUT as signed for error reporting, since printing
3238 -1 is more user-friendly than the maximum uint64_t value. */
3239 value_out
= tree_to_uhwi (arg
);
3243 /* Check that argument REL_ARGNO is an integer constant expression that
3244 has the value VALUE0 or VALUE1. REL_ARGNO counts from the end of the
3245 predication arguments. */
3247 function_checker::require_immediate_either_or (unsigned int rel_argno
,
3248 HOST_WIDE_INT value0
,
3249 HOST_WIDE_INT value1
)
3251 unsigned int argno
= m_base_arg
+ rel_argno
;
3252 if (!argument_exists_p (argno
))
3255 HOST_WIDE_INT actual
;
3256 if (!require_immediate (argno
, actual
))
3259 if (actual
!= value0
&& actual
!= value1
)
3261 report_neither_nor (location
, fndecl
, argno
, actual
, value0
, value1
);
3268 /* Check that argument REL_ARGNO is an integer constant expression that has
3269 a valid value for enumeration type TYPE. REL_ARGNO counts from the end
3270 of the predication arguments. */
3272 function_checker::require_immediate_enum (unsigned int rel_argno
, tree type
)
3274 unsigned int argno
= m_base_arg
+ rel_argno
;
3275 if (!argument_exists_p (argno
))
3278 HOST_WIDE_INT actual
;
3279 if (!require_immediate (argno
, actual
))
3282 for (tree entry
= TYPE_VALUES (type
); entry
; entry
= TREE_CHAIN (entry
))
3284 /* The value is an INTEGER_CST for C and a CONST_DECL wrapper
3285 around an INTEGER_CST for C++. */
3286 tree value
= TREE_VALUE (entry
);
3287 if (TREE_CODE (value
) == CONST_DECL
)
3288 value
= DECL_INITIAL (value
);
3289 if (wi::to_widest (value
) == actual
)
3293 report_not_enum (location
, fndecl
, argno
, actual
, type
);
3297 /* The intrinsic conceptually divides vector argument REL_VEC_ARGNO into
3298 groups of GROUP_SIZE elements. Return true if argument REL_ARGNO is
3299 a suitable constant index for selecting one of these groups. The
3300 selection happens within a 128-bit quadword, rather than the whole vector.
3302 REL_ARGNO and REL_VEC_ARGNO count from the end of the predication
3305 function_checker::require_immediate_lane_index (unsigned int rel_argno
,
3306 unsigned int rel_vec_argno
,
3307 unsigned int group_size
)
3309 unsigned int argno
= m_base_arg
+ rel_argno
;
3310 if (!argument_exists_p (argno
))
3313 /* Get the type of the vector argument. tree_argument_type wants a
3314 1-based number, whereas VEC_ARGNO is 0-based. */
3315 unsigned int vec_argno
= m_base_arg
+ rel_vec_argno
;
3316 machine_mode mode
= TYPE_MODE (type_argument_type (m_fntype
, vec_argno
+ 1));
3317 gcc_assert (VECTOR_MODE_P (mode
));
3318 unsigned int nlanes
= 128 / (group_size
* GET_MODE_UNIT_BITSIZE (mode
));
3319 return require_immediate_range (rel_argno
, 0, nlanes
- 1);
3322 /* Check that argument REL_ARGNO is an integer constant expression that
3323 has one of the given values. */
3325 function_checker::require_immediate_one_of (unsigned int rel_argno
,
3326 HOST_WIDE_INT value0
,
3327 HOST_WIDE_INT value1
,
3328 HOST_WIDE_INT value2
,
3329 HOST_WIDE_INT value3
)
3331 unsigned int argno
= m_base_arg
+ rel_argno
;
3332 if (!argument_exists_p (argno
))
3335 HOST_WIDE_INT actual
;
3336 if (!require_immediate (argno
, actual
))
3339 if (actual
!= value0
3342 && actual
!= value3
)
3344 report_not_one_of (location
, fndecl
, argno
, actual
,
3345 value0
, value1
, value2
, value3
);
3352 /* Check that argument REL_ARGNO is an integer constant expression in the
3353 range [MIN, MAX]. REL_ARGNO counts from the end of the predication
3356 function_checker::require_immediate_range (unsigned int rel_argno
,
3360 unsigned int argno
= m_base_arg
+ rel_argno
;
3361 if (!argument_exists_p (argno
))
3364 /* Required because of the tree_to_uhwi -> HOST_WIDE_INT conversion
3365 in require_immediate. */
3366 gcc_assert (min
>= 0 && min
<= max
);
3367 HOST_WIDE_INT actual
;
3368 if (!require_immediate (argno
, actual
))
3371 if (!IN_RANGE (actual
, min
, max
))
3373 report_out_of_range (location
, fndecl
, argno
, actual
, min
, max
);
3380 /* Perform semantic checks on the call. Return true if the call is valid,
3381 otherwise report a suitable error. */
3383 function_checker::check ()
3385 function_args_iterator iter
;
3388 FOREACH_FUNCTION_ARGS (m_fntype
, type
, iter
)
3390 if (type
== void_type_node
|| i
>= m_nargs
)
3394 && TREE_CODE (type
) == ENUMERAL_TYPE
3395 && !require_immediate_enum (i
- m_base_arg
, type
))
3401 return shape
->check (*this);
3404 /* Return true if V is a vector constant and if, for every in-range integer I,
3405 element STEP*I is equal to element 0. */
3407 vector_cst_all_same (tree v
, unsigned int step
)
3409 if (TREE_CODE (v
) != VECTOR_CST
)
3412 /* VECTOR_CST_NELTS_PER_PATTERN applies to any multiple of
3413 VECTOR_CST_NPATTERNS. */
3414 unsigned int lcm
= least_common_multiple (step
, VECTOR_CST_NPATTERNS (v
));
3415 unsigned int nelts
= lcm
* VECTOR_CST_NELTS_PER_PATTERN (v
);
3416 tree first_el
= VECTOR_CST_ENCODED_ELT (v
, 0);
3417 for (unsigned int i
= 0; i
< nelts
; i
+= step
)
3418 if (!operand_equal_p (VECTOR_CST_ELT (v
, i
), first_el
, 0))
3424 /* Return true if V is a constant predicate that acts as a ptrue when
3425 predicating STEP-byte elements. */
3427 is_ptrue (tree v
, unsigned int step
)
3429 return (TREE_CODE (v
) == VECTOR_CST
3430 && TYPE_MODE (TREE_TYPE (v
)) == VNx16BImode
3431 && integer_nonzerop (VECTOR_CST_ENCODED_ELT (v
, 0))
3432 && vector_cst_all_same (v
, step
));
3435 gimple_folder::gimple_folder (const function_instance
&instance
, tree fndecl
,
3436 gimple_stmt_iterator
*gsi_in
, gcall
*call_in
)
3437 : function_call_info (gimple_location (call_in
), instance
, fndecl
),
3438 gsi (gsi_in
), call (call_in
), lhs (gimple_call_lhs (call_in
))
3442 /* VALUE might be a vector of type VECTYPE or a single scalar element.
3443 Duplicate it into a vector of type VECTYPE in the latter case, adding any
3444 new statements to STMTS. */
3446 gimple_folder::force_vector (gimple_seq
&stmts
, tree vectype
, tree value
)
3448 if (!VECTOR_TYPE_P (TREE_TYPE (value
)))
3449 value
= gimple_build_vector_from_val (&stmts
, vectype
, value
);
3453 /* Convert predicate argument ARGNO so that it has the type appropriate for
3454 an operation on VECTYPE. Add any new statements to STMTS. */
3456 gimple_folder::convert_pred (gimple_seq
&stmts
, tree vectype
,
3459 tree pred
= gimple_call_arg (call
, argno
);
3460 if (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (pred
)),
3461 TYPE_VECTOR_SUBPARTS (vectype
)))
3464 return gimple_build (&stmts
, VIEW_CONVERT_EXPR
,
3465 truth_type_for (vectype
), pred
);
3468 /* Return a pointer to the address in a contiguous load or store,
3469 given that each memory vector has type VECTYPE. Add any new
3470 statements to STMTS. */
3472 gimple_folder::fold_contiguous_base (gimple_seq
&stmts
, tree vectype
)
3474 tree base
= gimple_call_arg (call
, 1);
3475 if (mode_suffix_id
== MODE_vnum
)
3477 tree offset
= gimple_call_arg (call
, 2);
3478 offset
= gimple_convert (&stmts
, sizetype
, offset
);
3479 offset
= gimple_build (&stmts
, MULT_EXPR
, sizetype
, offset
,
3480 TYPE_SIZE_UNIT (vectype
));
3481 base
= gimple_build (&stmts
, POINTER_PLUS_EXPR
, TREE_TYPE (base
),
3487 /* Return the alignment and TBAA argument to an internal load or store
3488 function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
3489 memory elements of type TYPE. */
3491 gimple_folder::load_store_cookie (tree type
)
3493 return build_int_cst (build_pointer_type (type
), TYPE_ALIGN (type
));
3496 /* Fold the call to a call to INSTANCE, with the same arguments. */
3498 gimple_folder::redirect_call (const function_instance
&instance
)
3500 registered_function
*rfn
3501 = function_table
->find_with_hash (instance
, instance
.hash ());
3505 gimple_call_set_fndecl (call
, rfn
->decl
);
3509 /* Redirect _z and _m calls to _x functions if the predicate is all-true.
3510 This allows us to use unpredicated instructions, where available. */
3512 gimple_folder::redirect_pred_x ()
3514 if (pred
!= PRED_z
&& pred
!= PRED_m
)
3517 if (gimple_call_num_args (call
) < 2)
3520 tree lhs_type
= TREE_TYPE (TREE_TYPE (fndecl
));
3521 tree arg0_type
= type_argument_type (TREE_TYPE (fndecl
), 1);
3522 tree arg1_type
= type_argument_type (TREE_TYPE (fndecl
), 2);
3523 if (!VECTOR_TYPE_P (lhs_type
)
3524 || !VECTOR_TYPE_P (arg0_type
)
3525 || !VECTOR_TYPE_P (arg1_type
))
3528 auto lhs_step
= element_precision (lhs_type
);
3529 auto rhs_step
= element_precision (arg1_type
);
3530 auto step
= MAX (lhs_step
, rhs_step
);
3531 if (!multiple_p (step
, BITS_PER_UNIT
)
3532 || !is_ptrue (gimple_call_arg (call
, 0), step
/ BITS_PER_UNIT
))
3535 function_instance
instance (*this);
3536 instance
.pred
= PRED_x
;
3537 return redirect_call (instance
);
3540 /* Fold the call to constant VAL. */
3542 gimple_folder::fold_to_cstu (poly_uint64 val
)
3544 return gimple_build_assign (lhs
, build_int_cstu (TREE_TYPE (lhs
), val
));
3547 /* Fold the call to a PTRUE, taking the element size from type suffix 0. */
3549 gimple_folder::fold_to_ptrue ()
3551 tree svbool_type
= TREE_TYPE (lhs
);
3552 tree bool_type
= TREE_TYPE (svbool_type
);
3553 unsigned int element_bytes
= type_suffix (0).element_bytes
;
3555 /* The return type is svbool_t for all type suffixes, thus for b8 we
3556 want { 1, 1, 1, 1, ... }, for b16 we want { 1, 0, 1, 0, ... }, etc. */
3557 tree_vector_builder
builder (svbool_type
, element_bytes
, 1);
3558 builder
.quick_push (build_all_ones_cst (bool_type
));
3559 for (unsigned int i
= 1; i
< element_bytes
; ++i
)
3560 builder
.quick_push (build_zero_cst (bool_type
));
3561 return gimple_build_assign (lhs
, builder
.build ());
3564 /* Fold the call to a PFALSE. */
3566 gimple_folder::fold_to_pfalse ()
3568 return gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
3571 /* Fold an operation to a constant predicate in which the first VL
3572 elements are set and the rest are clear. Take the element size
3573 from type suffix 0. */
3575 gimple_folder::fold_to_vl_pred (unsigned int vl
)
3577 tree vectype
= TREE_TYPE (lhs
);
3578 tree element_type
= TREE_TYPE (vectype
);
3579 tree minus_one
= build_all_ones_cst (element_type
);
3580 tree zero
= build_zero_cst (element_type
);
3581 unsigned int element_bytes
= type_suffix (0).element_bytes
;
3583 /* Construct COUNT elements that contain the ptrue followed by
3584 a repeating sequence of COUNT elements. */
3585 unsigned int count
= constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype
));
3586 gcc_assert (vl
* element_bytes
<= count
);
3587 tree_vector_builder
builder (vectype
, count
, 2);
3588 for (unsigned int i
= 0; i
< count
* 2; ++i
)
3590 bool bit
= (i
& (element_bytes
- 1)) == 0 && i
< vl
* element_bytes
;
3591 builder
.quick_push (bit
? minus_one
: zero
);
3593 return gimple_build_assign (lhs
, builder
.build ());
3596 /* Try to fold the call. Return the new statement on success and null
3599 gimple_folder::fold ()
3601 /* Don't fold anything when SVE is disabled; emit an error during
3602 expansion instead. */
3606 /* Punt if the function has a return type and no result location is
3607 provided. The attributes should allow target-independent code to
3608 remove the calls if appropriate. */
3609 if (!lhs
&& TREE_TYPE (gimple_call_fntype (call
)) != void_type_node
)
3612 /* First try some simplifications that are common to many functions. */
3613 if (auto *call
= redirect_pred_x ())
3616 return base
->fold (*this);
3619 function_expander::function_expander (const function_instance
&instance
,
3620 tree fndecl
, tree call_expr_in
,
3621 rtx possible_target_in
)
3622 : function_call_info (EXPR_LOCATION (call_expr_in
), instance
, fndecl
),
3623 call_expr (call_expr_in
), possible_target (possible_target_in
)
3627 /* Return the handler of direct optab OP for type suffix SUFFIX_I. */
3629 function_expander::direct_optab_handler (optab op
, unsigned int suffix_i
)
3631 return ::direct_optab_handler (op
, tuple_mode (suffix_i
));
3634 /* Choose between signed and unsigned direct optabs SIGNED_OP and
3635 UNSIGNED_OP based on the signedness of type suffix SUFFIX_I, then
3636 pick the appropriate optab handler for the mode. Use MODE as the
3637 mode if given, otherwise use the mode of type suffix SUFFIX_I. */
3639 function_expander::direct_optab_handler_for_sign (optab signed_op
,
3641 unsigned int suffix_i
,
3644 if (mode
== VOIDmode
)
3645 mode
= vector_mode (suffix_i
);
3646 optab op
= type_suffix (suffix_i
).unsigned_p
? unsigned_op
: signed_op
;
3647 return ::direct_optab_handler (op
, mode
);
3650 /* Return true if X overlaps any input. */
3652 function_expander::overlaps_input_p (rtx x
)
3654 for (unsigned int i
= 0; i
< args
.length (); ++i
)
3655 if (reg_overlap_mentioned_p (x
, args
[i
]))
3660 /* Convert ptr_mode value X to Pmode. */
3662 function_expander::convert_to_pmode (rtx x
)
3664 if (ptr_mode
== SImode
)
3665 x
= simplify_gen_unary (ZERO_EXTEND
, DImode
, x
, SImode
);
3669 /* Return the base address for a contiguous load or store function.
3670 MEM_MODE is the mode of the addressed memory, BASE_ARGNO is
3671 the index of the base argument, and VNUM_ARGNO is the index of
3672 the vnum offset argument (if any). VL_ISA_MODE is AARCH64_FL_SM_ON
3673 if the vnum argument is a factor of the SME vector length, 0 if it
3674 is a factor of the current prevailing vector length. */
3676 function_expander::get_contiguous_base (machine_mode mem_mode
,
3677 unsigned int base_argno
,
3678 unsigned int vnum_argno
,
3679 aarch64_feature_flags vl_isa_mode
)
3681 rtx base
= convert_to_pmode (args
[base_argno
]);
3682 if (mode_suffix_id
== MODE_vnum
)
3684 rtx vnum
= args
[vnum_argno
];
3685 if (vnum
!= const0_rtx
)
3687 /* Use the size of the memory mode for extending loads and truncating
3688 stores. Use the size of a full vector for non-extending loads
3689 and non-truncating stores (including svld[234] and svst[234]). */
3690 poly_int64 size
= ordered_min (GET_MODE_SIZE (mem_mode
),
3691 BYTES_PER_SVE_VECTOR
);
3693 if ((vl_isa_mode
& AARCH64_FL_SM_ON
)
3694 && !TARGET_STREAMING
3695 && !size
.is_constant ())
3697 gcc_assert (known_eq (size
, BYTES_PER_SVE_VECTOR
));
3698 if (CONST_INT_P (vnum
) && IN_RANGE (INTVAL (vnum
), -32, 31))
3699 offset
= aarch64_sme_vq_immediate (Pmode
, INTVAL (vnum
) * 16,
3703 offset
= aarch64_sme_vq_immediate (Pmode
, 16,
3705 offset
= simplify_gen_binary (MULT
, Pmode
, vnum
, offset
);
3710 offset
= gen_int_mode (size
, Pmode
);
3711 offset
= simplify_gen_binary (MULT
, Pmode
, vnum
, offset
);
3713 base
= simplify_gen_binary (PLUS
, Pmode
, base
, offset
);
3719 /* For a function that does the equivalent of:
3721 OUTPUT = COND ? FN (INPUTS) : FALLBACK;
3723 return the value of FALLBACK.
3725 MODE is the mode of OUTPUT. NOPS is the number of operands in INPUTS.
3726 MERGE_ARGNO is the argument that provides FALLBACK for _m functions,
3727 or DEFAULT_MERGE_ARGNO if we should apply the usual rules.
3729 ARGNO is the caller's index into args. If the returned value is
3730 argument 0 (as for unary _m operations), increment ARGNO past the
3731 returned argument. */
3733 function_expander::get_fallback_value (machine_mode mode
, unsigned int nops
,
3734 unsigned int merge_argno
,
3735 unsigned int &argno
)
3738 return CONST0_RTX (mode
);
3740 gcc_assert (pred
== PRED_m
|| pred
== PRED_x
);
3741 if (merge_argno
== DEFAULT_MERGE_ARGNO
)
3742 merge_argno
= shape
->has_merge_argument_p (*this, nops
) ? 0 : 1;
3744 if (merge_argno
== 0)
3745 return args
[argno
++];
3747 return args
[merge_argno
];
3750 /* Return a REG rtx that can be used for the result of the function,
3751 using the preferred target if suitable. */
3753 function_expander::get_reg_target ()
3755 machine_mode target_mode
= result_mode ();
3756 if (!possible_target
|| GET_MODE (possible_target
) != target_mode
)
3757 possible_target
= gen_reg_rtx (target_mode
);
3758 return possible_target
;
3761 /* As for get_reg_target, but make sure that the returned REG does not
3762 overlap any inputs. */
3764 function_expander::get_nonoverlapping_reg_target ()
3766 if (possible_target
&& overlaps_input_p (possible_target
))
3767 possible_target
= NULL_RTX
;
3768 return get_reg_target ();
3771 /* Add an output operand to the instruction we're building, which has
3772 code ICODE. Bind the output to the preferred target rtx if possible. */
3774 function_expander::add_output_operand (insn_code icode
)
3776 unsigned int opno
= m_ops
.length ();
3777 machine_mode mode
= insn_data
[icode
].operand
[opno
].mode
;
3778 m_ops
.safe_grow (opno
+ 1, true);
3779 create_output_operand (&m_ops
.last (), possible_target
, mode
);
3782 /* Add an input operand to the instruction we're building, which has
3783 code ICODE. Calculate the value of the operand as follows:
3785 - If the operand is a vector and X is not, broadcast X to fill a
3786 vector of the appropriate mode.
3788 - Otherwise, if the operand is a predicate, coerce X to have the
3789 mode that the instruction expects. In this case X is known to be
3790 VNx16BImode (the mode of svbool_t).
3792 - Otherwise use X directly. The expand machinery checks that X has
3793 the right mode for the instruction. */
3795 function_expander::add_input_operand (insn_code icode
, rtx x
)
3797 unsigned int opno
= m_ops
.length ();
3798 const insn_operand_data
&operand
= insn_data
[icode
].operand
[opno
];
3799 machine_mode mode
= operand
.mode
;
3800 if (mode
== VOIDmode
)
3802 /* The only allowable uses of VOIDmode are:
3804 - the wildcard aarch64_any_register_operand, which is used
3805 to avoid combinatorial explosion in the reinterpret patterns
3807 - pmode_register_operand, which always has mode Pmode. */
3808 if (operand
.predicate
== aarch64_any_register_operand
)
3809 mode
= GET_MODE (x
);
3810 else if (operand
.predicate
== pmode_register_operand
)
3815 else if (!VECTOR_MODE_P (GET_MODE (x
)) && VECTOR_MODE_P (mode
))
3816 x
= expand_vector_broadcast (mode
, x
);
3817 else if (GET_MODE_CLASS (mode
) == MODE_VECTOR_BOOL
)
3819 gcc_assert (GET_MODE (x
) == VNx16BImode
);
3820 x
= gen_lowpart (mode
, x
);
3822 m_ops
.safe_grow (m_ops
.length () + 1, true);
3823 create_input_operand (&m_ops
.last (), x
, mode
);
3826 /* Add an integer operand with value X to the instruction. */
3828 function_expander::add_integer_operand (poly_int64 x
)
3830 m_ops
.safe_grow (m_ops
.length () + 1, true);
3831 create_integer_operand (&m_ops
.last (), x
);
3834 /* Add a memory operand with mode MODE and address ADDR. */
3836 function_expander::add_mem_operand (machine_mode mode
, rtx addr
)
3838 /* Exception for OImode for the ld1ro intrinsics.
3839 They act on 256 bit octaword data, and it's just easier to use a scalar
3840 mode to represent that than add a new vector mode solely for the purpose
3841 of this intrinsic. */
3842 gcc_assert (VECTOR_MODE_P (mode
) || mode
== OImode
);
3843 rtx mem
= gen_rtx_MEM (mode
, memory_address (mode
, addr
));
3844 /* The memory is only guaranteed to be element-aligned. */
3845 set_mem_align (mem
, GET_MODE_ALIGNMENT (GET_MODE_INNER (mode
)));
3846 add_fixed_operand (mem
);
3849 /* Add an address operand with value X. The static operand data says
3850 what mode and form the address must have. */
3852 function_expander::add_address_operand (rtx x
)
3854 m_ops
.safe_grow (m_ops
.length () + 1, true);
3855 create_address_operand (&m_ops
.last (), x
);
3858 /* Add an operand that must be X. The only way of legitimizing an
3859 invalid X is to reload the address of a MEM. */
3861 function_expander::add_fixed_operand (rtx x
)
3863 m_ops
.safe_grow (m_ops
.length () + 1, true);
3864 create_fixed_operand (&m_ops
.last (), x
);
3867 /* Generate instruction ICODE, given that its operands have already
3868 been added to M_OPS. Return the value of the first operand. */
3870 function_expander::generate_insn (insn_code icode
)
3872 expand_insn (icode
, m_ops
.length (), m_ops
.address ());
3873 return function_returns_void_p () ? const0_rtx
: m_ops
[0].value
;
3876 /* Convert the arguments to a gather/scatter function into the
3877 associated md operands. Argument ARGNO is the scalar or vector base and
3878 argument ARGNO + 1 is the scalar or vector displacement (if applicable).
3879 The md pattern expects:
3882 - a vector displacement
3884 If SCALED_P is true, it also expects:
3886 - a const_int that is 1 if the displacement is zero-extended from 32 bits
3887 - a scaling multiplier (1 for bytes, 2 for .h indices, etc.).
3889 If SCALED_P is false, the displacement is implicitly zero-extended
3890 and the scaling multiplier is implicitly 1. */
3892 function_expander::prepare_gather_address_operands (unsigned int argno
,
3895 machine_mode mem_mode
= memory_vector_mode ();
3896 tree vector_type
= base_vector_type ();
3897 units_index units
= displacement_units ();
3899 if (units
== UNITS_none
)
3901 /* Vector base, no displacement. Convert to an integer zero base
3902 and a vector byte offset. */
3903 args
.quick_insert (argno
, const0_rtx
);
3904 units
= UNITS_bytes
;
3906 else if (vector_type
)
3908 /* Vector base, scalar displacement. Convert to a scalar base and
3909 a vector byte offset. */
3910 std::swap (args
[argno
], args
[argno
+ 1]);
3911 if (units
== UNITS_elements
)
3916 /* Scalar base, vector displacement. This is the order that the md
3918 args
[argno
] = convert_to_pmode (args
[argno
]);
3919 vector_type
= displacement_vector_type ();
3920 if (units
== UNITS_elements
&& !scaled_p
)
3921 shift_idx
= argno
+ 1;
3923 tree scalar_displacement_type
= TREE_TYPE (vector_type
);
3927 machine_mode arg_mode
= GET_MODE (args
[shift_idx
]);
3928 if (arg_mode
== VOIDmode
)
3930 unsigned int elt_bytes
= GET_MODE_UNIT_SIZE (mem_mode
);
3931 rtx shift
= gen_int_mode (exact_log2 (elt_bytes
), DImode
);
3932 args
[shift_idx
] = simplify_gen_binary (ASHIFT
, arg_mode
,
3933 args
[shift_idx
], shift
);
3934 units
= UNITS_bytes
;
3937 bool uxtw_p
= (TYPE_PRECISION (scalar_displacement_type
) == 64
3938 || TYPE_UNSIGNED (scalar_displacement_type
));
3939 unsigned int scale
= (units
== UNITS_bytes
3940 ? 1 : GET_MODE_UNIT_SIZE (mem_mode
));
3944 args
.quick_insert (argno
+ 2, GEN_INT (uxtw_p
));
3945 args
.quick_insert (argno
+ 3, GEN_INT (scale
));
3948 gcc_assert (uxtw_p
&& scale
== 1);
3951 /* The final argument is an immediate svprfop value. Add two fake arguments
3952 to represent the rw and locality operands of a PREFETCH rtx. */
3954 function_expander::prepare_prefetch_operands ()
3956 unsigned int prfop
= INTVAL (args
.last ());
3957 /* Bit 3 of the prfop selects stores over loads. */
3958 args
.quick_push (GEN_INT ((prfop
& 8) != 0));
3959 /* Bits 1 and 2 specify the locality; 0-based for svprfop but
3960 1-based for PREFETCH. */
3961 args
.quick_push (GEN_INT (((prfop
>> 1) & 3) + 1));
3964 /* Add a dummy argument to indicate whether predicate argument ARGNO
3965 is all-true when interpreted in mode PRED_MODE. The hint goes
3966 immediately after ARGNO. */
3968 function_expander::add_ptrue_hint (unsigned int argno
, machine_mode pred_mode
)
3970 rtx pred
= gen_lowpart (pred_mode
, args
[argno
]);
3971 int hint
= (pred
== CONSTM1_RTX (pred_mode
)
3972 ? SVE_KNOWN_PTRUE
: SVE_MAYBE_NOT_PTRUE
);
3973 args
.quick_insert (argno
+ 1, gen_int_mode (hint
, SImode
));
3976 /* Rotate inputs args[START:END] one position to the left, so that
3977 args[START] becomes args[END - 1]. */
3979 function_expander::rotate_inputs_left (unsigned int start
, unsigned int end
)
3981 rtx new_last
= args
[start
];
3982 for (unsigned int i
= start
; i
< end
- 1; ++i
)
3983 args
[i
] = args
[i
+ 1];
3984 args
[end
- 1] = new_last
;
3987 /* Return true if the negation of argument ARGNO can be folded away,
3988 replacing it with the negated value if so. MODE is the associated
3989 vector mode, but the argument could be a single element. The main
3990 case this handles is constant arguments. */
3992 function_expander::try_negating_argument (unsigned int argno
,
3995 rtx x
= args
[argno
];
3996 if (!VECTOR_MODE_P (GET_MODE (x
)))
3997 mode
= GET_MODE_INNER (mode
);
3999 x
= simplify_unary_operation (NEG
, mode
, x
, mode
);
4007 /* Implement the call using instruction ICODE, with a 1:1 mapping between
4008 arguments and input operands. */
4010 function_expander::use_exact_insn (insn_code icode
)
4012 unsigned int nops
= insn_data
[icode
].n_operands
;
4013 if (!function_returns_void_p ())
4015 add_output_operand (icode
);
4018 for (unsigned int i
= 0; i
< nops
; ++i
)
4019 add_input_operand (icode
, args
[i
]);
4020 return generate_insn (icode
);
4023 /* Implement the call using instruction ICODE, which does not use a
4024 governing predicate. We must therefore drop the GP from an _x call. */
4026 function_expander::use_unpred_insn (insn_code icode
)
4028 /* We can't drop the predicate for _z and _m. */
4029 gcc_assert (pred
== PRED_x
|| pred
== PRED_none
);
4030 /* Discount the output operand. */
4031 unsigned int nops
= insn_data
[icode
].n_operands
- 1;
4032 /* Drop the predicate argument in the case of _x predication. */
4033 unsigned int bias
= (pred
== PRED_x
? 1 : 0);
4036 add_output_operand (icode
);
4037 for (; i
< nops
; ++i
)
4038 add_input_operand (icode
, args
[i
+ bias
]);
4040 return generate_insn (icode
);
4043 /* Implement the call using instruction ICODE, which is a predicated
4044 operation that returns arbitrary values for inactive lanes. */
4046 function_expander::use_pred_x_insn (insn_code icode
)
4048 /* At present we never need to handle PRED_none, which would involve
4049 creating a new predicate rather than using one supplied by the user. */
4050 gcc_assert (pred
== PRED_x
);
4051 /* Discount the output operand. */
4052 unsigned int nops
= args
.length () - 1;
4054 bool has_float_operand_p
= FLOAT_MODE_P (insn_data
[icode
].operand
[0].mode
);
4056 /* Add the normal operands. */
4057 add_output_operand (icode
);
4058 add_input_operand (icode
, args
[0]);
4059 for (unsigned int i
= 0; i
< nops
; ++i
)
4061 add_input_operand (icode
, args
[i
+ 1]);
4062 if (FLOAT_MODE_P (GET_MODE (args
[i
+ 1])))
4063 has_float_operand_p
= true;
4066 if (has_float_operand_p
4067 && insn_data
[icode
].n_operands
> (int) nops
+ 2)
4069 /* Add a flag that indicates whether unpredicated instructions
4071 rtx pred
= m_ops
[1].value
;
4072 if (flag_trapping_math
&& pred
!= CONST1_RTX (GET_MODE (pred
)))
4073 add_integer_operand (SVE_STRICT_GP
);
4075 add_integer_operand (SVE_RELAXED_GP
);
4078 return generate_insn (icode
);
4081 /* Implement the call using instruction ICODE, which does the equivalent of:
4083 OUTPUT = COND ? FN (INPUTS) : FALLBACK;
4085 The instruction operands are in the order above: OUTPUT, COND, INPUTS
4086 and FALLBACK. MERGE_ARGNO is the argument that provides FALLBACK for _m
4087 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
4089 function_expander::use_cond_insn (insn_code icode
, unsigned int merge_argno
)
4091 /* At present we never need to handle PRED_none, which would involve
4092 creating a new predicate rather than using one supplied by the user. */
4093 gcc_assert (pred
!= PRED_none
);
4094 /* Discount the output, predicate and fallback value. */
4095 unsigned int nops
= insn_data
[icode
].n_operands
- 3;
4096 machine_mode mode
= insn_data
[icode
].operand
[0].mode
;
4098 unsigned int opno
= 0;
4099 rtx fallback_arg
= get_fallback_value (mode
, nops
, merge_argno
, opno
);
4100 rtx pred
= args
[opno
++];
4102 add_output_operand (icode
);
4103 add_input_operand (icode
, pred
);
4104 for (unsigned int i
= 0; i
< nops
; ++i
)
4105 add_input_operand (icode
, args
[opno
+ i
]);
4106 add_input_operand (icode
, fallback_arg
);
4107 return generate_insn (icode
);
4110 /* Implement the call using instruction ICODE, which is a select-like
4111 operation with the following operands:
4118 MERGE_ARGNO is the argument that provides the "false" value for _m
4119 functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
4121 function_expander::use_vcond_mask_insn (insn_code icode
,
4122 unsigned int merge_argno
)
4124 machine_mode mode
= vector_mode (0);
4126 unsigned int opno
= 0;
4127 rtx false_arg
= get_fallback_value (mode
, 1, merge_argno
, opno
);
4128 rtx pred_arg
= args
[opno
++];
4129 rtx true_arg
= args
[opno
++];
4131 add_output_operand (icode
);
4132 add_input_operand (icode
, true_arg
);
4133 add_input_operand (icode
, false_arg
);
4134 add_input_operand (icode
, pred_arg
);
4135 return generate_insn (icode
);
4138 /* Implement the call using instruction ICODE, which loads memory operand 1
4139 into register operand 0 under the control of predicate operand 2.
4140 Extending loads have a further predicate (operand 3) that nominally
4141 controls the extension. */
4143 function_expander::use_contiguous_load_insn (insn_code icode
)
4145 machine_mode mem_mode
= memory_vector_mode ();
4147 add_output_operand (icode
);
4148 add_mem_operand (mem_mode
, get_contiguous_base (mem_mode
));
4149 add_input_operand (icode
, args
[0]);
4150 if (GET_MODE_UNIT_BITSIZE (mem_mode
) < type_suffix (0).element_bits
)
4151 add_input_operand (icode
, CONSTM1_RTX (VNx16BImode
));
4152 return generate_insn (icode
);
4155 /* Implement the call using instruction ICODE, which prefetches from
4156 address operand 1 under the control of predicate operand 0.
4157 Operands 2, 3 and 4 respectively specify the svprfop value,
4158 the PREFETCH rw flag and the PREFETCH locality. */
4160 function_expander::use_contiguous_prefetch_insn (insn_code icode
)
4162 add_input_operand (icode
, args
[0]);
4163 add_address_operand (get_contiguous_base (VNx16QImode
));
4164 for (unsigned int i
= args
.length () - 3; i
< args
.length (); ++i
)
4165 add_input_operand (icode
, args
[i
]);
4166 return generate_insn (icode
);
4169 /* Implement the call using instruction ICODE, which stores register operand 1
4170 into memory operand 0 under the control of predicate operand 2. */
4172 function_expander::use_contiguous_store_insn (insn_code icode
)
4174 machine_mode mem_mode
= memory_vector_mode ();
4176 add_mem_operand (mem_mode
, get_contiguous_base (mem_mode
));
4177 add_input_operand (icode
, args
.last ());
4178 add_input_operand (icode
, args
[0]);
4179 return generate_insn (icode
);
4182 /* Implement the call using one of the following strategies, chosen in order:
4184 (1) "aarch64_pred_<optab><mode>_z" for PRED_z predicate functions
4186 (2) "aarch64_pred_<optab><mode>" for PRED_x functions
4188 (3) a normal unpredicated optab for PRED_none and PRED_x functions,
4189 dropping the predicate in the latter case
4191 (4) an unpredicated "aarch64_sve_<code_optab><mode>" for PRED_none and
4192 PRED_x functions, again dropping the predicate for PRED_x
4194 (5) "cond_<optab><mode>" otherwise
4196 where <optab> corresponds to:
4198 - CODE_FOR_SINT for signed integers
4199 - CODE_FOR_UINT for unsigned integers
4200 - UNSPEC_FOR_COND_FP for predicated floating-point
4201 - UNSPEC_FOR_UNCOND_FP for unpredicated floating-point
4203 and where <code_optab> is like <optab>, but uses CODE_FOR_SINT instead
4204 of UNSPEC_FOR_FP for floating-point values.
4206 MERGE_ARGNO is the argument that provides the values of inactive lanes for
4207 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
4209 function_expander::map_to_rtx_codes (rtx_code code_for_sint
,
4210 rtx_code code_for_uint
,
4211 int unspec_for_cond_fp
,
4212 int unspec_for_uncond_fp
,
4213 unsigned int merge_argno
)
4215 machine_mode mode
= tuple_mode (0);
4216 rtx_code code
= (type_suffix (0).unsigned_p
? code_for_uint
: code_for_sint
);
4219 if (mode_suffix_id
== MODE_single
)
4221 gcc_assert (pred
== PRED_none
);
4222 if (type_suffix (0).integer_p
)
4223 icode
= code_for_aarch64_sve_single (code
, mode
);
4225 icode
= code_for_aarch64_sve_single (unspec_for_uncond_fp
, mode
);
4226 return use_exact_insn (icode
);
4229 /* Handle predicate logic operations, which always use _z predication. */
4230 if (type_suffix (0).tclass
== TYPE_bool
)
4232 gcc_assert (pred
== PRED_z
&& code_for_uint
== code_for_sint
);
4233 return use_exact_insn (code_for_aarch64_pred_z (code
, mode
));
4236 /* First try using UNSPEC_PRED_X patterns for _x predication,
4240 if (type_suffix (0).integer_p
)
4241 icode
= maybe_code_for_aarch64_pred (code
, mode
);
4243 icode
= maybe_code_for_aarch64_pred (unspec_for_cond_fp
, mode
);
4244 if (icode
!= CODE_FOR_nothing
)
4245 return use_pred_x_insn (icode
);
4248 /* Otherwise expand PRED_none and PRED_x operations without a predicate.
4249 Floating-point operations conventionally use the signed rtx code. */
4250 if (pred
== PRED_none
|| pred
== PRED_x
)
4252 if (type_suffix (0).float_p
&& unspec_for_uncond_fp
>= 0)
4253 icode
= maybe_code_for_aarch64_sve (unspec_for_uncond_fp
, mode
);
4255 icode
= direct_optab_handler (code_to_optab (code
), 0);
4256 if (icode
== CODE_FOR_nothing
)
4257 icode
= code_for_aarch64_sve (code
, mode
);
4258 return use_unpred_insn (icode
);
4261 /* Don't use cond_*_optabs here, since not all codes have one yet. */
4262 if (type_suffix (0).integer_p
)
4263 icode
= code_for_cond (code
, mode
);
4265 icode
= code_for_cond (unspec_for_cond_fp
, mode
);
4266 return use_cond_insn (icode
, merge_argno
);
4269 /* Implement the call using one of the following strategies, chosen in order:
4271 (1) "aarch64_pred_<optab><mode>" for PRED_x functions; this is a
4274 (2) "aarch64_sve_<optab><mode>" for PRED_none and PRED_x functions;
4275 this is an unpredicated pattern
4277 (3) "cond_<optab><mode>" otherwise
4279 where <optab> corresponds to:
4281 - UNSPEC_FOR_SINT for signed integers
4282 - UNSPEC_FOR_UINT for unsigned integers
4283 - UNSPEC_FOR_FP for floating-point values
4285 MERGE_ARGNO is the argument that provides the values of inactive lanes for
4286 _m functions, or DEFAULT_MERGE_ARGNO if we should apply the usual rules. */
4288 function_expander::map_to_unspecs (int unspec_for_sint
, int unspec_for_uint
,
4289 int unspec_for_fp
, unsigned int merge_argno
)
4291 machine_mode mode
= tuple_mode (0);
4292 int unspec
= (!type_suffix (0).integer_p
? unspec_for_fp
4293 : type_suffix (0).unsigned_p
? unspec_for_uint
4296 if (mode_suffix_id
== MODE_single
)
4298 gcc_assert (pred
== PRED_none
);
4299 return use_exact_insn (code_for_aarch64_sve_single (unspec
, mode
));
4304 insn_code icode
= maybe_code_for_aarch64_pred (unspec
, mode
);
4305 if (icode
!= CODE_FOR_nothing
)
4306 return use_pred_x_insn (icode
);
4309 if (pred
== PRED_none
|| pred
== PRED_x
)
4311 insn_code icode
= maybe_code_for_aarch64_sve (unspec
, mode
);
4312 if (icode
!= CODE_FOR_nothing
)
4313 return use_unpred_insn (icode
);
4316 insn_code icode
= code_for_cond (unspec
, vector_mode (0));
4317 return use_cond_insn (icode
, merge_argno
);
4320 /* Expand the call and return its lhs. */
4322 function_expander::expand ()
4324 unsigned int nargs
= call_expr_nargs (call_expr
);
4325 args
.reserve (nargs
);
4326 for (unsigned int i
= 0; i
< nargs
; ++i
)
4327 args
.quick_push (expand_normal (CALL_EXPR_ARG (call_expr
, i
)));
4329 return base
->expand (*this);
4332 /* Return a structure type that contains a single field of type FIELD_TYPE.
4333 The field is called __val, but that's an internal detail rather than
4334 an exposed part of the API. */
4336 wrap_type_in_struct (tree field_type
)
4338 tree field
= build_decl (input_location
, FIELD_DECL
,
4339 get_identifier ("__val"), field_type
);
4340 tree struct_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
4341 DECL_FIELD_CONTEXT (field
) = struct_type
;
4342 TYPE_FIELDS (struct_type
) = field
;
4343 make_type_sizeless (struct_type
);
4344 layout_type (struct_type
);
4348 /* Register a built-in TYPE_DECL called NAME for TYPE. This is used/needed
4349 when TYPE is a structure type. */
4351 register_type_decl (tree type
, const char *name
)
4353 tree decl
= build_decl (input_location
, TYPE_DECL
,
4354 get_identifier (name
), type
);
4355 TYPE_NAME (type
) = decl
;
4356 TYPE_STUB_DECL (type
) = decl
;
4357 lang_hooks
.decls
.pushdecl (decl
);
4358 /* ??? Undo the effect of set_underlying_type for C. The C frontend
4359 doesn't recognize DECL as a built-in because (as intended) the decl has
4360 a real location instead of BUILTINS_LOCATION. The frontend therefore
4361 treats the decl like a normal C "typedef struct foo foo;", expecting
4362 the type for tag "struct foo" to have a dummy unnamed TYPE_DECL instead
4363 of the named one we attached above. It then sets DECL_ORIGINAL_TYPE
4364 on the supposedly unnamed decl, creating a circularity that upsets
4367 We don't want to follow the normal C model and create "struct foo"
4368 tags for tuple types since (a) the types are supposed to be opaque
4369 and (b) they couldn't be defined as a real struct anyway. Treating
4370 the TYPE_DECLs as "typedef struct foo foo;" without creating
4371 "struct foo" would lead to confusing error messages. */
4372 DECL_ORIGINAL_TYPE (decl
) = NULL_TREE
;
4375 /* Register the built-in SVE ABI types, such as __SVBool_t. */
4377 register_builtin_types ()
4379 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
4380 scalar_types[VECTOR_TYPE_ ## ACLE_NAME] = SCALAR_TYPE;
4381 #include "aarch64-sve-builtins.def"
4383 for (unsigned int i
= 0; i
< NUM_VECTOR_TYPES
; ++i
)
4386 unsigned int num_zr
= 0, num_pr
= 0;
4387 if (vector_type_index (i
) == VECTOR_TYPE_svcount_t
)
4389 vectype
= abi_vector_types
[VECTOR_TYPE_svbool_t
];
4390 vectype
= wrap_type_in_struct (vectype
);
4395 tree eltype
= scalar_types
[i
];
4396 if (eltype
== boolean_type_node
)
4398 vectype
= build_truth_vector_type_for_mode (BYTES_PER_SVE_VECTOR
,
4404 scalar_mode elmode
= SCALAR_TYPE_MODE (eltype
);
4405 unsigned int elbytes
= GET_MODE_SIZE (elmode
);
4406 poly_uint64 nunits
= exact_div (BYTES_PER_SVE_VECTOR
, elbytes
);
4408 = aarch64_sve_data_mode (elmode
, nunits
).require ();
4409 vectype
= build_vector_type_for_mode (eltype
, mode
);
4410 auto size
= wi::to_poly_offset (TYPE_SIZE (vectype
));
4411 gcc_assert (VECTOR_MODE_P (TYPE_MODE (vectype
))
4412 && TYPE_MODE (vectype
) == mode
4413 && TYPE_MODE_RAW (vectype
) == mode
4414 && TYPE_ALIGN (vectype
) == 128
4415 && known_eq (size
, BITS_PER_SVE_VECTOR
));
4418 vectype
= build_distinct_type_copy (vectype
);
4419 gcc_assert (vectype
== TYPE_MAIN_VARIANT (vectype
));
4420 SET_TYPE_STRUCTURAL_EQUALITY (vectype
);
4421 TYPE_ARTIFICIAL (vectype
) = 1;
4422 TYPE_INDIVISIBLE_P (vectype
) = 1;
4423 make_type_sizeless (vectype
);
4427 auto size
= wi::to_poly_offset (TYPE_SIZE (vectype
));
4428 gcc_assert (TYPE_MODE (vectype
) == VNx16BImode
4429 && TYPE_MODE (vectype
) == TYPE_MODE_RAW (vectype
)
4430 && TYPE_ALIGN (vectype
) == 16
4431 && known_eq (size
, BYTES_PER_SVE_VECTOR
));
4433 add_sve_type_attribute (vectype
, num_zr
, num_pr
,
4434 vector_types
[i
].mangled_name
,
4435 vector_types
[i
].acle_name
);
4436 abi_vector_types
[i
] = vectype
;
4437 if (TREE_CODE (vectype
) == RECORD_TYPE
)
4438 register_type_decl (vectype
, vector_types
[i
].abi_name
);
4440 lang_hooks
.types
.register_builtin_type (vectype
,
4441 vector_types
[i
].abi_name
);
4445 /* Initialize all compiler built-ins related to SVE that should be
4446 defined at start-up. */
4451 register_builtin_types ();
4454 handle_arm_sve_h (false);
4455 handle_arm_sme_h (false);
4456 handle_arm_neon_sve_bridge_h (false);
4460 /* Register vector type TYPE under its arm_sve.h name. */
4462 register_vector_type (vector_type_index type
)
4464 tree vectype
= abi_vector_types
[type
];
4465 tree id
= get_identifier (vector_types
[type
].acle_name
);
4466 tree decl
= build_decl (input_location
, TYPE_DECL
, id
, vectype
);
4467 decl
= lang_hooks
.decls
.pushdecl (decl
);
4469 /* Record the new ACLE type if pushdecl succeeded without error. Use
4470 the ABI type otherwise, so that the type we record at least has the
4471 right form, even if it doesn't have the right name. This should give
4472 better error recovery behavior than installing error_mark_node or
4473 installing an incorrect type. */
4475 && TREE_CODE (decl
) == TYPE_DECL
4476 && TREE_TYPE (decl
) != error_mark_node
4477 && TYPE_MAIN_VARIANT (TREE_TYPE (decl
)) == vectype
)
4478 vectype
= TREE_TYPE (decl
);
4479 acle_vector_types
[0][type
] = vectype
;
4482 /* Register the tuple type that contains NUM_VECTORS vectors of type TYPE. */
4484 register_tuple_type (unsigned int num_vectors
, vector_type_index type
)
4486 tree vector_type
= acle_vector_types
[0][type
];
4487 bool is_pred
= GET_MODE_CLASS (TYPE_MODE (vector_type
)) == MODE_VECTOR_BOOL
;
4489 /* Work out the structure name. */
4490 char buffer
[sizeof ("svbfloat16x4_t")];
4491 const char *vector_type_name
= vector_types
[type
].acle_name
;
4492 snprintf (buffer
, sizeof (buffer
), "%.*sx%d_t",
4493 (int) strlen (vector_type_name
) - 2, vector_type_name
,
4496 /* The contents of the type are opaque, so we can define them in any
4497 way that maps to the correct ABI type.
4499 Here we choose to use the same layout as for arm_neon.h, but with
4500 "__val" instead of "val":
4502 struct svfooxN_t { svfoo_t __val[N]; };
4504 (It wouldn't be possible to write that directly in C or C++ for
4505 sizeless types, but that's not a problem for this function.)
4507 Using arrays simplifies the handling of svget and svset for variable
4509 tree array_type
= build_array_type_nelts (vector_type
, num_vectors
);
4510 gcc_assert (VECTOR_MODE_P (TYPE_MODE (array_type
))
4511 && TYPE_MODE_RAW (array_type
) == TYPE_MODE (array_type
)
4512 && TYPE_ALIGN (array_type
) == (is_pred
? 16 : 128));
4514 tree tuple_type
= wrap_type_in_struct (array_type
);
4516 add_sve_type_attribute (tuple_type
, 0, num_vectors
, NULL
, buffer
);
4518 add_sve_type_attribute (tuple_type
, num_vectors
, 0, NULL
, buffer
);
4519 gcc_assert (VECTOR_MODE_P (TYPE_MODE (tuple_type
))
4520 && TYPE_MODE_RAW (tuple_type
) == TYPE_MODE (tuple_type
)
4521 && TYPE_ALIGN (tuple_type
) == TYPE_ALIGN (array_type
));
4523 register_type_decl (tuple_type
, buffer
);
4525 acle_vector_types
[num_vectors
- 1][type
] = tuple_type
;
4528 /* Register the svpattern enum. */
4530 register_svpattern ()
4532 auto_vec
<string_int_pair
, 32> values
;
4533 #define PUSH(UPPER, LOWER, VALUE) \
4534 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
4535 AARCH64_FOR_SVPATTERN (PUSH
)
4538 acle_svpattern
= lang_hooks
.types
.simulate_enum_decl (input_location
,
4539 "svpattern", &values
);
4542 /* Register the svprfop enum. */
4546 auto_vec
<string_int_pair
, 16> values
;
4547 #define PUSH(UPPER, LOWER, VALUE) \
4548 values.quick_push (string_int_pair ("SV_" #UPPER, VALUE));
4549 AARCH64_FOR_SVPRFOP (PUSH
)
4552 acle_svprfop
= lang_hooks
.types
.simulate_enum_decl (input_location
,
4553 "svprfop", &values
);
4556 /* Implement #pragma GCC aarch64 "arm_sve.h". */
4558 handle_arm_sve_h (bool function_nulls_p
)
4562 error ("duplicate definition of %qs", "arm_sve.h");
4568 /* Define the vector and tuple types. */
4569 for (unsigned int type_i
= 0; type_i
< NUM_VECTOR_TYPES
; ++type_i
)
4571 vector_type_index type
= vector_type_index (type_i
);
4572 register_vector_type (type
);
4573 if (type
!= VECTOR_TYPE_svcount_t
)
4574 for (unsigned int count
= 2; count
<= MAX_TUPLE_SIZE
; ++count
)
4575 if (type
!= VECTOR_TYPE_svbool_t
|| count
== 2)
4576 register_tuple_type (count
, type
);
4579 /* Define the enums. */
4580 register_svpattern ();
4581 register_svprfop ();
4583 /* Define the functions. */
4584 function_table
= new hash_table
<registered_function_hasher
> (1023);
4585 function_builder
builder (arm_sve_handle
, function_nulls_p
);
4586 for (unsigned int i
= 0; i
< ARRAY_SIZE (function_groups
); ++i
)
4587 builder
.register_function_group (function_groups
[i
]);
4590 /* Implement #pragma GCC aarch64 "arm_neon_sve_bridge.h". */
4592 handle_arm_neon_sve_bridge_h (bool function_nulls_p
)
4594 if (initial_indexes
[arm_sme_handle
] == 0)
4595 handle_arm_sme_h (true);
4597 /* Define the functions. */
4598 function_builder
builder (arm_neon_sve_handle
, function_nulls_p
);
4599 for (unsigned int i
= 0; i
< ARRAY_SIZE (neon_sve_function_groups
); ++i
)
4600 builder
.register_function_group (neon_sve_function_groups
[i
]);
4603 /* Return the function decl with SVE function subcode CODE, or error_mark_node
4604 if no such function exists. */
4606 builtin_decl (unsigned int code
, bool)
4608 if (code
>= vec_safe_length (registered_functions
))
4609 return error_mark_node
;
4610 return (*registered_functions
)[code
]->decl
;
4613 /* Implement #pragma GCC aarch64 "arm_sme.h". */
4615 handle_arm_sme_h (bool function_nulls_p
)
4617 if (!function_table
)
4619 error ("%qs defined without first defining %qs",
4620 "arm_sme.h", "arm_sve.h");
4626 function_builder
builder (arm_sme_handle
, function_nulls_p
);
4627 for (unsigned int i
= 0; i
< ARRAY_SIZE (sme_function_groups
); ++i
)
4628 builder
.register_function_group (sme_function_groups
[i
]);
4631 /* If we're implementing manual overloading, check whether the SVE
4632 function with subcode CODE is overloaded, and if so attempt to
4633 determine the corresponding non-overloaded function. The call
4634 occurs at location LOCATION and has the arguments given by ARGLIST.
4636 If the call is erroneous, report an appropriate error and return
4637 error_mark_node. Otherwise, if the function is overloaded, return
4638 the decl of the non-overloaded function. Return NULL_TREE otherwise,
4639 indicating that the call should be processed in the normal way. */
4641 resolve_overloaded_builtin (location_t location
, unsigned int code
,
4642 vec
<tree
, va_gc
> *arglist
)
4644 if (code
>= vec_safe_length (registered_functions
))
4647 registered_function
&rfn
= *(*registered_functions
)[code
];
4648 if (rfn
.overloaded_p
)
4649 return function_resolver (location
, rfn
.instance
, rfn
.decl
,
4650 *arglist
).resolve ();
4654 /* Perform any semantic checks needed for a call to the SVE function
4655 with subcode CODE, such as testing for integer constant expressions.
4656 The call occurs at location LOCATION and has NARGS arguments,
4657 given by ARGS. FNDECL is the original function decl, before
4658 overload resolution.
4660 Return true if the call is valid, otherwise report a suitable error. */
4662 check_builtin_call (location_t location
, vec
<location_t
>, unsigned int code
,
4663 tree fndecl
, unsigned int nargs
, tree
*args
)
4665 const registered_function
&rfn
= *(*registered_functions
)[code
];
4666 if (!aarch64_check_required_extensions (location
, rfn
.decl
,
4667 rfn
.required_extensions
))
4669 return function_checker (location
, rfn
.instance
, fndecl
,
4670 TREE_TYPE (rfn
.decl
), nargs
, args
).check ();
4673 /* Attempt to fold STMT, given that it's a call to the SVE function
4674 with subcode CODE. Return the new statement on success and null
4675 on failure. Insert any other new statements at GSI. */
4677 gimple_fold_builtin (unsigned int code
, gimple_stmt_iterator
*gsi
, gcall
*stmt
)
4679 registered_function
&rfn
= *(*registered_functions
)[code
];
4680 return gimple_folder (rfn
.instance
, rfn
.decl
, gsi
, stmt
).fold ();
4683 /* Expand a call to the SVE function with subcode CODE. EXP is the call
4684 expression and TARGET is the preferred location for the result.
4685 Return the value of the lhs. */
4687 expand_builtin (unsigned int code
, tree exp
, rtx target
)
4689 registered_function
&rfn
= *(*registered_functions
)[code
];
4690 if (!aarch64_check_required_extensions (EXPR_LOCATION (exp
), rfn
.decl
,
4691 rfn
.required_extensions
))
4693 return function_expander (rfn
.instance
, rfn
.decl
, exp
, target
).expand ();
4696 /* If TYPE is a built-in type defined by the SVE ABI, return the mangled name,
4697 otherwise return NULL. */
4699 mangle_builtin_type (const_tree type
)
4701 /* ??? The C++ frontend normally strips qualifiers and attributes before
4702 calling this hook, adding separate mangling for attributes that affect
4703 type identity. Fortunately the type copy will have the same TYPE_NAME
4704 as the original, so we can get the attributes from there. */
4705 if (TYPE_NAME (type
) && TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
)
4706 type
= TREE_TYPE (TYPE_NAME (type
));
4707 if (tree attr
= lookup_sve_type_attribute (type
))
4708 if (tree id
= TREE_VALUE (chain_index (2, TREE_VALUE (attr
))))
4709 return IDENTIFIER_POINTER (id
);
4713 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE. */
4715 builtin_type_p (const_tree type
)
4717 return lookup_sve_type_attribute (type
);
4720 /* Return true if TYPE is a built-in SVE type defined by the ABI or ACLE.
4721 If so, store the number of constituent SVE vectors in *NUM_ZR and the
4722 number of constituent SVE predicates in *NUM_PR. */
4724 builtin_type_p (const_tree type
, unsigned int *num_zr
, unsigned int *num_pr
)
4726 if (tree attr
= lookup_sve_type_attribute (type
))
4728 tree num_zr_node
= TREE_VALUE (attr
);
4729 tree num_pr_node
= TREE_CHAIN (num_zr_node
);
4730 *num_zr
= tree_to_uhwi (TREE_VALUE (num_zr_node
));
4731 *num_pr
= tree_to_uhwi (TREE_VALUE (num_pr_node
));
4737 /* ATTRS is the attribute list for a sizeless SVE type. Return the
4738 attributes of the associated fixed-length SVE type, taking the
4739 "SVE type" attributes from NEW_SVE_TYPE_ARGS. */
4741 get_arm_sve_vector_bits_attributes (tree old_attrs
, tree new_sve_type_args
)
4743 tree new_attrs
= NULL_TREE
;
4744 tree
*ptr
= &new_attrs
;
4745 for (tree attr
= old_attrs
; attr
; attr
= TREE_CHAIN (attr
))
4747 tree name
= get_attribute_name (attr
);
4748 if (is_attribute_p ("SVE sizeless type", name
))
4751 tree args
= TREE_VALUE (attr
);
4752 if (is_attribute_p ("SVE type", name
))
4753 args
= new_sve_type_args
;
4754 *ptr
= tree_cons (TREE_PURPOSE (attr
), args
, NULL_TREE
);
4755 ptr
= &TREE_CHAIN (*ptr
);
4760 /* An attribute callback for the "arm_sve_vector_bits" attribute. */
4762 handle_arm_sve_vector_bits_attribute (tree
*node
, tree
, tree args
, int,
4765 *no_add_attrs
= true;
4768 tree attr
= lookup_sve_type_attribute (type
);
4771 error ("%qs applied to non-SVE type %qT", "arm_sve_vector_bits", type
);
4775 if (!VECTOR_TYPE_P (type
))
4777 error ("%qs applied to non-vector type %qT",
4778 "arm_sve_vector_bits", type
);
4782 if (!sizeless_type_p (type
))
4784 error ("%qs applied to type %qT, which already has a size",
4785 "arm_sve_vector_bits", type
);
4789 tree size
= TREE_VALUE (args
);
4790 if (TREE_CODE (size
) != INTEGER_CST
)
4792 error ("%qs requires an integer constant expression",
4793 "arm_sve_vector_bits");
4797 unsigned HOST_WIDE_INT value
= tree_to_uhwi (size
);
4798 if (maybe_ne (value
, BITS_PER_SVE_VECTOR
))
4800 warning (OPT_Wattributes
, "unsupported SVE vector size");
4804 /* Construct a new list of "SVE type" attribute arguments. */
4805 tree new_sve_type_args
= copy_list (TREE_VALUE (attr
));
4807 /* Mangle the type as an instance of the imaginary template:
4809 __SVE_VLS<typename, unsigned>
4811 where the first parameter is the SVE type and where the second
4812 parameter is the SVE vector length in bits. */
4813 tree mangled_name_node
= chain_index (2, new_sve_type_args
);
4814 const char *old_mangled_name
4815 = IDENTIFIER_POINTER (TREE_VALUE (mangled_name_node
));
4816 char *new_mangled_name
4817 = xasprintf ("9__SVE_VLSI%sLj%dEE", old_mangled_name
, (int) value
);
4818 TREE_VALUE (mangled_name_node
) = get_identifier (new_mangled_name
);
4819 free (new_mangled_name
);
4821 /* FIXME: The type ought to be a distinct copy in all cases, but
4822 currently that makes the C frontend reject conversions between
4823 svbool_t and its fixed-length variants. Using a type variant
4824 avoids that but means that we treat some ambiguous combinations
4827 tree base_type
= TYPE_MAIN_VARIANT (type
);
4828 if (lang_GNU_C () && VECTOR_BOOLEAN_TYPE_P (type
))
4829 new_type
= build_variant_type_copy (base_type
);
4831 new_type
= build_distinct_type_copy (base_type
);
4833 /* Construct a TYPE_DECL for the new type. This serves two purposes:
4835 - It ensures we don't print the original TYPE_DECL in error messages.
4836 Printing the original name would be confusing because there are
4837 situations in which the distinction between the original type and
4838 the new type matters. For example:
4840 __SVInt8_t __attribute__((arm_sve_vector_bits(512))) *a;
4845 is invalid in C++, but without this, we'd print both types in
4848 - Having a separate TYPE_DECL is necessary to ensure that C++
4849 mangling works correctly. See mangle_builtin_type for details.
4851 The name of the decl is something like:
4853 svint8_t __attribute__((arm_sve_vector_bits(512)))
4855 This is a compromise. It would be more accurate to use something like:
4857 __SVInt8_t __attribute__((arm_sve_vector_bits(512)))
4859 but the <arm_sve.h> name is likely to be more meaningful. */
4860 tree acle_name_node
= TREE_CHAIN (mangled_name_node
);
4861 const char *old_type_name
= IDENTIFIER_POINTER (TREE_VALUE (acle_name_node
));
4863 = xasprintf ("%s __attribute__((arm_sve_vector_bits(%d)))",
4864 old_type_name
, (int) value
);
4865 tree decl
= build_decl (BUILTINS_LOCATION
, TYPE_DECL
,
4866 get_identifier (new_type_name
), new_type
);
4867 DECL_ARTIFICIAL (decl
) = 1;
4868 TYPE_NAME (new_type
) = decl
;
4869 free (new_type_name
);
4871 /* Allow the GNU vector extensions to be applied to vectors.
4872 The extensions aren't yet defined for packed predicates,
4873 so continue to treat them as abstract entities for now. */
4874 if (!VECTOR_BOOLEAN_TYPE_P (new_type
))
4875 TYPE_INDIVISIBLE_P (new_type
) = 0;
4877 /* The new type is a normal sized type; it doesn't have the same
4878 restrictions as sizeless types. */
4879 TYPE_ATTRIBUTES (new_type
)
4880 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (new_type
),
4883 /* Apply the relevant attributes, qualifiers and alignment of TYPE,
4884 if they differ from the original (sizeless) BASE_TYPE. */
4885 if (TYPE_ATTRIBUTES (base_type
) != TYPE_ATTRIBUTES (type
)
4886 || TYPE_QUALS (base_type
) != TYPE_QUALS (type
))
4889 = get_arm_sve_vector_bits_attributes (TYPE_ATTRIBUTES (type
),
4891 new_type
= build_type_attribute_qual_variant (new_type
, attrs
,
4894 if (TYPE_ALIGN (base_type
) != TYPE_ALIGN (type
))
4895 new_type
= build_aligned_type (new_type
, TYPE_ALIGN (type
));
4901 /* Implement TARGET_VERIFY_TYPE_CONTEXT for SVE types. */
4903 verify_type_context (location_t loc
, type_context_kind context
,
4904 const_tree type
, bool silent_p
)
4906 if (!sizeless_type_p (type
))
4912 case TCTX_STATIC_STORAGE
:
4914 error_at (loc
, "SVE type %qT does not have a fixed size", type
);
4919 error_at (loc
, "SVE type %qT does not have a defined alignment", type
);
4922 case TCTX_THREAD_STORAGE
:
4924 error_at (loc
, "variables of type %qT cannot have thread-local"
4925 " storage duration", type
);
4928 case TCTX_POINTER_ARITH
:
4930 error_at (loc
, "arithmetic on pointer to SVE type %qT", type
);
4936 else if (lang_GNU_CXX ())
4937 error_at (loc
, "member variables cannot have SVE type %qT", type
);
4939 error_at (loc
, "fields cannot have SVE type %qT", type
);
4942 case TCTX_ARRAY_ELEMENT
:
4944 error_at (loc
, "array elements cannot have SVE type %qT", type
);
4947 case TCTX_ALLOCATION
:
4949 error_at (loc
, "cannot allocate objects with SVE type %qT", type
);
4952 case TCTX_DEALLOCATION
:
4954 error_at (loc
, "cannot delete objects with SVE type %qT", type
);
4957 case TCTX_EXCEPTIONS
:
4959 error_at (loc
, "cannot throw or catch SVE type %qT", type
);
4962 case TCTX_CAPTURE_BY_COPY
:
4964 error_at (loc
, "capture by copy of SVE type %qT", type
);
4972 using namespace aarch64_sve
;
4975 gt_ggc_mx (function_instance
*)
4980 gt_pch_nx (function_instance
*)
4985 gt_pch_nx (function_instance
*, gt_pointer_operator
, void *)
4989 #include "gt-aarch64-sve-builtins.h"