1 /* Builtins' description for AArch64 SIMD architecture.
2 Copyright (C) 2011-2015 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
26 #include "basic-block.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "langhooks.h"
40 #include "gimple-iterator.h"
41 #include "case-cfn-macros.h"
43 #define v8qi_UP V8QImode
44 #define v4hi_UP V4HImode
45 #define v4hf_UP V4HFmode
46 #define v2si_UP V2SImode
47 #define v2sf_UP V2SFmode
48 #define v1df_UP V1DFmode
51 #define v16qi_UP V16QImode
52 #define v8hi_UP V8HImode
53 #define v8hf_UP V8HFmode
54 #define v4si_UP V4SImode
55 #define v4sf_UP V4SFmode
56 #define v2di_UP V2DImode
57 #define v2df_UP V2DFmode
68 #define SIMD_MAX_BUILTIN_ARGS 5
70 enum aarch64_type_qualifiers
75 qualifier_unsigned
= 0x1, /* 1 << 0 */
77 qualifier_const
= 0x2, /* 1 << 1 */
79 qualifier_pointer
= 0x4, /* 1 << 2 */
80 /* Used when expanding arguments if an operand could
82 qualifier_immediate
= 0x8, /* 1 << 3 */
83 qualifier_maybe_immediate
= 0x10, /* 1 << 4 */
85 qualifier_void
= 0x20, /* 1 << 5 */
86 /* Some patterns may have internal operands, this qualifier is an
87 instruction to the initialisation code to skip this operand. */
88 qualifier_internal
= 0x40, /* 1 << 6 */
89 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
90 rather than using the type of the operand. */
91 qualifier_map_mode
= 0x80, /* 1 << 7 */
92 /* qualifier_pointer | qualifier_map_mode */
93 qualifier_pointer_map_mode
= 0x84,
94 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
95 qualifier_const_pointer_map_mode
= 0x86,
96 /* Polynomial types. */
97 qualifier_poly
= 0x100,
98 /* Lane indices - must be in range, and flipped for bigendian. */
99 qualifier_lane_index
= 0x200,
100 /* Lane indices for single lane structure loads and stores. */
101 qualifier_struct_load_store_lane_index
= 0x400
108 const enum insn_code code
;
110 enum aarch64_type_qualifiers
*qualifiers
;
111 } aarch64_simd_builtin_datum
;
113 static enum aarch64_type_qualifiers
114 aarch64_types_unop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
115 = { qualifier_none
, qualifier_none
};
116 #define TYPES_UNOP (aarch64_types_unop_qualifiers)
117 static enum aarch64_type_qualifiers
118 aarch64_types_unopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
119 = { qualifier_unsigned
, qualifier_unsigned
};
120 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
121 static enum aarch64_type_qualifiers
122 aarch64_types_binop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
123 = { qualifier_none
, qualifier_none
, qualifier_maybe_immediate
};
124 #define TYPES_BINOP (aarch64_types_binop_qualifiers)
125 static enum aarch64_type_qualifiers
126 aarch64_types_binopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
127 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
};
128 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
129 static enum aarch64_type_qualifiers
130 aarch64_types_binop_uus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
131 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_none
};
132 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
133 static enum aarch64_type_qualifiers
134 aarch64_types_binop_ssu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
135 = { qualifier_none
, qualifier_none
, qualifier_unsigned
};
136 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
137 static enum aarch64_type_qualifiers
138 aarch64_types_binopp_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
139 = { qualifier_poly
, qualifier_poly
, qualifier_poly
};
140 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
142 static enum aarch64_type_qualifiers
143 aarch64_types_ternop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
144 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_none
};
145 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
146 static enum aarch64_type_qualifiers
147 aarch64_types_ternop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
148 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_lane_index
};
149 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
150 static enum aarch64_type_qualifiers
151 aarch64_types_ternopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
152 = { qualifier_unsigned
, qualifier_unsigned
,
153 qualifier_unsigned
, qualifier_unsigned
};
154 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
156 static enum aarch64_type_qualifiers
157 aarch64_types_quadop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
158 = { qualifier_none
, qualifier_none
, qualifier_none
,
159 qualifier_none
, qualifier_lane_index
};
160 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
162 static enum aarch64_type_qualifiers
163 aarch64_types_binop_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
164 = { qualifier_none
, qualifier_none
, qualifier_immediate
};
165 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
166 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
167 static enum aarch64_type_qualifiers
168 aarch64_types_shift_to_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
169 = { qualifier_unsigned
, qualifier_none
, qualifier_immediate
};
170 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
171 static enum aarch64_type_qualifiers
172 aarch64_types_unsigned_shift_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
173 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_immediate
};
174 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
176 static enum aarch64_type_qualifiers
177 aarch64_types_ternop_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
178 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_immediate
};
179 #define TYPES_SETREG (aarch64_types_ternop_imm_qualifiers)
180 #define TYPES_SHIFTINSERT (aarch64_types_ternop_imm_qualifiers)
181 #define TYPES_SHIFTACC (aarch64_types_ternop_imm_qualifiers)
183 static enum aarch64_type_qualifiers
184 aarch64_types_unsigned_shiftacc_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
185 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
186 qualifier_immediate
};
187 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
190 static enum aarch64_type_qualifiers
191 aarch64_types_combine_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
192 = { qualifier_none
, qualifier_none
, qualifier_none
};
193 #define TYPES_COMBINE (aarch64_types_combine_qualifiers)
195 static enum aarch64_type_qualifiers
196 aarch64_types_load1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
197 = { qualifier_none
, qualifier_const_pointer_map_mode
};
198 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
199 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
200 static enum aarch64_type_qualifiers
201 aarch64_types_loadstruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
202 = { qualifier_none
, qualifier_const_pointer_map_mode
,
203 qualifier_none
, qualifier_struct_load_store_lane_index
};
204 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
206 static enum aarch64_type_qualifiers
207 aarch64_types_bsl_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
208 = { qualifier_poly
, qualifier_unsigned
,
209 qualifier_poly
, qualifier_poly
};
210 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
211 static enum aarch64_type_qualifiers
212 aarch64_types_bsl_s_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
213 = { qualifier_none
, qualifier_unsigned
,
214 qualifier_none
, qualifier_none
};
215 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
216 static enum aarch64_type_qualifiers
217 aarch64_types_bsl_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
218 = { qualifier_unsigned
, qualifier_unsigned
,
219 qualifier_unsigned
, qualifier_unsigned
};
220 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
222 /* The first argument (return type) of a store should be void type,
223 which we represent with qualifier_void. Their first operand will be
224 a DImode pointer to the location to store to, so we must use
225 qualifier_map_mode | qualifier_pointer to build a pointer to the
226 element type of the vector. */
227 static enum aarch64_type_qualifiers
228 aarch64_types_store1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
229 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_none
};
230 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
231 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
232 static enum aarch64_type_qualifiers
233 aarch64_types_storestruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
234 = { qualifier_void
, qualifier_pointer_map_mode
,
235 qualifier_none
, qualifier_struct_load_store_lane_index
};
236 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
238 #define CF0(N, X) CODE_FOR_aarch64_##N##X
239 #define CF1(N, X) CODE_FOR_##N##X##1
240 #define CF2(N, X) CODE_FOR_##N##X##2
241 #define CF3(N, X) CODE_FOR_##N##X##3
242 #define CF4(N, X) CODE_FOR_##N##X##4
243 #define CF10(N, X) CODE_FOR_##N##X
245 #define VAR1(T, N, MAP, A) \
246 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T},
247 #define VAR2(T, N, MAP, A, B) \
248 VAR1 (T, N, MAP, A) \
250 #define VAR3(T, N, MAP, A, B, C) \
251 VAR2 (T, N, MAP, A, B) \
253 #define VAR4(T, N, MAP, A, B, C, D) \
254 VAR3 (T, N, MAP, A, B, C) \
256 #define VAR5(T, N, MAP, A, B, C, D, E) \
257 VAR4 (T, N, MAP, A, B, C, D) \
259 #define VAR6(T, N, MAP, A, B, C, D, E, F) \
260 VAR5 (T, N, MAP, A, B, C, D, E) \
262 #define VAR7(T, N, MAP, A, B, C, D, E, F, G) \
263 VAR6 (T, N, MAP, A, B, C, D, E, F) \
265 #define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \
266 VAR7 (T, N, MAP, A, B, C, D, E, F, G) \
268 #define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \
269 VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \
271 #define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
272 VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \
274 #define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
275 VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
277 #define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
278 VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
280 #define VAR13(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
281 VAR12 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
283 #define VAR14(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
284 VAR13 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
287 #include "aarch64-builtin-iterators.h"
289 static aarch64_simd_builtin_datum aarch64_simd_builtin_data
[] = {
290 #include "aarch64-simd-builtins.def"
293 /* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
294 #define AARCH64_CRC32_BUILTINS \
295 CRC32_BUILTIN (crc32b, QI) \
296 CRC32_BUILTIN (crc32h, HI) \
297 CRC32_BUILTIN (crc32w, SI) \
298 CRC32_BUILTIN (crc32x, DI) \
299 CRC32_BUILTIN (crc32cb, QI) \
300 CRC32_BUILTIN (crc32ch, HI) \
301 CRC32_BUILTIN (crc32cw, SI) \
302 CRC32_BUILTIN (crc32cx, DI)
308 const enum insn_code icode
;
310 } aarch64_crc_builtin_datum
;
312 #define CRC32_BUILTIN(N, M) \
316 #define VAR1(T, N, MAP, A) \
317 AARCH64_SIMD_BUILTIN_##T##_##N##A,
319 enum aarch64_builtins
323 AARCH64_BUILTIN_GET_FPCR
,
324 AARCH64_BUILTIN_SET_FPCR
,
325 AARCH64_BUILTIN_GET_FPSR
,
326 AARCH64_BUILTIN_SET_FPSR
,
328 AARCH64_BUILTIN_RSQRT_DF
,
329 AARCH64_BUILTIN_RSQRT_SF
,
330 AARCH64_BUILTIN_RSQRT_V2DF
,
331 AARCH64_BUILTIN_RSQRT_V2SF
,
332 AARCH64_BUILTIN_RSQRT_V4SF
,
333 AARCH64_SIMD_BUILTIN_BASE
,
334 AARCH64_SIMD_BUILTIN_LANE_CHECK
,
335 #include "aarch64-simd-builtins.def"
336 /* The first enum element which is based on an insn_data pattern. */
337 AARCH64_SIMD_PATTERN_START
= AARCH64_SIMD_BUILTIN_LANE_CHECK
+ 1,
338 AARCH64_SIMD_BUILTIN_MAX
= AARCH64_SIMD_PATTERN_START
339 + ARRAY_SIZE (aarch64_simd_builtin_data
) - 1,
340 AARCH64_CRC32_BUILTIN_BASE
,
341 AARCH64_CRC32_BUILTINS
342 AARCH64_CRC32_BUILTIN_MAX
,
347 #define CRC32_BUILTIN(N, M) \
348 {"__builtin_aarch64_"#N, M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
350 static aarch64_crc_builtin_datum aarch64_crc_builtin_data
[] = {
351 AARCH64_CRC32_BUILTINS
356 static GTY(()) tree aarch64_builtin_decls
[AARCH64_BUILTIN_MAX
];
358 #define NUM_DREG_TYPES 6
359 #define NUM_QREG_TYPES 6
361 /* Internal scalar builtin types. These types are used to support
362 neon intrinsic builtins. They are _not_ user-visible types. Therefore
363 the mangling for these types are implementation defined. */
364 const char *aarch64_scalar_builtin_types
[] = {
365 "__builtin_aarch64_simd_qi",
366 "__builtin_aarch64_simd_hi",
367 "__builtin_aarch64_simd_si",
368 "__builtin_aarch64_simd_hf",
369 "__builtin_aarch64_simd_sf",
370 "__builtin_aarch64_simd_di",
371 "__builtin_aarch64_simd_df",
372 "__builtin_aarch64_simd_poly8",
373 "__builtin_aarch64_simd_poly16",
374 "__builtin_aarch64_simd_poly64",
375 "__builtin_aarch64_simd_poly128",
376 "__builtin_aarch64_simd_ti",
377 "__builtin_aarch64_simd_uqi",
378 "__builtin_aarch64_simd_uhi",
379 "__builtin_aarch64_simd_usi",
380 "__builtin_aarch64_simd_udi",
381 "__builtin_aarch64_simd_ei",
382 "__builtin_aarch64_simd_oi",
383 "__builtin_aarch64_simd_ci",
384 "__builtin_aarch64_simd_xi",
388 #define ENTRY(E, M, Q, G) E,
389 enum aarch64_simd_type
391 #include "aarch64-simd-builtin-types.def"
392 ARM_NEON_H_TYPES_LAST
396 struct aarch64_simd_type_info
398 enum aarch64_simd_type type
;
400 /* Internal type name. */
403 /* Internal type name(mangled). The mangled names conform to the
404 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
405 Appendix A). To qualify for emission with the mangled names defined in
406 that document, a vector type must not only be of the correct mode but also
407 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
408 types are registered by aarch64_init_simd_builtin_types (). In other
409 words, vector types defined in other ways e.g. via vector_size attribute
410 will get default mangled names. */
419 /* Machine mode the internal type maps to. */
420 enum machine_mode mode
;
423 enum aarch64_type_qualifiers q
;
426 #define ENTRY(E, M, Q, G) \
427 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, M##mode, qualifier_##Q},
428 static struct aarch64_simd_type_info aarch64_simd_types
[] = {
429 #include "aarch64-simd-builtin-types.def"
433 /* This type is not SIMD-specific; it is the user-visible __fp16. */
434 static tree aarch64_fp16_type_node
= NULL_TREE
;
436 static tree aarch64_simd_intOI_type_node
= NULL_TREE
;
437 static tree aarch64_simd_intCI_type_node
= NULL_TREE
;
438 static tree aarch64_simd_intXI_type_node
= NULL_TREE
;
441 aarch64_mangle_builtin_scalar_type (const_tree type
)
445 while (aarch64_scalar_builtin_types
[i
] != NULL
)
447 const char *name
= aarch64_scalar_builtin_types
[i
];
449 if (TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
450 && DECL_NAME (TYPE_NAME (type
))
451 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type
))), name
))
452 return aarch64_scalar_builtin_types
[i
];
459 aarch64_mangle_builtin_vector_type (const_tree type
)
462 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
464 for (i
= 0; i
< nelts
; i
++)
465 if (aarch64_simd_types
[i
].mode
== TYPE_MODE (type
)
467 && TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
468 && DECL_NAME (TYPE_NAME (type
))
470 (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type
))),
471 aarch64_simd_types
[i
].name
))
472 return aarch64_simd_types
[i
].mangle
;
478 aarch64_mangle_builtin_type (const_tree type
)
481 /* Walk through all the AArch64 builtins types tables to filter out the
483 if ((mangle
= aarch64_mangle_builtin_vector_type (type
))
484 || (mangle
= aarch64_mangle_builtin_scalar_type (type
)))
491 aarch64_simd_builtin_std_type (enum machine_mode mode
,
492 enum aarch64_type_qualifiers q
)
494 #define QUAL_TYPE(M) \
495 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
499 return QUAL_TYPE (QI
);
501 return QUAL_TYPE (HI
);
503 return QUAL_TYPE (SI
);
505 return QUAL_TYPE (DI
);
507 return QUAL_TYPE (TI
);
509 return aarch64_simd_intOI_type_node
;
511 return aarch64_simd_intCI_type_node
;
513 return aarch64_simd_intXI_type_node
;
515 return aarch64_fp16_type_node
;
517 return float_type_node
;
519 return double_type_node
;
527 aarch64_lookup_simd_builtin_type (enum machine_mode mode
,
528 enum aarch64_type_qualifiers q
)
531 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
533 /* Non-poly scalar modes map to standard types not in the table. */
534 if (q
!= qualifier_poly
&& !VECTOR_MODE_P (mode
))
535 return aarch64_simd_builtin_std_type (mode
, q
);
537 for (i
= 0; i
< nelts
; i
++)
538 if (aarch64_simd_types
[i
].mode
== mode
539 && aarch64_simd_types
[i
].q
== q
)
540 return aarch64_simd_types
[i
].itype
;
546 aarch64_simd_builtin_type (enum machine_mode mode
,
547 bool unsigned_p
, bool poly_p
)
550 return aarch64_lookup_simd_builtin_type (mode
, qualifier_poly
);
552 return aarch64_lookup_simd_builtin_type (mode
, qualifier_unsigned
);
554 return aarch64_lookup_simd_builtin_type (mode
, qualifier_none
);
558 aarch64_init_simd_builtin_types (void)
561 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
564 /* Init all the element types built by the front-end. */
565 aarch64_simd_types
[Int8x8_t
].eltype
= intQI_type_node
;
566 aarch64_simd_types
[Int8x16_t
].eltype
= intQI_type_node
;
567 aarch64_simd_types
[Int16x4_t
].eltype
= intHI_type_node
;
568 aarch64_simd_types
[Int16x8_t
].eltype
= intHI_type_node
;
569 aarch64_simd_types
[Int32x2_t
].eltype
= intSI_type_node
;
570 aarch64_simd_types
[Int32x4_t
].eltype
= intSI_type_node
;
571 aarch64_simd_types
[Int64x1_t
].eltype
= intDI_type_node
;
572 aarch64_simd_types
[Int64x2_t
].eltype
= intDI_type_node
;
573 aarch64_simd_types
[Uint8x8_t
].eltype
= unsigned_intQI_type_node
;
574 aarch64_simd_types
[Uint8x16_t
].eltype
= unsigned_intQI_type_node
;
575 aarch64_simd_types
[Uint16x4_t
].eltype
= unsigned_intHI_type_node
;
576 aarch64_simd_types
[Uint16x8_t
].eltype
= unsigned_intHI_type_node
;
577 aarch64_simd_types
[Uint32x2_t
].eltype
= unsigned_intSI_type_node
;
578 aarch64_simd_types
[Uint32x4_t
].eltype
= unsigned_intSI_type_node
;
579 aarch64_simd_types
[Uint64x1_t
].eltype
= unsigned_intDI_type_node
;
580 aarch64_simd_types
[Uint64x2_t
].eltype
= unsigned_intDI_type_node
;
582 /* Poly types are a world of their own. */
583 aarch64_simd_types
[Poly8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
=
584 build_distinct_type_copy (unsigned_intQI_type_node
);
585 aarch64_simd_types
[Poly16_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
=
586 build_distinct_type_copy (unsigned_intHI_type_node
);
587 aarch64_simd_types
[Poly64_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
=
588 build_distinct_type_copy (unsigned_intDI_type_node
);
589 aarch64_simd_types
[Poly128_t
].eltype
= aarch64_simd_types
[Poly128_t
].itype
=
590 build_distinct_type_copy (unsigned_intTI_type_node
);
591 /* Init poly vector element types with scalar poly types. */
592 aarch64_simd_types
[Poly8x8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
593 aarch64_simd_types
[Poly8x16_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
594 aarch64_simd_types
[Poly16x4_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
595 aarch64_simd_types
[Poly16x8_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
596 aarch64_simd_types
[Poly64x1_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
597 aarch64_simd_types
[Poly64x2_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
599 /* Continue with standard types. */
600 aarch64_simd_types
[Float16x4_t
].eltype
= aarch64_fp16_type_node
;
601 aarch64_simd_types
[Float16x8_t
].eltype
= aarch64_fp16_type_node
;
602 aarch64_simd_types
[Float32x2_t
].eltype
= float_type_node
;
603 aarch64_simd_types
[Float32x4_t
].eltype
= float_type_node
;
604 aarch64_simd_types
[Float64x1_t
].eltype
= double_type_node
;
605 aarch64_simd_types
[Float64x2_t
].eltype
= double_type_node
;
607 for (i
= 0; i
< nelts
; i
++)
609 tree eltype
= aarch64_simd_types
[i
].eltype
;
610 enum machine_mode mode
= aarch64_simd_types
[i
].mode
;
612 if (aarch64_simd_types
[i
].itype
== NULL
)
613 aarch64_simd_types
[i
].itype
=
614 build_distinct_type_copy
615 (build_vector_type (eltype
, GET_MODE_NUNITS (mode
)));
617 tdecl
= add_builtin_type (aarch64_simd_types
[i
].name
,
618 aarch64_simd_types
[i
].itype
);
619 TYPE_NAME (aarch64_simd_types
[i
].itype
) = tdecl
;
620 SET_TYPE_STRUCTURAL_EQUALITY (aarch64_simd_types
[i
].itype
);
623 #define AARCH64_BUILD_SIGNED_TYPE(mode) \
624 make_signed_type (GET_MODE_PRECISION (mode));
625 aarch64_simd_intOI_type_node
= AARCH64_BUILD_SIGNED_TYPE (OImode
);
626 aarch64_simd_intCI_type_node
= AARCH64_BUILD_SIGNED_TYPE (CImode
);
627 aarch64_simd_intXI_type_node
= AARCH64_BUILD_SIGNED_TYPE (XImode
);
628 #undef AARCH64_BUILD_SIGNED_TYPE
630 tdecl
= add_builtin_type
631 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node
);
632 TYPE_NAME (aarch64_simd_intOI_type_node
) = tdecl
;
633 tdecl
= add_builtin_type
634 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node
);
635 TYPE_NAME (aarch64_simd_intCI_type_node
) = tdecl
;
636 tdecl
= add_builtin_type
637 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node
);
638 TYPE_NAME (aarch64_simd_intXI_type_node
) = tdecl
;
642 aarch64_init_simd_builtin_scalar_types (void)
644 /* Define typedefs for all the standard scalar types. */
645 (*lang_hooks
.types
.register_builtin_type
) (intQI_type_node
,
646 "__builtin_aarch64_simd_qi");
647 (*lang_hooks
.types
.register_builtin_type
) (intHI_type_node
,
648 "__builtin_aarch64_simd_hi");
649 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
,
650 "__builtin_aarch64_simd_hf");
651 (*lang_hooks
.types
.register_builtin_type
) (intSI_type_node
,
652 "__builtin_aarch64_simd_si");
653 (*lang_hooks
.types
.register_builtin_type
) (float_type_node
,
654 "__builtin_aarch64_simd_sf");
655 (*lang_hooks
.types
.register_builtin_type
) (intDI_type_node
,
656 "__builtin_aarch64_simd_di");
657 (*lang_hooks
.types
.register_builtin_type
) (double_type_node
,
658 "__builtin_aarch64_simd_df");
659 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
660 "__builtin_aarch64_simd_poly8");
661 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
662 "__builtin_aarch64_simd_poly16");
663 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
664 "__builtin_aarch64_simd_poly64");
665 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intTI_type_node
,
666 "__builtin_aarch64_simd_poly128");
667 (*lang_hooks
.types
.register_builtin_type
) (intTI_type_node
,
668 "__builtin_aarch64_simd_ti");
669 /* Unsigned integer types for various mode sizes. */
670 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
671 "__builtin_aarch64_simd_uqi");
672 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
673 "__builtin_aarch64_simd_uhi");
674 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intSI_type_node
,
675 "__builtin_aarch64_simd_usi");
676 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
677 "__builtin_aarch64_simd_udi");
680 static bool aarch64_simd_builtins_initialized_p
= false;
683 aarch64_init_simd_builtins (void)
685 unsigned int i
, fcode
= AARCH64_SIMD_PATTERN_START
;
687 if (aarch64_simd_builtins_initialized_p
)
690 aarch64_simd_builtins_initialized_p
= true;
692 aarch64_init_simd_builtin_types ();
694 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
695 Therefore we need to preserve the old __builtin scalar types. It can be
696 removed once all the intrinsics become strongly typed using the qualifier
698 aarch64_init_simd_builtin_scalar_types ();
700 tree lane_check_fpr
= build_function_type_list (void_type_node
,
705 aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_LANE_CHECK
] =
706 add_builtin_function ("__builtin_aarch64_im_lane_boundsi", lane_check_fpr
,
707 AARCH64_SIMD_BUILTIN_LANE_CHECK
, BUILT_IN_MD
,
710 for (i
= 0; i
< ARRAY_SIZE (aarch64_simd_builtin_data
); i
++, fcode
++)
712 bool print_type_signature_p
= false;
713 char type_signature
[SIMD_MAX_BUILTIN_ARGS
] = { 0 };
714 aarch64_simd_builtin_datum
*d
= &aarch64_simd_builtin_data
[i
];
721 /* We must track two variables here. op_num is
722 the operand number as in the RTL pattern. This is
723 required to access the mode (e.g. V4SF mode) of the
724 argument, from which the base type can be derived.
725 arg_num is an index in to the qualifiers data, which
726 gives qualifiers to the type (e.g. const unsigned).
727 The reason these two variables may differ by one is the
728 void return type. While all return types take the 0th entry
729 in the qualifiers array, there is no operand for them in the
731 int op_num
= insn_data
[d
->code
].n_operands
- 1;
732 int arg_num
= d
->qualifiers
[0] & qualifier_void
735 tree return_type
= void_type_node
, args
= void_list_node
;
738 /* Build a function type directly from the insn_data for this
739 builtin. The build_function_type () function takes care of
740 removing duplicates for us. */
741 for (; op_num
>= 0; arg_num
--, op_num
--)
743 machine_mode op_mode
= insn_data
[d
->code
].operand
[op_num
].mode
;
744 enum aarch64_type_qualifiers qualifiers
= d
->qualifiers
[arg_num
];
746 if (qualifiers
& qualifier_unsigned
)
748 type_signature
[arg_num
] = 'u';
749 print_type_signature_p
= true;
751 else if (qualifiers
& qualifier_poly
)
753 type_signature
[arg_num
] = 'p';
754 print_type_signature_p
= true;
757 type_signature
[arg_num
] = 's';
759 /* Skip an internal operand for vget_{low, high}. */
760 if (qualifiers
& qualifier_internal
)
763 /* Some builtins have different user-facing types
764 for certain arguments, encoded in d->mode. */
765 if (qualifiers
& qualifier_map_mode
)
768 /* For pointers, we want a pointer to the basic type
770 if (qualifiers
& qualifier_pointer
&& VECTOR_MODE_P (op_mode
))
771 op_mode
= GET_MODE_INNER (op_mode
);
773 eltype
= aarch64_simd_builtin_type
775 (qualifiers
& qualifier_unsigned
) != 0,
776 (qualifiers
& qualifier_poly
) != 0);
777 gcc_assert (eltype
!= NULL
);
779 /* Add qualifiers. */
780 if (qualifiers
& qualifier_const
)
781 eltype
= build_qualified_type (eltype
, TYPE_QUAL_CONST
);
783 if (qualifiers
& qualifier_pointer
)
784 eltype
= build_pointer_type (eltype
);
786 /* If we have reached arg_num == 0, we are at a non-void
787 return type. Otherwise, we are still processing
790 return_type
= eltype
;
792 args
= tree_cons (NULL_TREE
, eltype
, args
);
795 ftype
= build_function_type (return_type
, args
);
797 gcc_assert (ftype
!= NULL
);
799 if (print_type_signature_p
)
800 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s_%s",
801 d
->name
, type_signature
);
803 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s",
806 fndecl
= add_builtin_function (namebuf
, ftype
, fcode
, BUILT_IN_MD
,
808 aarch64_builtin_decls
[fcode
] = fndecl
;
813 aarch64_init_crc32_builtins ()
815 tree usi_type
= aarch64_simd_builtin_std_type (SImode
, qualifier_unsigned
);
818 for (i
= 0; i
< ARRAY_SIZE (aarch64_crc_builtin_data
); ++i
)
820 aarch64_crc_builtin_datum
* d
= &aarch64_crc_builtin_data
[i
];
821 tree argtype
= aarch64_simd_builtin_std_type (d
->mode
,
823 tree ftype
= build_function_type_list (usi_type
, usi_type
, argtype
, NULL_TREE
);
824 tree fndecl
= add_builtin_function (d
->name
, ftype
, d
->fcode
,
825 BUILT_IN_MD
, NULL
, NULL_TREE
);
827 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
831 /* Add builtins for reciprocal square root. */
834 aarch64_init_builtin_rsqrt (void)
839 tree V2SF_type_node
= build_vector_type (float_type_node
, 2);
840 tree V2DF_type_node
= build_vector_type (double_type_node
, 2);
841 tree V4SF_type_node
= build_vector_type (float_type_node
, 4);
843 struct builtin_decls_data
846 const char *builtin_name
;
850 builtin_decls_data bdda
[] =
852 { double_type_node
, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF
},
853 { float_type_node
, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF
},
854 { V2DF_type_node
, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF
},
855 { V2SF_type_node
, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF
},
856 { V4SF_type_node
, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF
}
859 builtin_decls_data
*bdd
= bdda
;
860 builtin_decls_data
*bdd_end
= bdd
+ (sizeof (bdda
) / sizeof (builtin_decls_data
));
862 for (; bdd
< bdd_end
; bdd
++)
864 ftype
= build_function_type_list (bdd
->type_node
, bdd
->type_node
, NULL_TREE
);
865 fndecl
= add_builtin_function (bdd
->builtin_name
,
866 ftype
, bdd
->function_code
, BUILT_IN_MD
, NULL
, NULL_TREE
);
867 aarch64_builtin_decls
[bdd
->function_code
] = fndecl
;
872 aarch64_init_builtins (void)
875 = build_function_type_list (void_type_node
, unsigned_type_node
, NULL
);
877 = build_function_type_list (unsigned_type_node
, NULL
);
879 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
]
880 = add_builtin_function ("__builtin_aarch64_get_fpcr", ftype_get_fpr
,
881 AARCH64_BUILTIN_GET_FPCR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
882 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
]
883 = add_builtin_function ("__builtin_aarch64_set_fpcr", ftype_set_fpr
,
884 AARCH64_BUILTIN_SET_FPCR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
885 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
]
886 = add_builtin_function ("__builtin_aarch64_get_fpsr", ftype_get_fpr
,
887 AARCH64_BUILTIN_GET_FPSR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
888 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
]
889 = add_builtin_function ("__builtin_aarch64_set_fpsr", ftype_set_fpr
,
890 AARCH64_BUILTIN_SET_FPSR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
892 aarch64_fp16_type_node
= make_node (REAL_TYPE
);
893 TYPE_PRECISION (aarch64_fp16_type_node
) = 16;
894 layout_type (aarch64_fp16_type_node
);
896 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
, "__fp16");
899 aarch64_init_simd_builtins ();
901 aarch64_init_crc32_builtins ();
902 aarch64_init_builtin_rsqrt ();
906 aarch64_builtin_decl (unsigned code
, bool initialize_p ATTRIBUTE_UNUSED
)
908 if (code
>= AARCH64_BUILTIN_MAX
)
909 return error_mark_node
;
911 return aarch64_builtin_decls
[code
];
916 SIMD_ARG_COPY_TO_REG
,
919 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
,
925 aarch64_simd_expand_args (rtx target
, int icode
, int have_retval
,
926 tree exp
, builtin_simd_arg
*args
,
927 enum machine_mode builtin_mode
)
930 rtx op
[SIMD_MAX_BUILTIN_ARGS
+ 1]; /* First element for result operand. */
935 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
937 || GET_MODE (target
) != tmode
938 || !(*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
939 target
= gen_reg_rtx (tmode
);
945 builtin_simd_arg thisarg
= args
[opc
- have_retval
];
947 if (thisarg
== SIMD_ARG_STOP
)
951 tree arg
= CALL_EXPR_ARG (exp
, opc
- have_retval
);
952 enum machine_mode mode
= insn_data
[icode
].operand
[opc
].mode
;
953 op
[opc
] = expand_normal (arg
);
957 case SIMD_ARG_COPY_TO_REG
:
958 if (POINTER_TYPE_P (TREE_TYPE (arg
)))
959 op
[opc
] = convert_memory_address (Pmode
, op
[opc
]);
960 /*gcc_assert (GET_MODE (op[opc]) == mode); */
961 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
963 op
[opc
] = copy_to_mode_reg (mode
, op
[opc
]);
966 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
:
967 gcc_assert (opc
> 1);
968 if (CONST_INT_P (op
[opc
]))
970 aarch64_simd_lane_bounds (op
[opc
], 0,
971 GET_MODE_NUNITS (builtin_mode
),
973 /* Keep to GCC-vector-extension lane indices in the RTL. */
975 GEN_INT (ENDIAN_LANE_N (builtin_mode
, INTVAL (op
[opc
])));
979 case SIMD_ARG_LANE_INDEX
:
980 /* Must be a previous operand into which this is an index. */
981 gcc_assert (opc
> 0);
982 if (CONST_INT_P (op
[opc
]))
984 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
985 aarch64_simd_lane_bounds (op
[opc
],
986 0, GET_MODE_NUNITS (vmode
), exp
);
987 /* Keep to GCC-vector-extension lane indices in the RTL. */
988 op
[opc
] = GEN_INT (ENDIAN_LANE_N (vmode
, INTVAL (op
[opc
])));
990 /* Fall through - if the lane index isn't a constant then
991 the next case will error. */
992 case SIMD_ARG_CONSTANT
:
994 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
997 error ("%Kargument %d must be a constant immediate",
998 exp
, opc
+ 1 - have_retval
);
1014 pat
= GEN_FCN (icode
) (op
[0]);
1018 pat
= GEN_FCN (icode
) (op
[0], op
[1]);
1022 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2]);
1026 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3]);
1030 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4]);
1034 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4], op
[5]);
1049 /* Expand an AArch64 AdvSIMD builtin(intrinsic). */
1051 aarch64_simd_expand_builtin (int fcode
, tree exp
, rtx target
)
1053 if (fcode
== AARCH64_SIMD_BUILTIN_LANE_CHECK
)
1055 rtx totalsize
= expand_normal (CALL_EXPR_ARG (exp
, 0));
1056 rtx elementsize
= expand_normal (CALL_EXPR_ARG (exp
, 1));
1057 if (CONST_INT_P (totalsize
) && CONST_INT_P (elementsize
)
1058 && UINTVAL (elementsize
) != 0
1059 && UINTVAL (totalsize
) != 0)
1061 rtx lane_idx
= expand_normal (CALL_EXPR_ARG (exp
, 2));
1062 if (CONST_INT_P (lane_idx
))
1063 aarch64_simd_lane_bounds (lane_idx
, 0,
1065 / UINTVAL (elementsize
),
1068 error ("%Klane index must be a constant immediate", exp
);
1071 error ("%Ktotal size and element size must be a non-zero constant immediate", exp
);
1072 /* Don't generate any RTL. */
1075 aarch64_simd_builtin_datum
*d
=
1076 &aarch64_simd_builtin_data
[fcode
- AARCH64_SIMD_PATTERN_START
];
1077 enum insn_code icode
= d
->code
;
1078 builtin_simd_arg args
[SIMD_MAX_BUILTIN_ARGS
+ 1];
1079 int num_args
= insn_data
[d
->code
].n_operands
;
1083 is_void
= !!(d
->qualifiers
[0] & qualifier_void
);
1085 num_args
+= is_void
;
1087 for (k
= 1; k
< num_args
; k
++)
1089 /* We have four arrays of data, each indexed in a different fashion.
1090 qualifiers - element 0 always describes the function return type.
1091 operands - element 0 is either the operand for return value (if
1092 the function has a non-void return type) or the operand for the
1094 expr_args - element 0 always holds the first argument.
1095 args - element 0 is always used for the return type. */
1096 int qualifiers_k
= k
;
1097 int operands_k
= k
- is_void
;
1098 int expr_args_k
= k
- 1;
1100 if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_index
)
1101 args
[k
] = SIMD_ARG_LANE_INDEX
;
1102 else if (d
->qualifiers
[qualifiers_k
] & qualifier_struct_load_store_lane_index
)
1103 args
[k
] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
;
1104 else if (d
->qualifiers
[qualifiers_k
] & qualifier_immediate
)
1105 args
[k
] = SIMD_ARG_CONSTANT
;
1106 else if (d
->qualifiers
[qualifiers_k
] & qualifier_maybe_immediate
)
1109 = expand_normal (CALL_EXPR_ARG (exp
,
1111 /* Handle constants only if the predicate allows it. */
1112 bool op_const_int_p
=
1114 && (*insn_data
[icode
].operand
[operands_k
].predicate
)
1115 (arg
, insn_data
[icode
].operand
[operands_k
].mode
));
1116 args
[k
] = op_const_int_p
? SIMD_ARG_CONSTANT
: SIMD_ARG_COPY_TO_REG
;
1119 args
[k
] = SIMD_ARG_COPY_TO_REG
;
1122 args
[k
] = SIMD_ARG_STOP
;
1124 /* The interface to aarch64_simd_expand_args expects a 0 if
1125 the function is void, and a 1 if it is not. */
1126 return aarch64_simd_expand_args
1127 (target
, icode
, !is_void
, exp
, &args
[1], d
->mode
);
1131 aarch64_crc32_expand_builtin (int fcode
, tree exp
, rtx target
)
1134 aarch64_crc_builtin_datum
*d
1135 = &aarch64_crc_builtin_data
[fcode
- (AARCH64_CRC32_BUILTIN_BASE
+ 1)];
1136 enum insn_code icode
= d
->icode
;
1137 tree arg0
= CALL_EXPR_ARG (exp
, 0);
1138 tree arg1
= CALL_EXPR_ARG (exp
, 1);
1139 rtx op0
= expand_normal (arg0
);
1140 rtx op1
= expand_normal (arg1
);
1141 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
1142 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
1143 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
1146 || GET_MODE (target
) != tmode
1147 || ! (*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
1148 target
= gen_reg_rtx (tmode
);
1150 gcc_assert ((GET_MODE (op0
) == mode0
|| GET_MODE (op0
) == VOIDmode
)
1151 && (GET_MODE (op1
) == mode1
|| GET_MODE (op1
) == VOIDmode
));
1153 if (! (*insn_data
[icode
].operand
[1].predicate
) (op0
, mode0
))
1154 op0
= copy_to_mode_reg (mode0
, op0
);
1155 if (! (*insn_data
[icode
].operand
[2].predicate
) (op1
, mode1
))
1156 op1
= copy_to_mode_reg (mode1
, op1
);
1158 pat
= GEN_FCN (icode
) (target
, op0
, op1
);
1166 /* Function to expand reciprocal square root builtins. */
1169 aarch64_expand_builtin_rsqrt (int fcode
, tree exp
, rtx target
)
1171 tree arg0
= CALL_EXPR_ARG (exp
, 0);
1172 rtx op0
= expand_normal (arg0
);
1174 rtx (*gen
) (rtx
, rtx
);
1178 case AARCH64_BUILTIN_RSQRT_DF
:
1181 case AARCH64_BUILTIN_RSQRT_SF
:
1184 case AARCH64_BUILTIN_RSQRT_V2DF
:
1185 gen
= gen_rsqrtv2df2
;
1187 case AARCH64_BUILTIN_RSQRT_V2SF
:
1188 gen
= gen_rsqrtv2sf2
;
1190 case AARCH64_BUILTIN_RSQRT_V4SF
:
1191 gen
= gen_rsqrtv4sf2
;
1193 default: gcc_unreachable ();
1197 target
= gen_reg_rtx (GET_MODE (op0
));
1199 emit_insn (gen (target
, op0
));
1204 /* Expand an expression EXP that calls a built-in function,
1205 with result going to TARGET if that's convenient. */
1207 aarch64_expand_builtin (tree exp
,
1209 rtx subtarget ATTRIBUTE_UNUSED
,
1210 machine_mode mode ATTRIBUTE_UNUSED
,
1211 int ignore ATTRIBUTE_UNUSED
)
1213 tree fndecl
= TREE_OPERAND (CALL_EXPR_FN (exp
), 0);
1214 int fcode
= DECL_FUNCTION_CODE (fndecl
);
1221 case AARCH64_BUILTIN_GET_FPCR
:
1222 case AARCH64_BUILTIN_SET_FPCR
:
1223 case AARCH64_BUILTIN_GET_FPSR
:
1224 case AARCH64_BUILTIN_SET_FPSR
:
1225 if ((fcode
== AARCH64_BUILTIN_GET_FPCR
)
1226 || (fcode
== AARCH64_BUILTIN_GET_FPSR
))
1228 icode
= (fcode
== AARCH64_BUILTIN_GET_FPSR
) ?
1229 CODE_FOR_get_fpsr
: CODE_FOR_get_fpcr
;
1230 target
= gen_reg_rtx (SImode
);
1231 pat
= GEN_FCN (icode
) (target
);
1236 icode
= (fcode
== AARCH64_BUILTIN_SET_FPSR
) ?
1237 CODE_FOR_set_fpsr
: CODE_FOR_set_fpcr
;
1238 arg0
= CALL_EXPR_ARG (exp
, 0);
1239 op0
= force_reg (SImode
, expand_normal (arg0
));
1240 pat
= GEN_FCN (icode
) (op0
);
1246 if (fcode
>= AARCH64_SIMD_BUILTIN_BASE
&& fcode
<= AARCH64_SIMD_BUILTIN_MAX
)
1247 return aarch64_simd_expand_builtin (fcode
, exp
, target
);
1248 else if (fcode
>= AARCH64_CRC32_BUILTIN_BASE
&& fcode
<= AARCH64_CRC32_BUILTIN_MAX
)
1249 return aarch64_crc32_expand_builtin (fcode
, exp
, target
);
1251 if (fcode
== AARCH64_BUILTIN_RSQRT_DF
1252 || fcode
== AARCH64_BUILTIN_RSQRT_SF
1253 || fcode
== AARCH64_BUILTIN_RSQRT_V2DF
1254 || fcode
== AARCH64_BUILTIN_RSQRT_V2SF
1255 || fcode
== AARCH64_BUILTIN_RSQRT_V4SF
)
1256 return aarch64_expand_builtin_rsqrt (fcode
, exp
, target
);
1262 aarch64_builtin_vectorized_function (unsigned int fn
, tree type_out
,
1265 machine_mode in_mode
, out_mode
;
1268 if (TREE_CODE (type_out
) != VECTOR_TYPE
1269 || TREE_CODE (type_in
) != VECTOR_TYPE
)
1272 out_mode
= TYPE_MODE (TREE_TYPE (type_out
));
1273 out_n
= TYPE_VECTOR_SUBPARTS (type_out
);
1274 in_mode
= TYPE_MODE (TREE_TYPE (type_in
));
1275 in_n
= TYPE_VECTOR_SUBPARTS (type_in
);
1277 #undef AARCH64_CHECK_BUILTIN_MODE
1278 #define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
1279 #define AARCH64_FIND_FRINT_VARIANT(N) \
1280 (AARCH64_CHECK_BUILTIN_MODE (2, D) \
1281 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
1282 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
1283 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
1284 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
1285 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
1289 #undef AARCH64_CHECK_BUILTIN_MODE
1290 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1291 (out_mode == N##Fmode && out_n == C \
1292 && in_mode == N##Fmode && in_n == C)
1294 return AARCH64_FIND_FRINT_VARIANT (floor
);
1296 return AARCH64_FIND_FRINT_VARIANT (ceil
);
1298 return AARCH64_FIND_FRINT_VARIANT (btrunc
);
1300 return AARCH64_FIND_FRINT_VARIANT (round
);
1302 return AARCH64_FIND_FRINT_VARIANT (nearbyint
);
1304 return AARCH64_FIND_FRINT_VARIANT (sqrt
);
1305 #undef AARCH64_CHECK_BUILTIN_MODE
1306 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1307 (out_mode == SImode && out_n == C \
1308 && in_mode == N##Imode && in_n == C)
1311 if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1312 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_clzv4si
];
1317 if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1318 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si
];
1319 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1320 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si
];
1323 #undef AARCH64_CHECK_BUILTIN_MODE
1324 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1325 (out_mode == N##Imode && out_n == C \
1326 && in_mode == N##Fmode && in_n == C)
1331 enum aarch64_builtins builtin
;
1332 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1333 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di
;
1334 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1335 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si
;
1336 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1337 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si
;
1341 return aarch64_builtin_decls
[builtin
];
1347 enum aarch64_builtins builtin
;
1348 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1349 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di
;
1350 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1351 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si
;
1352 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1353 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si
;
1357 return aarch64_builtin_decls
[builtin
];
1363 enum aarch64_builtins builtin
;
1364 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1365 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di
;
1366 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1367 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si
;
1368 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1369 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si
;
1373 return aarch64_builtin_decls
[builtin
];
1375 case CFN_BUILT_IN_BSWAP16
:
1376 #undef AARCH64_CHECK_BUILTIN_MODE
1377 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1378 (out_mode == N##Imode && out_n == C \
1379 && in_mode == N##Imode && in_n == C)
1380 if (AARCH64_CHECK_BUILTIN_MODE (4, H
))
1381 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4hi
];
1382 else if (AARCH64_CHECK_BUILTIN_MODE (8, H
))
1383 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv8hi
];
1386 case CFN_BUILT_IN_BSWAP32
:
1387 if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1388 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2si
];
1389 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1390 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4si
];
1393 case CFN_BUILT_IN_BSWAP64
:
1394 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1395 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2di
];
1405 /* Return builtin for reciprocal square root. */
1408 aarch64_builtin_rsqrt (unsigned int fn
)
1410 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df
)
1411 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2DF
];
1412 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf
)
1413 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2SF
];
1414 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf
)
1415 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V4SF
];
1420 #define VAR1(T, N, MAP, A) \
1421 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
1424 aarch64_fold_builtin (tree fndecl
, int n_args ATTRIBUTE_UNUSED
, tree
*args
,
1425 bool ignore ATTRIBUTE_UNUSED
)
1427 int fcode
= DECL_FUNCTION_CODE (fndecl
);
1428 tree type
= TREE_TYPE (TREE_TYPE (fndecl
));
1432 BUILTIN_VDQF (UNOP
, abs
, 2)
1433 return fold_build1 (ABS_EXPR
, type
, args
[0]);
1435 VAR1 (UNOP
, floatv2si
, 2, v2sf
)
1436 VAR1 (UNOP
, floatv4si
, 2, v4sf
)
1437 VAR1 (UNOP
, floatv2di
, 2, v2df
)
1438 return fold_build1 (FLOAT_EXPR
, type
, args
[0]);
1447 aarch64_gimple_fold_builtin (gimple_stmt_iterator
*gsi
)
1449 bool changed
= false;
1450 gimple
*stmt
= gsi_stmt (*gsi
);
1451 tree call
= gimple_call_fn (stmt
);
1453 gimple
*new_stmt
= NULL
;
1457 fndecl
= gimple_call_fndecl (stmt
);
1460 int fcode
= DECL_FUNCTION_CODE (fndecl
);
1461 unsigned nargs
= gimple_call_num_args (stmt
);
1462 tree
*args
= (nargs
> 0
1463 ? gimple_call_arg_ptr (stmt
, 0)
1464 : &error_mark_node
);
1466 /* We use gimple's REDUC_(PLUS|MIN|MAX)_EXPRs for float, signed int
1467 and unsigned int; it will distinguish according to the types of
1468 the arguments to the __builtin. */
1471 BUILTIN_VALL (UNOP
, reduc_plus_scal_
, 10)
1472 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
1473 REDUC_PLUS_EXPR
, args
[0]);
1475 BUILTIN_VDQIF (UNOP
, reduc_smax_scal_
, 10)
1476 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umax_scal_
, 10)
1477 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
1478 REDUC_MAX_EXPR
, args
[0]);
1480 BUILTIN_VDQIF (UNOP
, reduc_smin_scal_
, 10)
1481 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umin_scal_
, 10)
1482 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
1483 REDUC_MIN_EXPR
, args
[0]);
1485 BUILTIN_GPF (BINOP
, fmulx
, 0)
1487 gcc_assert (nargs
== 2);
1488 bool a0_cst_p
= TREE_CODE (args
[0]) == REAL_CST
;
1489 bool a1_cst_p
= TREE_CODE (args
[1]) == REAL_CST
;
1490 if (a0_cst_p
|| a1_cst_p
)
1492 if (a0_cst_p
&& a1_cst_p
)
1494 tree t0
= TREE_TYPE (args
[0]);
1495 real_value a0
= (TREE_REAL_CST (args
[0]));
1496 real_value a1
= (TREE_REAL_CST (args
[1]));
1497 if (real_equal (&a1
, &dconst0
))
1499 /* According to real_equal (), +0 equals -0. */
1500 if (real_equal (&a0
, &dconst0
) && real_isinf (&a1
))
1502 real_value res
= dconst2
;
1503 res
.sign
= a0
.sign
^ a1
.sign
;
1505 gimple_build_assign (gimple_call_lhs (stmt
),
1507 build_real (t0
, res
));
1511 gimple_build_assign (gimple_call_lhs (stmt
),
1515 else /* a0_cst_p ^ a1_cst_p. */
1517 real_value const_part
= a0_cst_p
1518 ? TREE_REAL_CST (args
[0]) : TREE_REAL_CST (args
[1]);
1519 if (!real_equal (&const_part
, &dconst0
)
1520 && !real_isinf (&const_part
))
1522 gimple_build_assign (gimple_call_lhs (stmt
),
1523 MULT_EXPR
, args
[0], args
[1]);
1528 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
1529 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
1541 gsi_replace (gsi
, new_stmt
, true);
1549 aarch64_atomic_assign_expand_fenv (tree
*hold
, tree
*clear
, tree
*update
)
1551 const unsigned AARCH64_FE_INVALID
= 1;
1552 const unsigned AARCH64_FE_DIVBYZERO
= 2;
1553 const unsigned AARCH64_FE_OVERFLOW
= 4;
1554 const unsigned AARCH64_FE_UNDERFLOW
= 8;
1555 const unsigned AARCH64_FE_INEXACT
= 16;
1556 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT
= (AARCH64_FE_INVALID
1557 | AARCH64_FE_DIVBYZERO
1558 | AARCH64_FE_OVERFLOW
1559 | AARCH64_FE_UNDERFLOW
1560 | AARCH64_FE_INEXACT
);
1561 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT
= 8;
1562 tree fenv_cr
, fenv_sr
, get_fpcr
, set_fpcr
, mask_cr
, mask_sr
;
1563 tree ld_fenv_cr
, ld_fenv_sr
, masked_fenv_cr
, masked_fenv_sr
, hold_fnclex_cr
;
1564 tree hold_fnclex_sr
, new_fenv_var
, reload_fenv
, restore_fnenv
, get_fpsr
, set_fpsr
;
1565 tree update_call
, atomic_feraiseexcept
, hold_fnclex
, masked_fenv
, ld_fenv
;
1567 /* Generate the equivalence of :
1568 unsigned int fenv_cr;
1569 fenv_cr = __builtin_aarch64_get_fpcr ();
1571 unsigned int fenv_sr;
1572 fenv_sr = __builtin_aarch64_get_fpsr ();
1574 Now set all exceptions to non-stop
1575 unsigned int mask_cr
1576 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
1577 unsigned int masked_cr;
1578 masked_cr = fenv_cr & mask_cr;
1580 And clear all exception flags
1581 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
1582 unsigned int masked_cr;
1583 masked_sr = fenv_sr & mask_sr;
1585 __builtin_aarch64_set_cr (masked_cr);
1586 __builtin_aarch64_set_sr (masked_sr); */
1588 fenv_cr
= create_tmp_var_raw (unsigned_type_node
);
1589 fenv_sr
= create_tmp_var_raw (unsigned_type_node
);
1591 get_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
];
1592 set_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
];
1593 get_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
];
1594 set_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
];
1596 mask_cr
= build_int_cst (unsigned_type_node
,
1597 ~(AARCH64_FE_ALL_EXCEPT
<< AARCH64_FE_EXCEPT_SHIFT
));
1598 mask_sr
= build_int_cst (unsigned_type_node
,
1599 ~(AARCH64_FE_ALL_EXCEPT
));
1601 ld_fenv_cr
= build2 (MODIFY_EXPR
, unsigned_type_node
,
1602 fenv_cr
, build_call_expr (get_fpcr
, 0));
1603 ld_fenv_sr
= build2 (MODIFY_EXPR
, unsigned_type_node
,
1604 fenv_sr
, build_call_expr (get_fpsr
, 0));
1606 masked_fenv_cr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_cr
, mask_cr
);
1607 masked_fenv_sr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_sr
, mask_sr
);
1609 hold_fnclex_cr
= build_call_expr (set_fpcr
, 1, masked_fenv_cr
);
1610 hold_fnclex_sr
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
1612 hold_fnclex
= build2 (COMPOUND_EXPR
, void_type_node
, hold_fnclex_cr
,
1614 masked_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv_cr
,
1616 ld_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, ld_fenv_cr
, ld_fenv_sr
);
1618 *hold
= build2 (COMPOUND_EXPR
, void_type_node
,
1619 build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv
, ld_fenv
),
1622 /* Store the value of masked_fenv to clear the exceptions:
1623 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
1625 *clear
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
1627 /* Generate the equivalent of :
1628 unsigned int new_fenv_var;
1629 new_fenv_var = __builtin_aarch64_get_fpsr ();
1631 __builtin_aarch64_set_fpsr (fenv_sr);
1633 __atomic_feraiseexcept (new_fenv_var); */
1635 new_fenv_var
= create_tmp_var_raw (unsigned_type_node
);
1636 reload_fenv
= build2 (MODIFY_EXPR
, unsigned_type_node
,
1637 new_fenv_var
, build_call_expr (get_fpsr
, 0));
1638 restore_fnenv
= build_call_expr (set_fpsr
, 1, fenv_sr
);
1639 atomic_feraiseexcept
= builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT
);
1640 update_call
= build_call_expr (atomic_feraiseexcept
, 1,
1641 fold_convert (integer_type_node
, new_fenv_var
));
1642 *update
= build2 (COMPOUND_EXPR
, void_type_node
,
1643 build2 (COMPOUND_EXPR
, void_type_node
,
1644 reload_fenv
, restore_fnenv
), update_call
);
1648 #undef AARCH64_CHECK_BUILTIN_MODE
1649 #undef AARCH64_FIND_FRINT_VARIANT
1668 #include "gt-aarch64-builtins.h"