2 Copyright (C) 2011-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* The names of each internal function, indexed by function number. */
60 const char *const internal_fn_name_array
[] = {
61 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62 #include "internal-fn.def"
66 /* The ECF_* flags of each internal function, indexed by function number. */
67 const int internal_fn_flags_array
[] = {
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69 #include "internal-fn.def"
73 /* Return the internal function called NAME, or IFN_LAST if there's
77 lookup_internal_fn (const char *name
)
79 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
80 static name_to_fn_map_type
*name_to_fn_map
;
84 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
85 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
86 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
89 internal_fn
*entry
= name_to_fn_map
->get (name
);
90 return entry
? *entry
: IFN_LAST
;
93 /* Geven an internal_fn IFN that is a widening function, return its
94 corresponding LO and HI internal_fns. */
97 lookup_hilo_internal_fn (internal_fn ifn
, internal_fn
*lo
, internal_fn
*hi
)
99 gcc_assert (widening_fn_p (ifn
));
105 #undef DEF_INTERNAL_FN
106 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
107 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
108 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
110 *lo = internal_fn (IFN_##NAME##_LO); \
111 *hi = internal_fn (IFN_##NAME##_HI); \
113 #include "internal-fn.def"
114 #undef DEF_INTERNAL_FN
115 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
119 /* Given an internal_fn IFN that is a widening function, return its
120 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
123 lookup_evenodd_internal_fn (internal_fn ifn
, internal_fn
*even
,
126 gcc_assert (widening_fn_p (ifn
));
132 #undef DEF_INTERNAL_FN
133 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
134 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
135 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
137 *even = internal_fn (IFN_##NAME##_EVEN); \
138 *odd = internal_fn (IFN_##NAME##_ODD); \
140 #include "internal-fn.def"
141 #undef DEF_INTERNAL_FN
142 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
147 /* Fnspec of each internal function, indexed by function number. */
148 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
153 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
154 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
155 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
156 #include "internal-fn.def"
157 internal_fn_fnspec_array
[IFN_LAST
] = 0;
160 /* Create static initializers for the information returned by
161 direct_internal_fn. */
162 #define not_direct { -2, -2, false }
163 #define mask_load_direct { -1, 2, false }
164 #define load_lanes_direct { -1, -1, false }
165 #define mask_load_lanes_direct { -1, -1, false }
166 #define gather_load_direct { 3, 1, false }
167 #define len_load_direct { -1, -1, false }
168 #define mask_len_load_direct { -1, 4, false }
169 #define mask_store_direct { 3, 2, false }
170 #define store_lanes_direct { 0, 0, false }
171 #define mask_store_lanes_direct { 0, 0, false }
172 #define vec_cond_mask_direct { 1, 0, false }
173 #define vec_cond_direct { 2, 0, false }
174 #define scatter_store_direct { 3, 1, false }
175 #define len_store_direct { 3, 3, false }
176 #define mask_len_store_direct { 4, 5, false }
177 #define vec_set_direct { 3, 3, false }
178 #define vec_extract_direct { 0, -1, false }
179 #define unary_direct { 0, 0, true }
180 #define unary_convert_direct { -1, 0, true }
181 #define binary_direct { 0, 0, true }
182 #define ternary_direct { 0, 0, true }
183 #define cond_unary_direct { 1, 1, true }
184 #define cond_binary_direct { 1, 1, true }
185 #define cond_ternary_direct { 1, 1, true }
186 #define cond_len_unary_direct { 1, 1, true }
187 #define cond_len_binary_direct { 1, 1, true }
188 #define cond_len_ternary_direct { 1, 1, true }
189 #define while_direct { 0, 2, false }
190 #define fold_extract_direct { 2, 2, false }
191 #define fold_len_extract_direct { 2, 2, false }
192 #define fold_left_direct { 1, 1, false }
193 #define mask_fold_left_direct { 1, 1, false }
194 #define mask_len_fold_left_direct { 1, 1, false }
195 #define check_ptrs_direct { 0, 0, false }
197 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
198 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
199 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
200 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
201 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
202 #include "internal-fn.def"
206 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
207 output operands and NINPUTS input operands, where NOUTPUTS is either
208 0 or 1. The output operand (if any) comes first, followed by the
209 NINPUTS input operands. */
212 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
213 unsigned int ninputs
)
215 gcc_assert (icode
!= CODE_FOR_nothing
);
217 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
218 unsigned int opno
= 0;
219 rtx lhs_rtx
= NULL_RTX
;
220 tree lhs
= gimple_call_lhs (stmt
);
224 gcc_assert (noutputs
== 1);
226 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
228 /* Do not assign directly to a promoted subreg, since there is no
229 guarantee that the instruction will leave the upper bits of the
230 register in the state required by SUBREG_PROMOTED_SIGN. */
232 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
234 create_output_operand (&ops
[opno
], dest
,
235 insn_data
[icode
].operand
[opno
].mode
);
241 for (unsigned int i
= 0; i
< ninputs
; ++i
)
243 tree rhs
= gimple_call_arg (stmt
, i
);
244 tree rhs_type
= TREE_TYPE (rhs
);
245 rtx rhs_rtx
= expand_normal (rhs
);
246 if (INTEGRAL_TYPE_P (rhs_type
))
247 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
248 TYPE_MODE (rhs_type
),
249 TYPE_UNSIGNED (rhs_type
));
250 else if (TREE_CODE (rhs
) == SSA_NAME
251 && SSA_NAME_IS_DEFAULT_DEF (rhs
)
252 && VAR_P (SSA_NAME_VAR (rhs
)))
253 create_undefined_input_operand (&ops
[opno
], TYPE_MODE (rhs_type
));
255 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
259 gcc_assert (opno
== noutputs
+ ninputs
);
260 expand_insn (icode
, opno
, ops
);
261 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
263 /* If the return value has an integral type, convert the instruction
264 result to that type. This is useful for things that return an
265 int regardless of the size of the input. If the instruction result
266 is smaller than required, assume that it is signed.
268 If the return value has a nonintegral type, its mode must match
269 the instruction result. */
270 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
272 /* If this is a scalar in a register that is stored in a wider
273 mode than the declared mode, compute the result into its
274 declared mode and then convert to the wider mode. */
275 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
276 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
277 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
278 SUBREG_PROMOTED_SIGN (lhs_rtx
));
280 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
281 emit_move_insn (lhs_rtx
, ops
[0].value
);
284 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
285 convert_move (lhs_rtx
, ops
[0].value
, 0);
290 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
291 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
293 static enum insn_code
294 get_multi_vector_move (tree array_type
, convert_optab optab
)
299 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
300 imode
= TYPE_MODE (array_type
);
301 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
303 return convert_optab_handler (optab
, imode
, vmode
);
306 /* Add mask and len arguments according to the STMT. */
309 add_mask_and_len_args (expand_operand
*ops
, unsigned int opno
, gcall
*stmt
)
311 internal_fn ifn
= gimple_call_internal_fn (stmt
);
312 int len_index
= internal_fn_len_index (ifn
);
313 /* BIAS is always consecutive next of LEN. */
314 int bias_index
= len_index
+ 1;
315 int mask_index
= internal_fn_mask_index (ifn
);
316 /* The order of arguments are always {len,bias,mask}. */
319 tree mask
= gimple_call_arg (stmt
, mask_index
);
320 rtx mask_rtx
= expand_normal (mask
);
321 create_input_operand (&ops
[opno
++], mask_rtx
,
322 TYPE_MODE (TREE_TYPE (mask
)));
326 tree len
= gimple_call_arg (stmt
, len_index
);
327 rtx len_rtx
= expand_normal (len
);
328 create_convert_operand_from (&ops
[opno
++], len_rtx
,
329 TYPE_MODE (TREE_TYPE (len
)),
330 TYPE_UNSIGNED (TREE_TYPE (len
)));
331 tree biast
= gimple_call_arg (stmt
, bias_index
);
332 rtx bias
= expand_normal (biast
);
333 create_input_operand (&ops
[opno
++], bias
, QImode
);
338 /* Expand LOAD_LANES call STMT using optab OPTAB. */
341 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
343 class expand_operand ops
[2];
347 lhs
= gimple_call_lhs (stmt
);
348 rhs
= gimple_call_arg (stmt
, 0);
349 type
= TREE_TYPE (lhs
);
351 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
352 mem
= expand_normal (rhs
);
354 gcc_assert (MEM_P (mem
));
355 PUT_MODE (mem
, TYPE_MODE (type
));
357 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
358 create_fixed_operand (&ops
[1], mem
);
359 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
360 if (!rtx_equal_p (target
, ops
[0].value
))
361 emit_move_insn (target
, ops
[0].value
);
364 /* Expand STORE_LANES call STMT using optab OPTAB. */
367 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
369 class expand_operand ops
[2];
373 lhs
= gimple_call_lhs (stmt
);
374 rhs
= gimple_call_arg (stmt
, 0);
375 type
= TREE_TYPE (rhs
);
377 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
378 reg
= expand_normal (rhs
);
380 gcc_assert (MEM_P (target
));
381 PUT_MODE (target
, TYPE_MODE (type
));
383 create_fixed_operand (&ops
[0], target
);
384 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
385 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
389 expand_ANNOTATE (internal_fn
, gcall
*)
394 /* This should get expanded in omp_device_lower pass. */
397 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
402 /* This should get expanded in omp_device_lower pass. */
405 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
410 /* Allocate per-lane storage and begin non-uniform execution region. */
413 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
416 tree lhs
= gimple_call_lhs (stmt
);
418 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
420 target
= gen_reg_rtx (Pmode
);
421 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
422 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
423 class expand_operand ops
[3];
424 create_output_operand (&ops
[0], target
, Pmode
);
425 create_input_operand (&ops
[1], size
, Pmode
);
426 create_input_operand (&ops
[2], align
, Pmode
);
427 gcc_assert (targetm
.have_omp_simt_enter ());
428 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
429 if (!rtx_equal_p (target
, ops
[0].value
))
430 emit_move_insn (target
, ops
[0].value
);
433 /* Deallocate per-lane storage and leave non-uniform execution region. */
436 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
438 gcc_checking_assert (!gimple_call_lhs (stmt
));
439 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
440 class expand_operand ops
[1];
441 create_input_operand (&ops
[0], arg
, Pmode
);
442 gcc_assert (targetm
.have_omp_simt_exit ());
443 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
446 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
447 without SIMT execution this should be expanded in omp_device_lower pass. */
450 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
452 tree lhs
= gimple_call_lhs (stmt
);
456 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
457 gcc_assert (targetm
.have_omp_simt_lane ());
458 emit_insn (targetm
.gen_omp_simt_lane (target
));
461 /* This should get expanded in omp_device_lower pass. */
464 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
469 /* This should get expanded in omp_device_lower pass. */
472 expand_GOMP_TARGET_REV (internal_fn
, gcall
*)
477 /* Lane index of the first SIMT lane that supplies a non-zero argument.
478 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
479 lane that executed the last iteration for handling OpenMP lastprivate. */
482 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
484 tree lhs
= gimple_call_lhs (stmt
);
488 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
489 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
490 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
491 class expand_operand ops
[2];
492 create_output_operand (&ops
[0], target
, mode
);
493 create_input_operand (&ops
[1], cond
, mode
);
494 gcc_assert (targetm
.have_omp_simt_last_lane ());
495 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
496 if (!rtx_equal_p (target
, ops
[0].value
))
497 emit_move_insn (target
, ops
[0].value
);
500 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
503 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
505 tree lhs
= gimple_call_lhs (stmt
);
509 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
510 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
511 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
512 class expand_operand ops
[2];
513 create_output_operand (&ops
[0], target
, mode
);
514 create_input_operand (&ops
[1], ctr
, mode
);
515 gcc_assert (targetm
.have_omp_simt_ordered ());
516 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
517 if (!rtx_equal_p (target
, ops
[0].value
))
518 emit_move_insn (target
, ops
[0].value
);
521 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
522 any lane supplies a non-zero argument. */
525 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
527 tree lhs
= gimple_call_lhs (stmt
);
531 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
532 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
533 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
534 class expand_operand ops
[2];
535 create_output_operand (&ops
[0], target
, mode
);
536 create_input_operand (&ops
[1], cond
, mode
);
537 gcc_assert (targetm
.have_omp_simt_vote_any ());
538 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
539 if (!rtx_equal_p (target
, ops
[0].value
))
540 emit_move_insn (target
, ops
[0].value
);
543 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
544 is destination lane index XOR given offset. */
547 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
549 tree lhs
= gimple_call_lhs (stmt
);
553 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
554 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
555 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
556 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
557 class expand_operand ops
[3];
558 create_output_operand (&ops
[0], target
, mode
);
559 create_input_operand (&ops
[1], src
, mode
);
560 create_input_operand (&ops
[2], idx
, SImode
);
561 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
562 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
563 if (!rtx_equal_p (target
, ops
[0].value
))
564 emit_move_insn (target
, ops
[0].value
);
567 /* Exchange between SIMT lanes according to given source lane index. */
570 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
572 tree lhs
= gimple_call_lhs (stmt
);
576 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
577 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
578 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
579 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
580 class expand_operand ops
[3];
581 create_output_operand (&ops
[0], target
, mode
);
582 create_input_operand (&ops
[1], src
, mode
);
583 create_input_operand (&ops
[2], idx
, SImode
);
584 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
585 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
586 if (!rtx_equal_p (target
, ops
[0].value
))
587 emit_move_insn (target
, ops
[0].value
);
590 /* This should get expanded in adjust_simduid_builtins. */
593 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
598 /* This should get expanded in adjust_simduid_builtins. */
601 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
606 /* This should get expanded in adjust_simduid_builtins. */
609 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
614 /* This should get expanded in adjust_simduid_builtins. */
617 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
622 /* This should get expanded in adjust_simduid_builtins. */
625 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
630 /* This should get expanded in the sanopt pass. */
633 expand_UBSAN_NULL (internal_fn
, gcall
*)
638 /* This should get expanded in the sanopt pass. */
641 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
646 /* This should get expanded in the sanopt pass. */
649 expand_UBSAN_VPTR (internal_fn
, gcall
*)
654 /* This should get expanded in the sanopt pass. */
657 expand_UBSAN_PTR (internal_fn
, gcall
*)
662 /* This should get expanded in the sanopt pass. */
665 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
670 /* This should get expanded in the sanopt pass. */
673 expand_HWASAN_CHECK (internal_fn
, gcall
*)
678 /* For hwasan stack tagging:
679 Clear tags on the dynamically allocated space.
680 For use after an object dynamically allocated on the stack goes out of
683 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
685 gcc_assert (Pmode
== ptr_mode
);
686 tree restored_position
= gimple_call_arg (gc
, 0);
687 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
689 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
690 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
691 stack_pointer_rtx
, NULL_RTX
, 0,
693 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
694 virtual_stack_dynamic_rtx
, Pmode
,
695 HWASAN_STACK_BACKGROUND
, QImode
,
699 /* For hwasan stack tagging:
700 Return a tag to be used for a dynamic allocation. */
702 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
704 tree tag
= gimple_call_lhs (gc
);
705 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
706 machine_mode mode
= GET_MODE (target
);
707 gcc_assert (mode
== QImode
);
709 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
710 gcc_assert (base_tag
);
711 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
712 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
713 target
, /* unsignedp = */1,
715 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
717 /* Really need to put the tag into the `target` RTX. */
718 if (chosen_tag
!= target
)
720 rtx temp
= chosen_tag
;
721 gcc_assert (GET_MODE (chosen_tag
) == mode
);
722 emit_move_insn (target
, temp
);
725 hwasan_increment_frame_tag ();
728 /* For hwasan stack tagging:
729 Tag a region of space in the shadow stack according to the base pointer of
730 an object on the stack. N.b. the length provided in the internal call is
731 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
733 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
735 gcc_assert (ptr_mode
== Pmode
);
736 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
737 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
739 tree base
= gimple_call_arg (gc
, 1);
740 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
741 rtx base_rtx
= expand_normal (base
);
743 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
744 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
745 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
747 tree len
= gimple_call_arg (gc
, 2);
748 rtx r_len
= expand_normal (len
);
750 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
751 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
752 tag
, QImode
, r_len
, Pmode
);
755 /* For hwasan stack tagging:
756 Store a tag into a pointer. */
758 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
760 gcc_assert (ptr_mode
== Pmode
);
761 tree g_target
= gimple_call_lhs (gc
);
762 tree g_ptr
= gimple_call_arg (gc
, 0);
763 tree g_tag
= gimple_call_arg (gc
, 1);
765 rtx ptr
= expand_normal (g_ptr
);
766 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
767 rtx target
= expand_normal (g_target
);
769 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
770 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
771 if (tagged_value
!= target
)
772 emit_move_insn (target
, tagged_value
);
775 /* This should get expanded in the sanopt pass. */
778 expand_ASAN_CHECK (internal_fn
, gcall
*)
783 /* This should get expanded in the sanopt pass. */
786 expand_ASAN_MARK (internal_fn
, gcall
*)
791 /* This should get expanded in the sanopt pass. */
794 expand_ASAN_POISON (internal_fn
, gcall
*)
799 /* This should get expanded in the sanopt pass. */
802 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
807 /* This should get expanded in the tsan pass. */
810 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
815 /* This should get expanded in the lower pass. */
818 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
820 error_at (gimple_location (call
),
821 "invalid use of attribute %<fallthrough%>");
824 /* Return minimum precision needed to represent all values
825 of ARG in SIGNed integral type. */
828 get_min_precision (tree arg
, signop sign
)
830 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
832 signop orig_sign
= sign
;
833 if (TREE_CODE (arg
) == INTEGER_CST
)
836 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
838 widest_int w
= wi::to_widest (arg
);
839 w
= wi::ext (w
, prec
, sign
);
840 p
= wi::min_precision (w
, sign
);
843 p
= wi::min_precision (wi::to_wide (arg
), sign
);
844 return MIN (p
, prec
);
846 while (CONVERT_EXPR_P (arg
)
847 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
848 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
850 arg
= TREE_OPERAND (arg
, 0);
851 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
853 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
855 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
856 return prec
+ (orig_sign
!= sign
);
857 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
860 return prec
+ (orig_sign
!= sign
);
862 if (CONVERT_EXPR_P (arg
)
863 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
864 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
866 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
867 If y_2's min precision is smaller than prec, return that. */
868 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
870 return oprec
+ (orig_sign
!= sign
);
872 if (TREE_CODE (arg
) != SSA_NAME
)
873 return prec
+ (orig_sign
!= sign
);
875 while (!get_global_range_query ()->range_of_expr (r
, arg
)
879 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
880 if (is_gimple_assign (g
)
881 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
883 tree t
= gimple_assign_rhs1 (g
);
884 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
885 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
888 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
890 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
892 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
893 return prec
+ (orig_sign
!= sign
);
894 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
897 return prec
+ (orig_sign
!= sign
);
901 return prec
+ (orig_sign
!= sign
);
903 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
905 int p1
= wi::min_precision (r
.lower_bound (), sign
);
906 int p2
= wi::min_precision (r
.upper_bound (), sign
);
908 prec
= MIN (prec
, p1
);
910 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
912 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
913 prec
= MIN (prec
, p
);
915 return prec
+ (orig_sign
!= sign
);
918 /* Helper for expand_*_overflow. Set the __imag__ part to true
919 (1 except for signed:1 type, in which case store -1). */
922 expand_arith_set_overflow (tree lhs
, rtx target
)
924 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
925 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
926 write_complex_part (target
, constm1_rtx
, true, false);
928 write_complex_part (target
, const1_rtx
, true, false);
931 /* Helper for expand_*_overflow. Store RES into the __real__ part
932 of TARGET. If RES has larger MODE than __real__ part of TARGET,
933 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
934 if LHS has smaller precision than its mode. */
937 expand_arith_overflow_result_store (tree lhs
, rtx target
,
938 scalar_int_mode mode
, rtx res
)
940 scalar_int_mode tgtmode
941 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
945 rtx_code_label
*done_label
= gen_label_rtx ();
946 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
947 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
948 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
949 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
950 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
951 profile_probability::very_likely ());
952 expand_arith_set_overflow (lhs
, target
);
953 emit_label (done_label
);
955 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
956 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
959 rtx_code_label
*done_label
= gen_label_rtx ();
960 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
965 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
967 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
968 true, OPTAB_LIB_WIDEN
);
972 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
974 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
977 do_compare_rtx_and_jump (res
, lres
,
978 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
979 profile_probability::very_likely ());
980 expand_arith_set_overflow (lhs
, target
);
981 emit_label (done_label
);
983 write_complex_part (target
, lres
, false, false);
986 /* Helper for expand_*_overflow. Store RES into TARGET. */
989 expand_ubsan_result_store (tree lhs
, rtx target
, scalar_int_mode mode
,
990 rtx res
, rtx_code_label
*do_error
)
992 if (TREE_CODE (TREE_TYPE (lhs
)) == BITINT_TYPE
993 && TYPE_PRECISION (TREE_TYPE (lhs
)) < GET_MODE_PRECISION (mode
))
995 int uns
= TYPE_UNSIGNED (TREE_TYPE (lhs
));
996 int prec
= TYPE_PRECISION (TREE_TYPE (lhs
));
997 int tgtprec
= GET_MODE_PRECISION (mode
);
998 rtx resc
= gen_reg_rtx (mode
), lres
;
999 emit_move_insn (resc
, res
);
1003 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
1005 lres
= expand_simple_binop (mode
, AND
, res
, mask
, NULL_RTX
,
1006 true, OPTAB_LIB_WIDEN
);
1010 lres
= expand_shift (LSHIFT_EXPR
, mode
, res
, tgtprec
- prec
,
1012 lres
= expand_shift (RSHIFT_EXPR
, mode
, lres
, tgtprec
- prec
,
1016 emit_move_insn (res
, lres
);
1017 do_compare_rtx_and_jump (res
, resc
,
1018 NE
, true, mode
, NULL_RTX
, NULL
, do_error
,
1019 profile_probability::very_unlikely ());
1021 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
1022 /* If this is a scalar in a register that is stored in a wider mode
1023 than the declared mode, compute the result into its declared mode
1024 and then convert to the wider mode. Our value is the computed
1026 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
1028 emit_move_insn (target
, res
);
1031 /* Add sub/add overflow checking to the statement STMT.
1032 CODE says whether the operation is +, or -. */
1035 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
1036 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
1037 bool uns1_p
, bool is_ubsan
, tree
*datap
)
1039 rtx res
, target
= NULL_RTX
;
1041 rtx_code_label
*done_label
= gen_label_rtx ();
1042 rtx_code_label
*do_error
= gen_label_rtx ();
1043 do_pending_stack_adjust ();
1044 rtx op0
= expand_normal (arg0
);
1045 rtx op1
= expand_normal (arg1
);
1046 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1047 int prec
= GET_MODE_PRECISION (mode
);
1048 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1049 bool do_xor
= false;
1052 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1056 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1058 write_complex_part (target
, const0_rtx
, true, false);
1061 /* We assume both operands and result have the same precision
1062 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1063 with that precision, U for unsigned type with that precision,
1064 sgn for unsigned most significant bit in that precision.
1065 s1 is signed first operand, u1 is unsigned first operand,
1066 s2 is signed second operand, u2 is unsigned second operand,
1067 sr is signed result, ur is unsigned result and the following
1068 rules say how to compute result (which is always result of
1069 the operands as if both were unsigned, cast to the right
1070 signedness) and how to compute whether operation overflowed.
1073 res = (S) ((U) s1 + (U) s2)
1074 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1076 res = (S) ((U) s1 - (U) s2)
1077 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1080 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1083 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1085 res = (S) ((U) s1 + u2)
1086 ovf = ((U) res ^ sgn) < u2
1091 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1093 res = (S) ((U) s1 - u2)
1094 ovf = u2 > ((U) s1 ^ sgn)
1097 ovf = s1 < 0 || u2 > (U) s1
1100 ovf = u1 >= ((U) s2 ^ sgn)
1105 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1107 res = (U) s1 + (U) s2
1108 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1111 ovf = (U) res < u2 || res < 0
1114 ovf = u1 >= u2 ? res < 0 : res >= 0
1116 res = (U) s1 - (U) s2
1117 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1119 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
1121 /* PLUS_EXPR is commutative, if operand signedness differs,
1122 canonicalize to the first operand being signed and second
1123 unsigned to simplify following code. */
1124 std::swap (op0
, op1
);
1125 std::swap (arg0
, arg1
);
1130 /* u1 +- u2 -> ur */
1131 if (uns0_p
&& uns1_p
&& unsr_p
)
1133 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
1134 : usubv4_optab
, mode
);
1135 if (icode
!= CODE_FOR_nothing
)
1137 class expand_operand ops
[4];
1138 rtx_insn
*last
= get_last_insn ();
1140 res
= gen_reg_rtx (mode
);
1141 create_output_operand (&ops
[0], res
, mode
);
1142 create_input_operand (&ops
[1], op0
, mode
);
1143 create_input_operand (&ops
[2], op1
, mode
);
1144 create_fixed_operand (&ops
[3], do_error
);
1145 if (maybe_expand_insn (icode
, 4, ops
))
1147 last
= get_last_insn ();
1148 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1150 && any_condjump_p (last
)
1151 && !find_reg_note (last
, REG_BR_PROB
, 0))
1152 add_reg_br_prob_note (last
,
1153 profile_probability::very_unlikely ());
1154 emit_jump (done_label
);
1155 goto do_error_label
;
1158 delete_insns_since (last
);
1161 /* Compute the operation. On RTL level, the addition is always
1163 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1164 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1166 /* For PLUS_EXPR, the operation is commutative, so we can pick
1167 operand to compare against. For prec <= BITS_PER_WORD, I think
1168 preferring REG operand is better over CONST_INT, because
1169 the CONST_INT might enlarge the instruction or CSE would need
1170 to figure out we'd already loaded it into a register before.
1171 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1172 as then the multi-word comparison can be perhaps simplified. */
1173 if (code
== PLUS_EXPR
1174 && (prec
<= BITS_PER_WORD
1175 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1176 : CONST_SCALAR_INT_P (op1
)))
1178 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1179 true, mode
, NULL_RTX
, NULL
, done_label
,
1180 profile_probability::very_likely ());
1181 goto do_error_label
;
1184 /* s1 +- u2 -> sr */
1185 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1187 /* Compute the operation. On RTL level, the addition is always
1189 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1190 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1191 rtx tem
= expand_binop (mode
, add_optab
,
1192 code
== PLUS_EXPR
? res
: op0
, sgn
,
1193 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1194 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1195 done_label
, profile_probability::very_likely ());
1196 goto do_error_label
;
1200 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1202 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1204 /* As we've changed op1, we have to avoid using the value range
1205 for the original argument. */
1206 arg1
= error_mark_node
;
1212 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1214 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1216 /* As we've changed op0, we have to avoid using the value range
1217 for the original argument. */
1218 arg0
= error_mark_node
;
1224 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1226 /* Compute the operation. On RTL level, the addition is always
1228 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1230 int pos_neg
= get_range_pos_neg (arg0
);
1232 /* If ARG0 is known to be always negative, this is always overflow. */
1233 emit_jump (do_error
);
1234 else if (pos_neg
== 3)
1235 /* If ARG0 is not known to be always positive, check at runtime. */
1236 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1237 NULL
, do_error
, profile_probability::very_unlikely ());
1238 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1239 done_label
, profile_probability::very_likely ());
1240 goto do_error_label
;
1244 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1246 /* Compute the operation. On RTL level, the addition is always
1248 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1250 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1252 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1253 done_label
, profile_probability::very_likely ());
1254 goto do_error_label
;
1258 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1260 /* Compute the operation. On RTL level, the addition is always
1262 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1264 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1265 NULL
, do_error
, profile_probability::very_unlikely ());
1267 /* The operation is commutative, so we can pick operand to compare
1268 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1269 is better over CONST_INT, because the CONST_INT might enlarge the
1270 instruction or CSE would need to figure out we'd already loaded it
1271 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1272 might be more beneficial, as then the multi-word comparison can be
1273 perhaps simplified. */
1274 if (prec
<= BITS_PER_WORD
1275 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1276 : CONST_SCALAR_INT_P (op0
))
1278 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1279 done_label
, profile_probability::very_likely ());
1280 goto do_error_label
;
1283 /* s1 +- s2 -> ur */
1284 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1286 /* Compute the operation. On RTL level, the addition is always
1288 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1289 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1290 int pos_neg
= get_range_pos_neg (arg1
);
1291 if (code
== PLUS_EXPR
)
1293 int pos_neg0
= get_range_pos_neg (arg0
);
1294 if (pos_neg0
!= 3 && pos_neg
== 3)
1296 std::swap (op0
, op1
);
1303 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1304 ? and_optab
: ior_optab
,
1305 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1306 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1307 NULL
, done_label
, profile_probability::very_likely ());
1311 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1312 do_compare_rtx_and_jump (op1
, const0_rtx
,
1313 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1314 NULL_RTX
, NULL
, do_ior_label
,
1315 profile_probability::even ());
1316 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1318 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1319 NULL
, done_label
, profile_probability::very_likely ());
1320 emit_jump (do_error
);
1321 emit_label (do_ior_label
);
1322 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1324 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1325 NULL
, done_label
, profile_probability::very_likely ());
1327 goto do_error_label
;
1331 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1333 /* Compute the operation. On RTL level, the addition is always
1335 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1337 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1338 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1339 op0_geu_op1
, profile_probability::even ());
1340 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1341 NULL
, done_label
, profile_probability::very_likely ());
1342 emit_jump (do_error
);
1343 emit_label (op0_geu_op1
);
1344 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1345 NULL
, done_label
, profile_probability::very_likely ());
1346 goto do_error_label
;
1349 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1351 /* s1 +- s2 -> sr */
1354 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1355 : subv4_optab
, mode
);
1356 if (icode
!= CODE_FOR_nothing
)
1358 class expand_operand ops
[4];
1359 rtx_insn
*last
= get_last_insn ();
1361 res
= gen_reg_rtx (mode
);
1362 create_output_operand (&ops
[0], res
, mode
);
1363 create_input_operand (&ops
[1], op0
, mode
);
1364 create_input_operand (&ops
[2], op1
, mode
);
1365 create_fixed_operand (&ops
[3], do_error
);
1366 if (maybe_expand_insn (icode
, 4, ops
))
1368 last
= get_last_insn ();
1369 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1371 && any_condjump_p (last
)
1372 && !find_reg_note (last
, REG_BR_PROB
, 0))
1373 add_reg_br_prob_note (last
,
1374 profile_probability::very_unlikely ());
1375 emit_jump (done_label
);
1376 goto do_error_label
;
1379 delete_insns_since (last
);
1382 /* Compute the operation. On RTL level, the addition is always
1384 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1385 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1387 /* If we can prove that one of the arguments (for MINUS_EXPR only
1388 the second operand, as subtraction is not commutative) is always
1389 non-negative or always negative, we can do just one comparison
1390 and conditional jump. */
1391 int pos_neg
= get_range_pos_neg (arg1
);
1392 if (code
== PLUS_EXPR
)
1394 int pos_neg0
= get_range_pos_neg (arg0
);
1395 if (pos_neg0
!= 3 && pos_neg
== 3)
1397 std::swap (op0
, op1
);
1402 /* Addition overflows if and only if the two operands have the same sign,
1403 and the result has the opposite sign. Subtraction overflows if and
1404 only if the two operands have opposite sign, and the subtrahend has
1405 the same sign as the result. Here 0 is counted as positive. */
1408 /* Compute op0 ^ op1 (operands have opposite sign). */
1409 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1412 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1413 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1417 if (code
== PLUS_EXPR
)
1419 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1420 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1421 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1426 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1427 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1428 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1432 /* No overflow if the result has bit sign cleared. */
1433 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1434 NULL
, done_label
, profile_probability::very_likely ());
1437 /* Compare the result of the operation with the first operand.
1438 No overflow for addition if second operand is positive and result
1439 is larger or second operand is negative and result is smaller.
1440 Likewise for subtraction with sign of second operand flipped. */
1442 do_compare_rtx_and_jump (res
, op0
,
1443 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1444 false, mode
, NULL_RTX
, NULL
, done_label
,
1445 profile_probability::very_likely ());
1449 emit_label (do_error
);
1452 /* Expand the ubsan builtin call. */
1454 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1458 do_pending_stack_adjust ();
1461 expand_arith_set_overflow (lhs
, target
);
1464 emit_label (done_label
);
1469 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1473 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1476 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1481 /* Add negate overflow checking to the statement STMT. */
1484 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1489 rtx_code_label
*done_label
, *do_error
;
1490 rtx target
= NULL_RTX
;
1492 done_label
= gen_label_rtx ();
1493 do_error
= gen_label_rtx ();
1495 do_pending_stack_adjust ();
1496 op1
= expand_normal (arg1
);
1498 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1501 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1503 write_complex_part (target
, const0_rtx
, true, false);
1506 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1507 if (icode
!= CODE_FOR_nothing
)
1509 class expand_operand ops
[3];
1510 rtx_insn
*last
= get_last_insn ();
1512 res
= gen_reg_rtx (mode
);
1513 create_output_operand (&ops
[0], res
, mode
);
1514 create_input_operand (&ops
[1], op1
, mode
);
1515 create_fixed_operand (&ops
[2], do_error
);
1516 if (maybe_expand_insn (icode
, 3, ops
))
1518 last
= get_last_insn ();
1519 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1521 && any_condjump_p (last
)
1522 && !find_reg_note (last
, REG_BR_PROB
, 0))
1523 add_reg_br_prob_note (last
,
1524 profile_probability::very_unlikely ());
1525 emit_jump (done_label
);
1529 delete_insns_since (last
);
1530 icode
= CODE_FOR_nothing
;
1534 if (icode
== CODE_FOR_nothing
)
1536 /* Compute the operation. On RTL level, the addition is always
1538 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1540 /* Compare the operand with the most negative value. */
1541 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1542 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1543 done_label
, profile_probability::very_likely ());
1546 emit_label (do_error
);
1549 /* Expand the ubsan builtin call. */
1551 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1552 arg1
, NULL_TREE
, datap
);
1555 do_pending_stack_adjust ();
1558 expand_arith_set_overflow (lhs
, target
);
1561 emit_label (done_label
);
1566 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1568 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1572 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1573 mode MODE can be expanded without using a libcall. */
1576 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1577 rtx op0
, rtx op1
, bool uns
)
1579 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1580 != CODE_FOR_nothing
)
1583 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1584 != CODE_FOR_nothing
)
1587 rtx_insn
*last
= get_last_insn ();
1588 if (CONSTANT_P (op0
))
1589 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1591 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1592 if (CONSTANT_P (op1
))
1593 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1595 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1596 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1597 delete_insns_since (last
);
1598 return ret
!= NULL_RTX
;
1601 /* Add mul overflow checking to the statement STMT. */
1604 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1605 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1610 rtx_code_label
*done_label
, *do_error
;
1611 rtx target
= NULL_RTX
;
1613 enum insn_code icode
;
1615 done_label
= gen_label_rtx ();
1616 do_error
= gen_label_rtx ();
1618 do_pending_stack_adjust ();
1619 op0
= expand_normal (arg0
);
1620 op1
= expand_normal (arg1
);
1622 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1626 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1628 write_complex_part (target
, const0_rtx
, true, false);
1632 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1634 /* We assume both operands and result have the same precision
1635 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1636 with that precision, U for unsigned type with that precision,
1637 sgn for unsigned most significant bit in that precision.
1638 s1 is signed first operand, u1 is unsigned first operand,
1639 s2 is signed second operand, u2 is unsigned second operand,
1640 sr is signed result, ur is unsigned result and the following
1641 rules say how to compute result (which is always result of
1642 the operands as if both were unsigned, cast to the right
1643 signedness) and how to compute whether operation overflowed.
1644 main_ovf (false) stands for jump on signed multiplication
1645 overflow or the main algorithm with uns == false.
1646 main_ovf (true) stands for jump on unsigned multiplication
1647 overflow or the main algorithm with uns == true.
1650 res = (S) ((U) s1 * (U) s2)
1651 ovf = main_ovf (false)
1654 ovf = main_ovf (true)
1657 ovf = (s1 < 0 && u2) || main_ovf (true)
1660 ovf = res < 0 || main_ovf (true)
1662 res = (S) ((U) s1 * u2)
1663 ovf = (S) u2 >= 0 ? main_ovf (false)
1664 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1666 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1667 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1669 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1671 if (uns0_p
&& !uns1_p
)
1673 /* Multiplication is commutative, if operand signedness differs,
1674 canonicalize to the first operand being signed and second
1675 unsigned to simplify following code. */
1676 std::swap (op0
, op1
);
1677 std::swap (arg0
, arg1
);
1682 int pos_neg0
= get_range_pos_neg (arg0
);
1683 int pos_neg1
= get_range_pos_neg (arg1
);
1684 /* Unsigned types with smaller than mode precision, even if they have most
1685 significant bit set, are still zero-extended. */
1686 if (uns0_p
&& TYPE_PRECISION (TREE_TYPE (arg0
)) < GET_MODE_PRECISION (mode
))
1688 if (uns1_p
&& TYPE_PRECISION (TREE_TYPE (arg1
)) < GET_MODE_PRECISION (mode
))
1692 if (!uns0_p
&& uns1_p
&& unsr_p
)
1697 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1700 /* If s1 is negative, avoid the main code, just multiply and
1701 signal overflow if op1 is not 0. */
1702 struct separate_ops ops
;
1703 ops
.code
= MULT_EXPR
;
1704 ops
.type
= TREE_TYPE (arg1
);
1705 ops
.op0
= make_tree (ops
.type
, op0
);
1706 ops
.op1
= make_tree (ops
.type
, op1
);
1707 ops
.op2
= NULL_TREE
;
1709 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1710 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1711 NULL
, done_label
, profile_probability::very_likely ());
1712 goto do_error_label
;
1714 if (get_min_precision (arg1
, UNSIGNED
)
1715 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1717 /* If the first operand is sign extended from narrower type, the
1718 second operand is zero extended from narrower type and
1719 the sum of the two precisions is smaller or equal to the
1720 result precision: if the first argument is at runtime
1721 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1722 and there will be no overflow, if the first argument is
1723 negative and the second argument zero, the result will be
1724 0 and there will be no overflow, if the first argument is
1725 negative and the second argument positive, the result when
1726 treated as signed will be negative (minimum -0x7f80 or
1727 -0x7f..f80..0) there will be always overflow. So, do
1729 ovf = (S) res < 0 */
1730 struct separate_ops ops
;
1731 ops
.code
= MULT_EXPR
;
1733 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1735 ops
.op0
= make_tree (ops
.type
, op0
);
1736 ops
.op1
= make_tree (ops
.type
, op1
);
1737 ops
.op2
= NULL_TREE
;
1739 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1740 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1741 mode
, NULL_RTX
, NULL
, done_label
,
1742 profile_probability::very_likely ());
1743 goto do_error_label
;
1745 rtx_code_label
*do_main_label
;
1746 do_main_label
= gen_label_rtx ();
1747 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1748 NULL
, do_main_label
, profile_probability::very_likely ());
1749 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1750 NULL
, do_main_label
, profile_probability::very_likely ());
1751 expand_arith_set_overflow (lhs
, target
);
1752 emit_label (do_main_label
);
1760 if (uns0_p
&& uns1_p
&& !unsr_p
)
1762 if ((pos_neg0
| pos_neg1
) == 1)
1764 /* If both arguments are zero extended from narrower types,
1765 the MSB will be clear on both and so we can pretend it is
1766 a normal s1 * s2 -> sr multiplication. */
1772 /* Rest of handling of this case after res is computed. */
1777 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1784 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1785 avoid the main code, just multiply and signal overflow
1786 unless 0 * u2 or -1 * ((U) Smin). */
1787 struct separate_ops ops
;
1788 ops
.code
= MULT_EXPR
;
1789 ops
.type
= TREE_TYPE (arg1
);
1790 ops
.op0
= make_tree (ops
.type
, op0
);
1791 ops
.op1
= make_tree (ops
.type
, op1
);
1792 ops
.op2
= NULL_TREE
;
1794 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1795 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1796 NULL
, done_label
, profile_probability::very_likely ());
1797 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1798 NULL
, do_error
, profile_probability::very_unlikely ());
1800 prec
= GET_MODE_PRECISION (mode
);
1802 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1803 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1804 NULL
, done_label
, profile_probability::very_likely ());
1805 goto do_error_label
;
1807 /* Rest of handling of this case after res is computed. */
1815 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1818 switch (pos_neg0
| pos_neg1
)
1820 case 1: /* Both operands known to be non-negative. */
1822 case 2: /* Both operands known to be negative. */
1823 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1824 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1825 /* Avoid looking at arg0/arg1 ranges, as we've changed
1827 arg0
= error_mark_node
;
1828 arg1
= error_mark_node
;
1831 if ((pos_neg0
^ pos_neg1
) == 3)
1833 /* If one operand is known to be negative and the other
1834 non-negative, this overflows always, unless the non-negative
1835 one is 0. Just do normal multiply and set overflow
1836 unless one of the operands is 0. */
1837 struct separate_ops ops
;
1838 ops
.code
= MULT_EXPR
;
1840 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1842 ops
.op0
= make_tree (ops
.type
, op0
);
1843 ops
.op1
= make_tree (ops
.type
, op1
);
1844 ops
.op2
= NULL_TREE
;
1846 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1847 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1848 true, mode
, NULL_RTX
, NULL
, done_label
,
1849 profile_probability::very_likely ());
1850 goto do_error_label
;
1852 if (get_min_precision (arg0
, SIGNED
)
1853 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1855 /* If both operands are sign extended from narrower types and
1856 the sum of the two precisions is smaller or equal to the
1857 result precision: if both arguments are at runtime
1858 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1859 and there will be no overflow, if both arguments are negative,
1860 maximum result will be 0x40..00 and there will be no overflow
1861 either, if one argument is positive and the other argument
1862 negative, the result when treated as signed will be negative
1863 and there will be always overflow, and if one argument is
1864 zero and the other negative the result will be zero and no
1867 ovf = (S) res < 0 */
1868 struct separate_ops ops
;
1869 ops
.code
= MULT_EXPR
;
1871 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1873 ops
.op0
= make_tree (ops
.type
, op0
);
1874 ops
.op1
= make_tree (ops
.type
, op1
);
1875 ops
.op2
= NULL_TREE
;
1877 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1878 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1879 mode
, NULL_RTX
, NULL
, done_label
,
1880 profile_probability::very_likely ());
1881 goto do_error_label
;
1883 /* The general case, do all the needed comparisons at runtime. */
1884 rtx_code_label
*do_main_label
, *after_negate_label
;
1886 rop0
= gen_reg_rtx (mode
);
1887 rop1
= gen_reg_rtx (mode
);
1888 emit_move_insn (rop0
, op0
);
1889 emit_move_insn (rop1
, op1
);
1892 do_main_label
= gen_label_rtx ();
1893 after_negate_label
= gen_label_rtx ();
1894 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1896 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1897 NULL
, after_negate_label
, profile_probability::very_likely ());
1898 /* Both arguments negative here, negate them and continue with
1899 normal unsigned overflow checking multiplication. */
1900 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1902 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1904 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1906 arg0
= error_mark_node
;
1907 arg1
= error_mark_node
;
1908 emit_jump (do_main_label
);
1909 emit_label (after_negate_label
);
1910 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1912 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1913 NULL
, do_main_label
,
1914 profile_probability::very_likely ());
1915 /* One argument is negative here, the other positive. This
1916 overflows always, unless one of the arguments is 0. But
1917 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1918 is, thus we can keep do_main code oring in overflow as is. */
1920 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1921 NULL
, do_main_label
,
1922 profile_probability::very_unlikely ());
1924 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1925 NULL
, do_main_label
,
1926 profile_probability::very_unlikely ());
1927 expand_arith_set_overflow (lhs
, target
);
1928 emit_label (do_main_label
);
1936 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1937 sign
= uns
? UNSIGNED
: SIGNED
;
1938 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1940 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1941 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1943 /* Optimize unsigned multiplication by power of 2 constant
1944 using 2 shifts, one for result, one to extract the shifted
1945 out bits to see if they are all zero.
1946 Don't do this if optimizing for size and we have umulv4_optab,
1947 in that case assume multiplication will be shorter.
1948 This is heuristics based on the single target that provides
1949 umulv4 right now (i?86/x86_64), if further targets add it, this
1950 might need to be revisited.
1951 Cases where both operands are constant should be folded already
1952 during GIMPLE, and cases where one operand is constant but not
1953 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1954 below can be done without multiplication, just by shifts and adds,
1955 or we'd need to divide the result (and hope it actually doesn't
1956 really divide nor multiply) and compare the result of the division
1957 with the original operand. */
1962 if (integer_pow2p (arg0
))
1964 std::swap (opn0
, opn1
);
1965 std::swap (argn0
, argn1
);
1967 int cnt
= tree_log2 (argn1
);
1968 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1970 rtx upper
= const0_rtx
;
1971 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1973 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1974 GET_MODE_PRECISION (mode
) - cnt
,
1976 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1977 NULL_RTX
, NULL
, done_label
,
1978 profile_probability::very_likely ());
1979 goto do_error_label
;
1982 if (icode
!= CODE_FOR_nothing
)
1984 class expand_operand ops
[4];
1985 rtx_insn
*last
= get_last_insn ();
1987 res
= gen_reg_rtx (mode
);
1988 create_output_operand (&ops
[0], res
, mode
);
1989 create_input_operand (&ops
[1], op0
, mode
);
1990 create_input_operand (&ops
[2], op1
, mode
);
1991 create_fixed_operand (&ops
[3], do_error
);
1992 if (maybe_expand_insn (icode
, 4, ops
))
1994 last
= get_last_insn ();
1995 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1997 && any_condjump_p (last
)
1998 && !find_reg_note (last
, REG_BR_PROB
, 0))
1999 add_reg_br_prob_note (last
,
2000 profile_probability::very_unlikely ());
2001 emit_jump (done_label
);
2005 delete_insns_since (last
);
2006 icode
= CODE_FOR_nothing
;
2010 if (icode
== CODE_FOR_nothing
)
2012 struct separate_ops ops
;
2013 int prec
= GET_MODE_PRECISION (mode
);
2014 scalar_int_mode hmode
, wmode
;
2015 ops
.op0
= make_tree (type
, op0
);
2016 ops
.op1
= make_tree (type
, op1
);
2017 ops
.op2
= NULL_TREE
;
2020 /* Optimize unsigned overflow check where we don't use the
2021 multiplication result, just whether overflow happened.
2022 If we can do MULT_HIGHPART_EXPR, that followed by
2023 comparison of the result against zero is cheapest.
2024 We'll still compute res, but it should be DCEd later. */
2030 && !(uns0_p
&& uns1_p
&& !unsr_p
)
2031 && can_mult_highpart_p (mode
, uns
) == 1
2032 && single_imm_use (lhs
, &use
, &use_stmt
)
2033 && is_gimple_assign (use_stmt
)
2034 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
2037 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2038 && targetm
.scalar_mode_supported_p (wmode
)
2039 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
2042 ops
.code
= WIDEN_MULT_EXPR
;
2044 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
2046 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
2047 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
2049 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
2050 res
= convert_modes (mode
, wmode
, res
, uns
);
2052 /* For the unsigned multiplication, there was overflow if
2053 HIPART is non-zero. */
2054 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2055 NULL_RTX
, NULL
, done_label
,
2056 profile_probability::very_likely ());
2059 /* RES is used more than once, place it in a pseudo. */
2060 res
= force_reg (mode
, res
);
2062 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2064 /* RES is low half of the double width result, HIPART
2065 the high half. There was overflow if
2066 HIPART is different from RES < 0 ? -1 : 0. */
2067 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2068 NULL_RTX
, NULL
, done_label
,
2069 profile_probability::very_likely ());
2072 else if (can_mult_highpart_p (mode
, uns
) == 1)
2075 ops
.code
= MULT_HIGHPART_EXPR
;
2078 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
2080 ops
.code
= MULT_EXPR
;
2081 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2083 /* For the unsigned multiplication, there was overflow if
2084 HIPART is non-zero. */
2085 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2086 NULL_RTX
, NULL
, done_label
,
2087 profile_probability::very_likely ());
2090 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2092 /* RES is low half of the double width result, HIPART
2093 the high half. There was overflow if
2094 HIPART is different from RES < 0 ? -1 : 0. */
2095 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2096 NULL_RTX
, NULL
, done_label
,
2097 profile_probability::very_likely ());
2101 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
2102 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
2104 rtx_code_label
*large_op0
= gen_label_rtx ();
2105 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
2106 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
2107 rtx_code_label
*both_ops_large
= gen_label_rtx ();
2108 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
2109 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
2110 rtx_code_label
*do_overflow
= gen_label_rtx ();
2111 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
2113 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
2114 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
2116 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
2117 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
2118 rtx signbit0
= const0_rtx
;
2120 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
2122 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
2124 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
2125 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
2126 rtx signbit1
= const0_rtx
;
2128 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
2131 res
= gen_reg_rtx (mode
);
2133 /* True if op0 resp. op1 are known to be in the range of
2135 bool op0_small_p
= false;
2136 bool op1_small_p
= false;
2137 /* True if op0 resp. op1 are known to have all zeros or all ones
2138 in the upper half of bits, but are not known to be
2140 bool op0_medium_p
= false;
2141 bool op1_medium_p
= false;
2142 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2143 nonnegative, 1 if unknown. */
2149 else if (pos_neg0
== 2)
2153 else if (pos_neg1
== 2)
2156 unsigned int mprec0
= prec
;
2157 if (arg0
!= error_mark_node
)
2158 mprec0
= get_min_precision (arg0
, sign
);
2159 if (mprec0
<= hprec
)
2161 else if (!uns
&& mprec0
<= hprec
+ 1)
2162 op0_medium_p
= true;
2163 unsigned int mprec1
= prec
;
2164 if (arg1
!= error_mark_node
)
2165 mprec1
= get_min_precision (arg1
, sign
);
2166 if (mprec1
<= hprec
)
2168 else if (!uns
&& mprec1
<= hprec
+ 1)
2169 op1_medium_p
= true;
2171 int smaller_sign
= 1;
2172 int larger_sign
= 1;
2175 smaller_sign
= op0_sign
;
2176 larger_sign
= op1_sign
;
2178 else if (op1_small_p
)
2180 smaller_sign
= op1_sign
;
2181 larger_sign
= op0_sign
;
2183 else if (op0_sign
== op1_sign
)
2185 smaller_sign
= op0_sign
;
2186 larger_sign
= op0_sign
;
2190 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2191 NULL_RTX
, NULL
, large_op0
,
2192 profile_probability::unlikely ());
2195 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2196 NULL_RTX
, NULL
, small_op0_large_op1
,
2197 profile_probability::unlikely ());
2199 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2200 hmode to mode, the multiplication will never overflow. We can
2201 do just one hmode x hmode => mode widening multiplication. */
2202 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2203 ops
.op0
= make_tree (halfstype
, lopart0
);
2204 ops
.op1
= make_tree (halfstype
, lopart1
);
2205 ops
.code
= WIDEN_MULT_EXPR
;
2208 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2209 emit_move_insn (res
, thisres
);
2210 emit_jump (done_label
);
2212 emit_label (small_op0_large_op1
);
2214 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2215 but op1 is not, just swap the arguments and handle it as op1
2216 sign/zero extended, op0 not. */
2217 rtx larger
= gen_reg_rtx (mode
);
2218 rtx hipart
= gen_reg_rtx (hmode
);
2219 rtx lopart
= gen_reg_rtx (hmode
);
2220 emit_move_insn (larger
, op1
);
2221 emit_move_insn (hipart
, hipart1
);
2222 emit_move_insn (lopart
, lopart0
);
2223 emit_jump (one_small_one_large
);
2225 emit_label (large_op0
);
2228 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2229 NULL_RTX
, NULL
, both_ops_large
,
2230 profile_probability::unlikely ());
2232 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2233 but op0 is not, prepare larger, hipart and lopart pseudos and
2234 handle it together with small_op0_large_op1. */
2235 emit_move_insn (larger
, op0
);
2236 emit_move_insn (hipart
, hipart0
);
2237 emit_move_insn (lopart
, lopart1
);
2239 emit_label (one_small_one_large
);
2241 /* lopart is the low part of the operand that is sign extended
2242 to mode, larger is the other operand, hipart is the
2243 high part of larger and lopart0 and lopart1 are the low parts
2245 We perform lopart0 * lopart1 and lopart * hipart widening
2247 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2248 ops
.op0
= make_tree (halfutype
, lopart0
);
2249 ops
.op1
= make_tree (halfutype
, lopart1
);
2251 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2253 ops
.op0
= make_tree (halfutype
, lopart
);
2254 ops
.op1
= make_tree (halfutype
, hipart
);
2255 rtx loxhi
= gen_reg_rtx (mode
);
2256 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2257 emit_move_insn (loxhi
, tem
);
2261 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2262 if (larger_sign
== 0)
2263 emit_jump (after_hipart_neg
);
2264 else if (larger_sign
!= -1)
2265 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2266 NULL_RTX
, NULL
, after_hipart_neg
,
2267 profile_probability::even ());
2269 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2270 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2271 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2273 emit_move_insn (loxhi
, tem
);
2275 emit_label (after_hipart_neg
);
2277 /* if (lopart < 0) loxhi -= larger; */
2278 if (smaller_sign
== 0)
2279 emit_jump (after_lopart_neg
);
2280 else if (smaller_sign
!= -1)
2281 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2282 NULL_RTX
, NULL
, after_lopart_neg
,
2283 profile_probability::even ());
2285 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2287 emit_move_insn (loxhi
, tem
);
2289 emit_label (after_lopart_neg
);
2292 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2293 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2294 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2296 emit_move_insn (loxhi
, tem
);
2298 /* if (loxhi >> (bitsize / 2)
2299 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2300 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2301 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2303 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2304 rtx signbitloxhi
= const0_rtx
;
2306 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2307 convert_modes (hmode
, mode
,
2309 hprec
- 1, NULL_RTX
, 0);
2311 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2312 NULL_RTX
, NULL
, do_overflow
,
2313 profile_probability::very_unlikely ());
2315 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2316 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2318 tem
= convert_modes (mode
, hmode
,
2319 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2321 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2324 emit_move_insn (res
, tem
);
2325 emit_jump (done_label
);
2327 emit_label (both_ops_large
);
2329 /* If both operands are large (not sign (!uns) or zero (uns)
2330 extended from hmode), then perform the full multiplication
2331 which will be the result of the operation.
2332 The only cases which don't overflow are for signed multiplication
2333 some cases where both hipart0 and highpart1 are 0 or -1.
2334 For unsigned multiplication when high parts are both non-zero
2335 this overflows always. */
2336 ops
.code
= MULT_EXPR
;
2337 ops
.op0
= make_tree (type
, op0
);
2338 ops
.op1
= make_tree (type
, op1
);
2339 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2340 emit_move_insn (res
, tem
);
2346 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2347 NULL_RTX
, 1, OPTAB_WIDEN
);
2348 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2349 NULL_RTX
, NULL
, do_error
,
2350 profile_probability::very_unlikely ());
2355 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2356 NULL_RTX
, 1, OPTAB_WIDEN
);
2357 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2358 NULL_RTX
, NULL
, do_error
,
2359 profile_probability::very_unlikely ());
2362 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2363 the same, overflow happened if res is non-positive, if they
2364 are different, overflow happened if res is positive. */
2365 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2366 emit_jump (hipart_different
);
2367 else if (op0_sign
== 1 || op1_sign
== 1)
2368 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2369 NULL_RTX
, NULL
, hipart_different
,
2370 profile_probability::even ());
2372 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2373 NULL_RTX
, NULL
, do_error
,
2374 profile_probability::very_unlikely ());
2375 emit_jump (done_label
);
2377 emit_label (hipart_different
);
2379 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2380 NULL_RTX
, NULL
, do_error
,
2381 profile_probability::very_unlikely ());
2382 emit_jump (done_label
);
2385 emit_label (do_overflow
);
2387 /* Overflow, do full multiplication and fallthru into do_error. */
2388 ops
.op0
= make_tree (type
, op0
);
2389 ops
.op1
= make_tree (type
, op1
);
2390 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2391 emit_move_insn (res
, tem
);
2393 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2394 && targetm
.scalar_mode_supported_p (wmode
))
2395 /* Even emitting a libcall is better than not detecting overflow
2400 gcc_assert (!is_ubsan
);
2401 ops
.code
= MULT_EXPR
;
2403 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2404 emit_jump (done_label
);
2409 emit_label (do_error
);
2412 /* Expand the ubsan builtin call. */
2414 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2418 do_pending_stack_adjust ();
2421 expand_arith_set_overflow (lhs
, target
);
2424 emit_label (done_label
);
2427 if (uns0_p
&& uns1_p
&& !unsr_p
)
2429 rtx_code_label
*all_done_label
= gen_label_rtx ();
2430 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2431 NULL
, all_done_label
, profile_probability::very_likely ());
2432 expand_arith_set_overflow (lhs
, target
);
2433 emit_label (all_done_label
);
2437 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2439 rtx_code_label
*all_done_label
= gen_label_rtx ();
2440 rtx_code_label
*set_noovf
= gen_label_rtx ();
2441 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2442 NULL
, all_done_label
, profile_probability::very_likely ());
2443 expand_arith_set_overflow (lhs
, target
);
2444 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2445 NULL
, set_noovf
, profile_probability::very_likely ());
2446 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2447 NULL
, all_done_label
, profile_probability::very_unlikely ());
2448 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2449 all_done_label
, profile_probability::very_unlikely ());
2450 emit_label (set_noovf
);
2451 write_complex_part (target
, const0_rtx
, true, false);
2452 emit_label (all_done_label
);
2458 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
2460 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2464 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2467 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2468 tree arg0
, tree arg1
)
2470 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2471 rtx_code_label
*loop_lab
= NULL
;
2472 rtx cntvar
= NULL_RTX
;
2473 tree cntv
= NULL_TREE
;
2474 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2475 tree sz
= TYPE_SIZE (eltype
);
2476 tree data
= NULL_TREE
;
2477 tree resv
= NULL_TREE
;
2478 rtx lhsr
= NULL_RTX
;
2479 rtx resvr
= NULL_RTX
;
2480 unsigned HOST_WIDE_INT const_cnt
= 0;
2481 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2486 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2487 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2488 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2489 optab_default
)) == unknown_optab
2490 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2491 == CODE_FOR_nothing
))
2494 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2497 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2498 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2504 do_pending_stack_adjust ();
2505 loop_lab
= gen_label_rtx ();
2506 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2507 cntv
= make_tree (sizetype
, cntvar
);
2508 emit_move_insn (cntvar
, const0_rtx
);
2509 emit_label (loop_lab
);
2511 if (TREE_CODE (arg0
) != VECTOR_CST
)
2513 rtx arg0r
= expand_normal (arg0
);
2514 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2516 if (TREE_CODE (arg1
) != VECTOR_CST
)
2518 rtx arg1r
= expand_normal (arg1
);
2519 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2521 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2523 tree op0
, op1
, res
= NULL_TREE
;
2526 tree atype
= build_array_type_nelts (eltype
, cnt
);
2527 op0
= uniform_vector_p (arg0
);
2528 if (op0
== NULL_TREE
)
2530 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2531 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2532 NULL_TREE
, NULL_TREE
);
2534 op1
= uniform_vector_p (arg1
);
2535 if (op1
== NULL_TREE
)
2537 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2538 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2539 NULL_TREE
, NULL_TREE
);
2543 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2544 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2545 NULL_TREE
, NULL_TREE
);
2550 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2551 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2552 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2554 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2560 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2561 false, false, false, true, &data
);
2564 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2565 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2567 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2568 false, false, false, true, &data
);
2571 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2580 struct separate_ops ops
;
2581 ops
.code
= PLUS_EXPR
;
2582 ops
.type
= TREE_TYPE (cntv
);
2584 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2585 ops
.op2
= NULL_TREE
;
2587 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2590 emit_move_insn (cntvar
, ret
);
2591 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2592 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2593 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2594 profile_probability::very_likely ());
2596 if (lhs
&& resv
== NULL_TREE
)
2598 struct separate_ops ops
;
2600 ops
.type
= TREE_TYPE (arg0
);
2603 ops
.op2
= NULL_TREE
;
2605 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2608 emit_move_insn (lhsr
, ret
);
2611 emit_move_insn (lhsr
, resvr
);
2614 /* Expand UBSAN_CHECK_ADD call STMT. */
2617 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2619 location_t loc
= gimple_location (stmt
);
2620 tree lhs
= gimple_call_lhs (stmt
);
2621 tree arg0
= gimple_call_arg (stmt
, 0);
2622 tree arg1
= gimple_call_arg (stmt
, 1);
2623 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2624 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2626 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2627 false, false, false, true, NULL
);
2630 /* Expand UBSAN_CHECK_SUB call STMT. */
2633 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2635 location_t loc
= gimple_location (stmt
);
2636 tree lhs
= gimple_call_lhs (stmt
);
2637 tree arg0
= gimple_call_arg (stmt
, 0);
2638 tree arg1
= gimple_call_arg (stmt
, 1);
2639 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2640 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2641 else if (integer_zerop (arg0
))
2642 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2644 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2645 false, false, false, true, NULL
);
2648 /* Expand UBSAN_CHECK_MUL call STMT. */
2651 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2653 location_t loc
= gimple_location (stmt
);
2654 tree lhs
= gimple_call_lhs (stmt
);
2655 tree arg0
= gimple_call_arg (stmt
, 0);
2656 tree arg1
= gimple_call_arg (stmt
, 1);
2657 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2658 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2660 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2664 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2667 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2669 tree lhs
= gimple_call_lhs (stmt
);
2670 if (lhs
== NULL_TREE
)
2672 tree arg0
= gimple_call_arg (stmt
, 0);
2673 tree arg1
= gimple_call_arg (stmt
, 1);
2674 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2675 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2676 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2677 int unsr_p
= TYPE_UNSIGNED (type
);
2678 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2679 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2680 int precres
= TYPE_PRECISION (type
);
2681 location_t loc
= gimple_location (stmt
);
2682 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2684 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2686 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2687 prec0
= MIN (prec0
, pr
);
2688 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2689 prec1
= MIN (prec1
, pr
);
2691 /* If uns0_p && uns1_p, precop is minimum needed precision
2692 of unsigned type to hold the exact result, otherwise
2693 precop is minimum needed precision of signed type to
2694 hold the exact result. */
2696 if (code
== MULT_EXPR
)
2697 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2700 if (uns0_p
== uns1_p
)
2701 precop
= MAX (prec0
, prec1
) + 1;
2703 precop
= MAX (prec0
+ 1, prec1
) + 1;
2705 precop
= MAX (prec0
, prec1
+ 1) + 1;
2707 int orig_precres
= precres
;
2711 if ((uns0_p
&& uns1_p
)
2712 ? ((precop
+ !unsr_p
) <= precres
2713 /* u1 - u2 -> ur can overflow, no matter what precision
2715 && (code
!= MINUS_EXPR
|| !unsr_p
))
2716 : (!unsr_p
&& precop
<= precres
))
2718 /* The infinity precision result will always fit into result. */
2719 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2720 write_complex_part (target
, const0_rtx
, true, false);
2721 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2722 struct separate_ops ops
;
2725 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2726 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2727 ops
.op2
= NULL_TREE
;
2729 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2730 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2734 /* For operations with low precision, if target doesn't have them, start
2735 with precres widening right away, otherwise do it only if the most
2736 simple cases can't be used. */
2737 const int min_precision
= targetm
.min_arithmetic_precision ();
2738 if (orig_precres
== precres
&& precres
< min_precision
)
2740 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2741 && prec1
<= precres
)
2742 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2743 && prec0
+ uns0_p
<= precres
2744 && prec1
+ uns1_p
<= precres
))
2746 arg0
= fold_convert_loc (loc
, type
, arg0
);
2747 arg1
= fold_convert_loc (loc
, type
, arg1
);
2751 if (integer_zerop (arg0
) && !unsr_p
)
2753 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2758 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2759 unsr_p
, unsr_p
, false, NULL
);
2762 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2763 unsr_p
, unsr_p
, false, NULL
);
2770 /* For sub-word operations, retry with a wider type first. */
2771 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2773 int p
= MAX (min_precision
, precop
);
2774 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2775 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2778 p
= TYPE_PRECISION (optype
);
2782 unsr_p
= TYPE_UNSIGNED (optype
);
2788 if (prec0
<= precres
&& prec1
<= precres
)
2793 types
[0] = build_nonstandard_integer_type (precres
, 0);
2799 types
[1] = build_nonstandard_integer_type (precres
, 1);
2801 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2802 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2803 if (code
!= MULT_EXPR
)
2804 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2805 uns0_p
, uns1_p
, false, NULL
);
2807 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2808 uns0_p
, uns1_p
, false, NULL
);
2812 /* Retry with a wider type. */
2813 if (orig_precres
== precres
)
2815 int p
= MAX (prec0
, prec1
);
2816 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2817 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2820 p
= TYPE_PRECISION (optype
);
2824 unsr_p
= TYPE_UNSIGNED (optype
);
2835 /* Expand ADD_OVERFLOW STMT. */
2838 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2840 expand_arith_overflow (PLUS_EXPR
, stmt
);
2843 /* Expand SUB_OVERFLOW STMT. */
2846 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2848 expand_arith_overflow (MINUS_EXPR
, stmt
);
2851 /* Expand MUL_OVERFLOW STMT. */
2854 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2856 expand_arith_overflow (MULT_EXPR
, stmt
);
2859 /* Expand UADDC STMT. */
2862 expand_UADDC (internal_fn ifn
, gcall
*stmt
)
2864 tree lhs
= gimple_call_lhs (stmt
);
2865 tree arg1
= gimple_call_arg (stmt
, 0);
2866 tree arg2
= gimple_call_arg (stmt
, 1);
2867 tree arg3
= gimple_call_arg (stmt
, 2);
2868 tree type
= TREE_TYPE (arg1
);
2869 machine_mode mode
= TYPE_MODE (type
);
2870 insn_code icode
= optab_handler (ifn
== IFN_UADDC
2871 ? uaddc5_optab
: usubc5_optab
, mode
);
2872 rtx op1
= expand_normal (arg1
);
2873 rtx op2
= expand_normal (arg2
);
2874 rtx op3
= expand_normal (arg3
);
2875 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2876 rtx re
= gen_reg_rtx (mode
);
2877 rtx im
= gen_reg_rtx (mode
);
2878 class expand_operand ops
[5];
2879 create_output_operand (&ops
[0], re
, mode
);
2880 create_output_operand (&ops
[1], im
, mode
);
2881 create_input_operand (&ops
[2], op1
, mode
);
2882 create_input_operand (&ops
[3], op2
, mode
);
2883 create_input_operand (&ops
[4], op3
, mode
);
2884 expand_insn (icode
, 5, ops
);
2885 write_complex_part (target
, re
, false, false);
2886 write_complex_part (target
, im
, true, false);
2889 /* Expand USUBC STMT. */
2892 expand_USUBC (internal_fn ifn
, gcall
*stmt
)
2894 expand_UADDC (ifn
, stmt
);
2897 /* This should get folded in tree-vectorizer.cc. */
2900 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2905 /* This should get folded in tree-vectorizer.cc. */
2908 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2913 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2914 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2917 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2919 tree addr
= gimple_call_arg (stmt
, index
);
2920 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2921 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2922 if (TYPE_ALIGN (type
) != align
)
2923 type
= build_aligned_type (type
, align
);
2926 if (TREE_CODE (tmp
) == SSA_NAME
)
2928 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2929 if (gimple_assign_single_p (def
))
2930 tmp
= gimple_assign_rhs1 (def
);
2933 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2935 tree mem
= TREE_OPERAND (tmp
, 0);
2936 if (TREE_CODE (mem
) == TARGET_MEM_REF
2937 && types_compatible_p (TREE_TYPE (mem
), type
))
2939 tree offset
= TMR_OFFSET (mem
);
2940 if (type
!= TREE_TYPE (mem
)
2941 || alias_ptr_type
!= TREE_TYPE (offset
)
2942 || !integer_zerop (offset
))
2944 mem
= copy_node (mem
);
2945 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2946 wi::to_poly_wide (offset
));
2947 TREE_TYPE (mem
) = type
;
2953 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2956 /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
2960 expand_partial_load_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
2963 class expand_operand ops
[5];
2964 tree type
, lhs
, rhs
, maskt
;
2968 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
2969 lhs
= gimple_call_lhs (stmt
);
2970 if (lhs
== NULL_TREE
)
2972 type
= TREE_TYPE (lhs
);
2973 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2975 if (optab
== vec_mask_load_lanes_optab
2976 || optab
== vec_mask_len_load_lanes_optab
)
2977 icode
= get_multi_vector_move (type
, optab
);
2978 else if (optab
== len_load_optab
)
2979 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2981 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2982 TYPE_MODE (TREE_TYPE (maskt
)));
2984 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2985 gcc_assert (MEM_P (mem
));
2986 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2987 create_output_operand (&ops
[i
++], target
, TYPE_MODE (type
));
2988 create_fixed_operand (&ops
[i
++], mem
);
2989 i
= add_mask_and_len_args (ops
, i
, stmt
);
2990 expand_insn (icode
, i
, ops
);
2992 if (!rtx_equal_p (target
, ops
[0].value
))
2993 emit_move_insn (target
, ops
[0].value
);
2996 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2997 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2998 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2999 #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3001 /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3005 expand_partial_store_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3008 class expand_operand ops
[5];
3009 tree type
, lhs
, rhs
, maskt
;
3013 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3014 rhs
= gimple_call_arg (stmt
, internal_fn_stored_value_index (ifn
));
3015 type
= TREE_TYPE (rhs
);
3016 lhs
= expand_call_mem_ref (type
, stmt
, 0);
3018 if (optab
== vec_mask_store_lanes_optab
3019 || optab
== vec_mask_len_store_lanes_optab
)
3020 icode
= get_multi_vector_move (type
, optab
);
3021 else if (optab
== len_store_optab
)
3022 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3024 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3025 TYPE_MODE (TREE_TYPE (maskt
)));
3027 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3028 gcc_assert (MEM_P (mem
));
3029 reg
= expand_normal (rhs
);
3030 create_fixed_operand (&ops
[i
++], mem
);
3031 create_input_operand (&ops
[i
++], reg
, TYPE_MODE (type
));
3032 i
= add_mask_and_len_args (ops
, i
, stmt
);
3033 expand_insn (icode
, i
, ops
);
3036 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3037 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3038 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3039 #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3041 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3042 The expansion of STMT happens based on OPTAB table associated. */
3045 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3047 class expand_operand ops
[6];
3049 tree lhs
= gimple_call_lhs (stmt
);
3050 tree op0a
= gimple_call_arg (stmt
, 0);
3051 tree op0b
= gimple_call_arg (stmt
, 1);
3052 tree op1
= gimple_call_arg (stmt
, 2);
3053 tree op2
= gimple_call_arg (stmt
, 3);
3054 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
3056 tree vec_cond_type
= TREE_TYPE (lhs
);
3057 tree op_mode
= TREE_TYPE (op0a
);
3058 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
3060 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3061 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
3063 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
3065 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
3066 /* vector_compare_rtx legitimizes operands, preserve equality when
3067 expanding op1/op2. */
3068 rtx rtx_op1
, rtx_op2
;
3069 if (operand_equal_p (op1
, op0a
))
3070 rtx_op1
= XEXP (comparison
, 0);
3071 else if (operand_equal_p (op1
, op0b
))
3072 rtx_op1
= XEXP (comparison
, 1);
3074 rtx_op1
= expand_normal (op1
);
3075 if (operand_equal_p (op2
, op0a
))
3076 rtx_op2
= XEXP (comparison
, 0);
3077 else if (operand_equal_p (op2
, op0b
))
3078 rtx_op2
= XEXP (comparison
, 1);
3080 rtx_op2
= expand_normal (op2
);
3082 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3083 create_output_operand (&ops
[0], target
, mode
);
3084 create_input_operand (&ops
[1], rtx_op1
, mode
);
3085 create_input_operand (&ops
[2], rtx_op2
, mode
);
3086 create_fixed_operand (&ops
[3], comparison
);
3087 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
3088 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
3089 expand_insn (icode
, 6, ops
);
3090 if (!rtx_equal_p (ops
[0].value
, target
))
3091 emit_move_insn (target
, ops
[0].value
);
3094 /* Expand VCOND_MASK optab internal function.
3095 The expansion of STMT happens based on OPTAB table associated. */
3098 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3100 class expand_operand ops
[4];
3102 tree lhs
= gimple_call_lhs (stmt
);
3103 tree op0
= gimple_call_arg (stmt
, 0);
3104 tree op1
= gimple_call_arg (stmt
, 1);
3105 tree op2
= gimple_call_arg (stmt
, 2);
3106 tree vec_cond_type
= TREE_TYPE (lhs
);
3108 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3109 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
3110 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
3111 rtx mask
, rtx_op1
, rtx_op2
;
3113 gcc_assert (icode
!= CODE_FOR_nothing
);
3115 mask
= expand_normal (op0
);
3116 rtx_op1
= expand_normal (op1
);
3117 rtx_op2
= expand_normal (op2
);
3119 mask
= force_reg (mask_mode
, mask
);
3120 rtx_op1
= force_reg (mode
, rtx_op1
);
3122 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3123 create_output_operand (&ops
[0], target
, mode
);
3124 create_input_operand (&ops
[1], rtx_op1
, mode
);
3125 create_input_operand (&ops
[2], rtx_op2
, mode
);
3126 create_input_operand (&ops
[3], mask
, mask_mode
);
3127 expand_insn (icode
, 4, ops
);
3128 if (!rtx_equal_p (ops
[0].value
, target
))
3129 emit_move_insn (target
, ops
[0].value
);
3132 /* Expand VEC_SET internal functions. */
3135 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3137 tree lhs
= gimple_call_lhs (stmt
);
3138 tree op0
= gimple_call_arg (stmt
, 0);
3139 tree op1
= gimple_call_arg (stmt
, 1);
3140 tree op2
= gimple_call_arg (stmt
, 2);
3141 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3142 rtx src
= expand_normal (op0
);
3144 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
3145 scalar_mode innermode
= GET_MODE_INNER (outermode
);
3147 rtx value
= expand_normal (op1
);
3148 rtx pos
= expand_normal (op2
);
3150 class expand_operand ops
[3];
3151 enum insn_code icode
= optab_handler (optab
, outermode
);
3153 if (icode
!= CODE_FOR_nothing
)
3155 rtx temp
= gen_reg_rtx (outermode
);
3156 emit_move_insn (temp
, src
);
3158 create_fixed_operand (&ops
[0], temp
);
3159 create_input_operand (&ops
[1], value
, innermode
);
3160 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
3162 if (maybe_expand_insn (icode
, 3, ops
))
3164 emit_move_insn (target
, temp
);
3172 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3177 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3179 /* When guessing was done, the hints should be already stripped away. */
3180 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3183 tree lhs
= gimple_call_lhs (stmt
);
3185 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3187 target
= const0_rtx
;
3188 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3189 if (lhs
&& val
!= target
)
3190 emit_move_insn (target
, val
);
3193 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3194 should never be called. */
3197 expand_VA_ARG (internal_fn
, gcall
*)
3202 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3203 dummy function should never be called. */
3206 expand_VEC_CONVERT (internal_fn
, gcall
*)
3211 /* Expand IFN_RAWMEMCHR internal function. */
3214 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3216 expand_operand ops
[3];
3218 tree lhs
= gimple_call_lhs (stmt
);
3221 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3222 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3223 create_output_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3225 tree mem
= gimple_call_arg (stmt
, 0);
3226 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3227 create_fixed_operand (&ops
[1], mem_rtx
);
3229 tree pattern
= gimple_call_arg (stmt
, 1);
3230 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3231 rtx pattern_rtx
= expand_normal (pattern
);
3232 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3234 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3236 expand_insn (icode
, 3, ops
);
3237 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3238 emit_move_insn (lhs_rtx
, ops
[0].value
);
3241 /* Expand the IFN_UNIQUE function according to its first argument. */
3244 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3246 rtx pattern
= NULL_RTX
;
3247 enum ifn_unique_kind kind
3248 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3255 case IFN_UNIQUE_UNSPEC
:
3256 if (targetm
.have_unique ())
3257 pattern
= targetm
.gen_unique ();
3260 case IFN_UNIQUE_OACC_FORK
:
3261 case IFN_UNIQUE_OACC_JOIN
:
3262 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3264 tree lhs
= gimple_call_lhs (stmt
);
3265 rtx target
= const0_rtx
;
3268 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3270 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3271 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3273 if (kind
== IFN_UNIQUE_OACC_FORK
)
3274 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3276 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3284 emit_insn (pattern
);
3287 /* Expand the IFN_DEFERRED_INIT function:
3288 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3290 Initialize the LHS with zero/pattern according to its second argument
3292 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3293 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3295 The LHS variable is initialized including paddings.
3296 The reasons to choose 0xFE for pattern initialization are:
3297 1. It is a non-canonical virtual address on x86_64, and at the
3298 high end of the i386 kernel address space.
3299 2. It is a very large float value (-1.694739530317379e+38).
3300 3. It is also an unusual number for integers. */
3301 #define INIT_PATTERN_VALUE 0xFE
3303 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3305 tree lhs
= gimple_call_lhs (stmt
);
3306 tree var_size
= gimple_call_arg (stmt
, 0);
3307 enum auto_init_type init_type
3308 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3309 bool reg_lhs
= true;
3311 tree var_type
= TREE_TYPE (lhs
);
3312 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3314 if (TREE_CODE (lhs
) == SSA_NAME
)
3318 tree lhs_base
= lhs
;
3319 while (handled_component_p (lhs_base
))
3320 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3321 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3322 || non_mem_decl_p (lhs_base
));
3323 /* If this expands to a register and the underlying decl is wrapped in
3324 a MEM_REF that just serves as an access type change expose the decl
3325 if it is of correct size. This avoids a situation as in PR103271
3326 if the target does not support a direct move to the registers mode. */
3328 && TREE_CODE (lhs_base
) == MEM_REF
3329 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3330 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3331 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3332 && tree_fits_uhwi_p (var_size
)
3333 && tree_int_cst_equal
3335 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3337 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3338 var_type
= TREE_TYPE (lhs
);
3344 /* If the variable is not in register, expand to a memset
3345 to initialize it. */
3346 mark_addressable (lhs
);
3347 tree var_addr
= build_fold_addr_expr (lhs
);
3349 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3350 ? build_int_cst (integer_type_node
,
3352 : integer_zero_node
;
3353 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3354 3, var_addr
, value
, var_size
);
3355 /* Expand this memset call. */
3356 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3360 /* If this variable is in a register use expand_assignment.
3361 For boolean scalars force zero-init. */
3363 scalar_int_mode var_mode
;
3364 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3365 && tree_fits_uhwi_p (var_size
)
3366 && (init_type
== AUTO_INIT_PATTERN
3367 || !is_gimple_reg_type (var_type
))
3368 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3369 0).exists (&var_mode
)
3370 && have_insn_for (SET
, var_mode
))
3372 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3373 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3374 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3375 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3376 tree itype
= build_nonstandard_integer_type
3377 (total_bytes
* BITS_PER_UNIT
, 1);
3378 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3379 init
= wide_int_to_tree (itype
, w
);
3380 /* Pun the LHS to make sure its type has constant size
3381 unless it is an SSA name where that's already known. */
3382 if (TREE_CODE (lhs
) != SSA_NAME
)
3383 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3385 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3388 /* Use zero-init also for variable-length sizes. */
3389 init
= build_zero_cst (var_type
);
3391 expand_assignment (lhs
, init
, false);
3395 /* The size of an OpenACC compute dimension. */
3398 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3400 tree lhs
= gimple_call_lhs (stmt
);
3405 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3406 if (targetm
.have_oacc_dim_size ())
3408 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3409 VOIDmode
, EXPAND_NORMAL
);
3410 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3413 emit_move_insn (target
, GEN_INT (1));
3416 /* The position of an OpenACC execution engine along one compute axis. */
3419 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3421 tree lhs
= gimple_call_lhs (stmt
);
3426 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3427 if (targetm
.have_oacc_dim_pos ())
3429 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3430 VOIDmode
, EXPAND_NORMAL
);
3431 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3434 emit_move_insn (target
, const0_rtx
);
3437 /* This is expanded by oacc_device_lower pass. */
3440 expand_GOACC_LOOP (internal_fn
, gcall
*)
3445 /* This is expanded by oacc_device_lower pass. */
3448 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3453 /* This is expanded by oacc_device_lower pass. */
3456 expand_GOACC_TILE (internal_fn
, gcall
*)
3461 /* Set errno to EDOM. */
3464 expand_SET_EDOM (internal_fn
, gcall
*)
3467 #ifdef GEN_ERRNO_RTX
3468 rtx errno_rtx
= GEN_ERRNO_RTX
;
3470 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3472 emit_move_insn (errno_rtx
,
3473 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3479 /* Expand atomic bit test and set. */
3482 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3484 expand_ifn_atomic_bit_test_and (call
);
3487 /* Expand atomic bit test and complement. */
3490 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3492 expand_ifn_atomic_bit_test_and (call
);
3495 /* Expand atomic bit test and reset. */
3498 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3500 expand_ifn_atomic_bit_test_and (call
);
3503 /* Expand atomic bit test and set. */
3506 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3508 expand_ifn_atomic_compare_exchange (call
);
3511 /* Expand atomic add fetch and cmp with 0. */
3514 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3516 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3519 /* Expand atomic sub fetch and cmp with 0. */
3522 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3524 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3527 /* Expand atomic and fetch and cmp with 0. */
3530 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3532 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3535 /* Expand atomic or fetch and cmp with 0. */
3538 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3540 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3543 /* Expand atomic xor fetch and cmp with 0. */
3546 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3548 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3551 /* Expand LAUNDER to assignment, lhs = arg0. */
3554 expand_LAUNDER (internal_fn
, gcall
*call
)
3556 tree lhs
= gimple_call_lhs (call
);
3561 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3564 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3567 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3569 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3570 int rhs_index
= internal_fn_stored_value_index (ifn
);
3571 tree base
= gimple_call_arg (stmt
, 0);
3572 tree offset
= gimple_call_arg (stmt
, 1);
3573 tree scale
= gimple_call_arg (stmt
, 2);
3574 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3576 rtx base_rtx
= expand_normal (base
);
3577 rtx offset_rtx
= expand_normal (offset
);
3578 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3579 rtx rhs_rtx
= expand_normal (rhs
);
3581 class expand_operand ops
[8];
3583 create_address_operand (&ops
[i
++], base_rtx
);
3584 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3585 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3586 create_integer_operand (&ops
[i
++], scale_int
);
3587 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3588 i
= add_mask_and_len_args (ops
, i
, stmt
);
3590 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3591 TYPE_MODE (TREE_TYPE (offset
)));
3592 expand_insn (icode
, i
, ops
);
3595 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3598 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3600 tree lhs
= gimple_call_lhs (stmt
);
3601 tree base
= gimple_call_arg (stmt
, 0);
3602 tree offset
= gimple_call_arg (stmt
, 1);
3603 tree scale
= gimple_call_arg (stmt
, 2);
3605 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3606 rtx base_rtx
= expand_normal (base
);
3607 rtx offset_rtx
= expand_normal (offset
);
3608 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3611 class expand_operand ops
[8];
3612 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3613 create_address_operand (&ops
[i
++], base_rtx
);
3614 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3615 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3616 create_integer_operand (&ops
[i
++], scale_int
);
3617 i
= add_mask_and_len_args (ops
, i
, stmt
);
3618 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3619 TYPE_MODE (TREE_TYPE (offset
)));
3620 expand_insn (icode
, i
, ops
);
3621 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3622 emit_move_insn (lhs_rtx
, ops
[0].value
);
3625 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3626 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3629 contains_call_div_mod (rtx_insn
*insn
)
3631 subrtx_iterator::array_type array
;
3632 for (; insn
; insn
= NEXT_INSN (insn
))
3635 else if (INSN_P (insn
))
3636 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3637 switch (GET_CODE (*iter
))
3651 /* Expand DIVMOD() using:
3652 a) optab handler for udivmod/sdivmod if it is available.
3653 b) If optab_handler doesn't exist, generate call to
3654 target-specific divmod libfunc. */
3657 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3659 tree lhs
= gimple_call_lhs (call_stmt
);
3660 tree arg0
= gimple_call_arg (call_stmt
, 0);
3661 tree arg1
= gimple_call_arg (call_stmt
, 1);
3663 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3664 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3665 machine_mode mode
= TYPE_MODE (type
);
3666 bool unsignedp
= TYPE_UNSIGNED (type
);
3667 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3669 rtx op0
= expand_normal (arg0
);
3670 rtx op1
= expand_normal (arg1
);
3671 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3673 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3674 rtx_insn
*insns
= NULL
;
3676 if (TREE_CODE (arg1
) == INTEGER_CST
)
3678 /* For DIVMOD by integral constants, there could be efficient code
3679 expanded inline e.g. using shifts and plus/minus. Try to expand
3680 the division and modulo and if it emits any library calls or any
3681 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3683 scalar_int_mode int_mode
;
3684 if (remainder
== NULL_RTX
3686 && CONST_INT_P (op1
)
3687 && !pow2p_hwi (INTVAL (op1
))
3688 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3689 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3690 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3691 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3692 && optimize_insn_for_speed_p ())
3694 rtx_insn
*last
= get_last_insn ();
3695 remainder
= NULL_RTX
;
3696 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3697 TYPE_UNSIGNED (type
));
3698 if (quotient
!= NULL_RTX
)
3700 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3702 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3703 set_dst_reg_note (move
, REG_EQUAL
,
3704 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3705 ? UDIV
: DIV
, int_mode
,
3706 copy_rtx (op0
), op1
),
3708 move
= emit_move_insn (remainder
, remainder
);
3709 set_dst_reg_note (move
, REG_EQUAL
,
3710 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3711 ? UMOD
: MOD
, int_mode
,
3712 copy_rtx (op0
), op1
),
3717 delete_insns_since (last
);
3720 if (remainder
== NULL_RTX
)
3722 struct separate_ops ops
;
3723 ops
.code
= TRUNC_DIV_EXPR
;
3725 ops
.op0
= make_tree (ops
.type
, op0
);
3727 ops
.op2
= NULL_TREE
;
3728 ops
.location
= gimple_location (call_stmt
);
3730 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3731 if (contains_call_div_mod (get_insns ()))
3732 quotient
= NULL_RTX
;
3735 ops
.code
= TRUNC_MOD_EXPR
;
3736 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3738 if (contains_call_div_mod (get_insns ()))
3739 remainder
= NULL_RTX
;
3742 insns
= get_insns ();
3750 /* Check if optab_handler exists for divmod_optab for given mode. */
3751 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3753 quotient
= gen_reg_rtx (mode
);
3754 remainder
= gen_reg_rtx (mode
);
3755 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3758 /* Generate call to divmod libfunc if it exists. */
3759 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3760 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3761 "ient
, &remainder
);
3766 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3767 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3768 make_tree (TREE_TYPE (arg0
), quotient
),
3769 make_tree (TREE_TYPE (arg1
), remainder
)),
3770 target
, VOIDmode
, EXPAND_NORMAL
);
3776 expand_NOP (internal_fn
, gcall
*)
3778 /* Nothing. But it shouldn't really prevail. */
3781 /* Coroutines, all should have been processed at this stage. */
3784 expand_CO_FRAME (internal_fn
, gcall
*)
3790 expand_CO_YIELD (internal_fn
, gcall
*)
3796 expand_CO_SUSPN (internal_fn
, gcall
*)
3802 expand_CO_ACTOR (internal_fn
, gcall
*)
3807 /* Expand a call to FN using the operands in STMT. FN has a single
3808 output operand and NARGS input operands. */
3811 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3814 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3815 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3816 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3819 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3822 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3824 expand_operand ops
[4];
3827 tree lhs
= gimple_call_lhs (stmt
);
3828 tree lhs_type
= TREE_TYPE (lhs
);
3829 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3830 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3832 for (unsigned int i
= 0; i
< 2; ++i
)
3834 tree rhs
= gimple_call_arg (stmt
, i
);
3835 rhs_type
[i
] = TREE_TYPE (rhs
);
3836 rtx rhs_rtx
= expand_normal (rhs
);
3837 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3841 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type
)))
3843 /* When the mask is an integer mode the exact vector length may not
3844 be clear to the backend, so we pass it in operand[3].
3845 Use the vector in arg2 for the most reliable intended size. */
3846 tree type
= TREE_TYPE (gimple_call_arg (stmt
, 2));
3847 create_integer_operand (&ops
[3], TYPE_VECTOR_SUBPARTS (type
));
3851 /* The mask has a vector type so the length operand is unnecessary. */
3854 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3855 TYPE_MODE (lhs_type
));
3857 expand_insn (icode
, opcnt
, ops
);
3858 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3859 emit_move_insn (lhs_rtx
, ops
[0].value
);
3862 /* Expand a call to a convert-like optab using the operands in STMT.
3863 FN has a single output operand and NARGS input operands. */
3866 expand_convert_optab_fn (internal_fn fn
, gcall
*stmt
, convert_optab optab
,
3869 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3870 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3871 TYPE_MODE (types
.second
));
3872 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3875 /* Expanders for optabs that can use expand_direct_optab_fn. */
3877 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3878 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3880 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3881 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3883 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3884 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3886 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3887 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3889 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3890 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3892 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3893 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3895 #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3896 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3898 #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3899 expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3901 #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3902 expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3904 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3905 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3907 #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3908 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3910 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3911 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3913 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3914 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3916 #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3917 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3919 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3920 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3922 /* Expanders for optabs that can use expand_convert_optab_fn. */
3924 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3925 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3927 #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3928 expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3930 /* RETURN_TYPE and ARGS are a return type and argument list that are
3931 in principle compatible with FN (which satisfies direct_internal_fn_p).
3932 Return the types that should be used to determine whether the
3933 target supports FN. */
3936 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3938 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3939 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3940 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3941 return tree_pair (type0
, type1
);
3944 /* CALL is a call whose return type and arguments are in principle
3945 compatible with FN (which satisfies direct_internal_fn_p). Return the
3946 types that should be used to determine whether the target supports FN. */
3949 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3951 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3952 tree op0
= (info
.type0
< 0
3953 ? gimple_call_lhs (call
)
3954 : gimple_call_arg (call
, info
.type0
));
3955 tree op1
= (info
.type1
< 0
3956 ? gimple_call_lhs (call
)
3957 : gimple_call_arg (call
, info
.type1
));
3958 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3961 /* Return true if OPTAB is supported for TYPES (whose modes should be
3962 the same) when the optimization type is OPT_TYPE. Used for simple
3966 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3967 optimization_type opt_type
)
3969 machine_mode mode
= TYPE_MODE (types
.first
);
3970 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3971 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3974 /* Return true if OPTAB is supported for TYPES, where the first type
3975 is the destination and the second type is the source. Used for
3979 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3980 optimization_type opt_type
)
3982 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3983 TYPE_MODE (types
.second
), opt_type
)
3984 != CODE_FOR_nothing
);
3987 /* Return true if load/store lanes optab OPTAB is supported for
3988 array type TYPES.first when the optimization type is OPT_TYPE. */
3991 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3992 optimization_type opt_type
)
3994 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3995 machine_mode imode
= TYPE_MODE (types
.first
);
3996 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3997 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3998 != CODE_FOR_nothing
);
4001 #define direct_unary_optab_supported_p direct_optab_supported_p
4002 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
4003 #define direct_binary_optab_supported_p direct_optab_supported_p
4004 #define direct_ternary_optab_supported_p direct_optab_supported_p
4005 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
4006 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
4007 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4008 #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4009 #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4010 #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4011 #define direct_mask_load_optab_supported_p convert_optab_supported_p
4012 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4013 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4014 #define direct_gather_load_optab_supported_p convert_optab_supported_p
4015 #define direct_len_load_optab_supported_p direct_optab_supported_p
4016 #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4017 #define direct_mask_store_optab_supported_p convert_optab_supported_p
4018 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4019 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4020 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4021 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
4022 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
4023 #define direct_len_store_optab_supported_p direct_optab_supported_p
4024 #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4025 #define direct_while_optab_supported_p convert_optab_supported_p
4026 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
4027 #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4028 #define direct_fold_left_optab_supported_p direct_optab_supported_p
4029 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4030 #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4031 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4032 #define direct_vec_set_optab_supported_p direct_optab_supported_p
4033 #define direct_vec_extract_optab_supported_p convert_optab_supported_p
4035 /* Return the optab used by internal function FN. */
4038 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
4042 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4043 case IFN_##CODE: break;
4044 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4045 case IFN_##CODE: return OPTAB##_optab;
4046 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4047 UNSIGNED_OPTAB, TYPE) \
4048 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4049 ? UNSIGNED_OPTAB ## _optab \
4050 : SIGNED_OPTAB ## _optab);
4051 #include "internal-fn.def"
4059 /* Return the optab used by internal function FN. */
4062 direct_internal_fn_optab (internal_fn fn
)
4066 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4067 case IFN_##CODE: break;
4068 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4069 case IFN_##CODE: return OPTAB##_optab;
4070 #include "internal-fn.def"
4078 /* Return true if FN is supported for the types in TYPES when the
4079 optimization type is OPT_TYPE. The types are those associated with
4080 the "type0" and "type1" fields of FN's direct_internal_fn_info
4084 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
4085 optimization_type opt_type
)
4089 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4090 case IFN_##CODE: break;
4091 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4093 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4095 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4096 UNSIGNED_OPTAB, TYPE) \
4099 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4100 ? UNSIGNED_OPTAB ## _optab \
4101 : SIGNED_OPTAB ## _optab); \
4102 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4105 #include "internal-fn.def"
4113 /* Return true if FN is supported for type TYPE when the optimization
4114 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4115 fields of FN's direct_internal_fn_info structure are the same. */
4118 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
4119 optimization_type opt_type
)
4121 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4122 gcc_checking_assert (info
.type0
== info
.type1
);
4123 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
4126 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4127 given that STMT is a call to a direct internal function. */
4130 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
4132 internal_fn fn
= gimple_call_internal_fn (stmt
);
4133 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
4134 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
4137 /* Return true if FN is a binary operation and if FN is commutative. */
4140 commutative_binary_fn_p (internal_fn fn
)
4151 case IFN_COMPLEX_MUL
:
4152 case IFN_UBSAN_CHECK_ADD
:
4153 case IFN_UBSAN_CHECK_MUL
:
4154 case IFN_ADD_OVERFLOW
:
4155 case IFN_MUL_OVERFLOW
:
4156 case IFN_VEC_WIDEN_PLUS
:
4157 case IFN_VEC_WIDEN_PLUS_LO
:
4158 case IFN_VEC_WIDEN_PLUS_HI
:
4159 case IFN_VEC_WIDEN_PLUS_EVEN
:
4160 case IFN_VEC_WIDEN_PLUS_ODD
:
4168 /* Return true if FN is a ternary operation and if its first two arguments
4172 commutative_ternary_fn_p (internal_fn fn
)
4188 /* Return true if FN is an associative binary operation. */
4191 associative_binary_fn_p (internal_fn fn
)
4204 /* If FN is commutative in two consecutive arguments, return the
4205 index of the first, otherwise return -1. */
4208 first_commutative_argument (internal_fn fn
)
4225 case IFN_COND_LEN_ADD
:
4226 case IFN_COND_LEN_MUL
:
4227 case IFN_COND_LEN_MIN
:
4228 case IFN_COND_LEN_MAX
:
4229 case IFN_COND_LEN_FMIN
:
4230 case IFN_COND_LEN_FMAX
:
4231 case IFN_COND_LEN_AND
:
4232 case IFN_COND_LEN_IOR
:
4233 case IFN_COND_LEN_XOR
:
4234 case IFN_COND_LEN_FMA
:
4235 case IFN_COND_LEN_FMS
:
4236 case IFN_COND_LEN_FNMA
:
4237 case IFN_COND_LEN_FNMS
:
4241 if (commutative_binary_fn_p (fn
)
4242 || commutative_ternary_fn_p (fn
))
4248 /* Return true if this CODE describes an internal_fn that returns a vector with
4249 elements twice as wide as the element size of the input vectors. */
4252 widening_fn_p (code_helper code
)
4254 if (!code
.is_fn_code ())
4257 if (!internal_fn_p ((combined_fn
) code
))
4260 internal_fn fn
= as_internal_fn ((combined_fn
) code
);
4263 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
4264 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4266 case IFN_##NAME##_HI: \
4267 case IFN_##NAME##_LO: \
4268 case IFN_##NAME##_EVEN: \
4269 case IFN_##NAME##_ODD: \
4271 #include "internal-fn.def"
4272 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
4279 /* Return true if IFN_SET_EDOM is supported. */
4282 set_edom_supported_p (void)
4291 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4293 expand_##CODE (internal_fn fn, gcall *stmt) \
4295 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4297 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4298 UNSIGNED_OPTAB, TYPE) \
4300 expand_##CODE (internal_fn fn, gcall *stmt) \
4302 tree_pair types = direct_internal_fn_types (fn, stmt); \
4303 optab which_optab = direct_internal_fn_optab (fn, types); \
4304 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4306 #include "internal-fn.def"
4307 #undef DEF_INTERNAL_OPTAB_FN
4308 #undef DEF_INTERNAL_SIGNED_OPTAB_FN
4310 /* Routines to expand each internal function, indexed by function number.
4311 Each routine has the prototype:
4313 expand_<NAME> (gcall *stmt)
4315 where STMT is the statement that performs the call. */
4316 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4318 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4319 #include "internal-fn.def"
4323 /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4324 that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4325 for each such IFN_COND_##SUFFIX. */
4326 #define FOR_EACH_CODE_MAPPING(T) \
4327 T (PLUS_EXPR, ADD) \
4328 T (MINUS_EXPR, SUB) \
4329 T (MULT_EXPR, MUL) \
4330 T (TRUNC_DIV_EXPR, DIV) \
4331 T (TRUNC_MOD_EXPR, MOD) \
4332 T (RDIV_EXPR, RDIV) \
4335 T (BIT_AND_EXPR, AND) \
4336 T (BIT_IOR_EXPR, IOR) \
4337 T (BIT_XOR_EXPR, XOR) \
4338 T (LSHIFT_EXPR, SHL) \
4339 T (RSHIFT_EXPR, SHR) \
4340 T (NEGATE_EXPR, NEG)
4342 /* Return a function that only performs CODE when a certain condition is met
4343 and that uses a given fallback value otherwise. For example, if CODE is
4344 a binary operation associated with conditional function FN:
4346 LHS = FN (COND, A, B, ELSE)
4348 is equivalent to the C expression:
4350 LHS = COND ? A CODE B : ELSE;
4352 operating elementwise if the operands are vectors.
4354 Return IFN_LAST if no such function exists. */
4357 get_conditional_internal_fn (tree_code code
)
4361 #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4362 FOR_EACH_CODE_MAPPING(CASE
)
4369 /* If IFN implements the conditional form of a tree code, return that
4370 tree code, otherwise return ERROR_MARK. */
4373 conditional_internal_fn_code (internal_fn ifn
)
4377 #define CASE(CODE, IFN) \
4378 case IFN_COND_##IFN: \
4379 case IFN_COND_LEN_##IFN: \
4381 FOR_EACH_CODE_MAPPING (CASE
)
4388 /* Like get_conditional_internal_fn, but return a function that
4389 additionally restricts the operation to the leading elements
4390 of a vector. The number of elements to process is given by a length
4391 and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4392 elements are taken from the fallback ("else") argument.
4394 For example, if CODE is a binary operation associated with FN:
4396 LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4398 is equivalent to the C code:
4400 for (int i = 0; i < NUNITS; i++)
4402 if (i < LEN + BIAS && COND[i])
4403 LHS[i] = A[i] CODE B[i];
4410 get_conditional_len_internal_fn (tree_code code
)
4414 #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4415 FOR_EACH_CODE_MAPPING(CASE
)
4422 /* Invoke T(IFN) for each internal function IFN that also has an
4424 #define FOR_EACH_COND_FN_PAIR(T) \
4432 /* Return a function that only performs internal function FN when a
4433 certain condition is met and that uses a given fallback value otherwise.
4434 In other words, the returned function FN' is such that:
4436 LHS = FN' (COND, A1, ... An, ELSE)
4438 is equivalent to the C expression:
4440 LHS = COND ? FN (A1, ..., An) : ELSE;
4442 operating elementwise if the operands are vectors.
4444 Return IFN_LAST if no such function exists. */
4447 get_conditional_internal_fn (internal_fn fn
)
4451 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4452 FOR_EACH_COND_FN_PAIR(CASE
)
4459 /* If there exists an internal function like IFN that operates on vectors,
4460 but with additional length and bias parameters, return the internal_fn
4461 for that function, otherwise return IFN_LAST. */
4463 get_len_internal_fn (internal_fn fn
)
4467 #undef DEF_INTERNAL_COND_FN
4468 #undef DEF_INTERNAL_SIGNED_COND_FN
4469 #define DEF_INTERNAL_COND_FN(NAME, ...) \
4470 case IFN_COND_##NAME: \
4471 return IFN_COND_LEN_##NAME;
4472 #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4473 case IFN_COND_##NAME: \
4474 return IFN_COND_LEN_##NAME;
4475 #include "internal-fn.def"
4476 #undef DEF_INTERNAL_COND_FN
4477 #undef DEF_INTERNAL_SIGNED_COND_FN
4483 /* If IFN implements the conditional form of an unconditional internal
4484 function, return that unconditional function, otherwise return IFN_LAST. */
4487 get_unconditional_internal_fn (internal_fn ifn
)
4491 #define CASE(NAME) \
4492 case IFN_COND_##NAME: \
4493 case IFN_COND_LEN_##NAME: \
4495 FOR_EACH_COND_FN_PAIR (CASE
)
4502 /* Return true if STMT can be interpreted as a conditional tree code
4503 operation of the form:
4505 LHS = COND ? OP (RHS1, ...) : ELSE;
4507 operating elementwise if the operands are vectors. This includes
4508 the case of an all-true COND, so that the operation always happens.
4510 There is an alternative approach to interpret the STMT when the operands
4511 are vectors which is the operation predicated by both conditional mask
4512 and loop control length, the equivalent C code:
4514 for (int i = 0; i < NUNTIS; i++)
4516 if (i < LEN + BIAS && COND[i])
4517 LHS[i] = A[i] CODE B[i];
4522 When returning true, set:
4524 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4525 is known to be all-true
4526 - *CODE_OUT to the tree code
4527 - OPS[I] to operand I of *CODE_OUT
4528 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4529 condition is known to be all true.
4530 - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4531 - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4534 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4535 tree_code
*code_out
,
4536 tree (&ops
)[3], tree
*else_out
,
4537 tree
*len
, tree
*bias
)
4541 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4543 *cond_out
= NULL_TREE
;
4544 *code_out
= gimple_assign_rhs_code (assign
);
4545 ops
[0] = gimple_assign_rhs1 (assign
);
4546 ops
[1] = gimple_assign_rhs2 (assign
);
4547 ops
[2] = gimple_assign_rhs3 (assign
);
4548 *else_out
= NULL_TREE
;
4551 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4552 if (gimple_call_internal_p (call
))
4554 internal_fn ifn
= gimple_call_internal_fn (call
);
4555 tree_code code
= conditional_internal_fn_code (ifn
);
4556 int len_index
= internal_fn_len_index (ifn
);
4557 int cond_nargs
= len_index
>= 0 ? 4 : 2;
4558 if (code
!= ERROR_MARK
)
4560 *cond_out
= gimple_call_arg (call
, 0);
4562 unsigned int nops
= gimple_call_num_args (call
) - cond_nargs
;
4563 for (unsigned int i
= 0; i
< 3; ++i
)
4564 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4565 *else_out
= gimple_call_arg (call
, nops
+ 1);
4568 if (integer_truep (*cond_out
))
4570 *cond_out
= NULL_TREE
;
4571 *else_out
= NULL_TREE
;
4576 *len
= gimple_call_arg (call
, len_index
);
4577 *bias
= gimple_call_arg (call
, len_index
+ 1);
4585 /* Return true if IFN is some form of load from memory. */
4588 internal_load_fn_p (internal_fn fn
)
4593 case IFN_LOAD_LANES
:
4594 case IFN_MASK_LOAD_LANES
:
4595 case IFN_MASK_LEN_LOAD_LANES
:
4596 case IFN_GATHER_LOAD
:
4597 case IFN_MASK_GATHER_LOAD
:
4598 case IFN_MASK_LEN_GATHER_LOAD
:
4600 case IFN_MASK_LEN_LOAD
:
4608 /* Return true if IFN is some form of store to memory. */
4611 internal_store_fn_p (internal_fn fn
)
4615 case IFN_MASK_STORE
:
4616 case IFN_STORE_LANES
:
4617 case IFN_MASK_STORE_LANES
:
4618 case IFN_MASK_LEN_STORE_LANES
:
4619 case IFN_SCATTER_STORE
:
4620 case IFN_MASK_SCATTER_STORE
:
4621 case IFN_MASK_LEN_SCATTER_STORE
:
4623 case IFN_MASK_LEN_STORE
:
4631 /* Return true if IFN is some form of gather load or scatter store. */
4634 internal_gather_scatter_fn_p (internal_fn fn
)
4638 case IFN_GATHER_LOAD
:
4639 case IFN_MASK_GATHER_LOAD
:
4640 case IFN_MASK_LEN_GATHER_LOAD
:
4641 case IFN_SCATTER_STORE
:
4642 case IFN_MASK_SCATTER_STORE
:
4643 case IFN_MASK_LEN_SCATTER_STORE
:
4651 /* If FN takes a vector len argument, return the index of that argument,
4652 otherwise return -1. */
4655 internal_fn_len_index (internal_fn fn
)
4663 case IFN_MASK_LEN_GATHER_LOAD
:
4664 case IFN_MASK_LEN_SCATTER_STORE
:
4665 case IFN_COND_LEN_FMA
:
4666 case IFN_COND_LEN_FMS
:
4667 case IFN_COND_LEN_FNMA
:
4668 case IFN_COND_LEN_FNMS
:
4671 case IFN_COND_LEN_ADD
:
4672 case IFN_COND_LEN_SUB
:
4673 case IFN_COND_LEN_MUL
:
4674 case IFN_COND_LEN_DIV
:
4675 case IFN_COND_LEN_MOD
:
4676 case IFN_COND_LEN_RDIV
:
4677 case IFN_COND_LEN_MIN
:
4678 case IFN_COND_LEN_MAX
:
4679 case IFN_COND_LEN_FMIN
:
4680 case IFN_COND_LEN_FMAX
:
4681 case IFN_COND_LEN_AND
:
4682 case IFN_COND_LEN_IOR
:
4683 case IFN_COND_LEN_XOR
:
4684 case IFN_COND_LEN_SHL
:
4685 case IFN_COND_LEN_SHR
:
4688 case IFN_COND_LEN_NEG
:
4689 case IFN_MASK_LEN_LOAD
:
4690 case IFN_MASK_LEN_STORE
:
4691 case IFN_MASK_LEN_LOAD_LANES
:
4692 case IFN_MASK_LEN_STORE_LANES
:
4700 /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4701 argument that is used when the condition is false. Return -1 otherwise. */
4704 internal_fn_else_index (internal_fn fn
)
4710 case IFN_COND_LEN_NEG
:
4711 case IFN_COND_LEN_NOT
:
4728 case IFN_COND_LEN_ADD
:
4729 case IFN_COND_LEN_SUB
:
4730 case IFN_COND_LEN_MUL
:
4731 case IFN_COND_LEN_DIV
:
4732 case IFN_COND_LEN_MOD
:
4733 case IFN_COND_LEN_MIN
:
4734 case IFN_COND_LEN_MAX
:
4735 case IFN_COND_LEN_FMIN
:
4736 case IFN_COND_LEN_FMAX
:
4737 case IFN_COND_LEN_AND
:
4738 case IFN_COND_LEN_IOR
:
4739 case IFN_COND_LEN_XOR
:
4740 case IFN_COND_LEN_SHL
:
4741 case IFN_COND_LEN_SHR
:
4748 case IFN_COND_LEN_FMA
:
4749 case IFN_COND_LEN_FMS
:
4750 case IFN_COND_LEN_FNMA
:
4751 case IFN_COND_LEN_FNMS
:
4761 /* If FN takes a vector mask argument, return the index of that argument,
4762 otherwise return -1. */
4765 internal_fn_mask_index (internal_fn fn
)
4770 case IFN_MASK_LOAD_LANES
:
4771 case IFN_MASK_LEN_LOAD_LANES
:
4772 case IFN_MASK_STORE
:
4773 case IFN_MASK_STORE_LANES
:
4774 case IFN_MASK_LEN_STORE_LANES
:
4775 case IFN_MASK_LEN_LOAD
:
4776 case IFN_MASK_LEN_STORE
:
4779 case IFN_MASK_GATHER_LOAD
:
4780 case IFN_MASK_SCATTER_STORE
:
4781 case IFN_MASK_LEN_GATHER_LOAD
:
4782 case IFN_MASK_LEN_SCATTER_STORE
:
4786 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4787 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4791 /* If FN takes a value that should be stored to memory, return the index
4792 of that argument, otherwise return -1. */
4795 internal_fn_stored_value_index (internal_fn fn
)
4799 case IFN_MASK_STORE
:
4800 case IFN_MASK_STORE_LANES
:
4801 case IFN_SCATTER_STORE
:
4802 case IFN_MASK_SCATTER_STORE
:
4803 case IFN_MASK_LEN_SCATTER_STORE
:
4809 case IFN_MASK_LEN_STORE
:
4810 case IFN_MASK_LEN_STORE_LANES
:
4818 /* Return true if the target supports gather load or scatter store function
4819 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4820 while for stores it is the vector type of the stored data argument.
4821 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4822 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4823 offset from the shared base address of each loaded or stored element.
4824 SCALE is the amount by which these offsets should be multiplied
4825 *after* they have been extended to address width. */
4828 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4829 tree memory_element_type
,
4830 tree offset_vector_type
, int scale
)
4832 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4833 TYPE_SIZE (memory_element_type
)))
4835 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4836 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4838 optab optab
= direct_internal_fn_optab (ifn
);
4839 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4840 TYPE_MODE (offset_vector_type
));
4841 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4842 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4843 return (icode
!= CODE_FOR_nothing
4844 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4845 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4848 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4849 for pointers of type TYPE when the accesses have LENGTH bytes and their
4850 common byte alignment is ALIGN. */
4853 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4854 poly_uint64 length
, unsigned int align
)
4856 machine_mode mode
= TYPE_MODE (type
);
4857 optab optab
= direct_internal_fn_optab (ifn
);
4858 insn_code icode
= direct_optab_handler (optab
, mode
);
4859 if (icode
== CODE_FOR_nothing
)
4861 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4862 return (insn_operand_matches (icode
, 3, length_rtx
)
4863 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4866 /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4867 or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4868 -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4869 {len_,mask_len_}store). If none of the biases match what the backend
4870 provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4873 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
4875 optab optab
= direct_internal_fn_optab (ifn
);
4876 insn_code icode
= direct_optab_handler (optab
, mode
);
4879 if (icode
== CODE_FOR_nothing
)
4881 machine_mode mask_mode
;
4882 if (!targetm
.vectorize
.get_mask_mode (mode
).exists (&mask_mode
))
4883 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4884 if (ifn
== IFN_LEN_LOAD
)
4886 /* Try MASK_LEN_LOAD. */
4887 optab
= direct_internal_fn_optab (IFN_MASK_LEN_LOAD
);
4891 /* Try MASK_LEN_STORE. */
4892 optab
= direct_internal_fn_optab (IFN_MASK_LEN_STORE
);
4894 icode
= convert_optab_handler (optab
, mode
, mask_mode
);
4898 if (icode
!= CODE_FOR_nothing
)
4900 /* For now we only support biases of 0 or -1. Try both of them. */
4901 if (insn_operand_matches (icode
, bias_no
, GEN_INT (0)))
4903 if (insn_operand_matches (icode
, bias_no
, GEN_INT (-1)))
4907 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4910 /* Expand STMT as though it were a call to internal function FN. */
4913 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4915 internal_fn_expanders
[fn
] (fn
, stmt
);
4918 /* Expand STMT, which is a call to internal function FN. */
4921 expand_internal_call (gcall
*stmt
)
4923 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4926 /* If TYPE is a vector type, return true if IFN is a direct internal
4927 function that is supported for that type. If TYPE is a scalar type,
4928 return true if IFN is a direct internal function that is supported for
4929 the target's preferred vector version of TYPE. */
4932 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4934 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4935 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4938 if (VECTOR_TYPE_P (type
)
4939 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4942 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4943 if (VECTOR_MODE_P (vmode
))
4945 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4946 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4950 auto_vector_modes vector_modes
;
4951 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4952 for (machine_mode base_mode
: vector_modes
)
4953 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
4955 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4956 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4964 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
4970 expand_PHI (internal_fn
, gcall
*)
4976 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
4978 tree lhs
= gimple_call_lhs (stmt
);
4979 tree rhs1
= gimple_call_arg (stmt
, 0);
4980 tree rhs2
= gimple_call_arg (stmt
, 1);
4981 tree type
= TREE_TYPE (rhs1
);
4983 do_pending_stack_adjust ();
4985 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
4986 rtx op1
= expand_normal (rhs1
);
4987 rtx op2
= expand_normal (rhs2
);
4989 class expand_operand ops
[3];
4990 create_output_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
4991 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
4992 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
4993 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
4994 expand_insn (icode
, 3, ops
);
4995 if (!rtx_equal_p (target
, ops
[0].value
))
4996 emit_move_insn (target
, ops
[0].value
);
5000 expand_ASSUME (internal_fn
, gcall
*)
5005 expand_MASK_CALL (internal_fn
, gcall
*)
5007 /* This IFN should only exist between ifcvt and vect passes. */
5012 expand_MULBITINT (internal_fn
, gcall
*stmt
)
5015 for (int i
= 0; i
< 6; i
++)
5016 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5017 (i
& 1) ? SImode
: ptr_mode
);
5018 rtx fun
= init_one_libfunc ("__mulbitint3");
5019 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 6, args
);
5023 expand_DIVMODBITINT (internal_fn
, gcall
*stmt
)
5026 for (int i
= 0; i
< 8; i
++)
5027 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5028 (i
& 1) ? SImode
: ptr_mode
);
5029 rtx fun
= init_one_libfunc ("__divmodbitint4");
5030 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 8, args
);
5034 expand_FLOATTOBITINT (internal_fn
, gcall
*stmt
)
5036 machine_mode mode
= TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt
, 2)));
5037 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5038 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5039 rtx arg2
= expand_normal (gimple_call_arg (stmt
, 2));
5040 const char *mname
= GET_MODE_NAME (mode
);
5041 unsigned mname_len
= strlen (mname
);
5042 int len
= 12 + mname_len
;
5043 if (DECIMAL_FLOAT_MODE_P (mode
))
5045 char *libfunc_name
= XALLOCAVEC (char, len
);
5046 char *p
= libfunc_name
;
5048 if (DECIMAL_FLOAT_MODE_P (mode
))
5050 #if ENABLE_DECIMAL_BID_FORMAT
5051 memcpy (p
, "__bid_fix", 9);
5053 memcpy (p
, "__dpd_fix", 9);
5059 memcpy (p
, "__fix", 5);
5062 for (q
= mname
; *q
; q
++)
5063 *p
++ = TOLOWER (*q
);
5064 memcpy (p
, "bitint", 7);
5065 rtx fun
= init_one_libfunc (libfunc_name
);
5066 emit_library_call (fun
, LCT_NORMAL
, VOIDmode
, arg0
, ptr_mode
, arg1
,
5067 SImode
, arg2
, mode
);
5071 expand_BITINTTOFLOAT (internal_fn
, gcall
*stmt
)
5073 tree lhs
= gimple_call_lhs (stmt
);
5076 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
5077 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5078 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5079 const char *mname
= GET_MODE_NAME (mode
);
5080 unsigned mname_len
= strlen (mname
);
5081 int len
= 14 + mname_len
;
5082 if (DECIMAL_FLOAT_MODE_P (mode
))
5084 char *libfunc_name
= XALLOCAVEC (char, len
);
5085 char *p
= libfunc_name
;
5087 if (DECIMAL_FLOAT_MODE_P (mode
))
5089 #if ENABLE_DECIMAL_BID_FORMAT
5090 memcpy (p
, "__bid_floatbitint", 17);
5092 memcpy (p
, "__dpd_floatbitint", 17);
5098 memcpy (p
, "__floatbitint", 13);
5101 for (q
= mname
; *q
; q
++)
5102 *p
++ = TOLOWER (*q
);
5104 rtx fun
= init_one_libfunc (libfunc_name
);
5105 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5106 rtx val
= emit_library_call_value (fun
, target
, LCT_PURE
, mode
,
5107 arg0
, ptr_mode
, arg1
, SImode
);
5109 emit_move_insn (target
, val
);