2 Copyright (C) 2011-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
55 #include "fold-const-call.h"
57 /* For lang_hooks.types.type_for_mode. */
58 #include "langhooks.h"
60 /* The names of each internal function, indexed by function number. */
61 const char *const internal_fn_name_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
63 #include "internal-fn.def"
67 /* The ECF_* flags of each internal function, indexed by function number. */
68 const int internal_fn_flags_array
[] = {
69 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
70 #include "internal-fn.def"
74 /* Return the internal function called NAME, or IFN_LAST if there's
78 lookup_internal_fn (const char *name
)
80 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
81 static name_to_fn_map_type
*name_to_fn_map
;
85 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
86 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
87 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
90 internal_fn
*entry
= name_to_fn_map
->get (name
);
91 return entry
? *entry
: IFN_LAST
;
94 /* Geven an internal_fn IFN that is a widening function, return its
95 corresponding LO and HI internal_fns. */
98 lookup_hilo_internal_fn (internal_fn ifn
, internal_fn
*lo
, internal_fn
*hi
)
100 gcc_assert (widening_fn_p (ifn
));
106 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
107 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
109 *lo = internal_fn (IFN_##NAME##_LO); \
110 *hi = internal_fn (IFN_##NAME##_HI); \
112 #include "internal-fn.def"
116 /* Given an internal_fn IFN that is a widening function, return its
117 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
120 lookup_evenodd_internal_fn (internal_fn ifn
, internal_fn
*even
,
123 gcc_assert (widening_fn_p (ifn
));
129 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
130 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
132 *even = internal_fn (IFN_##NAME##_EVEN); \
133 *odd = internal_fn (IFN_##NAME##_ODD); \
135 #include "internal-fn.def"
140 /* Fnspec of each internal function, indexed by function number. */
141 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
146 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
147 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
148 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
149 #include "internal-fn.def"
150 internal_fn_fnspec_array
[IFN_LAST
] = 0;
153 /* Create static initializers for the information returned by
154 direct_internal_fn. */
155 #define not_direct { -2, -2, false }
156 #define mask_load_direct { -1, 2, false }
157 #define load_lanes_direct { -1, -1, false }
158 #define mask_load_lanes_direct { -1, -1, false }
159 #define gather_load_direct { 3, 1, false }
160 #define len_load_direct { -1, -1, false }
161 #define mask_len_load_direct { -1, 4, false }
162 #define mask_store_direct { 3, 2, false }
163 #define store_lanes_direct { 0, 0, false }
164 #define mask_store_lanes_direct { 0, 0, false }
165 #define vec_cond_mask_direct { 1, 0, false }
166 #define vec_cond_mask_len_direct { 1, 1, false }
167 #define vec_cond_direct { 2, 0, false }
168 #define scatter_store_direct { 3, 1, false }
169 #define len_store_direct { 3, 3, false }
170 #define mask_len_store_direct { 4, 5, false }
171 #define vec_set_direct { 3, 3, false }
172 #define vec_extract_direct { 0, -1, false }
173 #define unary_direct { 0, 0, true }
174 #define unary_convert_direct { -1, 0, true }
175 #define binary_direct { 0, 0, true }
176 #define ternary_direct { 0, 0, true }
177 #define cond_unary_direct { 1, 1, true }
178 #define cond_binary_direct { 1, 1, true }
179 #define cond_ternary_direct { 1, 1, true }
180 #define cond_len_unary_direct { 1, 1, true }
181 #define cond_len_binary_direct { 1, 1, true }
182 #define cond_len_ternary_direct { 1, 1, true }
183 #define while_direct { 0, 2, false }
184 #define fold_extract_direct { 2, 2, false }
185 #define fold_len_extract_direct { 2, 2, false }
186 #define fold_left_direct { 1, 1, false }
187 #define mask_fold_left_direct { 1, 1, false }
188 #define mask_len_fold_left_direct { 1, 1, false }
189 #define check_ptrs_direct { 0, 0, false }
191 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
192 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
193 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
194 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
195 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
196 #include "internal-fn.def"
200 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
201 output operands and NINPUTS input operands, where NOUTPUTS is either
202 0 or 1. The output operand (if any) comes first, followed by the
203 NINPUTS input operands. */
206 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
207 unsigned int ninputs
)
209 gcc_assert (icode
!= CODE_FOR_nothing
);
211 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
212 unsigned int opno
= 0;
213 rtx lhs_rtx
= NULL_RTX
;
214 tree lhs
= gimple_call_lhs (stmt
);
218 gcc_assert (noutputs
== 1);
220 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
222 /* Do not assign directly to a promoted subreg, since there is no
223 guarantee that the instruction will leave the upper bits of the
224 register in the state required by SUBREG_PROMOTED_SIGN. */
226 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
228 create_output_operand (&ops
[opno
], dest
,
229 insn_data
[icode
].operand
[opno
].mode
);
235 for (unsigned int i
= 0; i
< ninputs
; ++i
)
237 tree rhs
= gimple_call_arg (stmt
, i
);
238 tree rhs_type
= TREE_TYPE (rhs
);
239 rtx rhs_rtx
= expand_normal (rhs
);
240 if (INTEGRAL_TYPE_P (rhs_type
))
241 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
242 TYPE_MODE (rhs_type
),
243 TYPE_UNSIGNED (rhs_type
));
244 else if (TREE_CODE (rhs
) == SSA_NAME
245 && SSA_NAME_IS_DEFAULT_DEF (rhs
)
246 && VAR_P (SSA_NAME_VAR (rhs
)))
247 create_undefined_input_operand (&ops
[opno
], TYPE_MODE (rhs_type
));
249 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
253 gcc_assert (opno
== noutputs
+ ninputs
);
254 expand_insn (icode
, opno
, ops
);
255 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
257 /* If the return value has an integral type, convert the instruction
258 result to that type. This is useful for things that return an
259 int regardless of the size of the input. If the instruction result
260 is smaller than required, assume that it is signed.
262 If the return value has a nonintegral type, its mode must match
263 the instruction result. */
264 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
266 /* If this is a scalar in a register that is stored in a wider
267 mode than the declared mode, compute the result into its
268 declared mode and then convert to the wider mode. */
269 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
270 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
271 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
272 SUBREG_PROMOTED_SIGN (lhs_rtx
));
274 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
275 emit_move_insn (lhs_rtx
, ops
[0].value
);
278 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
279 convert_move (lhs_rtx
, ops
[0].value
, 0);
284 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
285 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
287 static enum insn_code
288 get_multi_vector_move (tree array_type
, convert_optab optab
)
293 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
294 imode
= TYPE_MODE (array_type
);
295 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
297 return convert_optab_handler (optab
, imode
, vmode
);
300 /* Add mask and len arguments according to the STMT. */
303 add_mask_and_len_args (expand_operand
*ops
, unsigned int opno
, gcall
*stmt
)
305 internal_fn ifn
= gimple_call_internal_fn (stmt
);
306 int len_index
= internal_fn_len_index (ifn
);
307 /* BIAS is always consecutive next of LEN. */
308 int bias_index
= len_index
+ 1;
309 int mask_index
= internal_fn_mask_index (ifn
);
310 /* The order of arguments are always {len,bias,mask}. */
313 tree mask
= gimple_call_arg (stmt
, mask_index
);
314 rtx mask_rtx
= expand_normal (mask
);
315 create_input_operand (&ops
[opno
++], mask_rtx
,
316 TYPE_MODE (TREE_TYPE (mask
)));
320 tree len
= gimple_call_arg (stmt
, len_index
);
321 rtx len_rtx
= expand_normal (len
);
322 create_convert_operand_from (&ops
[opno
++], len_rtx
,
323 TYPE_MODE (TREE_TYPE (len
)),
324 TYPE_UNSIGNED (TREE_TYPE (len
)));
325 tree biast
= gimple_call_arg (stmt
, bias_index
);
326 rtx bias
= expand_normal (biast
);
327 create_input_operand (&ops
[opno
++], bias
, QImode
);
332 /* Expand LOAD_LANES call STMT using optab OPTAB. */
335 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
337 class expand_operand ops
[2];
341 lhs
= gimple_call_lhs (stmt
);
342 rhs
= gimple_call_arg (stmt
, 0);
343 type
= TREE_TYPE (lhs
);
345 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
346 mem
= expand_normal (rhs
);
348 gcc_assert (MEM_P (mem
));
349 PUT_MODE (mem
, TYPE_MODE (type
));
351 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
352 create_fixed_operand (&ops
[1], mem
);
353 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
354 if (!rtx_equal_p (target
, ops
[0].value
))
355 emit_move_insn (target
, ops
[0].value
);
358 /* Expand STORE_LANES call STMT using optab OPTAB. */
361 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
363 class expand_operand ops
[2];
367 lhs
= gimple_call_lhs (stmt
);
368 rhs
= gimple_call_arg (stmt
, 0);
369 type
= TREE_TYPE (rhs
);
371 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
372 reg
= expand_normal (rhs
);
374 gcc_assert (MEM_P (target
));
375 PUT_MODE (target
, TYPE_MODE (type
));
377 create_fixed_operand (&ops
[0], target
);
378 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
379 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
383 expand_ANNOTATE (internal_fn
, gcall
*)
388 /* This should get expanded in omp_device_lower pass. */
391 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
396 /* This should get expanded in omp_device_lower pass. */
399 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
404 /* Allocate per-lane storage and begin non-uniform execution region. */
407 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
410 tree lhs
= gimple_call_lhs (stmt
);
412 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
414 target
= gen_reg_rtx (Pmode
);
415 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
416 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
417 class expand_operand ops
[3];
418 create_output_operand (&ops
[0], target
, Pmode
);
419 create_input_operand (&ops
[1], size
, Pmode
);
420 create_input_operand (&ops
[2], align
, Pmode
);
421 gcc_assert (targetm
.have_omp_simt_enter ());
422 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
423 if (!rtx_equal_p (target
, ops
[0].value
))
424 emit_move_insn (target
, ops
[0].value
);
427 /* Deallocate per-lane storage and leave non-uniform execution region. */
430 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
432 gcc_checking_assert (!gimple_call_lhs (stmt
));
433 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
434 class expand_operand ops
[1];
435 create_input_operand (&ops
[0], arg
, Pmode
);
436 gcc_assert (targetm
.have_omp_simt_exit ());
437 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
440 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
441 without SIMT execution this should be expanded in omp_device_lower pass. */
444 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
446 tree lhs
= gimple_call_lhs (stmt
);
450 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
451 gcc_assert (targetm
.have_omp_simt_lane ());
452 emit_insn (targetm
.gen_omp_simt_lane (target
));
455 /* This should get expanded in omp_device_lower pass. */
458 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
463 /* This should get expanded in omp_device_lower pass. */
466 expand_GOMP_TARGET_REV (internal_fn
, gcall
*)
471 /* Lane index of the first SIMT lane that supplies a non-zero argument.
472 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
473 lane that executed the last iteration for handling OpenMP lastprivate. */
476 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
478 tree lhs
= gimple_call_lhs (stmt
);
482 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
483 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
484 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
485 class expand_operand ops
[2];
486 create_output_operand (&ops
[0], target
, mode
);
487 create_input_operand (&ops
[1], cond
, mode
);
488 gcc_assert (targetm
.have_omp_simt_last_lane ());
489 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
490 if (!rtx_equal_p (target
, ops
[0].value
))
491 emit_move_insn (target
, ops
[0].value
);
494 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
497 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
499 tree lhs
= gimple_call_lhs (stmt
);
503 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
504 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
505 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
506 class expand_operand ops
[2];
507 create_output_operand (&ops
[0], target
, mode
);
508 create_input_operand (&ops
[1], ctr
, mode
);
509 gcc_assert (targetm
.have_omp_simt_ordered ());
510 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
511 if (!rtx_equal_p (target
, ops
[0].value
))
512 emit_move_insn (target
, ops
[0].value
);
515 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
516 any lane supplies a non-zero argument. */
519 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
521 tree lhs
= gimple_call_lhs (stmt
);
525 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
526 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
527 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
528 class expand_operand ops
[2];
529 create_output_operand (&ops
[0], target
, mode
);
530 create_input_operand (&ops
[1], cond
, mode
);
531 gcc_assert (targetm
.have_omp_simt_vote_any ());
532 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
533 if (!rtx_equal_p (target
, ops
[0].value
))
534 emit_move_insn (target
, ops
[0].value
);
537 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
538 is destination lane index XOR given offset. */
541 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
543 tree lhs
= gimple_call_lhs (stmt
);
547 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
548 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
549 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
550 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
551 class expand_operand ops
[3];
552 create_output_operand (&ops
[0], target
, mode
);
553 create_input_operand (&ops
[1], src
, mode
);
554 create_input_operand (&ops
[2], idx
, SImode
);
555 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
556 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
557 if (!rtx_equal_p (target
, ops
[0].value
))
558 emit_move_insn (target
, ops
[0].value
);
561 /* Exchange between SIMT lanes according to given source lane index. */
564 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
566 tree lhs
= gimple_call_lhs (stmt
);
570 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
571 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
572 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
573 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
574 class expand_operand ops
[3];
575 create_output_operand (&ops
[0], target
, mode
);
576 create_input_operand (&ops
[1], src
, mode
);
577 create_input_operand (&ops
[2], idx
, SImode
);
578 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
579 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
580 if (!rtx_equal_p (target
, ops
[0].value
))
581 emit_move_insn (target
, ops
[0].value
);
584 /* This should get expanded in adjust_simduid_builtins. */
587 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
592 /* This should get expanded in adjust_simduid_builtins. */
595 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
600 /* This should get expanded in adjust_simduid_builtins. */
603 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
608 /* This should get expanded in adjust_simduid_builtins. */
611 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
616 /* This should get expanded in adjust_simduid_builtins. */
619 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
624 /* This should get expanded in the sanopt pass. */
627 expand_UBSAN_NULL (internal_fn
, gcall
*)
632 /* This should get expanded in the sanopt pass. */
635 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
640 /* This should get expanded in the sanopt pass. */
643 expand_UBSAN_VPTR (internal_fn
, gcall
*)
648 /* This should get expanded in the sanopt pass. */
651 expand_UBSAN_PTR (internal_fn
, gcall
*)
656 /* This should get expanded in the sanopt pass. */
659 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
664 /* This should get expanded in the sanopt pass. */
667 expand_HWASAN_CHECK (internal_fn
, gcall
*)
672 /* For hwasan stack tagging:
673 Clear tags on the dynamically allocated space.
674 For use after an object dynamically allocated on the stack goes out of
677 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
679 gcc_assert (Pmode
== ptr_mode
);
680 tree restored_position
= gimple_call_arg (gc
, 0);
681 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
683 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
684 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
685 stack_pointer_rtx
, NULL_RTX
, 0,
687 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
688 virtual_stack_dynamic_rtx
, Pmode
,
689 HWASAN_STACK_BACKGROUND
, QImode
,
693 /* For hwasan stack tagging:
694 Return a tag to be used for a dynamic allocation. */
696 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
698 tree tag
= gimple_call_lhs (gc
);
699 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
700 machine_mode mode
= GET_MODE (target
);
701 gcc_assert (mode
== QImode
);
703 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
704 gcc_assert (base_tag
);
705 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
706 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
707 target
, /* unsignedp = */1,
709 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
711 /* Really need to put the tag into the `target` RTX. */
712 if (chosen_tag
!= target
)
714 rtx temp
= chosen_tag
;
715 gcc_assert (GET_MODE (chosen_tag
) == mode
);
716 emit_move_insn (target
, temp
);
719 hwasan_increment_frame_tag ();
722 /* For hwasan stack tagging:
723 Tag a region of space in the shadow stack according to the base pointer of
724 an object on the stack. N.b. the length provided in the internal call is
725 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
727 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
729 gcc_assert (ptr_mode
== Pmode
);
730 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
731 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
733 tree base
= gimple_call_arg (gc
, 1);
734 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
735 rtx base_rtx
= expand_normal (base
);
737 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
738 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
739 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
741 tree len
= gimple_call_arg (gc
, 2);
742 rtx r_len
= expand_normal (len
);
744 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
745 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
746 tag
, QImode
, r_len
, Pmode
);
749 /* For hwasan stack tagging:
750 Store a tag into a pointer. */
752 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
754 gcc_assert (ptr_mode
== Pmode
);
755 tree g_target
= gimple_call_lhs (gc
);
756 tree g_ptr
= gimple_call_arg (gc
, 0);
757 tree g_tag
= gimple_call_arg (gc
, 1);
759 rtx ptr
= expand_normal (g_ptr
);
760 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
761 rtx target
= expand_normal (g_target
);
763 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
764 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
765 if (tagged_value
!= target
)
766 emit_move_insn (target
, tagged_value
);
769 /* This should get expanded in the sanopt pass. */
772 expand_ASAN_CHECK (internal_fn
, gcall
*)
777 /* This should get expanded in the sanopt pass. */
780 expand_ASAN_MARK (internal_fn
, gcall
*)
785 /* This should get expanded in the sanopt pass. */
788 expand_ASAN_POISON (internal_fn
, gcall
*)
793 /* This should get expanded in the sanopt pass. */
796 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
801 /* This should get expanded in the tsan pass. */
804 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
809 /* This should get expanded in the lower pass. */
812 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
814 error_at (gimple_location (call
),
815 "invalid use of attribute %<fallthrough%>");
818 /* Return minimum precision needed to represent all values
819 of ARG in SIGNed integral type. */
822 get_min_precision (tree arg
, signop sign
)
824 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
826 signop orig_sign
= sign
;
827 if (TREE_CODE (arg
) == INTEGER_CST
)
830 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
832 widest_int w
= wi::to_widest (arg
);
833 w
= wi::ext (w
, prec
, sign
);
834 p
= wi::min_precision (w
, sign
);
837 p
= wi::min_precision (wi::to_wide (arg
), sign
);
838 return MIN (p
, prec
);
840 while (CONVERT_EXPR_P (arg
)
841 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
842 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
844 arg
= TREE_OPERAND (arg
, 0);
845 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
847 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
849 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
850 return prec
+ (orig_sign
!= sign
);
851 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
854 return prec
+ (orig_sign
!= sign
);
856 if (CONVERT_EXPR_P (arg
)
857 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
858 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
860 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
861 If y_2's min precision is smaller than prec, return that. */
862 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
864 return oprec
+ (orig_sign
!= sign
);
866 if (TREE_CODE (arg
) != SSA_NAME
)
867 return prec
+ (orig_sign
!= sign
);
869 while (!get_global_range_query ()->range_of_expr (r
, arg
)
873 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
874 if (is_gimple_assign (g
)
875 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
877 tree t
= gimple_assign_rhs1 (g
);
878 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
879 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
882 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
884 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
886 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
887 return prec
+ (orig_sign
!= sign
);
888 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
891 return prec
+ (orig_sign
!= sign
);
895 return prec
+ (orig_sign
!= sign
);
897 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
899 int p1
= wi::min_precision (r
.lower_bound (), sign
);
900 int p2
= wi::min_precision (r
.upper_bound (), sign
);
902 prec
= MIN (prec
, p1
);
904 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
906 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
907 prec
= MIN (prec
, p
);
909 return prec
+ (orig_sign
!= sign
);
912 /* Helper for expand_*_overflow. Set the __imag__ part to true
913 (1 except for signed:1 type, in which case store -1). */
916 expand_arith_set_overflow (tree lhs
, rtx target
)
918 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
919 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
920 write_complex_part (target
, constm1_rtx
, true, false);
922 write_complex_part (target
, const1_rtx
, true, false);
925 /* Helper for expand_*_overflow. Store RES into the __real__ part
926 of TARGET. If RES has larger MODE than __real__ part of TARGET,
927 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
928 if LHS has smaller precision than its mode. */
931 expand_arith_overflow_result_store (tree lhs
, rtx target
,
932 scalar_int_mode mode
, rtx res
)
934 scalar_int_mode tgtmode
935 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
939 rtx_code_label
*done_label
= gen_label_rtx ();
940 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
941 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
942 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
943 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
944 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
945 profile_probability::very_likely ());
946 expand_arith_set_overflow (lhs
, target
);
947 emit_label (done_label
);
949 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
950 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
953 rtx_code_label
*done_label
= gen_label_rtx ();
954 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
959 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
961 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
962 true, OPTAB_LIB_WIDEN
);
966 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
968 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
971 do_compare_rtx_and_jump (res
, lres
,
972 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
973 profile_probability::very_likely ());
974 expand_arith_set_overflow (lhs
, target
);
975 emit_label (done_label
);
977 write_complex_part (target
, lres
, false, false);
980 /* Helper for expand_*_overflow. Store RES into TARGET. */
983 expand_ubsan_result_store (tree lhs
, rtx target
, scalar_int_mode mode
,
984 rtx res
, rtx_code_label
*do_error
)
986 if (TREE_CODE (TREE_TYPE (lhs
)) == BITINT_TYPE
987 && TYPE_PRECISION (TREE_TYPE (lhs
)) < GET_MODE_PRECISION (mode
))
989 int uns
= TYPE_UNSIGNED (TREE_TYPE (lhs
));
990 int prec
= TYPE_PRECISION (TREE_TYPE (lhs
));
991 int tgtprec
= GET_MODE_PRECISION (mode
);
992 rtx resc
= gen_reg_rtx (mode
), lres
;
993 emit_move_insn (resc
, res
);
997 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
999 lres
= expand_simple_binop (mode
, AND
, res
, mask
, NULL_RTX
,
1000 true, OPTAB_LIB_WIDEN
);
1004 lres
= expand_shift (LSHIFT_EXPR
, mode
, res
, tgtprec
- prec
,
1006 lres
= expand_shift (RSHIFT_EXPR
, mode
, lres
, tgtprec
- prec
,
1010 emit_move_insn (res
, lres
);
1011 do_compare_rtx_and_jump (res
, resc
,
1012 NE
, true, mode
, NULL_RTX
, NULL
, do_error
,
1013 profile_probability::very_unlikely ());
1015 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
1016 /* If this is a scalar in a register that is stored in a wider mode
1017 than the declared mode, compute the result into its declared mode
1018 and then convert to the wider mode. Our value is the computed
1020 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
1022 emit_move_insn (target
, res
);
1025 /* Add sub/add overflow checking to the statement STMT.
1026 CODE says whether the operation is +, or -. */
1029 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
1030 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
1031 bool uns1_p
, bool is_ubsan
, tree
*datap
)
1033 rtx res
, target
= NULL_RTX
;
1035 rtx_code_label
*done_label
= gen_label_rtx ();
1036 rtx_code_label
*do_error
= gen_label_rtx ();
1037 do_pending_stack_adjust ();
1038 rtx op0
= expand_normal (arg0
);
1039 rtx op1
= expand_normal (arg1
);
1040 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1041 int prec
= GET_MODE_PRECISION (mode
);
1042 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1043 bool do_xor
= false;
1046 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1050 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1052 write_complex_part (target
, const0_rtx
, true, false);
1055 /* We assume both operands and result have the same precision
1056 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1057 with that precision, U for unsigned type with that precision,
1058 sgn for unsigned most significant bit in that precision.
1059 s1 is signed first operand, u1 is unsigned first operand,
1060 s2 is signed second operand, u2 is unsigned second operand,
1061 sr is signed result, ur is unsigned result and the following
1062 rules say how to compute result (which is always result of
1063 the operands as if both were unsigned, cast to the right
1064 signedness) and how to compute whether operation overflowed.
1067 res = (S) ((U) s1 + (U) s2)
1068 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1070 res = (S) ((U) s1 - (U) s2)
1071 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1074 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1077 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1079 res = (S) ((U) s1 + u2)
1080 ovf = ((U) res ^ sgn) < u2
1085 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1087 res = (S) ((U) s1 - u2)
1088 ovf = u2 > ((U) s1 ^ sgn)
1091 ovf = s1 < 0 || u2 > (U) s1
1094 ovf = u1 >= ((U) s2 ^ sgn)
1099 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1101 res = (U) s1 + (U) s2
1102 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1105 ovf = (U) res < u2 || res < 0
1108 ovf = u1 >= u2 ? res < 0 : res >= 0
1110 res = (U) s1 - (U) s2
1111 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1113 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
1115 /* PLUS_EXPR is commutative, if operand signedness differs,
1116 canonicalize to the first operand being signed and second
1117 unsigned to simplify following code. */
1118 std::swap (op0
, op1
);
1119 std::swap (arg0
, arg1
);
1124 /* u1 +- u2 -> ur */
1125 if (uns0_p
&& uns1_p
&& unsr_p
)
1127 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
1128 : usubv4_optab
, mode
);
1129 if (icode
!= CODE_FOR_nothing
)
1131 class expand_operand ops
[4];
1132 rtx_insn
*last
= get_last_insn ();
1134 res
= gen_reg_rtx (mode
);
1135 create_output_operand (&ops
[0], res
, mode
);
1136 create_input_operand (&ops
[1], op0
, mode
);
1137 create_input_operand (&ops
[2], op1
, mode
);
1138 create_fixed_operand (&ops
[3], do_error
);
1139 if (maybe_expand_insn (icode
, 4, ops
))
1141 last
= get_last_insn ();
1142 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1144 && any_condjump_p (last
)
1145 && !find_reg_note (last
, REG_BR_PROB
, 0))
1146 add_reg_br_prob_note (last
,
1147 profile_probability::very_unlikely ());
1148 emit_jump (done_label
);
1149 goto do_error_label
;
1152 delete_insns_since (last
);
1155 /* Compute the operation. On RTL level, the addition is always
1157 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1158 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1160 /* For PLUS_EXPR, the operation is commutative, so we can pick
1161 operand to compare against. For prec <= BITS_PER_WORD, I think
1162 preferring REG operand is better over CONST_INT, because
1163 the CONST_INT might enlarge the instruction or CSE would need
1164 to figure out we'd already loaded it into a register before.
1165 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1166 as then the multi-word comparison can be perhaps simplified. */
1167 if (code
== PLUS_EXPR
1168 && (prec
<= BITS_PER_WORD
1169 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1170 : CONST_SCALAR_INT_P (op1
)))
1172 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1173 true, mode
, NULL_RTX
, NULL
, done_label
,
1174 profile_probability::very_likely ());
1175 goto do_error_label
;
1178 /* s1 +- u2 -> sr */
1179 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1181 /* Compute the operation. On RTL level, the addition is always
1183 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1184 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1185 rtx tem
= expand_binop (mode
, add_optab
,
1186 code
== PLUS_EXPR
? res
: op0
, sgn
,
1187 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1188 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1189 done_label
, profile_probability::very_likely ());
1190 goto do_error_label
;
1194 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1196 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1198 /* As we've changed op1, we have to avoid using the value range
1199 for the original argument. */
1200 arg1
= error_mark_node
;
1206 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1208 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1210 /* As we've changed op0, we have to avoid using the value range
1211 for the original argument. */
1212 arg0
= error_mark_node
;
1218 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1220 /* Compute the operation. On RTL level, the addition is always
1222 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1224 int pos_neg
= get_range_pos_neg (arg0
);
1226 /* If ARG0 is known to be always negative, this is always overflow. */
1227 emit_jump (do_error
);
1228 else if (pos_neg
== 3)
1229 /* If ARG0 is not known to be always positive, check at runtime. */
1230 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1231 NULL
, do_error
, profile_probability::very_unlikely ());
1232 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1233 done_label
, profile_probability::very_likely ());
1234 goto do_error_label
;
1238 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1240 /* Compute the operation. On RTL level, the addition is always
1242 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1244 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1246 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1247 done_label
, profile_probability::very_likely ());
1248 goto do_error_label
;
1252 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1254 /* Compute the operation. On RTL level, the addition is always
1256 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1258 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1259 NULL
, do_error
, profile_probability::very_unlikely ());
1261 /* The operation is commutative, so we can pick operand to compare
1262 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1263 is better over CONST_INT, because the CONST_INT might enlarge the
1264 instruction or CSE would need to figure out we'd already loaded it
1265 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1266 might be more beneficial, as then the multi-word comparison can be
1267 perhaps simplified. */
1268 if (prec
<= BITS_PER_WORD
1269 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1270 : CONST_SCALAR_INT_P (op0
))
1272 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1273 done_label
, profile_probability::very_likely ());
1274 goto do_error_label
;
1277 /* s1 +- s2 -> ur */
1278 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1280 /* Compute the operation. On RTL level, the addition is always
1282 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1283 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1284 int pos_neg
= get_range_pos_neg (arg1
);
1285 if (code
== PLUS_EXPR
)
1287 int pos_neg0
= get_range_pos_neg (arg0
);
1288 if (pos_neg0
!= 3 && pos_neg
== 3)
1290 std::swap (op0
, op1
);
1297 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1298 ? and_optab
: ior_optab
,
1299 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1300 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1301 NULL
, done_label
, profile_probability::very_likely ());
1305 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1306 do_compare_rtx_and_jump (op1
, const0_rtx
,
1307 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1308 NULL_RTX
, NULL
, do_ior_label
,
1309 profile_probability::even ());
1310 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1312 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1313 NULL
, done_label
, profile_probability::very_likely ());
1314 emit_jump (do_error
);
1315 emit_label (do_ior_label
);
1316 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1318 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1319 NULL
, done_label
, profile_probability::very_likely ());
1321 goto do_error_label
;
1325 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1327 /* Compute the operation. On RTL level, the addition is always
1329 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1331 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1332 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1333 op0_geu_op1
, profile_probability::even ());
1334 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1335 NULL
, done_label
, profile_probability::very_likely ());
1336 emit_jump (do_error
);
1337 emit_label (op0_geu_op1
);
1338 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1339 NULL
, done_label
, profile_probability::very_likely ());
1340 goto do_error_label
;
1343 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1345 /* s1 +- s2 -> sr */
1348 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1349 : subv4_optab
, mode
);
1350 if (icode
!= CODE_FOR_nothing
)
1352 class expand_operand ops
[4];
1353 rtx_insn
*last
= get_last_insn ();
1355 res
= gen_reg_rtx (mode
);
1356 create_output_operand (&ops
[0], res
, mode
);
1357 create_input_operand (&ops
[1], op0
, mode
);
1358 create_input_operand (&ops
[2], op1
, mode
);
1359 create_fixed_operand (&ops
[3], do_error
);
1360 if (maybe_expand_insn (icode
, 4, ops
))
1362 last
= get_last_insn ();
1363 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1365 && any_condjump_p (last
)
1366 && !find_reg_note (last
, REG_BR_PROB
, 0))
1367 add_reg_br_prob_note (last
,
1368 profile_probability::very_unlikely ());
1369 emit_jump (done_label
);
1370 goto do_error_label
;
1373 delete_insns_since (last
);
1376 /* Compute the operation. On RTL level, the addition is always
1378 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1379 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1381 /* If we can prove that one of the arguments (for MINUS_EXPR only
1382 the second operand, as subtraction is not commutative) is always
1383 non-negative or always negative, we can do just one comparison
1384 and conditional jump. */
1385 int pos_neg
= get_range_pos_neg (arg1
);
1386 if (code
== PLUS_EXPR
)
1388 int pos_neg0
= get_range_pos_neg (arg0
);
1389 if (pos_neg0
!= 3 && pos_neg
== 3)
1391 std::swap (op0
, op1
);
1396 /* Addition overflows if and only if the two operands have the same sign,
1397 and the result has the opposite sign. Subtraction overflows if and
1398 only if the two operands have opposite sign, and the subtrahend has
1399 the same sign as the result. Here 0 is counted as positive. */
1402 /* Compute op0 ^ op1 (operands have opposite sign). */
1403 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1406 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1407 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1411 if (code
== PLUS_EXPR
)
1413 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1414 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1415 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1420 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1421 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1422 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1426 /* No overflow if the result has bit sign cleared. */
1427 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1428 NULL
, done_label
, profile_probability::very_likely ());
1431 /* Compare the result of the operation with the first operand.
1432 No overflow for addition if second operand is positive and result
1433 is larger or second operand is negative and result is smaller.
1434 Likewise for subtraction with sign of second operand flipped. */
1436 do_compare_rtx_and_jump (res
, op0
,
1437 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1438 false, mode
, NULL_RTX
, NULL
, done_label
,
1439 profile_probability::very_likely ());
1443 emit_label (do_error
);
1446 /* Expand the ubsan builtin call. */
1448 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1452 do_pending_stack_adjust ();
1455 expand_arith_set_overflow (lhs
, target
);
1458 emit_label (done_label
);
1463 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1467 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1470 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1475 /* Add negate overflow checking to the statement STMT. */
1478 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1483 rtx_code_label
*done_label
, *do_error
;
1484 rtx target
= NULL_RTX
;
1486 done_label
= gen_label_rtx ();
1487 do_error
= gen_label_rtx ();
1489 do_pending_stack_adjust ();
1490 op1
= expand_normal (arg1
);
1492 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1495 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1497 write_complex_part (target
, const0_rtx
, true, false);
1500 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1501 if (icode
!= CODE_FOR_nothing
)
1503 class expand_operand ops
[3];
1504 rtx_insn
*last
= get_last_insn ();
1506 res
= gen_reg_rtx (mode
);
1507 create_output_operand (&ops
[0], res
, mode
);
1508 create_input_operand (&ops
[1], op1
, mode
);
1509 create_fixed_operand (&ops
[2], do_error
);
1510 if (maybe_expand_insn (icode
, 3, ops
))
1512 last
= get_last_insn ();
1513 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1515 && any_condjump_p (last
)
1516 && !find_reg_note (last
, REG_BR_PROB
, 0))
1517 add_reg_br_prob_note (last
,
1518 profile_probability::very_unlikely ());
1519 emit_jump (done_label
);
1523 delete_insns_since (last
);
1524 icode
= CODE_FOR_nothing
;
1528 if (icode
== CODE_FOR_nothing
)
1530 /* Compute the operation. On RTL level, the addition is always
1532 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1534 /* Compare the operand with the most negative value. */
1535 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1536 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1537 done_label
, profile_probability::very_likely ());
1540 emit_label (do_error
);
1543 /* Expand the ubsan builtin call. */
1545 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1546 arg1
, NULL_TREE
, datap
);
1549 do_pending_stack_adjust ();
1552 expand_arith_set_overflow (lhs
, target
);
1555 emit_label (done_label
);
1560 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1562 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1566 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1567 mode MODE can be expanded without using a libcall. */
1570 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1571 rtx op0
, rtx op1
, bool uns
)
1573 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1574 != CODE_FOR_nothing
)
1577 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1578 != CODE_FOR_nothing
)
1581 rtx_insn
*last
= get_last_insn ();
1582 if (CONSTANT_P (op0
))
1583 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1585 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1586 if (CONSTANT_P (op1
))
1587 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1589 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1590 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1591 delete_insns_since (last
);
1592 return ret
!= NULL_RTX
;
1595 /* Add mul overflow checking to the statement STMT. */
1598 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1599 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1604 rtx_code_label
*done_label
, *do_error
;
1605 rtx target
= NULL_RTX
;
1607 enum insn_code icode
;
1609 done_label
= gen_label_rtx ();
1610 do_error
= gen_label_rtx ();
1612 do_pending_stack_adjust ();
1613 op0
= expand_normal (arg0
);
1614 op1
= expand_normal (arg1
);
1616 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1620 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1622 write_complex_part (target
, const0_rtx
, true, false);
1626 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1628 /* We assume both operands and result have the same precision
1629 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1630 with that precision, U for unsigned type with that precision,
1631 sgn for unsigned most significant bit in that precision.
1632 s1 is signed first operand, u1 is unsigned first operand,
1633 s2 is signed second operand, u2 is unsigned second operand,
1634 sr is signed result, ur is unsigned result and the following
1635 rules say how to compute result (which is always result of
1636 the operands as if both were unsigned, cast to the right
1637 signedness) and how to compute whether operation overflowed.
1638 main_ovf (false) stands for jump on signed multiplication
1639 overflow or the main algorithm with uns == false.
1640 main_ovf (true) stands for jump on unsigned multiplication
1641 overflow or the main algorithm with uns == true.
1644 res = (S) ((U) s1 * (U) s2)
1645 ovf = main_ovf (false)
1648 ovf = main_ovf (true)
1651 ovf = (s1 < 0 && u2) || main_ovf (true)
1654 ovf = res < 0 || main_ovf (true)
1656 res = (S) ((U) s1 * u2)
1657 ovf = (S) u2 >= 0 ? main_ovf (false)
1658 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1660 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1661 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1663 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1665 if (uns0_p
&& !uns1_p
)
1667 /* Multiplication is commutative, if operand signedness differs,
1668 canonicalize to the first operand being signed and second
1669 unsigned to simplify following code. */
1670 std::swap (op0
, op1
);
1671 std::swap (arg0
, arg1
);
1676 int pos_neg0
= get_range_pos_neg (arg0
);
1677 int pos_neg1
= get_range_pos_neg (arg1
);
1678 /* Unsigned types with smaller than mode precision, even if they have most
1679 significant bit set, are still zero-extended. */
1680 if (uns0_p
&& TYPE_PRECISION (TREE_TYPE (arg0
)) < GET_MODE_PRECISION (mode
))
1682 if (uns1_p
&& TYPE_PRECISION (TREE_TYPE (arg1
)) < GET_MODE_PRECISION (mode
))
1686 if (!uns0_p
&& uns1_p
&& unsr_p
)
1691 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1694 /* If s1 is negative, avoid the main code, just multiply and
1695 signal overflow if op1 is not 0. */
1696 struct separate_ops ops
;
1697 ops
.code
= MULT_EXPR
;
1698 ops
.type
= TREE_TYPE (arg1
);
1699 ops
.op0
= make_tree (ops
.type
, op0
);
1700 ops
.op1
= make_tree (ops
.type
, op1
);
1701 ops
.op2
= NULL_TREE
;
1703 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1704 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1705 NULL
, done_label
, profile_probability::very_likely ());
1706 goto do_error_label
;
1708 if (get_min_precision (arg1
, UNSIGNED
)
1709 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1711 /* If the first operand is sign extended from narrower type, the
1712 second operand is zero extended from narrower type and
1713 the sum of the two precisions is smaller or equal to the
1714 result precision: if the first argument is at runtime
1715 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1716 and there will be no overflow, if the first argument is
1717 negative and the second argument zero, the result will be
1718 0 and there will be no overflow, if the first argument is
1719 negative and the second argument positive, the result when
1720 treated as signed will be negative (minimum -0x7f80 or
1721 -0x7f..f80..0) there will be always overflow. So, do
1723 ovf = (S) res < 0 */
1724 struct separate_ops ops
;
1725 ops
.code
= MULT_EXPR
;
1727 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1729 ops
.op0
= make_tree (ops
.type
, op0
);
1730 ops
.op1
= make_tree (ops
.type
, op1
);
1731 ops
.op2
= NULL_TREE
;
1733 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1734 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1735 mode
, NULL_RTX
, NULL
, done_label
,
1736 profile_probability::very_likely ());
1737 goto do_error_label
;
1739 rtx_code_label
*do_main_label
;
1740 do_main_label
= gen_label_rtx ();
1741 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1742 NULL
, do_main_label
, profile_probability::very_likely ());
1743 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1744 NULL
, do_main_label
, profile_probability::very_likely ());
1745 expand_arith_set_overflow (lhs
, target
);
1746 emit_label (do_main_label
);
1754 if (uns0_p
&& uns1_p
&& !unsr_p
)
1756 if ((pos_neg0
| pos_neg1
) == 1)
1758 /* If both arguments are zero extended from narrower types,
1759 the MSB will be clear on both and so we can pretend it is
1760 a normal s1 * s2 -> sr multiplication. */
1766 /* Rest of handling of this case after res is computed. */
1771 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1778 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1779 avoid the main code, just multiply and signal overflow
1780 unless 0 * u2 or -1 * ((U) Smin). */
1781 struct separate_ops ops
;
1782 ops
.code
= MULT_EXPR
;
1783 ops
.type
= TREE_TYPE (arg1
);
1784 ops
.op0
= make_tree (ops
.type
, op0
);
1785 ops
.op1
= make_tree (ops
.type
, op1
);
1786 ops
.op2
= NULL_TREE
;
1788 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1789 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1790 NULL
, done_label
, profile_probability::very_likely ());
1791 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1792 NULL
, do_error
, profile_probability::very_unlikely ());
1794 prec
= GET_MODE_PRECISION (mode
);
1796 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1797 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1798 NULL
, done_label
, profile_probability::very_likely ());
1799 goto do_error_label
;
1801 /* Rest of handling of this case after res is computed. */
1809 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1812 switch (pos_neg0
| pos_neg1
)
1814 case 1: /* Both operands known to be non-negative. */
1816 case 2: /* Both operands known to be negative. */
1817 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1818 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1819 /* Avoid looking at arg0/arg1 ranges, as we've changed
1821 arg0
= error_mark_node
;
1822 arg1
= error_mark_node
;
1825 if ((pos_neg0
^ pos_neg1
) == 3)
1827 /* If one operand is known to be negative and the other
1828 non-negative, this overflows always, unless the non-negative
1829 one is 0. Just do normal multiply and set overflow
1830 unless one of the operands is 0. */
1831 struct separate_ops ops
;
1832 ops
.code
= MULT_EXPR
;
1834 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1836 ops
.op0
= make_tree (ops
.type
, op0
);
1837 ops
.op1
= make_tree (ops
.type
, op1
);
1838 ops
.op2
= NULL_TREE
;
1840 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1841 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1842 true, mode
, NULL_RTX
, NULL
, done_label
,
1843 profile_probability::very_likely ());
1844 goto do_error_label
;
1846 if (get_min_precision (arg0
, SIGNED
)
1847 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1849 /* If both operands are sign extended from narrower types and
1850 the sum of the two precisions is smaller or equal to the
1851 result precision: if both arguments are at runtime
1852 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1853 and there will be no overflow, if both arguments are negative,
1854 maximum result will be 0x40..00 and there will be no overflow
1855 either, if one argument is positive and the other argument
1856 negative, the result when treated as signed will be negative
1857 and there will be always overflow, and if one argument is
1858 zero and the other negative the result will be zero and no
1861 ovf = (S) res < 0 */
1862 struct separate_ops ops
;
1863 ops
.code
= MULT_EXPR
;
1865 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1867 ops
.op0
= make_tree (ops
.type
, op0
);
1868 ops
.op1
= make_tree (ops
.type
, op1
);
1869 ops
.op2
= NULL_TREE
;
1871 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1872 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1873 mode
, NULL_RTX
, NULL
, done_label
,
1874 profile_probability::very_likely ());
1875 goto do_error_label
;
1877 /* The general case, do all the needed comparisons at runtime. */
1878 rtx_code_label
*do_main_label
, *after_negate_label
;
1880 rop0
= gen_reg_rtx (mode
);
1881 rop1
= gen_reg_rtx (mode
);
1882 emit_move_insn (rop0
, op0
);
1883 emit_move_insn (rop1
, op1
);
1886 do_main_label
= gen_label_rtx ();
1887 after_negate_label
= gen_label_rtx ();
1888 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1890 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1891 NULL
, after_negate_label
, profile_probability::very_likely ());
1892 /* Both arguments negative here, negate them and continue with
1893 normal unsigned overflow checking multiplication. */
1894 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1896 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1898 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1900 arg0
= error_mark_node
;
1901 arg1
= error_mark_node
;
1902 emit_jump (do_main_label
);
1903 emit_label (after_negate_label
);
1904 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1906 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1907 NULL
, do_main_label
,
1908 profile_probability::very_likely ());
1909 /* One argument is negative here, the other positive. This
1910 overflows always, unless one of the arguments is 0. But
1911 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1912 is, thus we can keep do_main code oring in overflow as is. */
1914 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1915 NULL
, do_main_label
,
1916 profile_probability::very_unlikely ());
1918 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1919 NULL
, do_main_label
,
1920 profile_probability::very_unlikely ());
1921 expand_arith_set_overflow (lhs
, target
);
1922 emit_label (do_main_label
);
1930 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1931 sign
= uns
? UNSIGNED
: SIGNED
;
1932 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1934 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1935 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1937 /* Optimize unsigned multiplication by power of 2 constant
1938 using 2 shifts, one for result, one to extract the shifted
1939 out bits to see if they are all zero.
1940 Don't do this if optimizing for size and we have umulv4_optab,
1941 in that case assume multiplication will be shorter.
1942 This is heuristics based on the single target that provides
1943 umulv4 right now (i?86/x86_64), if further targets add it, this
1944 might need to be revisited.
1945 Cases where both operands are constant should be folded already
1946 during GIMPLE, and cases where one operand is constant but not
1947 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1948 below can be done without multiplication, just by shifts and adds,
1949 or we'd need to divide the result (and hope it actually doesn't
1950 really divide nor multiply) and compare the result of the division
1951 with the original operand. */
1956 if (integer_pow2p (arg0
))
1958 std::swap (opn0
, opn1
);
1959 std::swap (argn0
, argn1
);
1961 int cnt
= tree_log2 (argn1
);
1962 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1964 rtx upper
= const0_rtx
;
1965 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1967 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1968 GET_MODE_PRECISION (mode
) - cnt
,
1970 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1971 NULL_RTX
, NULL
, done_label
,
1972 profile_probability::very_likely ());
1973 goto do_error_label
;
1976 if (icode
!= CODE_FOR_nothing
)
1978 class expand_operand ops
[4];
1979 rtx_insn
*last
= get_last_insn ();
1981 res
= gen_reg_rtx (mode
);
1982 create_output_operand (&ops
[0], res
, mode
);
1983 create_input_operand (&ops
[1], op0
, mode
);
1984 create_input_operand (&ops
[2], op1
, mode
);
1985 create_fixed_operand (&ops
[3], do_error
);
1986 if (maybe_expand_insn (icode
, 4, ops
))
1988 last
= get_last_insn ();
1989 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1991 && any_condjump_p (last
)
1992 && !find_reg_note (last
, REG_BR_PROB
, 0))
1993 add_reg_br_prob_note (last
,
1994 profile_probability::very_unlikely ());
1995 emit_jump (done_label
);
1999 delete_insns_since (last
);
2000 icode
= CODE_FOR_nothing
;
2004 if (icode
== CODE_FOR_nothing
)
2006 struct separate_ops ops
;
2007 int prec
= GET_MODE_PRECISION (mode
);
2008 scalar_int_mode hmode
, wmode
;
2009 ops
.op0
= make_tree (type
, op0
);
2010 ops
.op1
= make_tree (type
, op1
);
2011 ops
.op2
= NULL_TREE
;
2014 /* Optimize unsigned overflow check where we don't use the
2015 multiplication result, just whether overflow happened.
2016 If we can do MULT_HIGHPART_EXPR, that followed by
2017 comparison of the result against zero is cheapest.
2018 We'll still compute res, but it should be DCEd later. */
2024 && !(uns0_p
&& uns1_p
&& !unsr_p
)
2025 && can_mult_highpart_p (mode
, uns
) == 1
2026 && single_imm_use (lhs
, &use
, &use_stmt
)
2027 && is_gimple_assign (use_stmt
)
2028 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
2031 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2032 && targetm
.scalar_mode_supported_p (wmode
)
2033 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
2036 ops
.code
= WIDEN_MULT_EXPR
;
2038 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
2040 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
2041 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
2043 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
2044 res
= convert_modes (mode
, wmode
, res
, uns
);
2046 /* For the unsigned multiplication, there was overflow if
2047 HIPART is non-zero. */
2048 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2049 NULL_RTX
, NULL
, done_label
,
2050 profile_probability::very_likely ());
2053 /* RES is used more than once, place it in a pseudo. */
2054 res
= force_reg (mode
, res
);
2056 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2058 /* RES is low half of the double width result, HIPART
2059 the high half. There was overflow if
2060 HIPART is different from RES < 0 ? -1 : 0. */
2061 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2062 NULL_RTX
, NULL
, done_label
,
2063 profile_probability::very_likely ());
2066 else if (can_mult_highpart_p (mode
, uns
) == 1)
2069 ops
.code
= MULT_HIGHPART_EXPR
;
2072 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
2074 ops
.code
= MULT_EXPR
;
2075 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2077 /* For the unsigned multiplication, there was overflow if
2078 HIPART is non-zero. */
2079 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2080 NULL_RTX
, NULL
, done_label
,
2081 profile_probability::very_likely ());
2084 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2086 /* RES is low half of the double width result, HIPART
2087 the high half. There was overflow if
2088 HIPART is different from RES < 0 ? -1 : 0. */
2089 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2090 NULL_RTX
, NULL
, done_label
,
2091 profile_probability::very_likely ());
2095 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
2096 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
2098 rtx_code_label
*large_op0
= gen_label_rtx ();
2099 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
2100 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
2101 rtx_code_label
*both_ops_large
= gen_label_rtx ();
2102 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
2103 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
2104 rtx_code_label
*do_overflow
= gen_label_rtx ();
2105 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
2107 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
2108 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
2110 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
2111 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
2112 rtx signbit0
= const0_rtx
;
2114 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
2116 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
2118 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
2119 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
2120 rtx signbit1
= const0_rtx
;
2122 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
2125 res
= gen_reg_rtx (mode
);
2127 /* True if op0 resp. op1 are known to be in the range of
2129 bool op0_small_p
= false;
2130 bool op1_small_p
= false;
2131 /* True if op0 resp. op1 are known to have all zeros or all ones
2132 in the upper half of bits, but are not known to be
2134 bool op0_medium_p
= false;
2135 bool op1_medium_p
= false;
2136 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2137 nonnegative, 1 if unknown. */
2143 else if (pos_neg0
== 2)
2147 else if (pos_neg1
== 2)
2150 unsigned int mprec0
= prec
;
2151 if (arg0
!= error_mark_node
)
2152 mprec0
= get_min_precision (arg0
, sign
);
2153 if (mprec0
<= hprec
)
2155 else if (!uns
&& mprec0
<= hprec
+ 1)
2156 op0_medium_p
= true;
2157 unsigned int mprec1
= prec
;
2158 if (arg1
!= error_mark_node
)
2159 mprec1
= get_min_precision (arg1
, sign
);
2160 if (mprec1
<= hprec
)
2162 else if (!uns
&& mprec1
<= hprec
+ 1)
2163 op1_medium_p
= true;
2165 int smaller_sign
= 1;
2166 int larger_sign
= 1;
2169 smaller_sign
= op0_sign
;
2170 larger_sign
= op1_sign
;
2172 else if (op1_small_p
)
2174 smaller_sign
= op1_sign
;
2175 larger_sign
= op0_sign
;
2177 else if (op0_sign
== op1_sign
)
2179 smaller_sign
= op0_sign
;
2180 larger_sign
= op0_sign
;
2184 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2185 NULL_RTX
, NULL
, large_op0
,
2186 profile_probability::unlikely ());
2189 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2190 NULL_RTX
, NULL
, small_op0_large_op1
,
2191 profile_probability::unlikely ());
2193 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2194 hmode to mode, the multiplication will never overflow. We can
2195 do just one hmode x hmode => mode widening multiplication. */
2196 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2197 ops
.op0
= make_tree (halfstype
, lopart0
);
2198 ops
.op1
= make_tree (halfstype
, lopart1
);
2199 ops
.code
= WIDEN_MULT_EXPR
;
2202 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2203 emit_move_insn (res
, thisres
);
2204 emit_jump (done_label
);
2206 emit_label (small_op0_large_op1
);
2208 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2209 but op1 is not, just swap the arguments and handle it as op1
2210 sign/zero extended, op0 not. */
2211 rtx larger
= gen_reg_rtx (mode
);
2212 rtx hipart
= gen_reg_rtx (hmode
);
2213 rtx lopart
= gen_reg_rtx (hmode
);
2214 emit_move_insn (larger
, op1
);
2215 emit_move_insn (hipart
, hipart1
);
2216 emit_move_insn (lopart
, lopart0
);
2217 emit_jump (one_small_one_large
);
2219 emit_label (large_op0
);
2222 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2223 NULL_RTX
, NULL
, both_ops_large
,
2224 profile_probability::unlikely ());
2226 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2227 but op0 is not, prepare larger, hipart and lopart pseudos and
2228 handle it together with small_op0_large_op1. */
2229 emit_move_insn (larger
, op0
);
2230 emit_move_insn (hipart
, hipart0
);
2231 emit_move_insn (lopart
, lopart1
);
2233 emit_label (one_small_one_large
);
2235 /* lopart is the low part of the operand that is sign extended
2236 to mode, larger is the other operand, hipart is the
2237 high part of larger and lopart0 and lopart1 are the low parts
2239 We perform lopart0 * lopart1 and lopart * hipart widening
2241 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2242 ops
.op0
= make_tree (halfutype
, lopart0
);
2243 ops
.op1
= make_tree (halfutype
, lopart1
);
2245 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2247 ops
.op0
= make_tree (halfutype
, lopart
);
2248 ops
.op1
= make_tree (halfutype
, hipart
);
2249 rtx loxhi
= gen_reg_rtx (mode
);
2250 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2251 emit_move_insn (loxhi
, tem
);
2255 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2256 if (larger_sign
== 0)
2257 emit_jump (after_hipart_neg
);
2258 else if (larger_sign
!= -1)
2259 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2260 NULL_RTX
, NULL
, after_hipart_neg
,
2261 profile_probability::even ());
2263 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2264 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2265 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2267 emit_move_insn (loxhi
, tem
);
2269 emit_label (after_hipart_neg
);
2271 /* if (lopart < 0) loxhi -= larger; */
2272 if (smaller_sign
== 0)
2273 emit_jump (after_lopart_neg
);
2274 else if (smaller_sign
!= -1)
2275 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2276 NULL_RTX
, NULL
, after_lopart_neg
,
2277 profile_probability::even ());
2279 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2281 emit_move_insn (loxhi
, tem
);
2283 emit_label (after_lopart_neg
);
2286 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2287 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2288 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2290 emit_move_insn (loxhi
, tem
);
2292 /* if (loxhi >> (bitsize / 2)
2293 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2294 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2295 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2297 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2298 rtx signbitloxhi
= const0_rtx
;
2300 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2301 convert_modes (hmode
, mode
,
2303 hprec
- 1, NULL_RTX
, 0);
2305 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2306 NULL_RTX
, NULL
, do_overflow
,
2307 profile_probability::very_unlikely ());
2309 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2310 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2312 tem
= convert_modes (mode
, hmode
,
2313 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2315 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2318 emit_move_insn (res
, tem
);
2319 emit_jump (done_label
);
2321 emit_label (both_ops_large
);
2323 /* If both operands are large (not sign (!uns) or zero (uns)
2324 extended from hmode), then perform the full multiplication
2325 which will be the result of the operation.
2326 The only cases which don't overflow are for signed multiplication
2327 some cases where both hipart0 and highpart1 are 0 or -1.
2328 For unsigned multiplication when high parts are both non-zero
2329 this overflows always. */
2330 ops
.code
= MULT_EXPR
;
2331 ops
.op0
= make_tree (type
, op0
);
2332 ops
.op1
= make_tree (type
, op1
);
2333 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2334 emit_move_insn (res
, tem
);
2340 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2341 NULL_RTX
, 1, OPTAB_WIDEN
);
2342 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2343 NULL_RTX
, NULL
, do_error
,
2344 profile_probability::very_unlikely ());
2349 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2350 NULL_RTX
, 1, OPTAB_WIDEN
);
2351 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2352 NULL_RTX
, NULL
, do_error
,
2353 profile_probability::very_unlikely ());
2356 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2357 the same, overflow happened if res is non-positive, if they
2358 are different, overflow happened if res is positive. */
2359 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2360 emit_jump (hipart_different
);
2361 else if (op0_sign
== 1 || op1_sign
== 1)
2362 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2363 NULL_RTX
, NULL
, hipart_different
,
2364 profile_probability::even ());
2366 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2367 NULL_RTX
, NULL
, do_error
,
2368 profile_probability::very_unlikely ());
2369 emit_jump (done_label
);
2371 emit_label (hipart_different
);
2373 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2374 NULL_RTX
, NULL
, do_error
,
2375 profile_probability::very_unlikely ());
2376 emit_jump (done_label
);
2379 emit_label (do_overflow
);
2381 /* Overflow, do full multiplication and fallthru into do_error. */
2382 ops
.op0
= make_tree (type
, op0
);
2383 ops
.op1
= make_tree (type
, op1
);
2384 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2385 emit_move_insn (res
, tem
);
2387 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2388 && targetm
.scalar_mode_supported_p (wmode
))
2389 /* Even emitting a libcall is better than not detecting overflow
2394 gcc_assert (!is_ubsan
);
2395 ops
.code
= MULT_EXPR
;
2397 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2398 emit_jump (done_label
);
2403 emit_label (do_error
);
2406 /* Expand the ubsan builtin call. */
2408 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2412 do_pending_stack_adjust ();
2415 expand_arith_set_overflow (lhs
, target
);
2418 emit_label (done_label
);
2421 if (uns0_p
&& uns1_p
&& !unsr_p
)
2423 rtx_code_label
*all_done_label
= gen_label_rtx ();
2424 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2425 NULL
, all_done_label
, profile_probability::very_likely ());
2426 expand_arith_set_overflow (lhs
, target
);
2427 emit_label (all_done_label
);
2431 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2433 rtx_code_label
*all_done_label
= gen_label_rtx ();
2434 rtx_code_label
*set_noovf
= gen_label_rtx ();
2435 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2436 NULL
, all_done_label
, profile_probability::very_likely ());
2437 expand_arith_set_overflow (lhs
, target
);
2438 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2439 NULL
, set_noovf
, profile_probability::very_likely ());
2440 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2441 NULL
, all_done_label
, profile_probability::very_unlikely ());
2442 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2443 all_done_label
, profile_probability::very_unlikely ());
2444 emit_label (set_noovf
);
2445 write_complex_part (target
, const0_rtx
, true, false);
2446 emit_label (all_done_label
);
2452 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
2454 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2458 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2461 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2462 tree arg0
, tree arg1
)
2464 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2465 rtx_code_label
*loop_lab
= NULL
;
2466 rtx cntvar
= NULL_RTX
;
2467 tree cntv
= NULL_TREE
;
2468 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2469 tree sz
= TYPE_SIZE (eltype
);
2470 tree data
= NULL_TREE
;
2471 tree resv
= NULL_TREE
;
2472 rtx lhsr
= NULL_RTX
;
2473 rtx resvr
= NULL_RTX
;
2474 unsigned HOST_WIDE_INT const_cnt
= 0;
2475 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2480 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2481 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2482 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2483 optab_default
)) == unknown_optab
2484 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2485 == CODE_FOR_nothing
))
2488 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2491 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2492 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2498 do_pending_stack_adjust ();
2499 loop_lab
= gen_label_rtx ();
2500 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2501 cntv
= make_tree (sizetype
, cntvar
);
2502 emit_move_insn (cntvar
, const0_rtx
);
2503 emit_label (loop_lab
);
2505 if (TREE_CODE (arg0
) != VECTOR_CST
)
2507 rtx arg0r
= expand_normal (arg0
);
2508 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2510 if (TREE_CODE (arg1
) != VECTOR_CST
)
2512 rtx arg1r
= expand_normal (arg1
);
2513 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2515 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2517 tree op0
, op1
, res
= NULL_TREE
;
2520 tree atype
= build_array_type_nelts (eltype
, cnt
);
2521 op0
= uniform_vector_p (arg0
);
2522 if (op0
== NULL_TREE
)
2524 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2525 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2526 NULL_TREE
, NULL_TREE
);
2528 op1
= uniform_vector_p (arg1
);
2529 if (op1
== NULL_TREE
)
2531 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2532 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2533 NULL_TREE
, NULL_TREE
);
2537 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2538 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2539 NULL_TREE
, NULL_TREE
);
2544 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2545 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2546 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2548 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2554 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2555 false, false, false, true, &data
);
2558 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2559 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2561 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2562 false, false, false, true, &data
);
2565 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2574 struct separate_ops ops
;
2575 ops
.code
= PLUS_EXPR
;
2576 ops
.type
= TREE_TYPE (cntv
);
2578 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2579 ops
.op2
= NULL_TREE
;
2581 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2584 emit_move_insn (cntvar
, ret
);
2585 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2586 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2587 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2588 profile_probability::very_likely ());
2590 if (lhs
&& resv
== NULL_TREE
)
2592 struct separate_ops ops
;
2594 ops
.type
= TREE_TYPE (arg0
);
2597 ops
.op2
= NULL_TREE
;
2599 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2602 emit_move_insn (lhsr
, ret
);
2605 emit_move_insn (lhsr
, resvr
);
2608 /* Expand UBSAN_CHECK_ADD call STMT. */
2611 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2613 location_t loc
= gimple_location (stmt
);
2614 tree lhs
= gimple_call_lhs (stmt
);
2615 tree arg0
= gimple_call_arg (stmt
, 0);
2616 tree arg1
= gimple_call_arg (stmt
, 1);
2617 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2618 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2620 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2621 false, false, false, true, NULL
);
2624 /* Expand UBSAN_CHECK_SUB call STMT. */
2627 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2629 location_t loc
= gimple_location (stmt
);
2630 tree lhs
= gimple_call_lhs (stmt
);
2631 tree arg0
= gimple_call_arg (stmt
, 0);
2632 tree arg1
= gimple_call_arg (stmt
, 1);
2633 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2634 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2635 else if (integer_zerop (arg0
))
2636 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2638 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2639 false, false, false, true, NULL
);
2642 /* Expand UBSAN_CHECK_MUL call STMT. */
2645 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2647 location_t loc
= gimple_location (stmt
);
2648 tree lhs
= gimple_call_lhs (stmt
);
2649 tree arg0
= gimple_call_arg (stmt
, 0);
2650 tree arg1
= gimple_call_arg (stmt
, 1);
2651 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2652 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2654 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2658 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2661 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2663 tree lhs
= gimple_call_lhs (stmt
);
2664 if (lhs
== NULL_TREE
)
2666 tree arg0
= gimple_call_arg (stmt
, 0);
2667 tree arg1
= gimple_call_arg (stmt
, 1);
2668 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2669 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2670 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2671 int unsr_p
= TYPE_UNSIGNED (type
);
2672 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2673 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2674 int precres
= TYPE_PRECISION (type
);
2675 location_t loc
= gimple_location (stmt
);
2676 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2678 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2680 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2681 prec0
= MIN (prec0
, pr
);
2682 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2683 prec1
= MIN (prec1
, pr
);
2685 /* If uns0_p && uns1_p, precop is minimum needed precision
2686 of unsigned type to hold the exact result, otherwise
2687 precop is minimum needed precision of signed type to
2688 hold the exact result. */
2690 if (code
== MULT_EXPR
)
2691 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2694 if (uns0_p
== uns1_p
)
2695 precop
= MAX (prec0
, prec1
) + 1;
2697 precop
= MAX (prec0
+ 1, prec1
) + 1;
2699 precop
= MAX (prec0
, prec1
+ 1) + 1;
2701 int orig_precres
= precres
;
2705 if ((uns0_p
&& uns1_p
)
2706 ? ((precop
+ !unsr_p
) <= precres
2707 /* u1 - u2 -> ur can overflow, no matter what precision
2709 && (code
!= MINUS_EXPR
|| !unsr_p
))
2710 : (!unsr_p
&& precop
<= precres
))
2712 /* The infinity precision result will always fit into result. */
2713 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2714 write_complex_part (target
, const0_rtx
, true, false);
2715 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2716 struct separate_ops ops
;
2719 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2720 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2721 ops
.op2
= NULL_TREE
;
2723 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2724 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2728 /* For operations with low precision, if target doesn't have them, start
2729 with precres widening right away, otherwise do it only if the most
2730 simple cases can't be used. */
2731 const int min_precision
= targetm
.min_arithmetic_precision ();
2732 if (orig_precres
== precres
&& precres
< min_precision
)
2734 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2735 && prec1
<= precres
)
2736 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2737 && prec0
+ uns0_p
<= precres
2738 && prec1
+ uns1_p
<= precres
))
2740 arg0
= fold_convert_loc (loc
, type
, arg0
);
2741 arg1
= fold_convert_loc (loc
, type
, arg1
);
2745 if (integer_zerop (arg0
) && !unsr_p
)
2747 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2752 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2753 unsr_p
, unsr_p
, false, NULL
);
2756 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2757 unsr_p
, unsr_p
, false, NULL
);
2764 /* For sub-word operations, retry with a wider type first. */
2765 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2767 int p
= MAX (min_precision
, precop
);
2768 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2769 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2772 p
= TYPE_PRECISION (optype
);
2776 unsr_p
= TYPE_UNSIGNED (optype
);
2782 if (prec0
<= precres
&& prec1
<= precres
)
2787 types
[0] = build_nonstandard_integer_type (precres
, 0);
2793 types
[1] = build_nonstandard_integer_type (precres
, 1);
2795 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2796 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2797 if (code
!= MULT_EXPR
)
2798 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2799 uns0_p
, uns1_p
, false, NULL
);
2801 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2802 uns0_p
, uns1_p
, false, NULL
);
2806 /* Retry with a wider type. */
2807 if (orig_precres
== precres
)
2809 int p
= MAX (prec0
, prec1
);
2810 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2811 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2814 p
= TYPE_PRECISION (optype
);
2818 unsr_p
= TYPE_UNSIGNED (optype
);
2829 /* Expand ADD_OVERFLOW STMT. */
2832 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2834 expand_arith_overflow (PLUS_EXPR
, stmt
);
2837 /* Expand SUB_OVERFLOW STMT. */
2840 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2842 expand_arith_overflow (MINUS_EXPR
, stmt
);
2845 /* Expand MUL_OVERFLOW STMT. */
2848 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2850 expand_arith_overflow (MULT_EXPR
, stmt
);
2853 /* Expand UADDC STMT. */
2856 expand_UADDC (internal_fn ifn
, gcall
*stmt
)
2858 tree lhs
= gimple_call_lhs (stmt
);
2859 tree arg1
= gimple_call_arg (stmt
, 0);
2860 tree arg2
= gimple_call_arg (stmt
, 1);
2861 tree arg3
= gimple_call_arg (stmt
, 2);
2862 tree type
= TREE_TYPE (arg1
);
2863 machine_mode mode
= TYPE_MODE (type
);
2864 insn_code icode
= optab_handler (ifn
== IFN_UADDC
2865 ? uaddc5_optab
: usubc5_optab
, mode
);
2866 rtx op1
= expand_normal (arg1
);
2867 rtx op2
= expand_normal (arg2
);
2868 rtx op3
= expand_normal (arg3
);
2869 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2870 rtx re
= gen_reg_rtx (mode
);
2871 rtx im
= gen_reg_rtx (mode
);
2872 class expand_operand ops
[5];
2873 create_output_operand (&ops
[0], re
, mode
);
2874 create_output_operand (&ops
[1], im
, mode
);
2875 create_input_operand (&ops
[2], op1
, mode
);
2876 create_input_operand (&ops
[3], op2
, mode
);
2877 create_input_operand (&ops
[4], op3
, mode
);
2878 expand_insn (icode
, 5, ops
);
2879 write_complex_part (target
, re
, false, false);
2880 write_complex_part (target
, im
, true, false);
2883 /* Expand USUBC STMT. */
2886 expand_USUBC (internal_fn ifn
, gcall
*stmt
)
2888 expand_UADDC (ifn
, stmt
);
2891 /* This should get folded in tree-vectorizer.cc. */
2894 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2899 /* This should get folded in tree-vectorizer.cc. */
2902 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2907 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2908 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2911 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2913 tree addr
= gimple_call_arg (stmt
, index
);
2914 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2915 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2916 if (TYPE_ALIGN (type
) != align
)
2917 type
= build_aligned_type (type
, align
);
2920 if (TREE_CODE (tmp
) == SSA_NAME
)
2922 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2923 if (gimple_assign_single_p (def
))
2924 tmp
= gimple_assign_rhs1 (def
);
2927 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2929 tree mem
= TREE_OPERAND (tmp
, 0);
2930 if (TREE_CODE (mem
) == TARGET_MEM_REF
2931 && types_compatible_p (TREE_TYPE (mem
), type
))
2933 tree offset
= TMR_OFFSET (mem
);
2934 if (type
!= TREE_TYPE (mem
)
2935 || alias_ptr_type
!= TREE_TYPE (offset
)
2936 || !integer_zerop (offset
))
2938 mem
= copy_node (mem
);
2939 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2940 wi::to_poly_wide (offset
));
2941 TREE_TYPE (mem
) = type
;
2947 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2950 /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
2954 expand_partial_load_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
2957 class expand_operand ops
[5];
2958 tree type
, lhs
, rhs
, maskt
;
2962 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
2963 lhs
= gimple_call_lhs (stmt
);
2964 if (lhs
== NULL_TREE
)
2966 type
= TREE_TYPE (lhs
);
2967 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2969 if (optab
== vec_mask_load_lanes_optab
2970 || optab
== vec_mask_len_load_lanes_optab
)
2971 icode
= get_multi_vector_move (type
, optab
);
2972 else if (optab
== len_load_optab
)
2973 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2975 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2976 TYPE_MODE (TREE_TYPE (maskt
)));
2978 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2979 gcc_assert (MEM_P (mem
));
2980 /* The built MEM_REF does not accurately reflect that the load
2981 is only partial. Clear it. */
2982 set_mem_expr (mem
, NULL_TREE
);
2983 clear_mem_offset (mem
);
2984 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2985 create_output_operand (&ops
[i
++], target
, TYPE_MODE (type
));
2986 create_fixed_operand (&ops
[i
++], mem
);
2987 i
= add_mask_and_len_args (ops
, i
, stmt
);
2988 expand_insn (icode
, i
, ops
);
2990 if (!rtx_equal_p (target
, ops
[0].value
))
2991 emit_move_insn (target
, ops
[0].value
);
2994 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2995 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2996 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2997 #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
2999 /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3003 expand_partial_store_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3006 class expand_operand ops
[5];
3007 tree type
, lhs
, rhs
, maskt
;
3011 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3012 rhs
= gimple_call_arg (stmt
, internal_fn_stored_value_index (ifn
));
3013 type
= TREE_TYPE (rhs
);
3014 lhs
= expand_call_mem_ref (type
, stmt
, 0);
3016 if (optab
== vec_mask_store_lanes_optab
3017 || optab
== vec_mask_len_store_lanes_optab
)
3018 icode
= get_multi_vector_move (type
, optab
);
3019 else if (optab
== len_store_optab
)
3020 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3022 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3023 TYPE_MODE (TREE_TYPE (maskt
)));
3025 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3026 gcc_assert (MEM_P (mem
));
3027 /* The built MEM_REF does not accurately reflect that the store
3028 is only partial. Clear it. */
3029 set_mem_expr (mem
, NULL_TREE
);
3030 clear_mem_offset (mem
);
3031 reg
= expand_normal (rhs
);
3032 create_fixed_operand (&ops
[i
++], mem
);
3033 create_input_operand (&ops
[i
++], reg
, TYPE_MODE (type
));
3034 i
= add_mask_and_len_args (ops
, i
, stmt
);
3035 expand_insn (icode
, i
, ops
);
3038 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3039 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3040 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3041 #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3043 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3044 The expansion of STMT happens based on OPTAB table associated. */
3047 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3049 class expand_operand ops
[6];
3051 tree lhs
= gimple_call_lhs (stmt
);
3052 tree op0a
= gimple_call_arg (stmt
, 0);
3053 tree op0b
= gimple_call_arg (stmt
, 1);
3054 tree op1
= gimple_call_arg (stmt
, 2);
3055 tree op2
= gimple_call_arg (stmt
, 3);
3056 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
3058 tree vec_cond_type
= TREE_TYPE (lhs
);
3059 tree op_mode
= TREE_TYPE (op0a
);
3060 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
3062 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3063 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
3065 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
3067 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
3068 /* vector_compare_rtx legitimizes operands, preserve equality when
3069 expanding op1/op2. */
3070 rtx rtx_op1
, rtx_op2
;
3071 if (operand_equal_p (op1
, op0a
))
3072 rtx_op1
= XEXP (comparison
, 0);
3073 else if (operand_equal_p (op1
, op0b
))
3074 rtx_op1
= XEXP (comparison
, 1);
3076 rtx_op1
= expand_normal (op1
);
3077 if (operand_equal_p (op2
, op0a
))
3078 rtx_op2
= XEXP (comparison
, 0);
3079 else if (operand_equal_p (op2
, op0b
))
3080 rtx_op2
= XEXP (comparison
, 1);
3082 rtx_op2
= expand_normal (op2
);
3084 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3085 create_output_operand (&ops
[0], target
, mode
);
3086 create_input_operand (&ops
[1], rtx_op1
, mode
);
3087 create_input_operand (&ops
[2], rtx_op2
, mode
);
3088 create_fixed_operand (&ops
[3], comparison
);
3089 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
3090 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
3091 expand_insn (icode
, 6, ops
);
3092 if (!rtx_equal_p (ops
[0].value
, target
))
3093 emit_move_insn (target
, ops
[0].value
);
3096 /* Expand VCOND_MASK optab internal function.
3097 The expansion of STMT happens based on OPTAB table associated. */
3100 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3102 class expand_operand ops
[4];
3104 tree lhs
= gimple_call_lhs (stmt
);
3105 tree op0
= gimple_call_arg (stmt
, 0);
3106 tree op1
= gimple_call_arg (stmt
, 1);
3107 tree op2
= gimple_call_arg (stmt
, 2);
3108 tree vec_cond_type
= TREE_TYPE (lhs
);
3110 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3111 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
3112 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
3113 rtx mask
, rtx_op1
, rtx_op2
;
3115 gcc_assert (icode
!= CODE_FOR_nothing
);
3117 mask
= expand_normal (op0
);
3118 rtx_op1
= expand_normal (op1
);
3119 rtx_op2
= expand_normal (op2
);
3121 mask
= force_reg (mask_mode
, mask
);
3122 rtx_op1
= force_reg (mode
, rtx_op1
);
3124 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3125 create_output_operand (&ops
[0], target
, mode
);
3126 create_input_operand (&ops
[1], rtx_op1
, mode
);
3127 create_input_operand (&ops
[2], rtx_op2
, mode
);
3128 create_input_operand (&ops
[3], mask
, mask_mode
);
3129 expand_insn (icode
, 4, ops
);
3130 if (!rtx_equal_p (ops
[0].value
, target
))
3131 emit_move_insn (target
, ops
[0].value
);
3134 /* Expand VEC_SET internal functions. */
3137 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3139 tree lhs
= gimple_call_lhs (stmt
);
3140 tree op0
= gimple_call_arg (stmt
, 0);
3141 tree op1
= gimple_call_arg (stmt
, 1);
3142 tree op2
= gimple_call_arg (stmt
, 2);
3143 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3144 rtx src
= expand_normal (op0
);
3146 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
3147 scalar_mode innermode
= GET_MODE_INNER (outermode
);
3149 rtx value
= expand_normal (op1
);
3150 rtx pos
= expand_normal (op2
);
3152 class expand_operand ops
[3];
3153 enum insn_code icode
= optab_handler (optab
, outermode
);
3155 if (icode
!= CODE_FOR_nothing
)
3157 rtx temp
= gen_reg_rtx (outermode
);
3158 emit_move_insn (temp
, src
);
3160 create_fixed_operand (&ops
[0], temp
);
3161 create_input_operand (&ops
[1], value
, innermode
);
3162 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
3164 if (maybe_expand_insn (icode
, 3, ops
))
3166 emit_move_insn (target
, temp
);
3174 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3179 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3181 /* When guessing was done, the hints should be already stripped away. */
3182 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3185 tree lhs
= gimple_call_lhs (stmt
);
3187 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3189 target
= const0_rtx
;
3190 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3191 if (lhs
&& val
!= target
)
3192 emit_move_insn (target
, val
);
3195 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3196 should never be called. */
3199 expand_VA_ARG (internal_fn
, gcall
*)
3204 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3205 dummy function should never be called. */
3208 expand_VEC_CONVERT (internal_fn
, gcall
*)
3213 /* Expand IFN_RAWMEMCHR internal function. */
3216 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3218 expand_operand ops
[3];
3220 tree lhs
= gimple_call_lhs (stmt
);
3223 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3224 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3225 create_output_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3227 tree mem
= gimple_call_arg (stmt
, 0);
3228 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3229 create_fixed_operand (&ops
[1], mem_rtx
);
3231 tree pattern
= gimple_call_arg (stmt
, 1);
3232 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3233 rtx pattern_rtx
= expand_normal (pattern
);
3234 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3236 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3238 expand_insn (icode
, 3, ops
);
3239 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3240 emit_move_insn (lhs_rtx
, ops
[0].value
);
3243 /* Expand the IFN_UNIQUE function according to its first argument. */
3246 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3248 rtx pattern
= NULL_RTX
;
3249 enum ifn_unique_kind kind
3250 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3257 case IFN_UNIQUE_UNSPEC
:
3258 if (targetm
.have_unique ())
3259 pattern
= targetm
.gen_unique ();
3262 case IFN_UNIQUE_OACC_FORK
:
3263 case IFN_UNIQUE_OACC_JOIN
:
3264 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3266 tree lhs
= gimple_call_lhs (stmt
);
3267 rtx target
= const0_rtx
;
3270 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3272 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3273 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3275 if (kind
== IFN_UNIQUE_OACC_FORK
)
3276 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3278 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3286 emit_insn (pattern
);
3289 /* Expand the IFN_DEFERRED_INIT function:
3290 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3292 Initialize the LHS with zero/pattern according to its second argument
3294 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3295 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3297 The LHS variable is initialized including paddings.
3298 The reasons to choose 0xFE for pattern initialization are:
3299 1. It is a non-canonical virtual address on x86_64, and at the
3300 high end of the i386 kernel address space.
3301 2. It is a very large float value (-1.694739530317379e+38).
3302 3. It is also an unusual number for integers. */
3303 #define INIT_PATTERN_VALUE 0xFE
3305 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3307 tree lhs
= gimple_call_lhs (stmt
);
3308 tree var_size
= gimple_call_arg (stmt
, 0);
3309 enum auto_init_type init_type
3310 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3311 bool reg_lhs
= true;
3313 tree var_type
= TREE_TYPE (lhs
);
3314 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3316 if (TREE_CODE (lhs
) == SSA_NAME
)
3320 tree lhs_base
= lhs
;
3321 while (handled_component_p (lhs_base
))
3322 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3323 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3324 || non_mem_decl_p (lhs_base
));
3325 /* If this expands to a register and the underlying decl is wrapped in
3326 a MEM_REF that just serves as an access type change expose the decl
3327 if it is of correct size. This avoids a situation as in PR103271
3328 if the target does not support a direct move to the registers mode. */
3330 && TREE_CODE (lhs_base
) == MEM_REF
3331 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3332 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3333 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3334 && tree_fits_uhwi_p (var_size
)
3335 && tree_int_cst_equal
3337 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3339 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3340 var_type
= TREE_TYPE (lhs
);
3346 /* If the variable is not in register, expand to a memset
3347 to initialize it. */
3348 mark_addressable (lhs
);
3349 tree var_addr
= build_fold_addr_expr (lhs
);
3351 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3352 ? build_int_cst (integer_type_node
,
3354 : integer_zero_node
;
3355 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3356 3, var_addr
, value
, var_size
);
3357 /* Expand this memset call. */
3358 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3362 /* If this variable is in a register use expand_assignment.
3363 For boolean scalars force zero-init. */
3365 scalar_int_mode var_mode
;
3366 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3367 && tree_fits_uhwi_p (var_size
)
3368 && (init_type
== AUTO_INIT_PATTERN
3369 || !is_gimple_reg_type (var_type
))
3370 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3371 0).exists (&var_mode
)
3372 && have_insn_for (SET
, var_mode
))
3374 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3375 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3376 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3377 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3378 tree itype
= build_nonstandard_integer_type
3379 (total_bytes
* BITS_PER_UNIT
, 1);
3380 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3381 init
= wide_int_to_tree (itype
, w
);
3382 /* Pun the LHS to make sure its type has constant size
3383 unless it is an SSA name where that's already known. */
3384 if (TREE_CODE (lhs
) != SSA_NAME
)
3385 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3387 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3390 /* Use zero-init also for variable-length sizes. */
3391 init
= build_zero_cst (var_type
);
3393 expand_assignment (lhs
, init
, false);
3397 /* The size of an OpenACC compute dimension. */
3400 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3402 tree lhs
= gimple_call_lhs (stmt
);
3407 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3408 if (targetm
.have_oacc_dim_size ())
3410 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3411 VOIDmode
, EXPAND_NORMAL
);
3412 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3415 emit_move_insn (target
, GEN_INT (1));
3418 /* The position of an OpenACC execution engine along one compute axis. */
3421 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3423 tree lhs
= gimple_call_lhs (stmt
);
3428 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3429 if (targetm
.have_oacc_dim_pos ())
3431 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3432 VOIDmode
, EXPAND_NORMAL
);
3433 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3436 emit_move_insn (target
, const0_rtx
);
3439 /* This is expanded by oacc_device_lower pass. */
3442 expand_GOACC_LOOP (internal_fn
, gcall
*)
3447 /* This is expanded by oacc_device_lower pass. */
3450 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3455 /* This is expanded by oacc_device_lower pass. */
3458 expand_GOACC_TILE (internal_fn
, gcall
*)
3463 /* Set errno to EDOM. */
3466 expand_SET_EDOM (internal_fn
, gcall
*)
3469 #ifdef GEN_ERRNO_RTX
3470 rtx errno_rtx
= GEN_ERRNO_RTX
;
3472 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3474 emit_move_insn (errno_rtx
,
3475 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3481 /* Expand atomic bit test and set. */
3484 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3486 expand_ifn_atomic_bit_test_and (call
);
3489 /* Expand atomic bit test and complement. */
3492 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3494 expand_ifn_atomic_bit_test_and (call
);
3497 /* Expand atomic bit test and reset. */
3500 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3502 expand_ifn_atomic_bit_test_and (call
);
3505 /* Expand atomic bit test and set. */
3508 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3510 expand_ifn_atomic_compare_exchange (call
);
3513 /* Expand atomic add fetch and cmp with 0. */
3516 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3518 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3521 /* Expand atomic sub fetch and cmp with 0. */
3524 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3526 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3529 /* Expand atomic and fetch and cmp with 0. */
3532 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3534 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3537 /* Expand atomic or fetch and cmp with 0. */
3540 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3542 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3545 /* Expand atomic xor fetch and cmp with 0. */
3548 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3550 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3553 /* Expand LAUNDER to assignment, lhs = arg0. */
3556 expand_LAUNDER (internal_fn
, gcall
*call
)
3558 tree lhs
= gimple_call_lhs (call
);
3563 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3566 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3569 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3571 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3572 int rhs_index
= internal_fn_stored_value_index (ifn
);
3573 tree base
= gimple_call_arg (stmt
, 0);
3574 tree offset
= gimple_call_arg (stmt
, 1);
3575 tree scale
= gimple_call_arg (stmt
, 2);
3576 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3578 rtx base_rtx
= expand_normal (base
);
3579 rtx offset_rtx
= expand_normal (offset
);
3580 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3581 rtx rhs_rtx
= expand_normal (rhs
);
3583 class expand_operand ops
[8];
3585 create_address_operand (&ops
[i
++], base_rtx
);
3586 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3587 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3588 create_integer_operand (&ops
[i
++], scale_int
);
3589 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3590 i
= add_mask_and_len_args (ops
, i
, stmt
);
3592 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3593 TYPE_MODE (TREE_TYPE (offset
)));
3594 expand_insn (icode
, i
, ops
);
3597 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3600 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3602 tree lhs
= gimple_call_lhs (stmt
);
3603 tree base
= gimple_call_arg (stmt
, 0);
3604 tree offset
= gimple_call_arg (stmt
, 1);
3605 tree scale
= gimple_call_arg (stmt
, 2);
3607 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3608 rtx base_rtx
= expand_normal (base
);
3609 rtx offset_rtx
= expand_normal (offset
);
3610 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3613 class expand_operand ops
[8];
3614 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3615 create_address_operand (&ops
[i
++], base_rtx
);
3616 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3617 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3618 create_integer_operand (&ops
[i
++], scale_int
);
3619 i
= add_mask_and_len_args (ops
, i
, stmt
);
3620 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3621 TYPE_MODE (TREE_TYPE (offset
)));
3622 expand_insn (icode
, i
, ops
);
3623 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3624 emit_move_insn (lhs_rtx
, ops
[0].value
);
3627 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3628 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3631 contains_call_div_mod (rtx_insn
*insn
)
3633 subrtx_iterator::array_type array
;
3634 for (; insn
; insn
= NEXT_INSN (insn
))
3637 else if (INSN_P (insn
))
3638 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3639 switch (GET_CODE (*iter
))
3653 /* Expand DIVMOD() using:
3654 a) optab handler for udivmod/sdivmod if it is available.
3655 b) If optab_handler doesn't exist, generate call to
3656 target-specific divmod libfunc. */
3659 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3661 tree lhs
= gimple_call_lhs (call_stmt
);
3662 tree arg0
= gimple_call_arg (call_stmt
, 0);
3663 tree arg1
= gimple_call_arg (call_stmt
, 1);
3665 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3666 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3667 machine_mode mode
= TYPE_MODE (type
);
3668 bool unsignedp
= TYPE_UNSIGNED (type
);
3669 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3671 rtx op0
= expand_normal (arg0
);
3672 rtx op1
= expand_normal (arg1
);
3673 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3675 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3676 rtx_insn
*insns
= NULL
;
3678 if (TREE_CODE (arg1
) == INTEGER_CST
)
3680 /* For DIVMOD by integral constants, there could be efficient code
3681 expanded inline e.g. using shifts and plus/minus. Try to expand
3682 the division and modulo and if it emits any library calls or any
3683 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3685 scalar_int_mode int_mode
;
3686 if (remainder
== NULL_RTX
3688 && CONST_INT_P (op1
)
3689 && !pow2p_hwi (INTVAL (op1
))
3690 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3691 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3692 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3693 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3694 && optimize_insn_for_speed_p ())
3696 rtx_insn
*last
= get_last_insn ();
3697 remainder
= NULL_RTX
;
3698 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3699 TYPE_UNSIGNED (type
));
3700 if (quotient
!= NULL_RTX
)
3702 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3704 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3705 set_dst_reg_note (move
, REG_EQUAL
,
3706 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3707 ? UDIV
: DIV
, int_mode
,
3708 copy_rtx (op0
), op1
),
3710 move
= emit_move_insn (remainder
, remainder
);
3711 set_dst_reg_note (move
, REG_EQUAL
,
3712 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3713 ? UMOD
: MOD
, int_mode
,
3714 copy_rtx (op0
), op1
),
3719 delete_insns_since (last
);
3722 if (remainder
== NULL_RTX
)
3724 struct separate_ops ops
;
3725 ops
.code
= TRUNC_DIV_EXPR
;
3727 ops
.op0
= make_tree (ops
.type
, op0
);
3729 ops
.op2
= NULL_TREE
;
3730 ops
.location
= gimple_location (call_stmt
);
3732 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3733 if (contains_call_div_mod (get_insns ()))
3734 quotient
= NULL_RTX
;
3737 ops
.code
= TRUNC_MOD_EXPR
;
3738 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3740 if (contains_call_div_mod (get_insns ()))
3741 remainder
= NULL_RTX
;
3744 insns
= get_insns ();
3752 /* Check if optab_handler exists for divmod_optab for given mode. */
3753 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3755 quotient
= gen_reg_rtx (mode
);
3756 remainder
= gen_reg_rtx (mode
);
3757 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3760 /* Generate call to divmod libfunc if it exists. */
3761 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3762 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3763 "ient
, &remainder
);
3768 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3769 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3770 make_tree (TREE_TYPE (arg0
), quotient
),
3771 make_tree (TREE_TYPE (arg1
), remainder
)),
3772 target
, VOIDmode
, EXPAND_NORMAL
);
3778 expand_NOP (internal_fn
, gcall
*)
3780 /* Nothing. But it shouldn't really prevail. */
3783 /* Coroutines, all should have been processed at this stage. */
3786 expand_CO_FRAME (internal_fn
, gcall
*)
3792 expand_CO_YIELD (internal_fn
, gcall
*)
3798 expand_CO_SUSPN (internal_fn
, gcall
*)
3804 expand_CO_ACTOR (internal_fn
, gcall
*)
3809 /* Expand a call to FN using the operands in STMT. FN has a single
3810 output operand and NARGS input operands. */
3813 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3816 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3817 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3818 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3821 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3824 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3826 expand_operand ops
[4];
3829 tree lhs
= gimple_call_lhs (stmt
);
3830 tree lhs_type
= TREE_TYPE (lhs
);
3831 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3832 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3834 for (unsigned int i
= 0; i
< 2; ++i
)
3836 tree rhs
= gimple_call_arg (stmt
, i
);
3837 rhs_type
[i
] = TREE_TYPE (rhs
);
3838 rtx rhs_rtx
= expand_normal (rhs
);
3839 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3843 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type
)))
3845 /* When the mask is an integer mode the exact vector length may not
3846 be clear to the backend, so we pass it in operand[3].
3847 Use the vector in arg2 for the most reliable intended size. */
3848 tree type
= TREE_TYPE (gimple_call_arg (stmt
, 2));
3849 create_integer_operand (&ops
[3], TYPE_VECTOR_SUBPARTS (type
));
3853 /* The mask has a vector type so the length operand is unnecessary. */
3856 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3857 TYPE_MODE (lhs_type
));
3859 expand_insn (icode
, opcnt
, ops
);
3860 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3861 emit_move_insn (lhs_rtx
, ops
[0].value
);
3864 /* Expand a call to a convert-like optab using the operands in STMT.
3865 FN has a single output operand and NARGS input operands. */
3868 expand_convert_optab_fn (internal_fn fn
, gcall
*stmt
, convert_optab optab
,
3871 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3872 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3873 TYPE_MODE (types
.second
));
3874 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3877 /* Expanders for optabs that can use expand_direct_optab_fn. */
3879 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3880 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3882 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3883 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3885 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3886 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3888 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3889 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3891 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3892 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3894 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3895 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3897 #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3898 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3900 #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3901 expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3903 #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3904 expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3906 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3907 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3909 #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3910 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3912 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3913 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3915 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3916 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3918 #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3919 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3921 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3922 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3924 /* Expanders for optabs that can use expand_convert_optab_fn. */
3926 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3927 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3929 #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3930 expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3932 /* RETURN_TYPE and ARGS are a return type and argument list that are
3933 in principle compatible with FN (which satisfies direct_internal_fn_p).
3934 Return the types that should be used to determine whether the
3935 target supports FN. */
3938 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3940 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3941 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3942 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3943 return tree_pair (type0
, type1
);
3946 /* CALL is a call whose return type and arguments are in principle
3947 compatible with FN (which satisfies direct_internal_fn_p). Return the
3948 types that should be used to determine whether the target supports FN. */
3951 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3953 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3954 tree op0
= (info
.type0
< 0
3955 ? gimple_call_lhs (call
)
3956 : gimple_call_arg (call
, info
.type0
));
3957 tree op1
= (info
.type1
< 0
3958 ? gimple_call_lhs (call
)
3959 : gimple_call_arg (call
, info
.type1
));
3960 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3963 /* Return true if OPTAB is supported for TYPES (whose modes should be
3964 the same) when the optimization type is OPT_TYPE. Used for simple
3968 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3969 optimization_type opt_type
)
3971 machine_mode mode
= TYPE_MODE (types
.first
);
3972 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3973 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3976 /* Return true if OPTAB is supported for TYPES, where the first type
3977 is the destination and the second type is the source. Used for
3981 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3982 optimization_type opt_type
)
3984 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3985 TYPE_MODE (types
.second
), opt_type
)
3986 != CODE_FOR_nothing
);
3989 /* Return true if load/store lanes optab OPTAB is supported for
3990 array type TYPES.first when the optimization type is OPT_TYPE. */
3993 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3994 optimization_type opt_type
)
3996 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3997 machine_mode imode
= TYPE_MODE (types
.first
);
3998 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3999 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
4000 != CODE_FOR_nothing
);
4003 #define direct_unary_optab_supported_p direct_optab_supported_p
4004 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
4005 #define direct_binary_optab_supported_p direct_optab_supported_p
4006 #define direct_ternary_optab_supported_p direct_optab_supported_p
4007 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
4008 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
4009 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4010 #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4011 #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4012 #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4013 #define direct_mask_load_optab_supported_p convert_optab_supported_p
4014 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4015 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4016 #define direct_gather_load_optab_supported_p convert_optab_supported_p
4017 #define direct_len_load_optab_supported_p direct_optab_supported_p
4018 #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4019 #define direct_mask_store_optab_supported_p convert_optab_supported_p
4020 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4021 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4022 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4023 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
4024 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
4025 #define direct_len_store_optab_supported_p direct_optab_supported_p
4026 #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4027 #define direct_while_optab_supported_p convert_optab_supported_p
4028 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
4029 #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4030 #define direct_fold_left_optab_supported_p direct_optab_supported_p
4031 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4032 #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4033 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4034 #define direct_vec_set_optab_supported_p direct_optab_supported_p
4035 #define direct_vec_extract_optab_supported_p convert_optab_supported_p
4037 /* Return the optab used by internal function FN. */
4040 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
4044 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4045 case IFN_##CODE: break;
4046 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4047 case IFN_##CODE: return OPTAB##_optab;
4048 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4049 UNSIGNED_OPTAB, TYPE) \
4050 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4051 ? UNSIGNED_OPTAB ## _optab \
4052 : SIGNED_OPTAB ## _optab);
4053 #include "internal-fn.def"
4061 /* Return the optab used by internal function FN. */
4064 direct_internal_fn_optab (internal_fn fn
)
4068 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4069 case IFN_##CODE: break;
4070 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4071 case IFN_##CODE: return OPTAB##_optab;
4072 #include "internal-fn.def"
4080 /* Return true if FN is supported for the types in TYPES when the
4081 optimization type is OPT_TYPE. The types are those associated with
4082 the "type0" and "type1" fields of FN's direct_internal_fn_info
4086 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
4087 optimization_type opt_type
)
4091 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4092 case IFN_##CODE: break;
4093 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4095 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4097 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4098 UNSIGNED_OPTAB, TYPE) \
4101 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4102 ? UNSIGNED_OPTAB ## _optab \
4103 : SIGNED_OPTAB ## _optab); \
4104 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4107 #include "internal-fn.def"
4115 /* Return true if FN is supported for type TYPE when the optimization
4116 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4117 fields of FN's direct_internal_fn_info structure are the same. */
4120 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
4121 optimization_type opt_type
)
4123 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4124 gcc_checking_assert (info
.type0
== info
.type1
);
4125 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
4128 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4129 given that STMT is a call to a direct internal function. */
4132 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
4134 internal_fn fn
= gimple_call_internal_fn (stmt
);
4135 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
4136 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
4139 /* Return true if FN is a binary operation and if FN is commutative. */
4142 commutative_binary_fn_p (internal_fn fn
)
4153 case IFN_COMPLEX_MUL
:
4154 case IFN_UBSAN_CHECK_ADD
:
4155 case IFN_UBSAN_CHECK_MUL
:
4156 case IFN_ADD_OVERFLOW
:
4157 case IFN_MUL_OVERFLOW
:
4158 case IFN_VEC_WIDEN_PLUS
:
4159 case IFN_VEC_WIDEN_PLUS_LO
:
4160 case IFN_VEC_WIDEN_PLUS_HI
:
4161 case IFN_VEC_WIDEN_PLUS_EVEN
:
4162 case IFN_VEC_WIDEN_PLUS_ODD
:
4170 /* Return true if FN is a ternary operation and if its first two arguments
4174 commutative_ternary_fn_p (internal_fn fn
)
4190 /* Return true if FN is an associative binary operation. */
4193 associative_binary_fn_p (internal_fn fn
)
4206 /* If FN is commutative in two consecutive arguments, return the
4207 index of the first, otherwise return -1. */
4210 first_commutative_argument (internal_fn fn
)
4227 case IFN_COND_LEN_ADD
:
4228 case IFN_COND_LEN_MUL
:
4229 case IFN_COND_LEN_MIN
:
4230 case IFN_COND_LEN_MAX
:
4231 case IFN_COND_LEN_FMIN
:
4232 case IFN_COND_LEN_FMAX
:
4233 case IFN_COND_LEN_AND
:
4234 case IFN_COND_LEN_IOR
:
4235 case IFN_COND_LEN_XOR
:
4236 case IFN_COND_LEN_FMA
:
4237 case IFN_COND_LEN_FMS
:
4238 case IFN_COND_LEN_FNMA
:
4239 case IFN_COND_LEN_FNMS
:
4243 if (commutative_binary_fn_p (fn
)
4244 || commutative_ternary_fn_p (fn
))
4250 /* Return true if this CODE describes an internal_fn that returns a vector with
4251 elements twice as wide as the element size of the input vectors. */
4254 widening_fn_p (code_helper code
)
4256 if (!code
.is_fn_code ())
4259 if (!internal_fn_p ((combined_fn
) code
))
4262 internal_fn fn
= as_internal_fn ((combined_fn
) code
);
4265 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4267 case IFN_##NAME##_HI: \
4268 case IFN_##NAME##_LO: \
4269 case IFN_##NAME##_EVEN: \
4270 case IFN_##NAME##_ODD: \
4272 #include "internal-fn.def"
4279 /* Return true if IFN_SET_EDOM is supported. */
4282 set_edom_supported_p (void)
4291 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4293 expand_##CODE (internal_fn fn, gcall *stmt) \
4295 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4297 #define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4298 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4299 UNSIGNED_OPTAB, TYPE) \
4301 expand_##CODE (internal_fn fn, gcall *stmt) \
4303 tree_pair types = direct_internal_fn_types (fn, stmt); \
4304 optab which_optab = direct_internal_fn_optab (fn, types); \
4305 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4307 #include "internal-fn.def"
4309 /* Routines to expand each internal function, indexed by function number.
4310 Each routine has the prototype:
4312 expand_<NAME> (gcall *stmt)
4314 where STMT is the statement that performs the call. */
4315 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4317 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4318 #include "internal-fn.def"
4322 /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4323 that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4324 for each such IFN_COND_##SUFFIX. */
4325 #define FOR_EACH_CODE_MAPPING(T) \
4326 T (PLUS_EXPR, ADD) \
4327 T (MINUS_EXPR, SUB) \
4328 T (MULT_EXPR, MUL) \
4329 T (TRUNC_DIV_EXPR, DIV) \
4330 T (TRUNC_MOD_EXPR, MOD) \
4331 T (RDIV_EXPR, RDIV) \
4334 T (BIT_AND_EXPR, AND) \
4335 T (BIT_IOR_EXPR, IOR) \
4336 T (BIT_XOR_EXPR, XOR) \
4337 T (LSHIFT_EXPR, SHL) \
4338 T (RSHIFT_EXPR, SHR) \
4339 T (NEGATE_EXPR, NEG)
4341 /* Return a function that only performs CODE when a certain condition is met
4342 and that uses a given fallback value otherwise. For example, if CODE is
4343 a binary operation associated with conditional function FN:
4345 LHS = FN (COND, A, B, ELSE)
4347 is equivalent to the C expression:
4349 LHS = COND ? A CODE B : ELSE;
4351 operating elementwise if the operands are vectors.
4353 Return IFN_LAST if no such function exists. */
4356 get_conditional_internal_fn (tree_code code
)
4360 #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4361 FOR_EACH_CODE_MAPPING(CASE
)
4368 /* If IFN implements the conditional form of a tree code, return that
4369 tree code, otherwise return ERROR_MARK. */
4372 conditional_internal_fn_code (internal_fn ifn
)
4376 #define CASE(CODE, IFN) \
4377 case IFN_COND_##IFN: \
4378 case IFN_COND_LEN_##IFN: \
4380 FOR_EACH_CODE_MAPPING (CASE
)
4387 /* Like get_conditional_internal_fn, but return a function that
4388 additionally restricts the operation to the leading elements
4389 of a vector. The number of elements to process is given by a length
4390 and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4391 elements are taken from the fallback ("else") argument.
4393 For example, if CODE is a binary operation associated with FN:
4395 LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4397 is equivalent to the C code:
4399 for (int i = 0; i < NUNITS; i++)
4401 if (i < LEN + BIAS && COND[i])
4402 LHS[i] = A[i] CODE B[i];
4409 get_conditional_len_internal_fn (tree_code code
)
4413 #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4414 FOR_EACH_CODE_MAPPING(CASE
)
4421 /* Invoke T(IFN) for each internal function IFN that also has an
4423 #define FOR_EACH_COND_FN_PAIR(T) \
4431 /* Return a function that only performs internal function FN when a
4432 certain condition is met and that uses a given fallback value otherwise.
4433 In other words, the returned function FN' is such that:
4435 LHS = FN' (COND, A1, ... An, ELSE)
4437 is equivalent to the C expression:
4439 LHS = COND ? FN (A1, ..., An) : ELSE;
4441 operating elementwise if the operands are vectors.
4443 Return IFN_LAST if no such function exists. */
4446 get_conditional_internal_fn (internal_fn fn
)
4450 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4451 FOR_EACH_COND_FN_PAIR(CASE
)
4458 /* If there exists an internal function like IFN that operates on vectors,
4459 but with additional length and bias parameters, return the internal_fn
4460 for that function, otherwise return IFN_LAST. */
4462 get_len_internal_fn (internal_fn fn
)
4466 #define DEF_INTERNAL_COND_FN(NAME, ...) \
4467 case IFN_COND_##NAME: \
4468 return IFN_COND_LEN_##NAME;
4469 #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4470 case IFN_COND_##NAME: \
4471 return IFN_COND_LEN_##NAME;
4472 #include "internal-fn.def"
4478 /* If IFN implements the conditional form of an unconditional internal
4479 function, return that unconditional function, otherwise return IFN_LAST. */
4482 get_unconditional_internal_fn (internal_fn ifn
)
4486 #define CASE(NAME) \
4487 case IFN_COND_##NAME: \
4488 case IFN_COND_LEN_##NAME: \
4490 FOR_EACH_COND_FN_PAIR (CASE
)
4497 /* Return true if STMT can be interpreted as a conditional tree code
4498 operation of the form:
4500 LHS = COND ? OP (RHS1, ...) : ELSE;
4502 operating elementwise if the operands are vectors. This includes
4503 the case of an all-true COND, so that the operation always happens.
4505 There is an alternative approach to interpret the STMT when the operands
4506 are vectors which is the operation predicated by both conditional mask
4507 and loop control length, the equivalent C code:
4509 for (int i = 0; i < NUNTIS; i++)
4511 if (i < LEN + BIAS && COND[i])
4512 LHS[i] = A[i] CODE B[i];
4517 When returning true, set:
4519 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4520 is known to be all-true
4521 - *CODE_OUT to the tree code
4522 - OPS[I] to operand I of *CODE_OUT
4523 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4524 condition is known to be all true.
4525 - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4526 - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4529 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4530 tree_code
*code_out
,
4531 tree (&ops
)[3], tree
*else_out
,
4532 tree
*len
, tree
*bias
)
4536 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4538 *cond_out
= NULL_TREE
;
4539 *code_out
= gimple_assign_rhs_code (assign
);
4540 ops
[0] = gimple_assign_rhs1 (assign
);
4541 ops
[1] = gimple_assign_rhs2 (assign
);
4542 ops
[2] = gimple_assign_rhs3 (assign
);
4543 *else_out
= NULL_TREE
;
4546 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4547 if (gimple_call_internal_p (call
))
4549 internal_fn ifn
= gimple_call_internal_fn (call
);
4550 tree_code code
= conditional_internal_fn_code (ifn
);
4551 int len_index
= internal_fn_len_index (ifn
);
4552 int cond_nargs
= len_index
>= 0 ? 4 : 2;
4553 if (code
!= ERROR_MARK
)
4555 *cond_out
= gimple_call_arg (call
, 0);
4557 unsigned int nops
= gimple_call_num_args (call
) - cond_nargs
;
4558 for (unsigned int i
= 0; i
< 3; ++i
)
4559 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4560 *else_out
= gimple_call_arg (call
, nops
+ 1);
4563 if (integer_truep (*cond_out
))
4565 *cond_out
= NULL_TREE
;
4566 *else_out
= NULL_TREE
;
4571 *len
= gimple_call_arg (call
, len_index
);
4572 *bias
= gimple_call_arg (call
, len_index
+ 1);
4580 /* Return true if IFN is some form of load from memory. */
4583 internal_load_fn_p (internal_fn fn
)
4588 case IFN_LOAD_LANES
:
4589 case IFN_MASK_LOAD_LANES
:
4590 case IFN_MASK_LEN_LOAD_LANES
:
4591 case IFN_GATHER_LOAD
:
4592 case IFN_MASK_GATHER_LOAD
:
4593 case IFN_MASK_LEN_GATHER_LOAD
:
4595 case IFN_MASK_LEN_LOAD
:
4603 /* Return true if IFN is some form of store to memory. */
4606 internal_store_fn_p (internal_fn fn
)
4610 case IFN_MASK_STORE
:
4611 case IFN_STORE_LANES
:
4612 case IFN_MASK_STORE_LANES
:
4613 case IFN_MASK_LEN_STORE_LANES
:
4614 case IFN_SCATTER_STORE
:
4615 case IFN_MASK_SCATTER_STORE
:
4616 case IFN_MASK_LEN_SCATTER_STORE
:
4618 case IFN_MASK_LEN_STORE
:
4626 /* Return true if IFN is some form of gather load or scatter store. */
4629 internal_gather_scatter_fn_p (internal_fn fn
)
4633 case IFN_GATHER_LOAD
:
4634 case IFN_MASK_GATHER_LOAD
:
4635 case IFN_MASK_LEN_GATHER_LOAD
:
4636 case IFN_SCATTER_STORE
:
4637 case IFN_MASK_SCATTER_STORE
:
4638 case IFN_MASK_LEN_SCATTER_STORE
:
4646 /* If FN takes a vector len argument, return the index of that argument,
4647 otherwise return -1. */
4650 internal_fn_len_index (internal_fn fn
)
4658 case IFN_MASK_LEN_GATHER_LOAD
:
4659 case IFN_MASK_LEN_SCATTER_STORE
:
4660 case IFN_COND_LEN_FMA
:
4661 case IFN_COND_LEN_FMS
:
4662 case IFN_COND_LEN_FNMA
:
4663 case IFN_COND_LEN_FNMS
:
4666 case IFN_COND_LEN_ADD
:
4667 case IFN_COND_LEN_SUB
:
4668 case IFN_COND_LEN_MUL
:
4669 case IFN_COND_LEN_DIV
:
4670 case IFN_COND_LEN_MOD
:
4671 case IFN_COND_LEN_RDIV
:
4672 case IFN_COND_LEN_MIN
:
4673 case IFN_COND_LEN_MAX
:
4674 case IFN_COND_LEN_FMIN
:
4675 case IFN_COND_LEN_FMAX
:
4676 case IFN_COND_LEN_AND
:
4677 case IFN_COND_LEN_IOR
:
4678 case IFN_COND_LEN_XOR
:
4679 case IFN_COND_LEN_SHL
:
4680 case IFN_COND_LEN_SHR
:
4683 case IFN_COND_LEN_NEG
:
4684 case IFN_MASK_LEN_LOAD
:
4685 case IFN_MASK_LEN_STORE
:
4686 case IFN_MASK_LEN_LOAD_LANES
:
4687 case IFN_MASK_LEN_STORE_LANES
:
4688 case IFN_VCOND_MASK_LEN
:
4696 /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4697 argument that is used when the condition is false. Return -1 otherwise. */
4700 internal_fn_else_index (internal_fn fn
)
4706 case IFN_COND_LEN_NEG
:
4707 case IFN_COND_LEN_NOT
:
4724 case IFN_COND_LEN_ADD
:
4725 case IFN_COND_LEN_SUB
:
4726 case IFN_COND_LEN_MUL
:
4727 case IFN_COND_LEN_DIV
:
4728 case IFN_COND_LEN_MOD
:
4729 case IFN_COND_LEN_MIN
:
4730 case IFN_COND_LEN_MAX
:
4731 case IFN_COND_LEN_FMIN
:
4732 case IFN_COND_LEN_FMAX
:
4733 case IFN_COND_LEN_AND
:
4734 case IFN_COND_LEN_IOR
:
4735 case IFN_COND_LEN_XOR
:
4736 case IFN_COND_LEN_SHL
:
4737 case IFN_COND_LEN_SHR
:
4744 case IFN_COND_LEN_FMA
:
4745 case IFN_COND_LEN_FMS
:
4746 case IFN_COND_LEN_FNMA
:
4747 case IFN_COND_LEN_FNMS
:
4757 /* If FN takes a vector mask argument, return the index of that argument,
4758 otherwise return -1. */
4761 internal_fn_mask_index (internal_fn fn
)
4766 case IFN_MASK_LOAD_LANES
:
4767 case IFN_MASK_LEN_LOAD_LANES
:
4768 case IFN_MASK_STORE
:
4769 case IFN_MASK_STORE_LANES
:
4770 case IFN_MASK_LEN_STORE_LANES
:
4771 case IFN_MASK_LEN_LOAD
:
4772 case IFN_MASK_LEN_STORE
:
4775 case IFN_MASK_GATHER_LOAD
:
4776 case IFN_MASK_SCATTER_STORE
:
4777 case IFN_MASK_LEN_GATHER_LOAD
:
4778 case IFN_MASK_LEN_SCATTER_STORE
:
4781 case IFN_VCOND_MASK_LEN
:
4785 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4786 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4790 /* If FN takes a value that should be stored to memory, return the index
4791 of that argument, otherwise return -1. */
4794 internal_fn_stored_value_index (internal_fn fn
)
4798 case IFN_MASK_STORE
:
4799 case IFN_MASK_STORE_LANES
:
4800 case IFN_SCATTER_STORE
:
4801 case IFN_MASK_SCATTER_STORE
:
4802 case IFN_MASK_LEN_SCATTER_STORE
:
4808 case IFN_MASK_LEN_STORE
:
4809 case IFN_MASK_LEN_STORE_LANES
:
4817 /* Return true if the target supports gather load or scatter store function
4818 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4819 while for stores it is the vector type of the stored data argument.
4820 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4821 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4822 offset from the shared base address of each loaded or stored element.
4823 SCALE is the amount by which these offsets should be multiplied
4824 *after* they have been extended to address width. */
4827 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4828 tree memory_element_type
,
4829 tree offset_vector_type
, int scale
)
4831 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4832 TYPE_SIZE (memory_element_type
)))
4834 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4835 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4837 optab optab
= direct_internal_fn_optab (ifn
);
4838 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4839 TYPE_MODE (offset_vector_type
));
4840 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4841 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4842 return (icode
!= CODE_FOR_nothing
4843 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4844 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4847 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4848 for pointers of type TYPE when the accesses have LENGTH bytes and their
4849 common byte alignment is ALIGN. */
4852 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4853 poly_uint64 length
, unsigned int align
)
4855 machine_mode mode
= TYPE_MODE (type
);
4856 optab optab
= direct_internal_fn_optab (ifn
);
4857 insn_code icode
= direct_optab_handler (optab
, mode
);
4858 if (icode
== CODE_FOR_nothing
)
4860 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4861 return (insn_operand_matches (icode
, 3, length_rtx
)
4862 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4865 /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4866 or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4867 -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4868 {len_,mask_len_}store). If none of the biases match what the backend
4869 provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4872 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
4874 optab optab
= direct_internal_fn_optab (ifn
);
4875 insn_code icode
= direct_optab_handler (optab
, mode
);
4878 if (icode
== CODE_FOR_nothing
)
4880 machine_mode mask_mode
;
4881 if (!targetm
.vectorize
.get_mask_mode (mode
).exists (&mask_mode
))
4882 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4883 if (ifn
== IFN_LEN_LOAD
)
4885 /* Try MASK_LEN_LOAD. */
4886 optab
= direct_internal_fn_optab (IFN_MASK_LEN_LOAD
);
4890 /* Try MASK_LEN_STORE. */
4891 optab
= direct_internal_fn_optab (IFN_MASK_LEN_STORE
);
4893 icode
= convert_optab_handler (optab
, mode
, mask_mode
);
4897 if (icode
!= CODE_FOR_nothing
)
4899 /* For now we only support biases of 0 or -1. Try both of them. */
4900 if (insn_operand_matches (icode
, bias_no
, GEN_INT (0)))
4902 if (insn_operand_matches (icode
, bias_no
, GEN_INT (-1)))
4906 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4909 /* Expand STMT as though it were a call to internal function FN. */
4912 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4914 internal_fn_expanders
[fn
] (fn
, stmt
);
4917 /* Expand STMT, which is a call to internal function FN. */
4920 expand_internal_call (gcall
*stmt
)
4922 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4925 /* If TYPE is a vector type, return true if IFN is a direct internal
4926 function that is supported for that type. If TYPE is a scalar type,
4927 return true if IFN is a direct internal function that is supported for
4928 the target's preferred vector version of TYPE. */
4931 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4933 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4934 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4937 if (VECTOR_TYPE_P (type
)
4938 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4941 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4942 if (VECTOR_MODE_P (vmode
))
4944 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4945 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4949 auto_vector_modes vector_modes
;
4950 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4951 for (machine_mode base_mode
: vector_modes
)
4952 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
4954 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4955 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4963 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
4969 expand_PHI (internal_fn
, gcall
*)
4975 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
4977 tree lhs
= gimple_call_lhs (stmt
);
4978 tree rhs1
= gimple_call_arg (stmt
, 0);
4979 tree rhs2
= gimple_call_arg (stmt
, 1);
4980 tree type
= TREE_TYPE (rhs1
);
4982 do_pending_stack_adjust ();
4984 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
4985 rtx op1
= expand_normal (rhs1
);
4986 rtx op2
= expand_normal (rhs2
);
4988 class expand_operand ops
[3];
4989 create_output_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
4990 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
4991 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
4992 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
4993 expand_insn (icode
, 3, ops
);
4994 if (!rtx_equal_p (target
, ops
[0].value
))
4995 emit_move_insn (target
, ops
[0].value
);
4999 expand_ASSUME (internal_fn
, gcall
*)
5004 expand_MASK_CALL (internal_fn
, gcall
*)
5006 /* This IFN should only exist between ifcvt and vect passes. */
5011 expand_MULBITINT (internal_fn
, gcall
*stmt
)
5014 for (int i
= 0; i
< 6; i
++)
5015 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5016 (i
& 1) ? SImode
: ptr_mode
);
5017 rtx fun
= init_one_libfunc ("__mulbitint3");
5018 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 6, args
);
5022 expand_DIVMODBITINT (internal_fn
, gcall
*stmt
)
5025 for (int i
= 0; i
< 8; i
++)
5026 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5027 (i
& 1) ? SImode
: ptr_mode
);
5028 rtx fun
= init_one_libfunc ("__divmodbitint4");
5029 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 8, args
);
5033 expand_FLOATTOBITINT (internal_fn
, gcall
*stmt
)
5035 machine_mode mode
= TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt
, 2)));
5036 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5037 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5038 rtx arg2
= expand_normal (gimple_call_arg (stmt
, 2));
5039 const char *mname
= GET_MODE_NAME (mode
);
5040 unsigned mname_len
= strlen (mname
);
5041 int len
= 12 + mname_len
;
5042 if (DECIMAL_FLOAT_MODE_P (mode
))
5044 char *libfunc_name
= XALLOCAVEC (char, len
);
5045 char *p
= libfunc_name
;
5047 if (DECIMAL_FLOAT_MODE_P (mode
))
5049 #if ENABLE_DECIMAL_BID_FORMAT
5050 memcpy (p
, "__bid_fix", 9);
5052 memcpy (p
, "__dpd_fix", 9);
5058 memcpy (p
, "__fix", 5);
5061 for (q
= mname
; *q
; q
++)
5062 *p
++ = TOLOWER (*q
);
5063 memcpy (p
, "bitint", 7);
5064 rtx fun
= init_one_libfunc (libfunc_name
);
5065 emit_library_call (fun
, LCT_NORMAL
, VOIDmode
, arg0
, ptr_mode
, arg1
,
5066 SImode
, arg2
, mode
);
5070 expand_BITINTTOFLOAT (internal_fn
, gcall
*stmt
)
5072 tree lhs
= gimple_call_lhs (stmt
);
5075 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
5076 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5077 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5078 const char *mname
= GET_MODE_NAME (mode
);
5079 unsigned mname_len
= strlen (mname
);
5080 int len
= 14 + mname_len
;
5081 if (DECIMAL_FLOAT_MODE_P (mode
))
5083 char *libfunc_name
= XALLOCAVEC (char, len
);
5084 char *p
= libfunc_name
;
5086 if (DECIMAL_FLOAT_MODE_P (mode
))
5088 #if ENABLE_DECIMAL_BID_FORMAT
5089 memcpy (p
, "__bid_floatbitint", 17);
5091 memcpy (p
, "__dpd_floatbitint", 17);
5097 memcpy (p
, "__floatbitint", 13);
5100 for (q
= mname
; *q
; q
++)
5101 *p
++ = TOLOWER (*q
);
5103 rtx fun
= init_one_libfunc (libfunc_name
);
5104 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5105 rtx val
= emit_library_call_value (fun
, target
, LCT_PURE
, mode
,
5106 arg0
, ptr_mode
, arg1
, SImode
);
5108 emit_move_insn (target
, val
);
5112 expand_bitquery (internal_fn fn
, gcall
*stmt
)
5114 tree lhs
= gimple_call_lhs (stmt
);
5115 if (lhs
== NULL_TREE
)
5117 tree arg
= gimple_call_arg (stmt
, 0);
5118 if (TREE_CODE (arg
) == INTEGER_CST
)
5120 tree ret
= fold_const_call (as_combined_fn (fn
), TREE_TYPE (arg
), arg
);
5121 gcc_checking_assert (ret
&& TREE_CODE (ret
) == INTEGER_CST
);
5122 expand_assignment (lhs
, ret
, false);
5129 expand_CLRSB (internal_fn fn
, gcall
*stmt
)
5131 if (expand_bitquery (fn
, stmt
))
5132 expand_unary_optab_fn (fn
, stmt
, clrsb_optab
);
5136 expand_CLZ (internal_fn fn
, gcall
*stmt
)
5138 if (expand_bitquery (fn
, stmt
))
5139 expand_unary_optab_fn (fn
, stmt
, clz_optab
);
5143 expand_CTZ (internal_fn fn
, gcall
*stmt
)
5145 if (expand_bitquery (fn
, stmt
))
5146 expand_unary_optab_fn (fn
, stmt
, ctz_optab
);
5150 expand_FFS (internal_fn fn
, gcall
*stmt
)
5152 if (expand_bitquery (fn
, stmt
))
5153 expand_unary_optab_fn (fn
, stmt
, ffs_optab
);
5157 expand_PARITY (internal_fn fn
, gcall
*stmt
)
5159 if (expand_bitquery (fn
, stmt
))
5160 expand_unary_optab_fn (fn
, stmt
, parity_optab
);
5164 expand_POPCOUNT (internal_fn fn
, gcall
*stmt
)
5166 if (!expand_bitquery (fn
, stmt
))
5168 if (gimple_call_num_args (stmt
) == 1)
5170 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5173 /* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5174 because the result is only used in an equality comparison against 1.
5175 Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5176 or (arg ^ (arg - 1)) > arg - 1 is cheaper.
5177 If .POPCOUNT second argument is 0, we additionally know that arg
5178 is non-zero, so use arg & (arg - 1) == 0 instead. */
5179 bool speed_p
= optimize_insn_for_speed_p ();
5180 tree lhs
= gimple_call_lhs (stmt
);
5181 tree arg
= gimple_call_arg (stmt
, 0);
5182 bool nonzero_arg
= integer_zerop (gimple_call_arg (stmt
, 1));
5183 tree type
= TREE_TYPE (arg
);
5184 machine_mode mode
= TYPE_MODE (type
);
5185 do_pending_stack_adjust ();
5187 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5188 rtx_insn
*popcount_insns
= get_insns ();
5191 rtx plhs
= expand_normal (lhs
);
5192 rtx pcmp
= emit_store_flag (NULL_RTX
, EQ
, plhs
, const1_rtx
, mode
, 0, 0);
5193 if (pcmp
== NULL_RTX
)
5197 emit_insn (popcount_insns
);
5200 rtx_insn
*popcount_cmp_insns
= get_insns ();
5203 rtx op0
= expand_normal (arg
);
5204 rtx argm1
= expand_simple_binop (mode
, PLUS
, op0
, constm1_rtx
, NULL_RTX
,
5206 if (argm1
== NULL_RTX
)
5208 rtx argxorargm1
= expand_simple_binop (mode
, nonzero_arg
? AND
: XOR
, op0
,
5209 argm1
, NULL_RTX
, 1, OPTAB_DIRECT
);
5210 if (argxorargm1
== NULL_RTX
)
5214 cmp
= emit_store_flag (NULL_RTX
, EQ
, argxorargm1
, const0_rtx
, mode
, 1, 1);
5216 cmp
= emit_store_flag (NULL_RTX
, GTU
, argxorargm1
, argm1
, mode
, 1, 1);
5217 if (cmp
== NULL_RTX
)
5219 rtx_insn
*cmp_insns
= get_insns ();
5221 unsigned popcount_cost
= (seq_cost (popcount_insns
, speed_p
)
5222 + seq_cost (popcount_cmp_insns
, speed_p
));
5223 unsigned cmp_cost
= seq_cost (cmp_insns
, speed_p
);
5224 if (popcount_cost
<= cmp_cost
)
5225 emit_insn (popcount_insns
);
5228 emit_insn (cmp_insns
);
5229 plhs
= expand_normal (lhs
);
5230 if (GET_MODE (cmp
) != GET_MODE (plhs
))
5231 cmp
= convert_to_mode (GET_MODE (plhs
), cmp
, 1);
5232 emit_move_insn (plhs
, cmp
);