2 Copyright (C) 2011-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* The names of each internal function, indexed by function number. */
60 const char *const internal_fn_name_array
[] = {
61 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62 #include "internal-fn.def"
66 /* The ECF_* flags of each internal function, indexed by function number. */
67 const int internal_fn_flags_array
[] = {
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69 #include "internal-fn.def"
73 /* Return the internal function called NAME, or IFN_LAST if there's
77 lookup_internal_fn (const char *name
)
79 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
80 static name_to_fn_map_type
*name_to_fn_map
;
84 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
85 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
86 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
89 internal_fn
*entry
= name_to_fn_map
->get (name
);
90 return entry
? *entry
: IFN_LAST
;
93 /* Geven an internal_fn IFN that is a widening function, return its
94 corresponding LO and HI internal_fns. */
97 lookup_hilo_internal_fn (internal_fn ifn
, internal_fn
*lo
, internal_fn
*hi
)
99 gcc_assert (widening_fn_p (ifn
));
105 #undef DEF_INTERNAL_FN
106 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
107 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
108 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
110 *lo = internal_fn (IFN_##NAME##_LO); \
111 *hi = internal_fn (IFN_##NAME##_HI); \
113 #include "internal-fn.def"
114 #undef DEF_INTERNAL_FN
115 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
119 /* Given an internal_fn IFN that is a widening function, return its
120 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
123 lookup_evenodd_internal_fn (internal_fn ifn
, internal_fn
*even
,
126 gcc_assert (widening_fn_p (ifn
));
132 #undef DEF_INTERNAL_FN
133 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
134 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
135 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
137 *even = internal_fn (IFN_##NAME##_EVEN); \
138 *odd = internal_fn (IFN_##NAME##_ODD); \
140 #include "internal-fn.def"
141 #undef DEF_INTERNAL_FN
142 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
147 /* Fnspec of each internal function, indexed by function number. */
148 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
153 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
154 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
155 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
156 #include "internal-fn.def"
157 internal_fn_fnspec_array
[IFN_LAST
] = 0;
160 /* Create static initializers for the information returned by
161 direct_internal_fn. */
162 #define not_direct { -2, -2, false }
163 #define mask_load_direct { -1, 2, false }
164 #define load_lanes_direct { -1, -1, false }
165 #define mask_load_lanes_direct { -1, -1, false }
166 #define gather_load_direct { 3, 1, false }
167 #define len_load_direct { -1, -1, false }
168 #define len_maskload_direct { -1, 3, false }
169 #define mask_store_direct { 3, 2, false }
170 #define store_lanes_direct { 0, 0, false }
171 #define mask_store_lanes_direct { 0, 0, false }
172 #define vec_cond_mask_direct { 1, 0, false }
173 #define vec_cond_direct { 2, 0, false }
174 #define scatter_store_direct { 3, 1, false }
175 #define len_store_direct { 3, 3, false }
176 #define len_maskstore_direct { 4, 3, false }
177 #define vec_set_direct { 3, 3, false }
178 #define unary_direct { 0, 0, true }
179 #define unary_convert_direct { -1, 0, true }
180 #define binary_direct { 0, 0, true }
181 #define ternary_direct { 0, 0, true }
182 #define cond_unary_direct { 1, 1, true }
183 #define cond_binary_direct { 1, 1, true }
184 #define cond_ternary_direct { 1, 1, true }
185 #define while_direct { 0, 2, false }
186 #define fold_extract_direct { 2, 2, false }
187 #define fold_left_direct { 1, 1, false }
188 #define mask_fold_left_direct { 1, 1, false }
189 #define check_ptrs_direct { 0, 0, false }
191 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
192 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
193 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
194 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
195 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
196 #include "internal-fn.def"
200 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
201 output operands and NINPUTS input operands, where NOUTPUTS is either
202 0 or 1. The output operand (if any) comes first, followed by the
203 NINPUTS input operands. */
206 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
207 unsigned int ninputs
)
209 gcc_assert (icode
!= CODE_FOR_nothing
);
211 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
212 unsigned int opno
= 0;
213 rtx lhs_rtx
= NULL_RTX
;
214 tree lhs
= gimple_call_lhs (stmt
);
218 gcc_assert (noutputs
== 1);
220 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
222 /* Do not assign directly to a promoted subreg, since there is no
223 guarantee that the instruction will leave the upper bits of the
224 register in the state required by SUBREG_PROMOTED_SIGN. */
226 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
228 create_output_operand (&ops
[opno
], dest
,
229 insn_data
[icode
].operand
[opno
].mode
);
235 for (unsigned int i
= 0; i
< ninputs
; ++i
)
237 tree rhs
= gimple_call_arg (stmt
, i
);
238 tree rhs_type
= TREE_TYPE (rhs
);
239 rtx rhs_rtx
= expand_normal (rhs
);
240 if (INTEGRAL_TYPE_P (rhs_type
))
241 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
242 TYPE_MODE (rhs_type
),
243 TYPE_UNSIGNED (rhs_type
));
245 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
249 gcc_assert (opno
== noutputs
+ ninputs
);
250 expand_insn (icode
, opno
, ops
);
251 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
253 /* If the return value has an integral type, convert the instruction
254 result to that type. This is useful for things that return an
255 int regardless of the size of the input. If the instruction result
256 is smaller than required, assume that it is signed.
258 If the return value has a nonintegral type, its mode must match
259 the instruction result. */
260 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
262 /* If this is a scalar in a register that is stored in a wider
263 mode than the declared mode, compute the result into its
264 declared mode and then convert to the wider mode. */
265 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
266 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
267 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
268 SUBREG_PROMOTED_SIGN (lhs_rtx
));
270 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
271 emit_move_insn (lhs_rtx
, ops
[0].value
);
274 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
275 convert_move (lhs_rtx
, ops
[0].value
, 0);
280 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
281 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
283 static enum insn_code
284 get_multi_vector_move (tree array_type
, convert_optab optab
)
289 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
290 imode
= TYPE_MODE (array_type
);
291 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
293 return convert_optab_handler (optab
, imode
, vmode
);
296 /* Expand LOAD_LANES call STMT using optab OPTAB. */
299 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
301 class expand_operand ops
[2];
305 lhs
= gimple_call_lhs (stmt
);
306 rhs
= gimple_call_arg (stmt
, 0);
307 type
= TREE_TYPE (lhs
);
309 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
310 mem
= expand_normal (rhs
);
312 gcc_assert (MEM_P (mem
));
313 PUT_MODE (mem
, TYPE_MODE (type
));
315 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
316 create_fixed_operand (&ops
[1], mem
);
317 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
318 if (!rtx_equal_p (target
, ops
[0].value
))
319 emit_move_insn (target
, ops
[0].value
);
322 /* Expand STORE_LANES call STMT using optab OPTAB. */
325 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
327 class expand_operand ops
[2];
331 lhs
= gimple_call_lhs (stmt
);
332 rhs
= gimple_call_arg (stmt
, 0);
333 type
= TREE_TYPE (rhs
);
335 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
336 reg
= expand_normal (rhs
);
338 gcc_assert (MEM_P (target
));
339 PUT_MODE (target
, TYPE_MODE (type
));
341 create_fixed_operand (&ops
[0], target
);
342 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
343 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
347 expand_ANNOTATE (internal_fn
, gcall
*)
352 /* This should get expanded in omp_device_lower pass. */
355 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
360 /* This should get expanded in omp_device_lower pass. */
363 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
368 /* Allocate per-lane storage and begin non-uniform execution region. */
371 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
374 tree lhs
= gimple_call_lhs (stmt
);
376 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
378 target
= gen_reg_rtx (Pmode
);
379 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
380 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
381 class expand_operand ops
[3];
382 create_output_operand (&ops
[0], target
, Pmode
);
383 create_input_operand (&ops
[1], size
, Pmode
);
384 create_input_operand (&ops
[2], align
, Pmode
);
385 gcc_assert (targetm
.have_omp_simt_enter ());
386 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
387 if (!rtx_equal_p (target
, ops
[0].value
))
388 emit_move_insn (target
, ops
[0].value
);
391 /* Deallocate per-lane storage and leave non-uniform execution region. */
394 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
396 gcc_checking_assert (!gimple_call_lhs (stmt
));
397 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
398 class expand_operand ops
[1];
399 create_input_operand (&ops
[0], arg
, Pmode
);
400 gcc_assert (targetm
.have_omp_simt_exit ());
401 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
404 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
405 without SIMT execution this should be expanded in omp_device_lower pass. */
408 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
410 tree lhs
= gimple_call_lhs (stmt
);
414 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
415 gcc_assert (targetm
.have_omp_simt_lane ());
416 emit_insn (targetm
.gen_omp_simt_lane (target
));
419 /* This should get expanded in omp_device_lower pass. */
422 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
427 /* This should get expanded in omp_device_lower pass. */
430 expand_GOMP_TARGET_REV (internal_fn
, gcall
*)
435 /* Lane index of the first SIMT lane that supplies a non-zero argument.
436 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
437 lane that executed the last iteration for handling OpenMP lastprivate. */
440 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
442 tree lhs
= gimple_call_lhs (stmt
);
446 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
447 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
448 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
449 class expand_operand ops
[2];
450 create_output_operand (&ops
[0], target
, mode
);
451 create_input_operand (&ops
[1], cond
, mode
);
452 gcc_assert (targetm
.have_omp_simt_last_lane ());
453 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
454 if (!rtx_equal_p (target
, ops
[0].value
))
455 emit_move_insn (target
, ops
[0].value
);
458 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
461 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
463 tree lhs
= gimple_call_lhs (stmt
);
467 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
468 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
469 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
470 class expand_operand ops
[2];
471 create_output_operand (&ops
[0], target
, mode
);
472 create_input_operand (&ops
[1], ctr
, mode
);
473 gcc_assert (targetm
.have_omp_simt_ordered ());
474 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
475 if (!rtx_equal_p (target
, ops
[0].value
))
476 emit_move_insn (target
, ops
[0].value
);
479 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
480 any lane supplies a non-zero argument. */
483 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
485 tree lhs
= gimple_call_lhs (stmt
);
489 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
490 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
491 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
492 class expand_operand ops
[2];
493 create_output_operand (&ops
[0], target
, mode
);
494 create_input_operand (&ops
[1], cond
, mode
);
495 gcc_assert (targetm
.have_omp_simt_vote_any ());
496 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
497 if (!rtx_equal_p (target
, ops
[0].value
))
498 emit_move_insn (target
, ops
[0].value
);
501 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
502 is destination lane index XOR given offset. */
505 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
507 tree lhs
= gimple_call_lhs (stmt
);
511 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
512 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
513 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
514 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
515 class expand_operand ops
[3];
516 create_output_operand (&ops
[0], target
, mode
);
517 create_input_operand (&ops
[1], src
, mode
);
518 create_input_operand (&ops
[2], idx
, SImode
);
519 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
520 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
521 if (!rtx_equal_p (target
, ops
[0].value
))
522 emit_move_insn (target
, ops
[0].value
);
525 /* Exchange between SIMT lanes according to given source lane index. */
528 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
530 tree lhs
= gimple_call_lhs (stmt
);
534 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
535 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
536 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
537 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
538 class expand_operand ops
[3];
539 create_output_operand (&ops
[0], target
, mode
);
540 create_input_operand (&ops
[1], src
, mode
);
541 create_input_operand (&ops
[2], idx
, SImode
);
542 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
543 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
544 if (!rtx_equal_p (target
, ops
[0].value
))
545 emit_move_insn (target
, ops
[0].value
);
548 /* This should get expanded in adjust_simduid_builtins. */
551 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
556 /* This should get expanded in adjust_simduid_builtins. */
559 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
564 /* This should get expanded in adjust_simduid_builtins. */
567 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
572 /* This should get expanded in adjust_simduid_builtins. */
575 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
580 /* This should get expanded in adjust_simduid_builtins. */
583 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
588 /* This should get expanded in the sanopt pass. */
591 expand_UBSAN_NULL (internal_fn
, gcall
*)
596 /* This should get expanded in the sanopt pass. */
599 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
604 /* This should get expanded in the sanopt pass. */
607 expand_UBSAN_VPTR (internal_fn
, gcall
*)
612 /* This should get expanded in the sanopt pass. */
615 expand_UBSAN_PTR (internal_fn
, gcall
*)
620 /* This should get expanded in the sanopt pass. */
623 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
628 /* This should get expanded in the sanopt pass. */
631 expand_HWASAN_CHECK (internal_fn
, gcall
*)
636 /* For hwasan stack tagging:
637 Clear tags on the dynamically allocated space.
638 For use after an object dynamically allocated on the stack goes out of
641 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
643 gcc_assert (Pmode
== ptr_mode
);
644 tree restored_position
= gimple_call_arg (gc
, 0);
645 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
647 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
648 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
649 stack_pointer_rtx
, NULL_RTX
, 0,
651 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
652 virtual_stack_dynamic_rtx
, Pmode
,
653 HWASAN_STACK_BACKGROUND
, QImode
,
657 /* For hwasan stack tagging:
658 Return a tag to be used for a dynamic allocation. */
660 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
662 tree tag
= gimple_call_lhs (gc
);
663 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
664 machine_mode mode
= GET_MODE (target
);
665 gcc_assert (mode
== QImode
);
667 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
668 gcc_assert (base_tag
);
669 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
670 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
671 target
, /* unsignedp = */1,
673 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
675 /* Really need to put the tag into the `target` RTX. */
676 if (chosen_tag
!= target
)
678 rtx temp
= chosen_tag
;
679 gcc_assert (GET_MODE (chosen_tag
) == mode
);
680 emit_move_insn (target
, temp
);
683 hwasan_increment_frame_tag ();
686 /* For hwasan stack tagging:
687 Tag a region of space in the shadow stack according to the base pointer of
688 an object on the stack. N.b. the length provided in the internal call is
689 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
691 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
693 gcc_assert (ptr_mode
== Pmode
);
694 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
695 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
697 tree base
= gimple_call_arg (gc
, 1);
698 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
699 rtx base_rtx
= expand_normal (base
);
701 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
702 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
703 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
705 tree len
= gimple_call_arg (gc
, 2);
706 rtx r_len
= expand_normal (len
);
708 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
709 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
710 tag
, QImode
, r_len
, Pmode
);
713 /* For hwasan stack tagging:
714 Store a tag into a pointer. */
716 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
718 gcc_assert (ptr_mode
== Pmode
);
719 tree g_target
= gimple_call_lhs (gc
);
720 tree g_ptr
= gimple_call_arg (gc
, 0);
721 tree g_tag
= gimple_call_arg (gc
, 1);
723 rtx ptr
= expand_normal (g_ptr
);
724 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
725 rtx target
= expand_normal (g_target
);
727 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
728 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
729 if (tagged_value
!= target
)
730 emit_move_insn (target
, tagged_value
);
733 /* This should get expanded in the sanopt pass. */
736 expand_ASAN_CHECK (internal_fn
, gcall
*)
741 /* This should get expanded in the sanopt pass. */
744 expand_ASAN_MARK (internal_fn
, gcall
*)
749 /* This should get expanded in the sanopt pass. */
752 expand_ASAN_POISON (internal_fn
, gcall
*)
757 /* This should get expanded in the sanopt pass. */
760 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
765 /* This should get expanded in the tsan pass. */
768 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
773 /* This should get expanded in the lower pass. */
776 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
778 error_at (gimple_location (call
),
779 "invalid use of attribute %<fallthrough%>");
782 /* Return minimum precision needed to represent all values
783 of ARG in SIGNed integral type. */
786 get_min_precision (tree arg
, signop sign
)
788 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
790 signop orig_sign
= sign
;
791 if (TREE_CODE (arg
) == INTEGER_CST
)
794 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
796 widest_int w
= wi::to_widest (arg
);
797 w
= wi::ext (w
, prec
, sign
);
798 p
= wi::min_precision (w
, sign
);
801 p
= wi::min_precision (wi::to_wide (arg
), sign
);
802 return MIN (p
, prec
);
804 while (CONVERT_EXPR_P (arg
)
805 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
806 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
808 arg
= TREE_OPERAND (arg
, 0);
809 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
811 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
813 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
814 return prec
+ (orig_sign
!= sign
);
815 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
818 return prec
+ (orig_sign
!= sign
);
820 if (CONVERT_EXPR_P (arg
)
821 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
822 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
824 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
825 If y_2's min precision is smaller than prec, return that. */
826 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
828 return oprec
+ (orig_sign
!= sign
);
830 if (TREE_CODE (arg
) != SSA_NAME
)
831 return prec
+ (orig_sign
!= sign
);
833 while (!get_global_range_query ()->range_of_expr (r
, arg
)
837 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
838 if (is_gimple_assign (g
)
839 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
841 tree t
= gimple_assign_rhs1 (g
);
842 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
843 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
846 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
848 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
850 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
851 return prec
+ (orig_sign
!= sign
);
852 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
855 return prec
+ (orig_sign
!= sign
);
859 return prec
+ (orig_sign
!= sign
);
861 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
863 int p1
= wi::min_precision (r
.lower_bound (), sign
);
864 int p2
= wi::min_precision (r
.upper_bound (), sign
);
866 prec
= MIN (prec
, p1
);
868 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
870 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
871 prec
= MIN (prec
, p
);
873 return prec
+ (orig_sign
!= sign
);
876 /* Helper for expand_*_overflow. Set the __imag__ part to true
877 (1 except for signed:1 type, in which case store -1). */
880 expand_arith_set_overflow (tree lhs
, rtx target
)
882 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
883 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
884 write_complex_part (target
, constm1_rtx
, true, false);
886 write_complex_part (target
, const1_rtx
, true, false);
889 /* Helper for expand_*_overflow. Store RES into the __real__ part
890 of TARGET. If RES has larger MODE than __real__ part of TARGET,
891 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
892 if LHS has smaller precision than its mode. */
895 expand_arith_overflow_result_store (tree lhs
, rtx target
,
896 scalar_int_mode mode
, rtx res
)
898 scalar_int_mode tgtmode
899 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
903 rtx_code_label
*done_label
= gen_label_rtx ();
904 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
905 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
906 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
907 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
908 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
909 profile_probability::very_likely ());
910 expand_arith_set_overflow (lhs
, target
);
911 emit_label (done_label
);
913 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
914 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
917 rtx_code_label
*done_label
= gen_label_rtx ();
918 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
923 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
925 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
926 true, OPTAB_LIB_WIDEN
);
930 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
932 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
935 do_compare_rtx_and_jump (res
, lres
,
936 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
937 profile_probability::very_likely ());
938 expand_arith_set_overflow (lhs
, target
);
939 emit_label (done_label
);
941 write_complex_part (target
, lres
, false, false);
944 /* Helper for expand_*_overflow. Store RES into TARGET. */
947 expand_ubsan_result_store (rtx target
, rtx res
)
949 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
950 /* If this is a scalar in a register that is stored in a wider mode
951 than the declared mode, compute the result into its declared mode
952 and then convert to the wider mode. Our value is the computed
954 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
956 emit_move_insn (target
, res
);
959 /* Add sub/add overflow checking to the statement STMT.
960 CODE says whether the operation is +, or -. */
963 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
964 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
965 bool uns1_p
, bool is_ubsan
, tree
*datap
)
967 rtx res
, target
= NULL_RTX
;
969 rtx_code_label
*done_label
= gen_label_rtx ();
970 rtx_code_label
*do_error
= gen_label_rtx ();
971 do_pending_stack_adjust ();
972 rtx op0
= expand_normal (arg0
);
973 rtx op1
= expand_normal (arg1
);
974 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
975 int prec
= GET_MODE_PRECISION (mode
);
976 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
980 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
984 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
986 write_complex_part (target
, const0_rtx
, true, false);
989 /* We assume both operands and result have the same precision
990 here (GET_MODE_BITSIZE (mode)), S stands for signed type
991 with that precision, U for unsigned type with that precision,
992 sgn for unsigned most significant bit in that precision.
993 s1 is signed first operand, u1 is unsigned first operand,
994 s2 is signed second operand, u2 is unsigned second operand,
995 sr is signed result, ur is unsigned result and the following
996 rules say how to compute result (which is always result of
997 the operands as if both were unsigned, cast to the right
998 signedness) and how to compute whether operation overflowed.
1001 res = (S) ((U) s1 + (U) s2)
1002 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1004 res = (S) ((U) s1 - (U) s2)
1005 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1008 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1011 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1013 res = (S) ((U) s1 + u2)
1014 ovf = ((U) res ^ sgn) < u2
1019 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1021 res = (S) ((U) s1 - u2)
1022 ovf = u2 > ((U) s1 ^ sgn)
1025 ovf = s1 < 0 || u2 > (U) s1
1028 ovf = u1 >= ((U) s2 ^ sgn)
1033 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1035 res = (U) s1 + (U) s2
1036 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1039 ovf = (U) res < u2 || res < 0
1042 ovf = u1 >= u2 ? res < 0 : res >= 0
1044 res = (U) s1 - (U) s2
1045 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1047 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
1049 /* PLUS_EXPR is commutative, if operand signedness differs,
1050 canonicalize to the first operand being signed and second
1051 unsigned to simplify following code. */
1052 std::swap (op0
, op1
);
1053 std::swap (arg0
, arg1
);
1058 /* u1 +- u2 -> ur */
1059 if (uns0_p
&& uns1_p
&& unsr_p
)
1061 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
1062 : usubv4_optab
, mode
);
1063 if (icode
!= CODE_FOR_nothing
)
1065 class expand_operand ops
[4];
1066 rtx_insn
*last
= get_last_insn ();
1068 res
= gen_reg_rtx (mode
);
1069 create_output_operand (&ops
[0], res
, mode
);
1070 create_input_operand (&ops
[1], op0
, mode
);
1071 create_input_operand (&ops
[2], op1
, mode
);
1072 create_fixed_operand (&ops
[3], do_error
);
1073 if (maybe_expand_insn (icode
, 4, ops
))
1075 last
= get_last_insn ();
1076 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1078 && any_condjump_p (last
)
1079 && !find_reg_note (last
, REG_BR_PROB
, 0))
1080 add_reg_br_prob_note (last
,
1081 profile_probability::very_unlikely ());
1082 emit_jump (done_label
);
1083 goto do_error_label
;
1086 delete_insns_since (last
);
1089 /* Compute the operation. On RTL level, the addition is always
1091 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1092 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1094 /* For PLUS_EXPR, the operation is commutative, so we can pick
1095 operand to compare against. For prec <= BITS_PER_WORD, I think
1096 preferring REG operand is better over CONST_INT, because
1097 the CONST_INT might enlarge the instruction or CSE would need
1098 to figure out we'd already loaded it into a register before.
1099 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1100 as then the multi-word comparison can be perhaps simplified. */
1101 if (code
== PLUS_EXPR
1102 && (prec
<= BITS_PER_WORD
1103 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1104 : CONST_SCALAR_INT_P (op1
)))
1106 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1107 true, mode
, NULL_RTX
, NULL
, done_label
,
1108 profile_probability::very_likely ());
1109 goto do_error_label
;
1112 /* s1 +- u2 -> sr */
1113 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1115 /* Compute the operation. On RTL level, the addition is always
1117 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1118 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1119 rtx tem
= expand_binop (mode
, add_optab
,
1120 code
== PLUS_EXPR
? res
: op0
, sgn
,
1121 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1122 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1123 done_label
, profile_probability::very_likely ());
1124 goto do_error_label
;
1128 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1130 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1132 /* As we've changed op1, we have to avoid using the value range
1133 for the original argument. */
1134 arg1
= error_mark_node
;
1140 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1142 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1144 /* As we've changed op0, we have to avoid using the value range
1145 for the original argument. */
1146 arg0
= error_mark_node
;
1152 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1154 /* Compute the operation. On RTL level, the addition is always
1156 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1158 int pos_neg
= get_range_pos_neg (arg0
);
1160 /* If ARG0 is known to be always negative, this is always overflow. */
1161 emit_jump (do_error
);
1162 else if (pos_neg
== 3)
1163 /* If ARG0 is not known to be always positive, check at runtime. */
1164 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1165 NULL
, do_error
, profile_probability::very_unlikely ());
1166 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1167 done_label
, profile_probability::very_likely ());
1168 goto do_error_label
;
1172 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1174 /* Compute the operation. On RTL level, the addition is always
1176 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1178 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1180 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1181 done_label
, profile_probability::very_likely ());
1182 goto do_error_label
;
1186 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1188 /* Compute the operation. On RTL level, the addition is always
1190 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1192 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1193 NULL
, do_error
, profile_probability::very_unlikely ());
1195 /* The operation is commutative, so we can pick operand to compare
1196 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1197 is better over CONST_INT, because the CONST_INT might enlarge the
1198 instruction or CSE would need to figure out we'd already loaded it
1199 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1200 might be more beneficial, as then the multi-word comparison can be
1201 perhaps simplified. */
1202 if (prec
<= BITS_PER_WORD
1203 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1204 : CONST_SCALAR_INT_P (op0
))
1206 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1207 done_label
, profile_probability::very_likely ());
1208 goto do_error_label
;
1211 /* s1 +- s2 -> ur */
1212 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1214 /* Compute the operation. On RTL level, the addition is always
1216 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1217 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1218 int pos_neg
= get_range_pos_neg (arg1
);
1219 if (code
== PLUS_EXPR
)
1221 int pos_neg0
= get_range_pos_neg (arg0
);
1222 if (pos_neg0
!= 3 && pos_neg
== 3)
1224 std::swap (op0
, op1
);
1231 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1232 ? and_optab
: ior_optab
,
1233 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1234 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1235 NULL
, done_label
, profile_probability::very_likely ());
1239 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1240 do_compare_rtx_and_jump (op1
, const0_rtx
,
1241 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1242 NULL_RTX
, NULL
, do_ior_label
,
1243 profile_probability::even ());
1244 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1246 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1247 NULL
, done_label
, profile_probability::very_likely ());
1248 emit_jump (do_error
);
1249 emit_label (do_ior_label
);
1250 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1252 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1253 NULL
, done_label
, profile_probability::very_likely ());
1255 goto do_error_label
;
1259 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1261 /* Compute the operation. On RTL level, the addition is always
1263 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1265 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1266 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1267 op0_geu_op1
, profile_probability::even ());
1268 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1269 NULL
, done_label
, profile_probability::very_likely ());
1270 emit_jump (do_error
);
1271 emit_label (op0_geu_op1
);
1272 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1273 NULL
, done_label
, profile_probability::very_likely ());
1274 goto do_error_label
;
1277 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1279 /* s1 +- s2 -> sr */
1282 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1283 : subv4_optab
, mode
);
1284 if (icode
!= CODE_FOR_nothing
)
1286 class expand_operand ops
[4];
1287 rtx_insn
*last
= get_last_insn ();
1289 res
= gen_reg_rtx (mode
);
1290 create_output_operand (&ops
[0], res
, mode
);
1291 create_input_operand (&ops
[1], op0
, mode
);
1292 create_input_operand (&ops
[2], op1
, mode
);
1293 create_fixed_operand (&ops
[3], do_error
);
1294 if (maybe_expand_insn (icode
, 4, ops
))
1296 last
= get_last_insn ();
1297 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1299 && any_condjump_p (last
)
1300 && !find_reg_note (last
, REG_BR_PROB
, 0))
1301 add_reg_br_prob_note (last
,
1302 profile_probability::very_unlikely ());
1303 emit_jump (done_label
);
1304 goto do_error_label
;
1307 delete_insns_since (last
);
1310 /* Compute the operation. On RTL level, the addition is always
1312 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1313 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1315 /* If we can prove that one of the arguments (for MINUS_EXPR only
1316 the second operand, as subtraction is not commutative) is always
1317 non-negative or always negative, we can do just one comparison
1318 and conditional jump. */
1319 int pos_neg
= get_range_pos_neg (arg1
);
1320 if (code
== PLUS_EXPR
)
1322 int pos_neg0
= get_range_pos_neg (arg0
);
1323 if (pos_neg0
!= 3 && pos_neg
== 3)
1325 std::swap (op0
, op1
);
1330 /* Addition overflows if and only if the two operands have the same sign,
1331 and the result has the opposite sign. Subtraction overflows if and
1332 only if the two operands have opposite sign, and the subtrahend has
1333 the same sign as the result. Here 0 is counted as positive. */
1336 /* Compute op0 ^ op1 (operands have opposite sign). */
1337 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1340 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1341 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1345 if (code
== PLUS_EXPR
)
1347 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1348 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1349 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1354 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1355 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1356 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1360 /* No overflow if the result has bit sign cleared. */
1361 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1362 NULL
, done_label
, profile_probability::very_likely ());
1365 /* Compare the result of the operation with the first operand.
1366 No overflow for addition if second operand is positive and result
1367 is larger or second operand is negative and result is smaller.
1368 Likewise for subtraction with sign of second operand flipped. */
1370 do_compare_rtx_and_jump (res
, op0
,
1371 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1372 false, mode
, NULL_RTX
, NULL
, done_label
,
1373 profile_probability::very_likely ());
1377 emit_label (do_error
);
1380 /* Expand the ubsan builtin call. */
1382 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1386 do_pending_stack_adjust ();
1389 expand_arith_set_overflow (lhs
, target
);
1392 emit_label (done_label
);
1397 expand_ubsan_result_store (target
, res
);
1401 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1404 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1409 /* Add negate overflow checking to the statement STMT. */
1412 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1417 rtx_code_label
*done_label
, *do_error
;
1418 rtx target
= NULL_RTX
;
1420 done_label
= gen_label_rtx ();
1421 do_error
= gen_label_rtx ();
1423 do_pending_stack_adjust ();
1424 op1
= expand_normal (arg1
);
1426 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1429 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1431 write_complex_part (target
, const0_rtx
, true, false);
1434 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1435 if (icode
!= CODE_FOR_nothing
)
1437 class expand_operand ops
[3];
1438 rtx_insn
*last
= get_last_insn ();
1440 res
= gen_reg_rtx (mode
);
1441 create_output_operand (&ops
[0], res
, mode
);
1442 create_input_operand (&ops
[1], op1
, mode
);
1443 create_fixed_operand (&ops
[2], do_error
);
1444 if (maybe_expand_insn (icode
, 3, ops
))
1446 last
= get_last_insn ();
1447 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1449 && any_condjump_p (last
)
1450 && !find_reg_note (last
, REG_BR_PROB
, 0))
1451 add_reg_br_prob_note (last
,
1452 profile_probability::very_unlikely ());
1453 emit_jump (done_label
);
1457 delete_insns_since (last
);
1458 icode
= CODE_FOR_nothing
;
1462 if (icode
== CODE_FOR_nothing
)
1464 /* Compute the operation. On RTL level, the addition is always
1466 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1468 /* Compare the operand with the most negative value. */
1469 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1470 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1471 done_label
, profile_probability::very_likely ());
1474 emit_label (do_error
);
1477 /* Expand the ubsan builtin call. */
1479 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1480 arg1
, NULL_TREE
, datap
);
1483 do_pending_stack_adjust ();
1486 expand_arith_set_overflow (lhs
, target
);
1489 emit_label (done_label
);
1494 expand_ubsan_result_store (target
, res
);
1496 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1500 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1501 mode MODE can be expanded without using a libcall. */
1504 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1505 rtx op0
, rtx op1
, bool uns
)
1507 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1508 != CODE_FOR_nothing
)
1511 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1512 != CODE_FOR_nothing
)
1515 rtx_insn
*last
= get_last_insn ();
1516 if (CONSTANT_P (op0
))
1517 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1519 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1520 if (CONSTANT_P (op1
))
1521 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1523 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1524 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1525 delete_insns_since (last
);
1526 return ret
!= NULL_RTX
;
1529 /* Add mul overflow checking to the statement STMT. */
1532 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1533 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1538 rtx_code_label
*done_label
, *do_error
;
1539 rtx target
= NULL_RTX
;
1541 enum insn_code icode
;
1543 done_label
= gen_label_rtx ();
1544 do_error
= gen_label_rtx ();
1546 do_pending_stack_adjust ();
1547 op0
= expand_normal (arg0
);
1548 op1
= expand_normal (arg1
);
1550 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1554 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1556 write_complex_part (target
, const0_rtx
, true, false);
1560 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1562 /* We assume both operands and result have the same precision
1563 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1564 with that precision, U for unsigned type with that precision,
1565 sgn for unsigned most significant bit in that precision.
1566 s1 is signed first operand, u1 is unsigned first operand,
1567 s2 is signed second operand, u2 is unsigned second operand,
1568 sr is signed result, ur is unsigned result and the following
1569 rules say how to compute result (which is always result of
1570 the operands as if both were unsigned, cast to the right
1571 signedness) and how to compute whether operation overflowed.
1572 main_ovf (false) stands for jump on signed multiplication
1573 overflow or the main algorithm with uns == false.
1574 main_ovf (true) stands for jump on unsigned multiplication
1575 overflow or the main algorithm with uns == true.
1578 res = (S) ((U) s1 * (U) s2)
1579 ovf = main_ovf (false)
1582 ovf = main_ovf (true)
1585 ovf = (s1 < 0 && u2) || main_ovf (true)
1588 ovf = res < 0 || main_ovf (true)
1590 res = (S) ((U) s1 * u2)
1591 ovf = (S) u2 >= 0 ? main_ovf (false)
1592 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1594 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1595 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1597 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1599 if (uns0_p
&& !uns1_p
)
1601 /* Multiplication is commutative, if operand signedness differs,
1602 canonicalize to the first operand being signed and second
1603 unsigned to simplify following code. */
1604 std::swap (op0
, op1
);
1605 std::swap (arg0
, arg1
);
1610 int pos_neg0
= get_range_pos_neg (arg0
);
1611 int pos_neg1
= get_range_pos_neg (arg1
);
1614 if (!uns0_p
&& uns1_p
&& unsr_p
)
1619 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1622 /* If s1 is negative, avoid the main code, just multiply and
1623 signal overflow if op1 is not 0. */
1624 struct separate_ops ops
;
1625 ops
.code
= MULT_EXPR
;
1626 ops
.type
= TREE_TYPE (arg1
);
1627 ops
.op0
= make_tree (ops
.type
, op0
);
1628 ops
.op1
= make_tree (ops
.type
, op1
);
1629 ops
.op2
= NULL_TREE
;
1631 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1632 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1633 NULL
, done_label
, profile_probability::very_likely ());
1634 goto do_error_label
;
1636 if (get_min_precision (arg1
, UNSIGNED
)
1637 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1639 /* If the first operand is sign extended from narrower type, the
1640 second operand is zero extended from narrower type and
1641 the sum of the two precisions is smaller or equal to the
1642 result precision: if the first argument is at runtime
1643 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1644 and there will be no overflow, if the first argument is
1645 negative and the second argument zero, the result will be
1646 0 and there will be no overflow, if the first argument is
1647 negative and the second argument positive, the result when
1648 treated as signed will be negative (minimum -0x7f80 or
1649 -0x7f..f80..0) there will be always overflow. So, do
1651 ovf = (S) res < 0 */
1652 struct separate_ops ops
;
1653 ops
.code
= MULT_EXPR
;
1655 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1657 ops
.op0
= make_tree (ops
.type
, op0
);
1658 ops
.op1
= make_tree (ops
.type
, op1
);
1659 ops
.op2
= NULL_TREE
;
1661 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1662 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1663 mode
, NULL_RTX
, NULL
, done_label
,
1664 profile_probability::very_likely ());
1665 goto do_error_label
;
1667 rtx_code_label
*do_main_label
;
1668 do_main_label
= gen_label_rtx ();
1669 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1670 NULL
, do_main_label
, profile_probability::very_likely ());
1671 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1672 NULL
, do_main_label
, profile_probability::very_likely ());
1673 expand_arith_set_overflow (lhs
, target
);
1674 emit_label (do_main_label
);
1682 if (uns0_p
&& uns1_p
&& !unsr_p
)
1684 if ((pos_neg0
| pos_neg1
) == 1)
1686 /* If both arguments are zero extended from narrower types,
1687 the MSB will be clear on both and so we can pretend it is
1688 a normal s1 * s2 -> sr multiplication. */
1694 /* Rest of handling of this case after res is computed. */
1699 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1706 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1707 avoid the main code, just multiply and signal overflow
1708 unless 0 * u2 or -1 * ((U) Smin). */
1709 struct separate_ops ops
;
1710 ops
.code
= MULT_EXPR
;
1711 ops
.type
= TREE_TYPE (arg1
);
1712 ops
.op0
= make_tree (ops
.type
, op0
);
1713 ops
.op1
= make_tree (ops
.type
, op1
);
1714 ops
.op2
= NULL_TREE
;
1716 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1717 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1718 NULL
, done_label
, profile_probability::very_likely ());
1719 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1720 NULL
, do_error
, profile_probability::very_unlikely ());
1722 prec
= GET_MODE_PRECISION (mode
);
1724 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1725 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1726 NULL
, done_label
, profile_probability::very_likely ());
1727 goto do_error_label
;
1729 /* Rest of handling of this case after res is computed. */
1737 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1740 switch (pos_neg0
| pos_neg1
)
1742 case 1: /* Both operands known to be non-negative. */
1744 case 2: /* Both operands known to be negative. */
1745 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1746 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1747 /* Avoid looking at arg0/arg1 ranges, as we've changed
1749 arg0
= error_mark_node
;
1750 arg1
= error_mark_node
;
1753 if ((pos_neg0
^ pos_neg1
) == 3)
1755 /* If one operand is known to be negative and the other
1756 non-negative, this overflows always, unless the non-negative
1757 one is 0. Just do normal multiply and set overflow
1758 unless one of the operands is 0. */
1759 struct separate_ops ops
;
1760 ops
.code
= MULT_EXPR
;
1762 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1764 ops
.op0
= make_tree (ops
.type
, op0
);
1765 ops
.op1
= make_tree (ops
.type
, op1
);
1766 ops
.op2
= NULL_TREE
;
1768 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1769 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1770 true, mode
, NULL_RTX
, NULL
, done_label
,
1771 profile_probability::very_likely ());
1772 goto do_error_label
;
1774 if (get_min_precision (arg0
, SIGNED
)
1775 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1777 /* If both operands are sign extended from narrower types and
1778 the sum of the two precisions is smaller or equal to the
1779 result precision: if both arguments are at runtime
1780 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1781 and there will be no overflow, if both arguments are negative,
1782 maximum result will be 0x40..00 and there will be no overflow
1783 either, if one argument is positive and the other argument
1784 negative, the result when treated as signed will be negative
1785 and there will be always overflow, and if one argument is
1786 zero and the other negative the result will be zero and no
1789 ovf = (S) res < 0 */
1790 struct separate_ops ops
;
1791 ops
.code
= MULT_EXPR
;
1793 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1795 ops
.op0
= make_tree (ops
.type
, op0
);
1796 ops
.op1
= make_tree (ops
.type
, op1
);
1797 ops
.op2
= NULL_TREE
;
1799 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1800 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1801 mode
, NULL_RTX
, NULL
, done_label
,
1802 profile_probability::very_likely ());
1803 goto do_error_label
;
1805 /* The general case, do all the needed comparisons at runtime. */
1806 rtx_code_label
*do_main_label
, *after_negate_label
;
1808 rop0
= gen_reg_rtx (mode
);
1809 rop1
= gen_reg_rtx (mode
);
1810 emit_move_insn (rop0
, op0
);
1811 emit_move_insn (rop1
, op1
);
1814 do_main_label
= gen_label_rtx ();
1815 after_negate_label
= gen_label_rtx ();
1816 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1818 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1819 NULL
, after_negate_label
, profile_probability::very_likely ());
1820 /* Both arguments negative here, negate them and continue with
1821 normal unsigned overflow checking multiplication. */
1822 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1824 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1826 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1828 arg0
= error_mark_node
;
1829 arg1
= error_mark_node
;
1830 emit_jump (do_main_label
);
1831 emit_label (after_negate_label
);
1832 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1834 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1835 NULL
, do_main_label
,
1836 profile_probability::very_likely ());
1837 /* One argument is negative here, the other positive. This
1838 overflows always, unless one of the arguments is 0. But
1839 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1840 is, thus we can keep do_main code oring in overflow as is. */
1842 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1843 NULL
, do_main_label
,
1844 profile_probability::very_unlikely ());
1846 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1847 NULL
, do_main_label
,
1848 profile_probability::very_unlikely ());
1849 expand_arith_set_overflow (lhs
, target
);
1850 emit_label (do_main_label
);
1858 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1859 sign
= uns
? UNSIGNED
: SIGNED
;
1860 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1862 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1863 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1865 /* Optimize unsigned multiplication by power of 2 constant
1866 using 2 shifts, one for result, one to extract the shifted
1867 out bits to see if they are all zero.
1868 Don't do this if optimizing for size and we have umulv4_optab,
1869 in that case assume multiplication will be shorter.
1870 This is heuristics based on the single target that provides
1871 umulv4 right now (i?86/x86_64), if further targets add it, this
1872 might need to be revisited.
1873 Cases where both operands are constant should be folded already
1874 during GIMPLE, and cases where one operand is constant but not
1875 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1876 below can be done without multiplication, just by shifts and adds,
1877 or we'd need to divide the result (and hope it actually doesn't
1878 really divide nor multiply) and compare the result of the division
1879 with the original operand. */
1884 if (integer_pow2p (arg0
))
1886 std::swap (opn0
, opn1
);
1887 std::swap (argn0
, argn1
);
1889 int cnt
= tree_log2 (argn1
);
1890 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1892 rtx upper
= const0_rtx
;
1893 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1895 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1896 GET_MODE_PRECISION (mode
) - cnt
,
1898 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1899 NULL_RTX
, NULL
, done_label
,
1900 profile_probability::very_likely ());
1901 goto do_error_label
;
1904 if (icode
!= CODE_FOR_nothing
)
1906 class expand_operand ops
[4];
1907 rtx_insn
*last
= get_last_insn ();
1909 res
= gen_reg_rtx (mode
);
1910 create_output_operand (&ops
[0], res
, mode
);
1911 create_input_operand (&ops
[1], op0
, mode
);
1912 create_input_operand (&ops
[2], op1
, mode
);
1913 create_fixed_operand (&ops
[3], do_error
);
1914 if (maybe_expand_insn (icode
, 4, ops
))
1916 last
= get_last_insn ();
1917 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1919 && any_condjump_p (last
)
1920 && !find_reg_note (last
, REG_BR_PROB
, 0))
1921 add_reg_br_prob_note (last
,
1922 profile_probability::very_unlikely ());
1923 emit_jump (done_label
);
1927 delete_insns_since (last
);
1928 icode
= CODE_FOR_nothing
;
1932 if (icode
== CODE_FOR_nothing
)
1934 struct separate_ops ops
;
1935 int prec
= GET_MODE_PRECISION (mode
);
1936 scalar_int_mode hmode
, wmode
;
1937 ops
.op0
= make_tree (type
, op0
);
1938 ops
.op1
= make_tree (type
, op1
);
1939 ops
.op2
= NULL_TREE
;
1942 /* Optimize unsigned overflow check where we don't use the
1943 multiplication result, just whether overflow happened.
1944 If we can do MULT_HIGHPART_EXPR, that followed by
1945 comparison of the result against zero is cheapest.
1946 We'll still compute res, but it should be DCEd later. */
1952 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1953 && can_mult_highpart_p (mode
, uns
) == 1
1954 && single_imm_use (lhs
, &use
, &use_stmt
)
1955 && is_gimple_assign (use_stmt
)
1956 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1959 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1960 && targetm
.scalar_mode_supported_p (wmode
)
1961 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1964 ops
.code
= WIDEN_MULT_EXPR
;
1966 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1968 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1969 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1971 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1972 res
= convert_modes (mode
, wmode
, res
, uns
);
1974 /* For the unsigned multiplication, there was overflow if
1975 HIPART is non-zero. */
1976 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1977 NULL_RTX
, NULL
, done_label
,
1978 profile_probability::very_likely ());
1981 /* RES is used more than once, place it in a pseudo. */
1982 res
= force_reg (mode
, res
);
1984 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1986 /* RES is low half of the double width result, HIPART
1987 the high half. There was overflow if
1988 HIPART is different from RES < 0 ? -1 : 0. */
1989 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1990 NULL_RTX
, NULL
, done_label
,
1991 profile_probability::very_likely ());
1994 else if (can_mult_highpart_p (mode
, uns
) == 1)
1997 ops
.code
= MULT_HIGHPART_EXPR
;
2000 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
2002 ops
.code
= MULT_EXPR
;
2003 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2005 /* For the unsigned multiplication, there was overflow if
2006 HIPART is non-zero. */
2007 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2008 NULL_RTX
, NULL
, done_label
,
2009 profile_probability::very_likely ());
2012 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2014 /* RES is low half of the double width result, HIPART
2015 the high half. There was overflow if
2016 HIPART is different from RES < 0 ? -1 : 0. */
2017 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2018 NULL_RTX
, NULL
, done_label
,
2019 profile_probability::very_likely ());
2023 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
2024 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
2026 rtx_code_label
*large_op0
= gen_label_rtx ();
2027 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
2028 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
2029 rtx_code_label
*both_ops_large
= gen_label_rtx ();
2030 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
2031 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
2032 rtx_code_label
*do_overflow
= gen_label_rtx ();
2033 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
2035 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
2036 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
2038 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
2039 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
2040 rtx signbit0
= const0_rtx
;
2042 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
2044 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
2046 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
2047 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
2048 rtx signbit1
= const0_rtx
;
2050 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
2053 res
= gen_reg_rtx (mode
);
2055 /* True if op0 resp. op1 are known to be in the range of
2057 bool op0_small_p
= false;
2058 bool op1_small_p
= false;
2059 /* True if op0 resp. op1 are known to have all zeros or all ones
2060 in the upper half of bits, but are not known to be
2062 bool op0_medium_p
= false;
2063 bool op1_medium_p
= false;
2064 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2065 nonnegative, 1 if unknown. */
2071 else if (pos_neg0
== 2)
2075 else if (pos_neg1
== 2)
2078 unsigned int mprec0
= prec
;
2079 if (arg0
!= error_mark_node
)
2080 mprec0
= get_min_precision (arg0
, sign
);
2081 if (mprec0
<= hprec
)
2083 else if (!uns
&& mprec0
<= hprec
+ 1)
2084 op0_medium_p
= true;
2085 unsigned int mprec1
= prec
;
2086 if (arg1
!= error_mark_node
)
2087 mprec1
= get_min_precision (arg1
, sign
);
2088 if (mprec1
<= hprec
)
2090 else if (!uns
&& mprec1
<= hprec
+ 1)
2091 op1_medium_p
= true;
2093 int smaller_sign
= 1;
2094 int larger_sign
= 1;
2097 smaller_sign
= op0_sign
;
2098 larger_sign
= op1_sign
;
2100 else if (op1_small_p
)
2102 smaller_sign
= op1_sign
;
2103 larger_sign
= op0_sign
;
2105 else if (op0_sign
== op1_sign
)
2107 smaller_sign
= op0_sign
;
2108 larger_sign
= op0_sign
;
2112 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2113 NULL_RTX
, NULL
, large_op0
,
2114 profile_probability::unlikely ());
2117 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2118 NULL_RTX
, NULL
, small_op0_large_op1
,
2119 profile_probability::unlikely ());
2121 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2122 hmode to mode, the multiplication will never overflow. We can
2123 do just one hmode x hmode => mode widening multiplication. */
2124 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2125 ops
.op0
= make_tree (halfstype
, lopart0
);
2126 ops
.op1
= make_tree (halfstype
, lopart1
);
2127 ops
.code
= WIDEN_MULT_EXPR
;
2130 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2131 emit_move_insn (res
, thisres
);
2132 emit_jump (done_label
);
2134 emit_label (small_op0_large_op1
);
2136 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2137 but op1 is not, just swap the arguments and handle it as op1
2138 sign/zero extended, op0 not. */
2139 rtx larger
= gen_reg_rtx (mode
);
2140 rtx hipart
= gen_reg_rtx (hmode
);
2141 rtx lopart
= gen_reg_rtx (hmode
);
2142 emit_move_insn (larger
, op1
);
2143 emit_move_insn (hipart
, hipart1
);
2144 emit_move_insn (lopart
, lopart0
);
2145 emit_jump (one_small_one_large
);
2147 emit_label (large_op0
);
2150 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2151 NULL_RTX
, NULL
, both_ops_large
,
2152 profile_probability::unlikely ());
2154 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2155 but op0 is not, prepare larger, hipart and lopart pseudos and
2156 handle it together with small_op0_large_op1. */
2157 emit_move_insn (larger
, op0
);
2158 emit_move_insn (hipart
, hipart0
);
2159 emit_move_insn (lopart
, lopart1
);
2161 emit_label (one_small_one_large
);
2163 /* lopart is the low part of the operand that is sign extended
2164 to mode, larger is the other operand, hipart is the
2165 high part of larger and lopart0 and lopart1 are the low parts
2167 We perform lopart0 * lopart1 and lopart * hipart widening
2169 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2170 ops
.op0
= make_tree (halfutype
, lopart0
);
2171 ops
.op1
= make_tree (halfutype
, lopart1
);
2173 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2175 ops
.op0
= make_tree (halfutype
, lopart
);
2176 ops
.op1
= make_tree (halfutype
, hipart
);
2177 rtx loxhi
= gen_reg_rtx (mode
);
2178 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2179 emit_move_insn (loxhi
, tem
);
2183 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2184 if (larger_sign
== 0)
2185 emit_jump (after_hipart_neg
);
2186 else if (larger_sign
!= -1)
2187 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2188 NULL_RTX
, NULL
, after_hipart_neg
,
2189 profile_probability::even ());
2191 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2192 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2193 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2195 emit_move_insn (loxhi
, tem
);
2197 emit_label (after_hipart_neg
);
2199 /* if (lopart < 0) loxhi -= larger; */
2200 if (smaller_sign
== 0)
2201 emit_jump (after_lopart_neg
);
2202 else if (smaller_sign
!= -1)
2203 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2204 NULL_RTX
, NULL
, after_lopart_neg
,
2205 profile_probability::even ());
2207 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2209 emit_move_insn (loxhi
, tem
);
2211 emit_label (after_lopart_neg
);
2214 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2215 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2216 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2218 emit_move_insn (loxhi
, tem
);
2220 /* if (loxhi >> (bitsize / 2)
2221 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2222 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2223 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2225 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2226 rtx signbitloxhi
= const0_rtx
;
2228 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2229 convert_modes (hmode
, mode
,
2231 hprec
- 1, NULL_RTX
, 0);
2233 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2234 NULL_RTX
, NULL
, do_overflow
,
2235 profile_probability::very_unlikely ());
2237 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2238 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2240 tem
= convert_modes (mode
, hmode
,
2241 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2243 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2246 emit_move_insn (res
, tem
);
2247 emit_jump (done_label
);
2249 emit_label (both_ops_large
);
2251 /* If both operands are large (not sign (!uns) or zero (uns)
2252 extended from hmode), then perform the full multiplication
2253 which will be the result of the operation.
2254 The only cases which don't overflow are for signed multiplication
2255 some cases where both hipart0 and highpart1 are 0 or -1.
2256 For unsigned multiplication when high parts are both non-zero
2257 this overflows always. */
2258 ops
.code
= MULT_EXPR
;
2259 ops
.op0
= make_tree (type
, op0
);
2260 ops
.op1
= make_tree (type
, op1
);
2261 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2262 emit_move_insn (res
, tem
);
2268 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2269 NULL_RTX
, 1, OPTAB_WIDEN
);
2270 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2271 NULL_RTX
, NULL
, do_error
,
2272 profile_probability::very_unlikely ());
2277 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2278 NULL_RTX
, 1, OPTAB_WIDEN
);
2279 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2280 NULL_RTX
, NULL
, do_error
,
2281 profile_probability::very_unlikely ());
2284 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2285 the same, overflow happened if res is non-positive, if they
2286 are different, overflow happened if res is positive. */
2287 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2288 emit_jump (hipart_different
);
2289 else if (op0_sign
== 1 || op1_sign
== 1)
2290 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2291 NULL_RTX
, NULL
, hipart_different
,
2292 profile_probability::even ());
2294 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2295 NULL_RTX
, NULL
, do_error
,
2296 profile_probability::very_unlikely ());
2297 emit_jump (done_label
);
2299 emit_label (hipart_different
);
2301 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2302 NULL_RTX
, NULL
, do_error
,
2303 profile_probability::very_unlikely ());
2304 emit_jump (done_label
);
2307 emit_label (do_overflow
);
2309 /* Overflow, do full multiplication and fallthru into do_error. */
2310 ops
.op0
= make_tree (type
, op0
);
2311 ops
.op1
= make_tree (type
, op1
);
2312 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2313 emit_move_insn (res
, tem
);
2315 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2316 && targetm
.scalar_mode_supported_p (wmode
))
2317 /* Even emitting a libcall is better than not detecting overflow
2322 gcc_assert (!is_ubsan
);
2323 ops
.code
= MULT_EXPR
;
2325 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2326 emit_jump (done_label
);
2331 emit_label (do_error
);
2334 /* Expand the ubsan builtin call. */
2336 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2340 do_pending_stack_adjust ();
2343 expand_arith_set_overflow (lhs
, target
);
2346 emit_label (done_label
);
2349 if (uns0_p
&& uns1_p
&& !unsr_p
)
2351 rtx_code_label
*all_done_label
= gen_label_rtx ();
2352 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2353 NULL
, all_done_label
, profile_probability::very_likely ());
2354 expand_arith_set_overflow (lhs
, target
);
2355 emit_label (all_done_label
);
2359 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2361 rtx_code_label
*all_done_label
= gen_label_rtx ();
2362 rtx_code_label
*set_noovf
= gen_label_rtx ();
2363 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2364 NULL
, all_done_label
, profile_probability::very_likely ());
2365 expand_arith_set_overflow (lhs
, target
);
2366 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2367 NULL
, set_noovf
, profile_probability::very_likely ());
2368 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2369 NULL
, all_done_label
, profile_probability::very_unlikely ());
2370 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2371 all_done_label
, profile_probability::very_unlikely ());
2372 emit_label (set_noovf
);
2373 write_complex_part (target
, const0_rtx
, true, false);
2374 emit_label (all_done_label
);
2380 expand_ubsan_result_store (target
, res
);
2382 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2386 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2389 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2390 tree arg0
, tree arg1
)
2392 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2393 rtx_code_label
*loop_lab
= NULL
;
2394 rtx cntvar
= NULL_RTX
;
2395 tree cntv
= NULL_TREE
;
2396 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2397 tree sz
= TYPE_SIZE (eltype
);
2398 tree data
= NULL_TREE
;
2399 tree resv
= NULL_TREE
;
2400 rtx lhsr
= NULL_RTX
;
2401 rtx resvr
= NULL_RTX
;
2402 unsigned HOST_WIDE_INT const_cnt
= 0;
2403 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2408 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2409 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2410 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2411 optab_default
)) == unknown_optab
2412 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2413 == CODE_FOR_nothing
))
2416 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2419 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2420 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2426 do_pending_stack_adjust ();
2427 loop_lab
= gen_label_rtx ();
2428 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2429 cntv
= make_tree (sizetype
, cntvar
);
2430 emit_move_insn (cntvar
, const0_rtx
);
2431 emit_label (loop_lab
);
2433 if (TREE_CODE (arg0
) != VECTOR_CST
)
2435 rtx arg0r
= expand_normal (arg0
);
2436 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2438 if (TREE_CODE (arg1
) != VECTOR_CST
)
2440 rtx arg1r
= expand_normal (arg1
);
2441 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2443 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2445 tree op0
, op1
, res
= NULL_TREE
;
2448 tree atype
= build_array_type_nelts (eltype
, cnt
);
2449 op0
= uniform_vector_p (arg0
);
2450 if (op0
== NULL_TREE
)
2452 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2453 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2454 NULL_TREE
, NULL_TREE
);
2456 op1
= uniform_vector_p (arg1
);
2457 if (op1
== NULL_TREE
)
2459 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2460 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2461 NULL_TREE
, NULL_TREE
);
2465 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2466 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2467 NULL_TREE
, NULL_TREE
);
2472 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2473 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2474 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2476 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2482 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2483 false, false, false, true, &data
);
2486 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2487 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2489 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2490 false, false, false, true, &data
);
2493 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2502 struct separate_ops ops
;
2503 ops
.code
= PLUS_EXPR
;
2504 ops
.type
= TREE_TYPE (cntv
);
2506 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2507 ops
.op2
= NULL_TREE
;
2509 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2512 emit_move_insn (cntvar
, ret
);
2513 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2514 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2515 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2516 profile_probability::very_likely ());
2518 if (lhs
&& resv
== NULL_TREE
)
2520 struct separate_ops ops
;
2522 ops
.type
= TREE_TYPE (arg0
);
2525 ops
.op2
= NULL_TREE
;
2527 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2530 emit_move_insn (lhsr
, ret
);
2533 emit_move_insn (lhsr
, resvr
);
2536 /* Expand UBSAN_CHECK_ADD call STMT. */
2539 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2541 location_t loc
= gimple_location (stmt
);
2542 tree lhs
= gimple_call_lhs (stmt
);
2543 tree arg0
= gimple_call_arg (stmt
, 0);
2544 tree arg1
= gimple_call_arg (stmt
, 1);
2545 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2546 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2548 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2549 false, false, false, true, NULL
);
2552 /* Expand UBSAN_CHECK_SUB call STMT. */
2555 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2557 location_t loc
= gimple_location (stmt
);
2558 tree lhs
= gimple_call_lhs (stmt
);
2559 tree arg0
= gimple_call_arg (stmt
, 0);
2560 tree arg1
= gimple_call_arg (stmt
, 1);
2561 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2562 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2563 else if (integer_zerop (arg0
))
2564 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2566 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2567 false, false, false, true, NULL
);
2570 /* Expand UBSAN_CHECK_MUL call STMT. */
2573 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2575 location_t loc
= gimple_location (stmt
);
2576 tree lhs
= gimple_call_lhs (stmt
);
2577 tree arg0
= gimple_call_arg (stmt
, 0);
2578 tree arg1
= gimple_call_arg (stmt
, 1);
2579 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2580 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2582 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2586 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2589 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2591 tree lhs
= gimple_call_lhs (stmt
);
2592 if (lhs
== NULL_TREE
)
2594 tree arg0
= gimple_call_arg (stmt
, 0);
2595 tree arg1
= gimple_call_arg (stmt
, 1);
2596 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2597 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2598 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2599 int unsr_p
= TYPE_UNSIGNED (type
);
2600 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2601 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2602 int precres
= TYPE_PRECISION (type
);
2603 location_t loc
= gimple_location (stmt
);
2604 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2606 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2608 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2609 prec0
= MIN (prec0
, pr
);
2610 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2611 prec1
= MIN (prec1
, pr
);
2613 /* If uns0_p && uns1_p, precop is minimum needed precision
2614 of unsigned type to hold the exact result, otherwise
2615 precop is minimum needed precision of signed type to
2616 hold the exact result. */
2618 if (code
== MULT_EXPR
)
2619 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2622 if (uns0_p
== uns1_p
)
2623 precop
= MAX (prec0
, prec1
) + 1;
2625 precop
= MAX (prec0
+ 1, prec1
) + 1;
2627 precop
= MAX (prec0
, prec1
+ 1) + 1;
2629 int orig_precres
= precres
;
2633 if ((uns0_p
&& uns1_p
)
2634 ? ((precop
+ !unsr_p
) <= precres
2635 /* u1 - u2 -> ur can overflow, no matter what precision
2637 && (code
!= MINUS_EXPR
|| !unsr_p
))
2638 : (!unsr_p
&& precop
<= precres
))
2640 /* The infinity precision result will always fit into result. */
2641 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2642 write_complex_part (target
, const0_rtx
, true, false);
2643 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2644 struct separate_ops ops
;
2647 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2648 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2649 ops
.op2
= NULL_TREE
;
2651 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2652 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2656 /* For operations with low precision, if target doesn't have them, start
2657 with precres widening right away, otherwise do it only if the most
2658 simple cases can't be used. */
2659 const int min_precision
= targetm
.min_arithmetic_precision ();
2660 if (orig_precres
== precres
&& precres
< min_precision
)
2662 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2663 && prec1
<= precres
)
2664 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2665 && prec0
+ uns0_p
<= precres
2666 && prec1
+ uns1_p
<= precres
))
2668 arg0
= fold_convert_loc (loc
, type
, arg0
);
2669 arg1
= fold_convert_loc (loc
, type
, arg1
);
2673 if (integer_zerop (arg0
) && !unsr_p
)
2675 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2680 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2681 unsr_p
, unsr_p
, false, NULL
);
2684 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2685 unsr_p
, unsr_p
, false, NULL
);
2692 /* For sub-word operations, retry with a wider type first. */
2693 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2695 int p
= MAX (min_precision
, precop
);
2696 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2697 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2700 p
= TYPE_PRECISION (optype
);
2704 unsr_p
= TYPE_UNSIGNED (optype
);
2710 if (prec0
<= precres
&& prec1
<= precres
)
2715 types
[0] = build_nonstandard_integer_type (precres
, 0);
2721 types
[1] = build_nonstandard_integer_type (precres
, 1);
2723 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2724 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2725 if (code
!= MULT_EXPR
)
2726 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2727 uns0_p
, uns1_p
, false, NULL
);
2729 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2730 uns0_p
, uns1_p
, false, NULL
);
2734 /* Retry with a wider type. */
2735 if (orig_precres
== precres
)
2737 int p
= MAX (prec0
, prec1
);
2738 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2739 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2742 p
= TYPE_PRECISION (optype
);
2746 unsr_p
= TYPE_UNSIGNED (optype
);
2757 /* Expand ADD_OVERFLOW STMT. */
2760 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2762 expand_arith_overflow (PLUS_EXPR
, stmt
);
2765 /* Expand SUB_OVERFLOW STMT. */
2768 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2770 expand_arith_overflow (MINUS_EXPR
, stmt
);
2773 /* Expand MUL_OVERFLOW STMT. */
2776 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2778 expand_arith_overflow (MULT_EXPR
, stmt
);
2781 /* Expand UADDC STMT. */
2784 expand_UADDC (internal_fn ifn
, gcall
*stmt
)
2786 tree lhs
= gimple_call_lhs (stmt
);
2787 tree arg1
= gimple_call_arg (stmt
, 0);
2788 tree arg2
= gimple_call_arg (stmt
, 1);
2789 tree arg3
= gimple_call_arg (stmt
, 2);
2790 tree type
= TREE_TYPE (arg1
);
2791 machine_mode mode
= TYPE_MODE (type
);
2792 insn_code icode
= optab_handler (ifn
== IFN_UADDC
2793 ? uaddc5_optab
: usubc5_optab
, mode
);
2794 rtx op1
= expand_normal (arg1
);
2795 rtx op2
= expand_normal (arg2
);
2796 rtx op3
= expand_normal (arg3
);
2797 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2798 rtx re
= gen_reg_rtx (mode
);
2799 rtx im
= gen_reg_rtx (mode
);
2800 class expand_operand ops
[5];
2801 create_output_operand (&ops
[0], re
, mode
);
2802 create_output_operand (&ops
[1], im
, mode
);
2803 create_input_operand (&ops
[2], op1
, mode
);
2804 create_input_operand (&ops
[3], op2
, mode
);
2805 create_input_operand (&ops
[4], op3
, mode
);
2806 expand_insn (icode
, 5, ops
);
2807 write_complex_part (target
, re
, false, false);
2808 write_complex_part (target
, im
, true, false);
2811 /* Expand USUBC STMT. */
2814 expand_USUBC (internal_fn ifn
, gcall
*stmt
)
2816 expand_UADDC (ifn
, stmt
);
2819 /* This should get folded in tree-vectorizer.cc. */
2822 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2827 /* This should get folded in tree-vectorizer.cc. */
2830 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2835 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2836 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2839 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2841 tree addr
= gimple_call_arg (stmt
, index
);
2842 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2843 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2844 if (TYPE_ALIGN (type
) != align
)
2845 type
= build_aligned_type (type
, align
);
2848 if (TREE_CODE (tmp
) == SSA_NAME
)
2850 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2851 if (gimple_assign_single_p (def
))
2852 tmp
= gimple_assign_rhs1 (def
);
2855 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2857 tree mem
= TREE_OPERAND (tmp
, 0);
2858 if (TREE_CODE (mem
) == TARGET_MEM_REF
2859 && types_compatible_p (TREE_TYPE (mem
), type
))
2861 tree offset
= TMR_OFFSET (mem
);
2862 if (type
!= TREE_TYPE (mem
)
2863 || alias_ptr_type
!= TREE_TYPE (offset
)
2864 || !integer_zerop (offset
))
2866 mem
= copy_node (mem
);
2867 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2868 wi::to_poly_wide (offset
));
2869 TREE_TYPE (mem
) = type
;
2875 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2878 /* Expand MASK_LOAD{,_LANES}, LEN_MASK_LOAD or LEN_LOAD call STMT using optab
2882 expand_partial_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2884 class expand_operand ops
[5];
2885 tree type
, lhs
, rhs
, maskt
, biast
;
2886 rtx mem
, target
, mask
, bias
;
2889 maskt
= gimple_call_arg (stmt
, 2);
2890 lhs
= gimple_call_lhs (stmt
);
2891 if (lhs
== NULL_TREE
)
2893 type
= TREE_TYPE (lhs
);
2894 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2896 if (optab
== vec_mask_load_lanes_optab
)
2897 icode
= get_multi_vector_move (type
, optab
);
2898 else if (optab
== len_load_optab
)
2899 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2901 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2902 TYPE_MODE (TREE_TYPE (maskt
)));
2904 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2905 gcc_assert (MEM_P (mem
));
2906 mask
= expand_normal (maskt
);
2907 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2908 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2909 create_fixed_operand (&ops
[1], mem
);
2910 if (optab
== len_load_optab
)
2912 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2913 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2914 biast
= gimple_call_arg (stmt
, 3);
2915 bias
= expand_normal (biast
);
2916 create_input_operand (&ops
[3], bias
, QImode
);
2917 expand_insn (icode
, 4, ops
);
2919 else if (optab
== len_maskload_optab
)
2921 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2922 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2923 maskt
= gimple_call_arg (stmt
, 3);
2924 mask
= expand_normal (maskt
);
2925 create_input_operand (&ops
[3], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2926 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2927 TYPE_MODE (TREE_TYPE (maskt
)));
2928 biast
= gimple_call_arg (stmt
, 4);
2929 bias
= expand_normal (biast
);
2930 create_input_operand (&ops
[4], bias
, QImode
);
2931 expand_insn (icode
, 5, ops
);
2935 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2936 expand_insn (icode
, 3, ops
);
2939 if (!rtx_equal_p (target
, ops
[0].value
))
2940 emit_move_insn (target
, ops
[0].value
);
2943 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2944 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2945 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2946 #define expand_len_maskload_optab_fn expand_partial_load_optab_fn
2948 /* Expand MASK_STORE{,_LANES}, LEN_MASK_STORE or LEN_STORE call STMT using optab
2952 expand_partial_store_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
2954 class expand_operand ops
[5];
2955 tree type
, lhs
, rhs
, maskt
, biast
;
2956 rtx mem
, reg
, mask
, bias
;
2959 maskt
= gimple_call_arg (stmt
, 2);
2960 rhs
= gimple_call_arg (stmt
, internal_fn_stored_value_index (ifn
));
2961 type
= TREE_TYPE (rhs
);
2962 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2964 if (optab
== vec_mask_store_lanes_optab
)
2965 icode
= get_multi_vector_move (type
, optab
);
2966 else if (optab
== len_store_optab
)
2967 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2969 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2970 TYPE_MODE (TREE_TYPE (maskt
)));
2972 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2973 gcc_assert (MEM_P (mem
));
2974 mask
= expand_normal (maskt
);
2975 reg
= expand_normal (rhs
);
2976 create_fixed_operand (&ops
[0], mem
);
2977 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2978 if (optab
== len_store_optab
)
2980 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2981 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2982 biast
= gimple_call_arg (stmt
, 4);
2983 bias
= expand_normal (biast
);
2984 create_input_operand (&ops
[3], bias
, QImode
);
2985 expand_insn (icode
, 4, ops
);
2987 else if (optab
== len_maskstore_optab
)
2989 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2990 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2991 maskt
= gimple_call_arg (stmt
, 3);
2992 mask
= expand_normal (maskt
);
2993 create_input_operand (&ops
[3], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2994 biast
= gimple_call_arg (stmt
, 5);
2995 bias
= expand_normal (biast
);
2996 create_input_operand (&ops
[4], bias
, QImode
);
2997 icode
= convert_optab_handler (optab
, TYPE_MODE (type
), GET_MODE (mask
));
2998 expand_insn (icode
, 5, ops
);
3002 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
3003 expand_insn (icode
, 3, ops
);
3007 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3008 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3009 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3010 #define expand_len_maskstore_optab_fn expand_partial_store_optab_fn
3012 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3013 The expansion of STMT happens based on OPTAB table associated. */
3016 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3018 class expand_operand ops
[6];
3020 tree lhs
= gimple_call_lhs (stmt
);
3021 tree op0a
= gimple_call_arg (stmt
, 0);
3022 tree op0b
= gimple_call_arg (stmt
, 1);
3023 tree op1
= gimple_call_arg (stmt
, 2);
3024 tree op2
= gimple_call_arg (stmt
, 3);
3025 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
3027 tree vec_cond_type
= TREE_TYPE (lhs
);
3028 tree op_mode
= TREE_TYPE (op0a
);
3029 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
3031 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3032 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
3034 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
3036 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
3037 rtx rtx_op1
= expand_normal (op1
);
3038 rtx rtx_op2
= expand_normal (op2
);
3040 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3041 create_output_operand (&ops
[0], target
, mode
);
3042 create_input_operand (&ops
[1], rtx_op1
, mode
);
3043 create_input_operand (&ops
[2], rtx_op2
, mode
);
3044 create_fixed_operand (&ops
[3], comparison
);
3045 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
3046 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
3047 expand_insn (icode
, 6, ops
);
3048 if (!rtx_equal_p (ops
[0].value
, target
))
3049 emit_move_insn (target
, ops
[0].value
);
3052 /* Expand VCOND_MASK optab internal function.
3053 The expansion of STMT happens based on OPTAB table associated. */
3056 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3058 class expand_operand ops
[4];
3060 tree lhs
= gimple_call_lhs (stmt
);
3061 tree op0
= gimple_call_arg (stmt
, 0);
3062 tree op1
= gimple_call_arg (stmt
, 1);
3063 tree op2
= gimple_call_arg (stmt
, 2);
3064 tree vec_cond_type
= TREE_TYPE (lhs
);
3066 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3067 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
3068 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
3069 rtx mask
, rtx_op1
, rtx_op2
;
3071 gcc_assert (icode
!= CODE_FOR_nothing
);
3073 mask
= expand_normal (op0
);
3074 rtx_op1
= expand_normal (op1
);
3075 rtx_op2
= expand_normal (op2
);
3077 mask
= force_reg (mask_mode
, mask
);
3078 rtx_op1
= force_reg (mode
, rtx_op1
);
3080 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3081 create_output_operand (&ops
[0], target
, mode
);
3082 create_input_operand (&ops
[1], rtx_op1
, mode
);
3083 create_input_operand (&ops
[2], rtx_op2
, mode
);
3084 create_input_operand (&ops
[3], mask
, mask_mode
);
3085 expand_insn (icode
, 4, ops
);
3086 if (!rtx_equal_p (ops
[0].value
, target
))
3087 emit_move_insn (target
, ops
[0].value
);
3090 /* Expand VEC_SET internal functions. */
3093 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3095 tree lhs
= gimple_call_lhs (stmt
);
3096 tree op0
= gimple_call_arg (stmt
, 0);
3097 tree op1
= gimple_call_arg (stmt
, 1);
3098 tree op2
= gimple_call_arg (stmt
, 2);
3099 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3100 rtx src
= expand_normal (op0
);
3102 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
3103 scalar_mode innermode
= GET_MODE_INNER (outermode
);
3105 rtx value
= expand_normal (op1
);
3106 rtx pos
= expand_normal (op2
);
3108 class expand_operand ops
[3];
3109 enum insn_code icode
= optab_handler (optab
, outermode
);
3111 if (icode
!= CODE_FOR_nothing
)
3113 rtx temp
= gen_reg_rtx (outermode
);
3114 emit_move_insn (temp
, src
);
3116 create_fixed_operand (&ops
[0], temp
);
3117 create_input_operand (&ops
[1], value
, innermode
);
3118 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
3120 if (maybe_expand_insn (icode
, 3, ops
))
3122 emit_move_insn (target
, temp
);
3130 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3135 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3137 /* When guessing was done, the hints should be already stripped away. */
3138 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3141 tree lhs
= gimple_call_lhs (stmt
);
3143 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3145 target
= const0_rtx
;
3146 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3147 if (lhs
&& val
!= target
)
3148 emit_move_insn (target
, val
);
3151 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3152 should never be called. */
3155 expand_VA_ARG (internal_fn
, gcall
*)
3160 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3161 dummy function should never be called. */
3164 expand_VEC_CONVERT (internal_fn
, gcall
*)
3169 /* Expand IFN_RAWMEMCHAR internal function. */
3172 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3174 expand_operand ops
[3];
3176 tree lhs
= gimple_call_lhs (stmt
);
3179 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3180 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3181 create_output_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3183 tree mem
= gimple_call_arg (stmt
, 0);
3184 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3185 create_fixed_operand (&ops
[1], mem_rtx
);
3187 tree pattern
= gimple_call_arg (stmt
, 1);
3188 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3189 rtx pattern_rtx
= expand_normal (pattern
);
3190 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3192 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3194 expand_insn (icode
, 3, ops
);
3195 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3196 emit_move_insn (lhs_rtx
, ops
[0].value
);
3199 /* Expand the IFN_UNIQUE function according to its first argument. */
3202 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3204 rtx pattern
= NULL_RTX
;
3205 enum ifn_unique_kind kind
3206 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3213 case IFN_UNIQUE_UNSPEC
:
3214 if (targetm
.have_unique ())
3215 pattern
= targetm
.gen_unique ();
3218 case IFN_UNIQUE_OACC_FORK
:
3219 case IFN_UNIQUE_OACC_JOIN
:
3220 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3222 tree lhs
= gimple_call_lhs (stmt
);
3223 rtx target
= const0_rtx
;
3226 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3228 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3229 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3231 if (kind
== IFN_UNIQUE_OACC_FORK
)
3232 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3234 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3242 emit_insn (pattern
);
3245 /* Expand the IFN_DEFERRED_INIT function:
3246 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3248 Initialize the LHS with zero/pattern according to its second argument
3250 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3251 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3253 The LHS variable is initialized including paddings.
3254 The reasons to choose 0xFE for pattern initialization are:
3255 1. It is a non-canonical virtual address on x86_64, and at the
3256 high end of the i386 kernel address space.
3257 2. It is a very large float value (-1.694739530317379e+38).
3258 3. It is also an unusual number for integers. */
3259 #define INIT_PATTERN_VALUE 0xFE
3261 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3263 tree lhs
= gimple_call_lhs (stmt
);
3264 tree var_size
= gimple_call_arg (stmt
, 0);
3265 enum auto_init_type init_type
3266 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3267 bool reg_lhs
= true;
3269 tree var_type
= TREE_TYPE (lhs
);
3270 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3272 if (TREE_CODE (lhs
) == SSA_NAME
)
3276 tree lhs_base
= lhs
;
3277 while (handled_component_p (lhs_base
))
3278 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3279 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3280 || non_mem_decl_p (lhs_base
));
3281 /* If this expands to a register and the underlying decl is wrapped in
3282 a MEM_REF that just serves as an access type change expose the decl
3283 if it is of correct size. This avoids a situation as in PR103271
3284 if the target does not support a direct move to the registers mode. */
3286 && TREE_CODE (lhs_base
) == MEM_REF
3287 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3288 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3289 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3290 && tree_fits_uhwi_p (var_size
)
3291 && tree_int_cst_equal
3293 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3295 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3296 var_type
= TREE_TYPE (lhs
);
3302 /* If the variable is not in register, expand to a memset
3303 to initialize it. */
3304 mark_addressable (lhs
);
3305 tree var_addr
= build_fold_addr_expr (lhs
);
3307 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3308 ? build_int_cst (integer_type_node
,
3310 : integer_zero_node
;
3311 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3312 3, var_addr
, value
, var_size
);
3313 /* Expand this memset call. */
3314 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3318 /* If this variable is in a register use expand_assignment.
3319 For boolean scalars force zero-init. */
3321 scalar_int_mode var_mode
;
3322 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3323 && tree_fits_uhwi_p (var_size
)
3324 && (init_type
== AUTO_INIT_PATTERN
3325 || !is_gimple_reg_type (var_type
))
3326 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3327 0).exists (&var_mode
)
3328 && have_insn_for (SET
, var_mode
))
3330 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3331 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3332 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3333 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3334 tree itype
= build_nonstandard_integer_type
3335 (total_bytes
* BITS_PER_UNIT
, 1);
3336 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3337 init
= wide_int_to_tree (itype
, w
);
3338 /* Pun the LHS to make sure its type has constant size
3339 unless it is an SSA name where that's already known. */
3340 if (TREE_CODE (lhs
) != SSA_NAME
)
3341 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3343 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3346 /* Use zero-init also for variable-length sizes. */
3347 init
= build_zero_cst (var_type
);
3349 expand_assignment (lhs
, init
, false);
3353 /* The size of an OpenACC compute dimension. */
3356 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3358 tree lhs
= gimple_call_lhs (stmt
);
3363 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3364 if (targetm
.have_oacc_dim_size ())
3366 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3367 VOIDmode
, EXPAND_NORMAL
);
3368 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3371 emit_move_insn (target
, GEN_INT (1));
3374 /* The position of an OpenACC execution engine along one compute axis. */
3377 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3379 tree lhs
= gimple_call_lhs (stmt
);
3384 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3385 if (targetm
.have_oacc_dim_pos ())
3387 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3388 VOIDmode
, EXPAND_NORMAL
);
3389 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3392 emit_move_insn (target
, const0_rtx
);
3395 /* This is expanded by oacc_device_lower pass. */
3398 expand_GOACC_LOOP (internal_fn
, gcall
*)
3403 /* This is expanded by oacc_device_lower pass. */
3406 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3411 /* This is expanded by oacc_device_lower pass. */
3414 expand_GOACC_TILE (internal_fn
, gcall
*)
3419 /* Set errno to EDOM. */
3422 expand_SET_EDOM (internal_fn
, gcall
*)
3425 #ifdef GEN_ERRNO_RTX
3426 rtx errno_rtx
= GEN_ERRNO_RTX
;
3428 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3430 emit_move_insn (errno_rtx
,
3431 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3437 /* Expand atomic bit test and set. */
3440 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3442 expand_ifn_atomic_bit_test_and (call
);
3445 /* Expand atomic bit test and complement. */
3448 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3450 expand_ifn_atomic_bit_test_and (call
);
3453 /* Expand atomic bit test and reset. */
3456 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3458 expand_ifn_atomic_bit_test_and (call
);
3461 /* Expand atomic bit test and set. */
3464 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3466 expand_ifn_atomic_compare_exchange (call
);
3469 /* Expand atomic add fetch and cmp with 0. */
3472 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3474 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3477 /* Expand atomic sub fetch and cmp with 0. */
3480 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3482 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3485 /* Expand atomic and fetch and cmp with 0. */
3488 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3490 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3493 /* Expand atomic or fetch and cmp with 0. */
3496 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3498 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3501 /* Expand atomic xor fetch and cmp with 0. */
3504 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3506 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3509 /* Expand LAUNDER to assignment, lhs = arg0. */
3512 expand_LAUNDER (internal_fn
, gcall
*call
)
3514 tree lhs
= gimple_call_lhs (call
);
3519 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3522 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3525 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3527 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3528 int rhs_index
= internal_fn_stored_value_index (ifn
);
3529 int mask_index
= internal_fn_mask_index (ifn
);
3530 tree base
= gimple_call_arg (stmt
, 0);
3531 tree offset
= gimple_call_arg (stmt
, 1);
3532 tree scale
= gimple_call_arg (stmt
, 2);
3533 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3535 rtx base_rtx
= expand_normal (base
);
3536 rtx offset_rtx
= expand_normal (offset
);
3537 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3538 rtx rhs_rtx
= expand_normal (rhs
);
3540 class expand_operand ops
[6];
3542 create_address_operand (&ops
[i
++], base_rtx
);
3543 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3544 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3545 create_integer_operand (&ops
[i
++], scale_int
);
3546 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3547 if (mask_index
>= 0)
3549 tree mask
= gimple_call_arg (stmt
, mask_index
);
3550 rtx mask_rtx
= expand_normal (mask
);
3551 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3554 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3555 TYPE_MODE (TREE_TYPE (offset
)));
3556 expand_insn (icode
, i
, ops
);
3559 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3562 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3564 tree lhs
= gimple_call_lhs (stmt
);
3565 tree base
= gimple_call_arg (stmt
, 0);
3566 tree offset
= gimple_call_arg (stmt
, 1);
3567 tree scale
= gimple_call_arg (stmt
, 2);
3569 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3570 rtx base_rtx
= expand_normal (base
);
3571 rtx offset_rtx
= expand_normal (offset
);
3572 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3575 class expand_operand ops
[6];
3576 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3577 create_address_operand (&ops
[i
++], base_rtx
);
3578 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3579 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3580 create_integer_operand (&ops
[i
++], scale_int
);
3581 if (optab
== mask_gather_load_optab
)
3583 tree mask
= gimple_call_arg (stmt
, 4);
3584 rtx mask_rtx
= expand_normal (mask
);
3585 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3587 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3588 TYPE_MODE (TREE_TYPE (offset
)));
3589 expand_insn (icode
, i
, ops
);
3590 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3591 emit_move_insn (lhs_rtx
, ops
[0].value
);
3594 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3595 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3598 contains_call_div_mod (rtx_insn
*insn
)
3600 subrtx_iterator::array_type array
;
3601 for (; insn
; insn
= NEXT_INSN (insn
))
3604 else if (INSN_P (insn
))
3605 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3606 switch (GET_CODE (*iter
))
3620 /* Expand DIVMOD() using:
3621 a) optab handler for udivmod/sdivmod if it is available.
3622 b) If optab_handler doesn't exist, generate call to
3623 target-specific divmod libfunc. */
3626 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3628 tree lhs
= gimple_call_lhs (call_stmt
);
3629 tree arg0
= gimple_call_arg (call_stmt
, 0);
3630 tree arg1
= gimple_call_arg (call_stmt
, 1);
3632 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3633 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3634 machine_mode mode
= TYPE_MODE (type
);
3635 bool unsignedp
= TYPE_UNSIGNED (type
);
3636 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3638 rtx op0
= expand_normal (arg0
);
3639 rtx op1
= expand_normal (arg1
);
3640 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3642 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3643 rtx_insn
*insns
= NULL
;
3645 if (TREE_CODE (arg1
) == INTEGER_CST
)
3647 /* For DIVMOD by integral constants, there could be efficient code
3648 expanded inline e.g. using shifts and plus/minus. Try to expand
3649 the division and modulo and if it emits any library calls or any
3650 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3652 scalar_int_mode int_mode
;
3653 if (remainder
== NULL_RTX
3655 && CONST_INT_P (op1
)
3656 && !pow2p_hwi (INTVAL (op1
))
3657 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3658 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3659 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3660 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3661 && optimize_insn_for_speed_p ())
3663 rtx_insn
*last
= get_last_insn ();
3664 remainder
= NULL_RTX
;
3665 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3666 TYPE_UNSIGNED (type
));
3667 if (quotient
!= NULL_RTX
)
3669 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3671 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3672 set_dst_reg_note (move
, REG_EQUAL
,
3673 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3674 ? UDIV
: DIV
, int_mode
,
3675 copy_rtx (op0
), op1
),
3677 move
= emit_move_insn (remainder
, remainder
);
3678 set_dst_reg_note (move
, REG_EQUAL
,
3679 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3680 ? UMOD
: MOD
, int_mode
,
3681 copy_rtx (op0
), op1
),
3686 delete_insns_since (last
);
3689 if (remainder
== NULL_RTX
)
3691 struct separate_ops ops
;
3692 ops
.code
= TRUNC_DIV_EXPR
;
3694 ops
.op0
= make_tree (ops
.type
, op0
);
3696 ops
.op2
= NULL_TREE
;
3697 ops
.location
= gimple_location (call_stmt
);
3699 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3700 if (contains_call_div_mod (get_insns ()))
3701 quotient
= NULL_RTX
;
3704 ops
.code
= TRUNC_MOD_EXPR
;
3705 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3707 if (contains_call_div_mod (get_insns ()))
3708 remainder
= NULL_RTX
;
3711 insns
= get_insns ();
3719 /* Check if optab_handler exists for divmod_optab for given mode. */
3720 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3722 quotient
= gen_reg_rtx (mode
);
3723 remainder
= gen_reg_rtx (mode
);
3724 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3727 /* Generate call to divmod libfunc if it exists. */
3728 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3729 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3730 "ient
, &remainder
);
3735 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3736 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3737 make_tree (TREE_TYPE (arg0
), quotient
),
3738 make_tree (TREE_TYPE (arg1
), remainder
)),
3739 target
, VOIDmode
, EXPAND_NORMAL
);
3745 expand_NOP (internal_fn
, gcall
*)
3747 /* Nothing. But it shouldn't really prevail. */
3750 /* Coroutines, all should have been processed at this stage. */
3753 expand_CO_FRAME (internal_fn
, gcall
*)
3759 expand_CO_YIELD (internal_fn
, gcall
*)
3765 expand_CO_SUSPN (internal_fn
, gcall
*)
3771 expand_CO_ACTOR (internal_fn
, gcall
*)
3776 /* Expand a call to FN using the operands in STMT. FN has a single
3777 output operand and NARGS input operands. */
3780 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3783 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3784 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3785 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3788 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3791 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3793 expand_operand ops
[4];
3796 tree lhs
= gimple_call_lhs (stmt
);
3797 tree lhs_type
= TREE_TYPE (lhs
);
3798 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3799 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3801 for (unsigned int i
= 0; i
< 2; ++i
)
3803 tree rhs
= gimple_call_arg (stmt
, i
);
3804 rhs_type
[i
] = TREE_TYPE (rhs
);
3805 rtx rhs_rtx
= expand_normal (rhs
);
3806 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3810 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type
)))
3812 /* When the mask is an integer mode the exact vector length may not
3813 be clear to the backend, so we pass it in operand[3].
3814 Use the vector in arg2 for the most reliable intended size. */
3815 tree type
= TREE_TYPE (gimple_call_arg (stmt
, 2));
3816 create_integer_operand (&ops
[3], TYPE_VECTOR_SUBPARTS (type
));
3820 /* The mask has a vector type so the length operand is unnecessary. */
3823 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3824 TYPE_MODE (lhs_type
));
3826 expand_insn (icode
, opcnt
, ops
);
3827 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3828 emit_move_insn (lhs_rtx
, ops
[0].value
);
3831 /* Expand a call to a convert-like optab using the operands in STMT.
3832 FN has a single output operand and NARGS input operands. */
3835 expand_convert_optab_fn (internal_fn fn
, gcall
*stmt
, convert_optab optab
,
3838 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3839 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3840 TYPE_MODE (types
.second
));
3841 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3844 /* Expanders for optabs that can use expand_direct_optab_fn. */
3846 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3847 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3849 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3850 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3852 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3853 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3855 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3856 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3858 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3859 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3861 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3862 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3864 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3865 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3867 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3868 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3870 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3871 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3873 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3874 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3876 /* Expanders for optabs that can use expand_convert_optab_fn. */
3878 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3879 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3881 /* RETURN_TYPE and ARGS are a return type and argument list that are
3882 in principle compatible with FN (which satisfies direct_internal_fn_p).
3883 Return the types that should be used to determine whether the
3884 target supports FN. */
3887 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3889 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3890 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3891 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3892 return tree_pair (type0
, type1
);
3895 /* CALL is a call whose return type and arguments are in principle
3896 compatible with FN (which satisfies direct_internal_fn_p). Return the
3897 types that should be used to determine whether the target supports FN. */
3900 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3902 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3903 tree op0
= (info
.type0
< 0
3904 ? gimple_call_lhs (call
)
3905 : gimple_call_arg (call
, info
.type0
));
3906 tree op1
= (info
.type1
< 0
3907 ? gimple_call_lhs (call
)
3908 : gimple_call_arg (call
, info
.type1
));
3909 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3912 /* Return true if OPTAB is supported for TYPES (whose modes should be
3913 the same) when the optimization type is OPT_TYPE. Used for simple
3917 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3918 optimization_type opt_type
)
3920 machine_mode mode
= TYPE_MODE (types
.first
);
3921 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3922 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3925 /* Return true if OPTAB is supported for TYPES, where the first type
3926 is the destination and the second type is the source. Used for
3930 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3931 optimization_type opt_type
)
3933 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3934 TYPE_MODE (types
.second
), opt_type
)
3935 != CODE_FOR_nothing
);
3938 /* Return true if load/store lanes optab OPTAB is supported for
3939 array type TYPES.first when the optimization type is OPT_TYPE. */
3942 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3943 optimization_type opt_type
)
3945 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3946 machine_mode imode
= TYPE_MODE (types
.first
);
3947 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3948 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3949 != CODE_FOR_nothing
);
3952 #define direct_unary_optab_supported_p direct_optab_supported_p
3953 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
3954 #define direct_binary_optab_supported_p direct_optab_supported_p
3955 #define direct_ternary_optab_supported_p direct_optab_supported_p
3956 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3957 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3958 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3959 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3960 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3961 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3962 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3963 #define direct_len_load_optab_supported_p direct_optab_supported_p
3964 #define direct_len_maskload_optab_supported_p convert_optab_supported_p
3965 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3966 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3967 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3968 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3969 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3970 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3971 #define direct_len_store_optab_supported_p direct_optab_supported_p
3972 #define direct_len_maskstore_optab_supported_p convert_optab_supported_p
3973 #define direct_while_optab_supported_p convert_optab_supported_p
3974 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3975 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3976 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3977 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3978 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3980 /* Return the optab used by internal function FN. */
3983 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3987 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3988 case IFN_##CODE: break;
3989 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3990 case IFN_##CODE: return OPTAB##_optab;
3991 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3992 UNSIGNED_OPTAB, TYPE) \
3993 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3994 ? UNSIGNED_OPTAB ## _optab \
3995 : SIGNED_OPTAB ## _optab);
3996 #include "internal-fn.def"
4004 /* Return the optab used by internal function FN. */
4007 direct_internal_fn_optab (internal_fn fn
)
4011 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4012 case IFN_##CODE: break;
4013 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4014 case IFN_##CODE: return OPTAB##_optab;
4015 #include "internal-fn.def"
4023 /* Return true if FN is supported for the types in TYPES when the
4024 optimization type is OPT_TYPE. The types are those associated with
4025 the "type0" and "type1" fields of FN's direct_internal_fn_info
4029 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
4030 optimization_type opt_type
)
4034 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4035 case IFN_##CODE: break;
4036 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4038 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4040 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4041 UNSIGNED_OPTAB, TYPE) \
4044 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4045 ? UNSIGNED_OPTAB ## _optab \
4046 : SIGNED_OPTAB ## _optab); \
4047 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4050 #include "internal-fn.def"
4058 /* Return true if FN is supported for type TYPE when the optimization
4059 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4060 fields of FN's direct_internal_fn_info structure are the same. */
4063 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
4064 optimization_type opt_type
)
4066 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4067 gcc_checking_assert (info
.type0
== info
.type1
);
4068 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
4071 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4072 given that STMT is a call to a direct internal function. */
4075 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
4077 internal_fn fn
= gimple_call_internal_fn (stmt
);
4078 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
4079 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
4082 /* Return true if FN is a binary operation and if FN is commutative. */
4085 commutative_binary_fn_p (internal_fn fn
)
4096 case IFN_COMPLEX_MUL
:
4097 case IFN_UBSAN_CHECK_ADD
:
4098 case IFN_UBSAN_CHECK_MUL
:
4099 case IFN_ADD_OVERFLOW
:
4100 case IFN_MUL_OVERFLOW
:
4101 case IFN_VEC_WIDEN_PLUS
:
4102 case IFN_VEC_WIDEN_PLUS_LO
:
4103 case IFN_VEC_WIDEN_PLUS_HI
:
4104 case IFN_VEC_WIDEN_PLUS_EVEN
:
4105 case IFN_VEC_WIDEN_PLUS_ODD
:
4113 /* Return true if FN is a ternary operation and if its first two arguments
4117 commutative_ternary_fn_p (internal_fn fn
)
4133 /* Return true if FN is an associative binary operation. */
4136 associative_binary_fn_p (internal_fn fn
)
4149 /* If FN is commutative in two consecutive arguments, return the
4150 index of the first, otherwise return -1. */
4153 first_commutative_argument (internal_fn fn
)
4173 if (commutative_binary_fn_p (fn
)
4174 || commutative_ternary_fn_p (fn
))
4180 /* Return true if this CODE describes an internal_fn that returns a vector with
4181 elements twice as wide as the element size of the input vectors. */
4184 widening_fn_p (code_helper code
)
4186 if (!code
.is_fn_code ())
4189 if (!internal_fn_p ((combined_fn
) code
))
4192 internal_fn fn
= as_internal_fn ((combined_fn
) code
);
4195 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
4196 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4198 case IFN_##NAME##_HI: \
4199 case IFN_##NAME##_LO: \
4200 case IFN_##NAME##_EVEN: \
4201 case IFN_##NAME##_ODD: \
4203 #include "internal-fn.def"
4204 #undef DEF_INTERNAL_WIDENING_OPTAB_FN
4211 /* Return true if IFN_SET_EDOM is supported. */
4214 set_edom_supported_p (void)
4223 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4225 expand_##CODE (internal_fn fn, gcall *stmt) \
4227 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4229 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4230 UNSIGNED_OPTAB, TYPE) \
4232 expand_##CODE (internal_fn fn, gcall *stmt) \
4234 tree_pair types = direct_internal_fn_types (fn, stmt); \
4235 optab which_optab = direct_internal_fn_optab (fn, types); \
4236 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4238 #include "internal-fn.def"
4239 #undef DEF_INTERNAL_OPTAB_FN
4240 #undef DEF_INTERNAL_SIGNED_OPTAB_FN
4242 /* Routines to expand each internal function, indexed by function number.
4243 Each routine has the prototype:
4245 expand_<NAME> (gcall *stmt)
4247 where STMT is the statement that performs the call. */
4248 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4250 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4251 #include "internal-fn.def"
4255 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
4257 #define FOR_EACH_CODE_MAPPING(T) \
4258 T (PLUS_EXPR, IFN_COND_ADD) \
4259 T (MINUS_EXPR, IFN_COND_SUB) \
4260 T (MULT_EXPR, IFN_COND_MUL) \
4261 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
4262 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
4263 T (RDIV_EXPR, IFN_COND_RDIV) \
4264 T (MIN_EXPR, IFN_COND_MIN) \
4265 T (MAX_EXPR, IFN_COND_MAX) \
4266 T (BIT_AND_EXPR, IFN_COND_AND) \
4267 T (BIT_IOR_EXPR, IFN_COND_IOR) \
4268 T (BIT_XOR_EXPR, IFN_COND_XOR) \
4269 T (LSHIFT_EXPR, IFN_COND_SHL) \
4270 T (RSHIFT_EXPR, IFN_COND_SHR) \
4271 T (NEGATE_EXPR, IFN_COND_NEG)
4273 /* Return a function that only performs CODE when a certain condition is met
4274 and that uses a given fallback value otherwise. For example, if CODE is
4275 a binary operation associated with conditional function FN:
4277 LHS = FN (COND, A, B, ELSE)
4279 is equivalent to the C expression:
4281 LHS = COND ? A CODE B : ELSE;
4283 operating elementwise if the operands are vectors.
4285 Return IFN_LAST if no such function exists. */
4288 get_conditional_internal_fn (tree_code code
)
4292 #define CASE(CODE, IFN) case CODE: return IFN;
4293 FOR_EACH_CODE_MAPPING(CASE
)
4300 /* If IFN implements the conditional form of a tree code, return that
4301 tree code, otherwise return ERROR_MARK. */
4304 conditional_internal_fn_code (internal_fn ifn
)
4308 #define CASE(CODE, IFN) case IFN: return CODE;
4309 FOR_EACH_CODE_MAPPING(CASE
)
4316 /* Invoke T(IFN) for each internal function IFN that also has an
4318 #define FOR_EACH_COND_FN_PAIR(T) \
4326 /* Return a function that only performs internal function FN when a
4327 certain condition is met and that uses a given fallback value otherwise.
4328 In other words, the returned function FN' is such that:
4330 LHS = FN' (COND, A1, ... An, ELSE)
4332 is equivalent to the C expression:
4334 LHS = COND ? FN (A1, ..., An) : ELSE;
4336 operating elementwise if the operands are vectors.
4338 Return IFN_LAST if no such function exists. */
4341 get_conditional_internal_fn (internal_fn fn
)
4345 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4346 FOR_EACH_COND_FN_PAIR(CASE
)
4353 /* If IFN implements the conditional form of an unconditional internal
4354 function, return that unconditional function, otherwise return IFN_LAST. */
4357 get_unconditional_internal_fn (internal_fn ifn
)
4361 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
4362 FOR_EACH_COND_FN_PAIR(CASE
)
4369 /* Return true if STMT can be interpreted as a conditional tree code
4370 operation of the form:
4372 LHS = COND ? OP (RHS1, ...) : ELSE;
4374 operating elementwise if the operands are vectors. This includes
4375 the case of an all-true COND, so that the operation always happens.
4377 When returning true, set:
4379 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4380 is known to be all-true
4381 - *CODE_OUT to the tree code
4382 - OPS[I] to operand I of *CODE_OUT
4383 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4384 condition is known to be all true. */
4387 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4388 tree_code
*code_out
,
4389 tree (&ops
)[3], tree
*else_out
)
4391 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4393 *cond_out
= NULL_TREE
;
4394 *code_out
= gimple_assign_rhs_code (assign
);
4395 ops
[0] = gimple_assign_rhs1 (assign
);
4396 ops
[1] = gimple_assign_rhs2 (assign
);
4397 ops
[2] = gimple_assign_rhs3 (assign
);
4398 *else_out
= NULL_TREE
;
4401 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4402 if (gimple_call_internal_p (call
))
4404 internal_fn ifn
= gimple_call_internal_fn (call
);
4405 tree_code code
= conditional_internal_fn_code (ifn
);
4406 if (code
!= ERROR_MARK
)
4408 *cond_out
= gimple_call_arg (call
, 0);
4410 unsigned int nops
= gimple_call_num_args (call
) - 2;
4411 for (unsigned int i
= 0; i
< 3; ++i
)
4412 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4413 *else_out
= gimple_call_arg (call
, nops
+ 1);
4414 if (integer_truep (*cond_out
))
4416 *cond_out
= NULL_TREE
;
4417 *else_out
= NULL_TREE
;
4425 /* Return true if IFN is some form of load from memory. */
4428 internal_load_fn_p (internal_fn fn
)
4433 case IFN_LOAD_LANES
:
4434 case IFN_MASK_LOAD_LANES
:
4435 case IFN_GATHER_LOAD
:
4436 case IFN_MASK_GATHER_LOAD
:
4438 case IFN_LEN_MASK_LOAD
:
4446 /* Return true if IFN is some form of store to memory. */
4449 internal_store_fn_p (internal_fn fn
)
4453 case IFN_MASK_STORE
:
4454 case IFN_STORE_LANES
:
4455 case IFN_MASK_STORE_LANES
:
4456 case IFN_SCATTER_STORE
:
4457 case IFN_MASK_SCATTER_STORE
:
4459 case IFN_LEN_MASK_STORE
:
4467 /* Return true if IFN is some form of gather load or scatter store. */
4470 internal_gather_scatter_fn_p (internal_fn fn
)
4474 case IFN_GATHER_LOAD
:
4475 case IFN_MASK_GATHER_LOAD
:
4476 case IFN_SCATTER_STORE
:
4477 case IFN_MASK_SCATTER_STORE
:
4485 /* If FN takes a vector mask argument, return the index of that argument,
4486 otherwise return -1. */
4489 internal_fn_mask_index (internal_fn fn
)
4494 case IFN_MASK_LOAD_LANES
:
4495 case IFN_MASK_STORE
:
4496 case IFN_MASK_STORE_LANES
:
4499 case IFN_MASK_GATHER_LOAD
:
4500 case IFN_MASK_SCATTER_STORE
:
4503 case IFN_LEN_MASK_LOAD
:
4504 case IFN_LEN_MASK_STORE
:
4508 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4509 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4513 /* If FN takes a value that should be stored to memory, return the index
4514 of that argument, otherwise return -1. */
4517 internal_fn_stored_value_index (internal_fn fn
)
4521 case IFN_MASK_STORE
:
4522 case IFN_MASK_STORE_LANES
:
4523 case IFN_SCATTER_STORE
:
4524 case IFN_MASK_SCATTER_STORE
:
4528 case IFN_LEN_MASK_STORE
:
4536 /* Return true if the target supports gather load or scatter store function
4537 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4538 while for stores it is the vector type of the stored data argument.
4539 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4540 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4541 offset from the shared base address of each loaded or stored element.
4542 SCALE is the amount by which these offsets should be multiplied
4543 *after* they have been extended to address width. */
4546 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4547 tree memory_element_type
,
4548 tree offset_vector_type
, int scale
)
4550 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4551 TYPE_SIZE (memory_element_type
)))
4553 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4554 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4556 optab optab
= direct_internal_fn_optab (ifn
);
4557 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4558 TYPE_MODE (offset_vector_type
));
4559 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4560 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4561 return (icode
!= CODE_FOR_nothing
4562 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4563 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4566 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4567 for pointers of type TYPE when the accesses have LENGTH bytes and their
4568 common byte alignment is ALIGN. */
4571 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4572 poly_uint64 length
, unsigned int align
)
4574 machine_mode mode
= TYPE_MODE (type
);
4575 optab optab
= direct_internal_fn_optab (ifn
);
4576 insn_code icode
= direct_optab_handler (optab
, mode
);
4577 if (icode
== CODE_FOR_nothing
)
4579 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4580 return (insn_operand_matches (icode
, 3, length_rtx
)
4581 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4584 /* Return the supported bias for IFN which is either IFN_LEN_LOAD
4585 or IFN_LEN_STORE. For now we only support the biases of 0 and -1
4586 (in case 0 is not an allowable length for len_load or len_store).
4587 If none of the biases match what the backend provides, return
4588 VECT_PARTIAL_BIAS_UNSUPPORTED. */
4591 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
4593 optab optab
= direct_internal_fn_optab (ifn
);
4594 insn_code icode
= direct_optab_handler (optab
, mode
);
4597 if (icode
== CODE_FOR_nothing
)
4599 machine_mode mask_mode
;
4600 if (!targetm
.vectorize
.get_mask_mode (mode
).exists (&mask_mode
))
4601 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4602 if (ifn
== IFN_LEN_LOAD
)
4604 /* Try LEN_MASK_LOAD. */
4605 optab
= direct_internal_fn_optab (IFN_LEN_MASK_LOAD
);
4609 /* Try LEN_MASK_STORE. */
4610 optab
= direct_internal_fn_optab (IFN_LEN_MASK_STORE
);
4612 icode
= convert_optab_handler (optab
, mode
, mask_mode
);
4616 if (icode
!= CODE_FOR_nothing
)
4618 /* For now we only support biases of 0 or -1. Try both of them. */
4619 if (insn_operand_matches (icode
, bias_opno
, GEN_INT (0)))
4621 if (insn_operand_matches (icode
, bias_opno
, GEN_INT (-1)))
4625 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4628 /* Expand STMT as though it were a call to internal function FN. */
4631 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4633 internal_fn_expanders
[fn
] (fn
, stmt
);
4636 /* Expand STMT, which is a call to internal function FN. */
4639 expand_internal_call (gcall
*stmt
)
4641 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4644 /* If TYPE is a vector type, return true if IFN is a direct internal
4645 function that is supported for that type. If TYPE is a scalar type,
4646 return true if IFN is a direct internal function that is supported for
4647 the target's preferred vector version of TYPE. */
4650 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4652 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4653 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4656 if (VECTOR_TYPE_P (type
)
4657 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4660 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4661 if (VECTOR_MODE_P (vmode
))
4663 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4664 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4668 auto_vector_modes vector_modes
;
4669 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4670 for (machine_mode base_mode
: vector_modes
)
4671 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
4673 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4674 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4682 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
4688 expand_PHI (internal_fn
, gcall
*)
4694 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
4696 tree lhs
= gimple_call_lhs (stmt
);
4697 tree rhs1
= gimple_call_arg (stmt
, 0);
4698 tree rhs2
= gimple_call_arg (stmt
, 1);
4699 tree type
= TREE_TYPE (rhs1
);
4701 do_pending_stack_adjust ();
4703 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
4704 rtx op1
= expand_normal (rhs1
);
4705 rtx op2
= expand_normal (rhs2
);
4707 class expand_operand ops
[3];
4708 create_output_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
4709 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
4710 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
4711 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
4712 expand_insn (icode
, 3, ops
);
4713 if (!rtx_equal_p (target
, ops
[0].value
))
4714 emit_move_insn (target
, ops
[0].value
);
4718 expand_ASSUME (internal_fn
, gcall
*)
4723 expand_MASK_CALL (internal_fn
, gcall
*)
4725 /* This IFN should only exist between ifcvt and vect passes. */