2 Copyright (C) 2011-2022 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* The names of each internal function, indexed by function number. */
60 const char *const internal_fn_name_array
[] = {
61 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62 #include "internal-fn.def"
66 /* The ECF_* flags of each internal function, indexed by function number. */
67 const int internal_fn_flags_array
[] = {
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69 #include "internal-fn.def"
73 /* Return the internal function called NAME, or IFN_LAST if there's
77 lookup_internal_fn (const char *name
)
79 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
80 static name_to_fn_map_type
*name_to_fn_map
;
84 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
85 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
86 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
89 internal_fn
*entry
= name_to_fn_map
->get (name
);
90 return entry
? *entry
: IFN_LAST
;
93 /* Fnspec of each internal function, indexed by function number. */
94 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
99 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
100 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
101 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
102 #include "internal-fn.def"
103 internal_fn_fnspec_array
[IFN_LAST
] = 0;
106 /* Create static initializers for the information returned by
107 direct_internal_fn. */
108 #define not_direct { -2, -2, false }
109 #define mask_load_direct { -1, 2, false }
110 #define load_lanes_direct { -1, -1, false }
111 #define mask_load_lanes_direct { -1, -1, false }
112 #define gather_load_direct { 3, 1, false }
113 #define len_load_direct { -1, -1, false }
114 #define mask_store_direct { 3, 2, false }
115 #define store_lanes_direct { 0, 0, false }
116 #define mask_store_lanes_direct { 0, 0, false }
117 #define vec_cond_mask_direct { 1, 0, false }
118 #define vec_cond_direct { 2, 0, false }
119 #define scatter_store_direct { 3, 1, false }
120 #define len_store_direct { 3, 3, false }
121 #define vec_set_direct { 3, 3, false }
122 #define unary_direct { 0, 0, true }
123 #define binary_direct { 0, 0, true }
124 #define ternary_direct { 0, 0, true }
125 #define cond_unary_direct { 1, 1, true }
126 #define cond_binary_direct { 1, 1, true }
127 #define cond_ternary_direct { 1, 1, true }
128 #define while_direct { 0, 2, false }
129 #define fold_extract_direct { 2, 2, false }
130 #define fold_left_direct { 1, 1, false }
131 #define mask_fold_left_direct { 1, 1, false }
132 #define check_ptrs_direct { 0, 0, false }
134 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
135 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
136 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
137 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
138 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
139 #include "internal-fn.def"
143 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
144 output operands and NINPUTS input operands, where NOUTPUTS is either
145 0 or 1. The output operand (if any) comes first, followed by the
146 NINPUTS input operands. */
149 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
150 unsigned int ninputs
)
152 gcc_assert (icode
!= CODE_FOR_nothing
);
154 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
155 unsigned int opno
= 0;
156 rtx lhs_rtx
= NULL_RTX
;
157 tree lhs
= gimple_call_lhs (stmt
);
161 gcc_assert (noutputs
== 1);
163 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
165 /* Do not assign directly to a promoted subreg, since there is no
166 guarantee that the instruction will leave the upper bits of the
167 register in the state required by SUBREG_PROMOTED_SIGN. */
169 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
171 create_output_operand (&ops
[opno
], dest
,
172 insn_data
[icode
].operand
[opno
].mode
);
178 for (unsigned int i
= 0; i
< ninputs
; ++i
)
180 tree rhs
= gimple_call_arg (stmt
, i
);
181 tree rhs_type
= TREE_TYPE (rhs
);
182 rtx rhs_rtx
= expand_normal (rhs
);
183 if (INTEGRAL_TYPE_P (rhs_type
))
184 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
185 TYPE_MODE (rhs_type
),
186 TYPE_UNSIGNED (rhs_type
));
188 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
192 gcc_assert (opno
== noutputs
+ ninputs
);
193 expand_insn (icode
, opno
, ops
);
194 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
196 /* If the return value has an integral type, convert the instruction
197 result to that type. This is useful for things that return an
198 int regardless of the size of the input. If the instruction result
199 is smaller than required, assume that it is signed.
201 If the return value has a nonintegral type, its mode must match
202 the instruction result. */
203 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
205 /* If this is a scalar in a register that is stored in a wider
206 mode than the declared mode, compute the result into its
207 declared mode and then convert to the wider mode. */
208 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
209 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
210 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
211 SUBREG_PROMOTED_SIGN (lhs_rtx
));
213 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
214 emit_move_insn (lhs_rtx
, ops
[0].value
);
217 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
218 convert_move (lhs_rtx
, ops
[0].value
, 0);
223 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
224 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
226 static enum insn_code
227 get_multi_vector_move (tree array_type
, convert_optab optab
)
232 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
233 imode
= TYPE_MODE (array_type
);
234 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
236 return convert_optab_handler (optab
, imode
, vmode
);
239 /* Expand LOAD_LANES call STMT using optab OPTAB. */
242 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
244 class expand_operand ops
[2];
248 lhs
= gimple_call_lhs (stmt
);
249 rhs
= gimple_call_arg (stmt
, 0);
250 type
= TREE_TYPE (lhs
);
252 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
253 mem
= expand_normal (rhs
);
255 gcc_assert (MEM_P (mem
));
256 PUT_MODE (mem
, TYPE_MODE (type
));
258 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
259 create_fixed_operand (&ops
[1], mem
);
260 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
261 if (!rtx_equal_p (target
, ops
[0].value
))
262 emit_move_insn (target
, ops
[0].value
);
265 /* Expand STORE_LANES call STMT using optab OPTAB. */
268 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
270 class expand_operand ops
[2];
274 lhs
= gimple_call_lhs (stmt
);
275 rhs
= gimple_call_arg (stmt
, 0);
276 type
= TREE_TYPE (rhs
);
278 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
279 reg
= expand_normal (rhs
);
281 gcc_assert (MEM_P (target
));
282 PUT_MODE (target
, TYPE_MODE (type
));
284 create_fixed_operand (&ops
[0], target
);
285 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
286 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
290 expand_ANNOTATE (internal_fn
, gcall
*)
295 /* This should get expanded in omp_device_lower pass. */
298 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
303 /* This should get expanded in omp_device_lower pass. */
306 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
311 /* Allocate per-lane storage and begin non-uniform execution region. */
314 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
317 tree lhs
= gimple_call_lhs (stmt
);
319 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
321 target
= gen_reg_rtx (Pmode
);
322 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
323 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
324 class expand_operand ops
[3];
325 create_output_operand (&ops
[0], target
, Pmode
);
326 create_input_operand (&ops
[1], size
, Pmode
);
327 create_input_operand (&ops
[2], align
, Pmode
);
328 gcc_assert (targetm
.have_omp_simt_enter ());
329 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
330 if (!rtx_equal_p (target
, ops
[0].value
))
331 emit_move_insn (target
, ops
[0].value
);
334 /* Deallocate per-lane storage and leave non-uniform execution region. */
337 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
339 gcc_checking_assert (!gimple_call_lhs (stmt
));
340 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
341 class expand_operand ops
[1];
342 create_input_operand (&ops
[0], arg
, Pmode
);
343 gcc_assert (targetm
.have_omp_simt_exit ());
344 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
347 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
348 without SIMT execution this should be expanded in omp_device_lower pass. */
351 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
353 tree lhs
= gimple_call_lhs (stmt
);
357 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
358 gcc_assert (targetm
.have_omp_simt_lane ());
359 emit_insn (targetm
.gen_omp_simt_lane (target
));
362 /* This should get expanded in omp_device_lower pass. */
365 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
370 /* Lane index of the first SIMT lane that supplies a non-zero argument.
371 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
372 lane that executed the last iteration for handling OpenMP lastprivate. */
375 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
377 tree lhs
= gimple_call_lhs (stmt
);
381 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
382 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
383 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
384 class expand_operand ops
[2];
385 create_output_operand (&ops
[0], target
, mode
);
386 create_input_operand (&ops
[1], cond
, mode
);
387 gcc_assert (targetm
.have_omp_simt_last_lane ());
388 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
389 if (!rtx_equal_p (target
, ops
[0].value
))
390 emit_move_insn (target
, ops
[0].value
);
393 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
396 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
398 tree lhs
= gimple_call_lhs (stmt
);
402 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
403 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
404 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
405 class expand_operand ops
[2];
406 create_output_operand (&ops
[0], target
, mode
);
407 create_input_operand (&ops
[1], ctr
, mode
);
408 gcc_assert (targetm
.have_omp_simt_ordered ());
409 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
410 if (!rtx_equal_p (target
, ops
[0].value
))
411 emit_move_insn (target
, ops
[0].value
);
414 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
415 any lane supplies a non-zero argument. */
418 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
420 tree lhs
= gimple_call_lhs (stmt
);
424 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
425 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
426 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
427 class expand_operand ops
[2];
428 create_output_operand (&ops
[0], target
, mode
);
429 create_input_operand (&ops
[1], cond
, mode
);
430 gcc_assert (targetm
.have_omp_simt_vote_any ());
431 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
432 if (!rtx_equal_p (target
, ops
[0].value
))
433 emit_move_insn (target
, ops
[0].value
);
436 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
437 is destination lane index XOR given offset. */
440 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
442 tree lhs
= gimple_call_lhs (stmt
);
446 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
447 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
448 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
449 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
450 class expand_operand ops
[3];
451 create_output_operand (&ops
[0], target
, mode
);
452 create_input_operand (&ops
[1], src
, mode
);
453 create_input_operand (&ops
[2], idx
, SImode
);
454 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
455 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
456 if (!rtx_equal_p (target
, ops
[0].value
))
457 emit_move_insn (target
, ops
[0].value
);
460 /* Exchange between SIMT lanes according to given source lane index. */
463 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
465 tree lhs
= gimple_call_lhs (stmt
);
469 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
470 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
471 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
472 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
473 class expand_operand ops
[3];
474 create_output_operand (&ops
[0], target
, mode
);
475 create_input_operand (&ops
[1], src
, mode
);
476 create_input_operand (&ops
[2], idx
, SImode
);
477 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
478 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
479 if (!rtx_equal_p (target
, ops
[0].value
))
480 emit_move_insn (target
, ops
[0].value
);
483 /* This should get expanded in adjust_simduid_builtins. */
486 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
491 /* This should get expanded in adjust_simduid_builtins. */
494 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
499 /* This should get expanded in adjust_simduid_builtins. */
502 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
507 /* This should get expanded in adjust_simduid_builtins. */
510 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
515 /* This should get expanded in adjust_simduid_builtins. */
518 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
523 /* This should get expanded in the sanopt pass. */
526 expand_UBSAN_NULL (internal_fn
, gcall
*)
531 /* This should get expanded in the sanopt pass. */
534 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
539 /* This should get expanded in the sanopt pass. */
542 expand_UBSAN_VPTR (internal_fn
, gcall
*)
547 /* This should get expanded in the sanopt pass. */
550 expand_UBSAN_PTR (internal_fn
, gcall
*)
555 /* This should get expanded in the sanopt pass. */
558 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
563 /* This should get expanded in the sanopt pass. */
566 expand_HWASAN_CHECK (internal_fn
, gcall
*)
571 /* For hwasan stack tagging:
572 Clear tags on the dynamically allocated space.
573 For use after an object dynamically allocated on the stack goes out of
576 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
578 gcc_assert (Pmode
== ptr_mode
);
579 tree restored_position
= gimple_call_arg (gc
, 0);
580 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
582 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
583 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
584 stack_pointer_rtx
, NULL_RTX
, 0,
586 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
587 virtual_stack_dynamic_rtx
, Pmode
,
588 HWASAN_STACK_BACKGROUND
, QImode
,
592 /* For hwasan stack tagging:
593 Return a tag to be used for a dynamic allocation. */
595 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
597 tree tag
= gimple_call_lhs (gc
);
598 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
599 machine_mode mode
= GET_MODE (target
);
600 gcc_assert (mode
== QImode
);
602 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
603 gcc_assert (base_tag
);
604 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
605 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
606 target
, /* unsignedp = */1,
608 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
610 /* Really need to put the tag into the `target` RTX. */
611 if (chosen_tag
!= target
)
613 rtx temp
= chosen_tag
;
614 gcc_assert (GET_MODE (chosen_tag
) == mode
);
615 emit_move_insn (target
, temp
);
618 hwasan_increment_frame_tag ();
621 /* For hwasan stack tagging:
622 Tag a region of space in the shadow stack according to the base pointer of
623 an object on the stack. N.b. the length provided in the internal call is
624 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
626 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
628 gcc_assert (ptr_mode
== Pmode
);
629 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
630 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
632 tree base
= gimple_call_arg (gc
, 1);
633 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
634 rtx base_rtx
= expand_normal (base
);
636 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
637 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
638 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
640 tree len
= gimple_call_arg (gc
, 2);
641 rtx r_len
= expand_normal (len
);
643 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
644 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
645 tag
, QImode
, r_len
, Pmode
);
648 /* For hwasan stack tagging:
649 Store a tag into a pointer. */
651 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
653 gcc_assert (ptr_mode
== Pmode
);
654 tree g_target
= gimple_call_lhs (gc
);
655 tree g_ptr
= gimple_call_arg (gc
, 0);
656 tree g_tag
= gimple_call_arg (gc
, 1);
658 rtx ptr
= expand_normal (g_ptr
);
659 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
660 rtx target
= expand_normal (g_target
);
662 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
663 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
664 if (tagged_value
!= target
)
665 emit_move_insn (target
, tagged_value
);
668 /* This should get expanded in the sanopt pass. */
671 expand_ASAN_CHECK (internal_fn
, gcall
*)
676 /* This should get expanded in the sanopt pass. */
679 expand_ASAN_MARK (internal_fn
, gcall
*)
684 /* This should get expanded in the sanopt pass. */
687 expand_ASAN_POISON (internal_fn
, gcall
*)
692 /* This should get expanded in the sanopt pass. */
695 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
700 /* This should get expanded in the tsan pass. */
703 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
708 /* This should get expanded in the lower pass. */
711 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
713 error_at (gimple_location (call
),
714 "invalid use of attribute %<fallthrough%>");
717 /* Return minimum precision needed to represent all values
718 of ARG in SIGNed integral type. */
721 get_min_precision (tree arg
, signop sign
)
723 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
725 signop orig_sign
= sign
;
726 if (TREE_CODE (arg
) == INTEGER_CST
)
729 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
731 widest_int w
= wi::to_widest (arg
);
732 w
= wi::ext (w
, prec
, sign
);
733 p
= wi::min_precision (w
, sign
);
736 p
= wi::min_precision (wi::to_wide (arg
), sign
);
737 return MIN (p
, prec
);
739 while (CONVERT_EXPR_P (arg
)
740 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
741 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
743 arg
= TREE_OPERAND (arg
, 0);
744 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
746 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
748 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
749 return prec
+ (orig_sign
!= sign
);
750 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
753 return prec
+ (orig_sign
!= sign
);
755 if (CONVERT_EXPR_P (arg
)
756 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
757 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
759 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
760 If y_2's min precision is smaller than prec, return that. */
761 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
763 return oprec
+ (orig_sign
!= sign
);
765 if (TREE_CODE (arg
) != SSA_NAME
)
766 return prec
+ (orig_sign
!= sign
);
768 while (!get_global_range_query ()->range_of_expr (r
, arg
)
769 || r
.kind () != VR_RANGE
)
771 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
772 if (is_gimple_assign (g
)
773 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
775 tree t
= gimple_assign_rhs1 (g
);
776 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
777 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
780 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
782 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
784 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
785 return prec
+ (orig_sign
!= sign
);
786 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
789 return prec
+ (orig_sign
!= sign
);
793 return prec
+ (orig_sign
!= sign
);
795 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
797 int p1
= wi::min_precision (r
.lower_bound (), sign
);
798 int p2
= wi::min_precision (r
.upper_bound (), sign
);
800 prec
= MIN (prec
, p1
);
802 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
804 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
805 prec
= MIN (prec
, p
);
807 return prec
+ (orig_sign
!= sign
);
810 /* Helper for expand_*_overflow. Set the __imag__ part to true
811 (1 except for signed:1 type, in which case store -1). */
814 expand_arith_set_overflow (tree lhs
, rtx target
)
816 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
817 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
818 write_complex_part (target
, constm1_rtx
, true);
820 write_complex_part (target
, const1_rtx
, true);
823 /* Helper for expand_*_overflow. Store RES into the __real__ part
824 of TARGET. If RES has larger MODE than __real__ part of TARGET,
825 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
826 if LHS has smaller precision than its mode. */
829 expand_arith_overflow_result_store (tree lhs
, rtx target
,
830 scalar_int_mode mode
, rtx res
)
832 scalar_int_mode tgtmode
833 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
837 rtx_code_label
*done_label
= gen_label_rtx ();
838 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
839 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
840 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
841 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
842 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
843 profile_probability::very_likely ());
844 expand_arith_set_overflow (lhs
, target
);
845 emit_label (done_label
);
847 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
848 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
851 rtx_code_label
*done_label
= gen_label_rtx ();
852 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
857 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
859 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
860 true, OPTAB_LIB_WIDEN
);
864 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
866 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
869 do_compare_rtx_and_jump (res
, lres
,
870 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
871 profile_probability::very_likely ());
872 expand_arith_set_overflow (lhs
, target
);
873 emit_label (done_label
);
875 write_complex_part (target
, lres
, false);
878 /* Helper for expand_*_overflow. Store RES into TARGET. */
881 expand_ubsan_result_store (rtx target
, rtx res
)
883 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
884 /* If this is a scalar in a register that is stored in a wider mode
885 than the declared mode, compute the result into its declared mode
886 and then convert to the wider mode. Our value is the computed
888 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
890 emit_move_insn (target
, res
);
893 /* Add sub/add overflow checking to the statement STMT.
894 CODE says whether the operation is +, or -. */
897 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
898 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
899 bool uns1_p
, bool is_ubsan
, tree
*datap
)
901 rtx res
, target
= NULL_RTX
;
903 rtx_code_label
*done_label
= gen_label_rtx ();
904 rtx_code_label
*do_error
= gen_label_rtx ();
905 do_pending_stack_adjust ();
906 rtx op0
= expand_normal (arg0
);
907 rtx op1
= expand_normal (arg1
);
908 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
909 int prec
= GET_MODE_PRECISION (mode
);
910 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
914 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
918 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
920 write_complex_part (target
, const0_rtx
, true);
923 /* We assume both operands and result have the same precision
924 here (GET_MODE_BITSIZE (mode)), S stands for signed type
925 with that precision, U for unsigned type with that precision,
926 sgn for unsigned most significant bit in that precision.
927 s1 is signed first operand, u1 is unsigned first operand,
928 s2 is signed second operand, u2 is unsigned second operand,
929 sr is signed result, ur is unsigned result and the following
930 rules say how to compute result (which is always result of
931 the operands as if both were unsigned, cast to the right
932 signedness) and how to compute whether operation overflowed.
935 res = (S) ((U) s1 + (U) s2)
936 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
938 res = (S) ((U) s1 - (U) s2)
939 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
942 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
945 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
947 res = (S) ((U) s1 + u2)
948 ovf = ((U) res ^ sgn) < u2
953 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
955 res = (S) ((U) s1 - u2)
956 ovf = u2 > ((U) s1 ^ sgn)
959 ovf = s1 < 0 || u2 > (U) s1
962 ovf = u1 >= ((U) s2 ^ sgn)
967 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
969 res = (U) s1 + (U) s2
970 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
973 ovf = (U) res < u2 || res < 0
976 ovf = u1 >= u2 ? res < 0 : res >= 0
978 res = (U) s1 - (U) s2
979 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
981 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
983 /* PLUS_EXPR is commutative, if operand signedness differs,
984 canonicalize to the first operand being signed and second
985 unsigned to simplify following code. */
986 std::swap (op0
, op1
);
987 std::swap (arg0
, arg1
);
993 if (uns0_p
&& uns1_p
&& unsr_p
)
995 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
996 : usubv4_optab
, mode
);
997 if (icode
!= CODE_FOR_nothing
)
999 class expand_operand ops
[4];
1000 rtx_insn
*last
= get_last_insn ();
1002 res
= gen_reg_rtx (mode
);
1003 create_output_operand (&ops
[0], res
, mode
);
1004 create_input_operand (&ops
[1], op0
, mode
);
1005 create_input_operand (&ops
[2], op1
, mode
);
1006 create_fixed_operand (&ops
[3], do_error
);
1007 if (maybe_expand_insn (icode
, 4, ops
))
1009 last
= get_last_insn ();
1010 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1012 && any_condjump_p (last
)
1013 && !find_reg_note (last
, REG_BR_PROB
, 0))
1014 add_reg_br_prob_note (last
,
1015 profile_probability::very_unlikely ());
1016 emit_jump (done_label
);
1017 goto do_error_label
;
1020 delete_insns_since (last
);
1023 /* Compute the operation. On RTL level, the addition is always
1025 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1026 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1028 /* For PLUS_EXPR, the operation is commutative, so we can pick
1029 operand to compare against. For prec <= BITS_PER_WORD, I think
1030 preferring REG operand is better over CONST_INT, because
1031 the CONST_INT might enlarge the instruction or CSE would need
1032 to figure out we'd already loaded it into a register before.
1033 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1034 as then the multi-word comparison can be perhaps simplified. */
1035 if (code
== PLUS_EXPR
1036 && (prec
<= BITS_PER_WORD
1037 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1038 : CONST_SCALAR_INT_P (op1
)))
1040 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1041 true, mode
, NULL_RTX
, NULL
, done_label
,
1042 profile_probability::very_likely ());
1043 goto do_error_label
;
1046 /* s1 +- u2 -> sr */
1047 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1049 /* Compute the operation. On RTL level, the addition is always
1051 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1052 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1053 rtx tem
= expand_binop (mode
, add_optab
,
1054 code
== PLUS_EXPR
? res
: op0
, sgn
,
1055 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1056 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1057 done_label
, profile_probability::very_likely ());
1058 goto do_error_label
;
1062 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1064 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1066 /* As we've changed op1, we have to avoid using the value range
1067 for the original argument. */
1068 arg1
= error_mark_node
;
1074 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1076 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1078 /* As we've changed op0, we have to avoid using the value range
1079 for the original argument. */
1080 arg0
= error_mark_node
;
1086 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1088 /* Compute the operation. On RTL level, the addition is always
1090 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1092 int pos_neg
= get_range_pos_neg (arg0
);
1094 /* If ARG0 is known to be always negative, this is always overflow. */
1095 emit_jump (do_error
);
1096 else if (pos_neg
== 3)
1097 /* If ARG0 is not known to be always positive, check at runtime. */
1098 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1099 NULL
, do_error
, profile_probability::very_unlikely ());
1100 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1101 done_label
, profile_probability::very_likely ());
1102 goto do_error_label
;
1106 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1108 /* Compute the operation. On RTL level, the addition is always
1110 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1112 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1114 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1115 done_label
, profile_probability::very_likely ());
1116 goto do_error_label
;
1120 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1122 /* Compute the operation. On RTL level, the addition is always
1124 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1126 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1127 NULL
, do_error
, profile_probability::very_unlikely ());
1129 /* The operation is commutative, so we can pick operand to compare
1130 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1131 is better over CONST_INT, because the CONST_INT might enlarge the
1132 instruction or CSE would need to figure out we'd already loaded it
1133 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1134 might be more beneficial, as then the multi-word comparison can be
1135 perhaps simplified. */
1136 if (prec
<= BITS_PER_WORD
1137 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1138 : CONST_SCALAR_INT_P (op0
))
1140 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1141 done_label
, profile_probability::very_likely ());
1142 goto do_error_label
;
1145 /* s1 +- s2 -> ur */
1146 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1148 /* Compute the operation. On RTL level, the addition is always
1150 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1151 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1152 int pos_neg
= get_range_pos_neg (arg1
);
1153 if (code
== PLUS_EXPR
)
1155 int pos_neg0
= get_range_pos_neg (arg0
);
1156 if (pos_neg0
!= 3 && pos_neg
== 3)
1158 std::swap (op0
, op1
);
1165 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1166 ? and_optab
: ior_optab
,
1167 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1168 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1169 NULL
, done_label
, profile_probability::very_likely ());
1173 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1174 do_compare_rtx_and_jump (op1
, const0_rtx
,
1175 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1176 NULL_RTX
, NULL
, do_ior_label
,
1177 profile_probability::even ());
1178 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1180 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1181 NULL
, done_label
, profile_probability::very_likely ());
1182 emit_jump (do_error
);
1183 emit_label (do_ior_label
);
1184 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1186 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1187 NULL
, done_label
, profile_probability::very_likely ());
1189 goto do_error_label
;
1193 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1195 /* Compute the operation. On RTL level, the addition is always
1197 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1199 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1200 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1201 op0_geu_op1
, profile_probability::even ());
1202 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1203 NULL
, done_label
, profile_probability::very_likely ());
1204 emit_jump (do_error
);
1205 emit_label (op0_geu_op1
);
1206 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1207 NULL
, done_label
, profile_probability::very_likely ());
1208 goto do_error_label
;
1211 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1213 /* s1 +- s2 -> sr */
1216 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1217 : subv4_optab
, mode
);
1218 if (icode
!= CODE_FOR_nothing
)
1220 class expand_operand ops
[4];
1221 rtx_insn
*last
= get_last_insn ();
1223 res
= gen_reg_rtx (mode
);
1224 create_output_operand (&ops
[0], res
, mode
);
1225 create_input_operand (&ops
[1], op0
, mode
);
1226 create_input_operand (&ops
[2], op1
, mode
);
1227 create_fixed_operand (&ops
[3], do_error
);
1228 if (maybe_expand_insn (icode
, 4, ops
))
1230 last
= get_last_insn ();
1231 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1233 && any_condjump_p (last
)
1234 && !find_reg_note (last
, REG_BR_PROB
, 0))
1235 add_reg_br_prob_note (last
,
1236 profile_probability::very_unlikely ());
1237 emit_jump (done_label
);
1238 goto do_error_label
;
1241 delete_insns_since (last
);
1244 /* Compute the operation. On RTL level, the addition is always
1246 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1247 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1249 /* If we can prove that one of the arguments (for MINUS_EXPR only
1250 the second operand, as subtraction is not commutative) is always
1251 non-negative or always negative, we can do just one comparison
1252 and conditional jump. */
1253 int pos_neg
= get_range_pos_neg (arg1
);
1254 if (code
== PLUS_EXPR
)
1256 int pos_neg0
= get_range_pos_neg (arg0
);
1257 if (pos_neg0
!= 3 && pos_neg
== 3)
1259 std::swap (op0
, op1
);
1264 /* Addition overflows if and only if the two operands have the same sign,
1265 and the result has the opposite sign. Subtraction overflows if and
1266 only if the two operands have opposite sign, and the subtrahend has
1267 the same sign as the result. Here 0 is counted as positive. */
1270 /* Compute op0 ^ op1 (operands have opposite sign). */
1271 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1274 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1275 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1279 if (code
== PLUS_EXPR
)
1281 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1282 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1283 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1288 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1289 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1290 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1294 /* No overflow if the result has bit sign cleared. */
1295 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1296 NULL
, done_label
, profile_probability::very_likely ());
1299 /* Compare the result of the operation with the first operand.
1300 No overflow for addition if second operand is positive and result
1301 is larger or second operand is negative and result is smaller.
1302 Likewise for subtraction with sign of second operand flipped. */
1304 do_compare_rtx_and_jump (res
, op0
,
1305 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1306 false, mode
, NULL_RTX
, NULL
, done_label
,
1307 profile_probability::very_likely ());
1311 emit_label (do_error
);
1314 /* Expand the ubsan builtin call. */
1316 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1320 do_pending_stack_adjust ();
1323 expand_arith_set_overflow (lhs
, target
);
1326 emit_label (done_label
);
1331 expand_ubsan_result_store (target
, res
);
1335 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1338 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1343 /* Add negate overflow checking to the statement STMT. */
1346 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1351 rtx_code_label
*done_label
, *do_error
;
1352 rtx target
= NULL_RTX
;
1354 done_label
= gen_label_rtx ();
1355 do_error
= gen_label_rtx ();
1357 do_pending_stack_adjust ();
1358 op1
= expand_normal (arg1
);
1360 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1363 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1365 write_complex_part (target
, const0_rtx
, true);
1368 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1369 if (icode
!= CODE_FOR_nothing
)
1371 class expand_operand ops
[3];
1372 rtx_insn
*last
= get_last_insn ();
1374 res
= gen_reg_rtx (mode
);
1375 create_output_operand (&ops
[0], res
, mode
);
1376 create_input_operand (&ops
[1], op1
, mode
);
1377 create_fixed_operand (&ops
[2], do_error
);
1378 if (maybe_expand_insn (icode
, 3, ops
))
1380 last
= get_last_insn ();
1381 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1383 && any_condjump_p (last
)
1384 && !find_reg_note (last
, REG_BR_PROB
, 0))
1385 add_reg_br_prob_note (last
,
1386 profile_probability::very_unlikely ());
1387 emit_jump (done_label
);
1391 delete_insns_since (last
);
1392 icode
= CODE_FOR_nothing
;
1396 if (icode
== CODE_FOR_nothing
)
1398 /* Compute the operation. On RTL level, the addition is always
1400 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1402 /* Compare the operand with the most negative value. */
1403 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1404 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1405 done_label
, profile_probability::very_likely ());
1408 emit_label (do_error
);
1411 /* Expand the ubsan builtin call. */
1413 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1414 arg1
, NULL_TREE
, datap
);
1417 do_pending_stack_adjust ();
1420 expand_arith_set_overflow (lhs
, target
);
1423 emit_label (done_label
);
1428 expand_ubsan_result_store (target
, res
);
1430 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1434 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1435 mode MODE can be expanded without using a libcall. */
1438 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1439 rtx op0
, rtx op1
, bool uns
)
1441 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1442 != CODE_FOR_nothing
)
1445 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1446 != CODE_FOR_nothing
)
1449 rtx_insn
*last
= get_last_insn ();
1450 if (CONSTANT_P (op0
))
1451 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1453 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1454 if (CONSTANT_P (op1
))
1455 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1457 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1458 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1459 delete_insns_since (last
);
1460 return ret
!= NULL_RTX
;
1463 /* Add mul overflow checking to the statement STMT. */
1466 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1467 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1472 rtx_code_label
*done_label
, *do_error
;
1473 rtx target
= NULL_RTX
;
1475 enum insn_code icode
;
1477 done_label
= gen_label_rtx ();
1478 do_error
= gen_label_rtx ();
1480 do_pending_stack_adjust ();
1481 op0
= expand_normal (arg0
);
1482 op1
= expand_normal (arg1
);
1484 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1488 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1490 write_complex_part (target
, const0_rtx
, true);
1494 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1496 /* We assume both operands and result have the same precision
1497 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1498 with that precision, U for unsigned type with that precision,
1499 sgn for unsigned most significant bit in that precision.
1500 s1 is signed first operand, u1 is unsigned first operand,
1501 s2 is signed second operand, u2 is unsigned second operand,
1502 sr is signed result, ur is unsigned result and the following
1503 rules say how to compute result (which is always result of
1504 the operands as if both were unsigned, cast to the right
1505 signedness) and how to compute whether operation overflowed.
1506 main_ovf (false) stands for jump on signed multiplication
1507 overflow or the main algorithm with uns == false.
1508 main_ovf (true) stands for jump on unsigned multiplication
1509 overflow or the main algorithm with uns == true.
1512 res = (S) ((U) s1 * (U) s2)
1513 ovf = main_ovf (false)
1516 ovf = main_ovf (true)
1519 ovf = (s1 < 0 && u2) || main_ovf (true)
1522 ovf = res < 0 || main_ovf (true)
1524 res = (S) ((U) s1 * u2)
1525 ovf = (S) u2 >= 0 ? main_ovf (false)
1526 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1528 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1529 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1531 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1533 if (uns0_p
&& !uns1_p
)
1535 /* Multiplication is commutative, if operand signedness differs,
1536 canonicalize to the first operand being signed and second
1537 unsigned to simplify following code. */
1538 std::swap (op0
, op1
);
1539 std::swap (arg0
, arg1
);
1544 int pos_neg0
= get_range_pos_neg (arg0
);
1545 int pos_neg1
= get_range_pos_neg (arg1
);
1548 if (!uns0_p
&& uns1_p
&& unsr_p
)
1553 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1556 /* If s1 is negative, avoid the main code, just multiply and
1557 signal overflow if op1 is not 0. */
1558 struct separate_ops ops
;
1559 ops
.code
= MULT_EXPR
;
1560 ops
.type
= TREE_TYPE (arg1
);
1561 ops
.op0
= make_tree (ops
.type
, op0
);
1562 ops
.op1
= make_tree (ops
.type
, op1
);
1563 ops
.op2
= NULL_TREE
;
1565 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1566 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1567 NULL
, done_label
, profile_probability::very_likely ());
1568 goto do_error_label
;
1570 if (get_min_precision (arg1
, UNSIGNED
)
1571 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1573 /* If the first operand is sign extended from narrower type, the
1574 second operand is zero extended from narrower type and
1575 the sum of the two precisions is smaller or equal to the
1576 result precision: if the first argument is at runtime
1577 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1578 and there will be no overflow, if the first argument is
1579 negative and the second argument zero, the result will be
1580 0 and there will be no overflow, if the first argument is
1581 negative and the second argument positive, the result when
1582 treated as signed will be negative (minimum -0x7f80 or
1583 -0x7f..f80..0) there will be always overflow. So, do
1585 ovf = (S) res < 0 */
1586 struct separate_ops ops
;
1587 ops
.code
= MULT_EXPR
;
1589 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1591 ops
.op0
= make_tree (ops
.type
, op0
);
1592 ops
.op1
= make_tree (ops
.type
, op1
);
1593 ops
.op2
= NULL_TREE
;
1595 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1596 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1597 mode
, NULL_RTX
, NULL
, done_label
,
1598 profile_probability::very_likely ());
1599 goto do_error_label
;
1601 rtx_code_label
*do_main_label
;
1602 do_main_label
= gen_label_rtx ();
1603 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1604 NULL
, do_main_label
, profile_probability::very_likely ());
1605 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1606 NULL
, do_main_label
, profile_probability::very_likely ());
1607 expand_arith_set_overflow (lhs
, target
);
1608 emit_label (do_main_label
);
1616 if (uns0_p
&& uns1_p
&& !unsr_p
)
1618 if ((pos_neg0
| pos_neg1
) == 1)
1620 /* If both arguments are zero extended from narrower types,
1621 the MSB will be clear on both and so we can pretend it is
1622 a normal s1 * s2 -> sr multiplication. */
1628 /* Rest of handling of this case after res is computed. */
1633 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1640 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1641 avoid the main code, just multiply and signal overflow
1642 unless 0 * u2 or -1 * ((U) Smin). */
1643 struct separate_ops ops
;
1644 ops
.code
= MULT_EXPR
;
1645 ops
.type
= TREE_TYPE (arg1
);
1646 ops
.op0
= make_tree (ops
.type
, op0
);
1647 ops
.op1
= make_tree (ops
.type
, op1
);
1648 ops
.op2
= NULL_TREE
;
1650 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1651 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1652 NULL
, done_label
, profile_probability::very_likely ());
1653 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1654 NULL
, do_error
, profile_probability::very_unlikely ());
1656 prec
= GET_MODE_PRECISION (mode
);
1658 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1659 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1660 NULL
, done_label
, profile_probability::very_likely ());
1661 goto do_error_label
;
1663 /* Rest of handling of this case after res is computed. */
1671 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1674 switch (pos_neg0
| pos_neg1
)
1676 case 1: /* Both operands known to be non-negative. */
1678 case 2: /* Both operands known to be negative. */
1679 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1680 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1681 /* Avoid looking at arg0/arg1 ranges, as we've changed
1683 arg0
= error_mark_node
;
1684 arg1
= error_mark_node
;
1687 if ((pos_neg0
^ pos_neg1
) == 3)
1689 /* If one operand is known to be negative and the other
1690 non-negative, this overflows always, unless the non-negative
1691 one is 0. Just do normal multiply and set overflow
1692 unless one of the operands is 0. */
1693 struct separate_ops ops
;
1694 ops
.code
= MULT_EXPR
;
1696 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1698 ops
.op0
= make_tree (ops
.type
, op0
);
1699 ops
.op1
= make_tree (ops
.type
, op1
);
1700 ops
.op2
= NULL_TREE
;
1702 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1703 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1704 true, mode
, NULL_RTX
, NULL
, done_label
,
1705 profile_probability::very_likely ());
1706 goto do_error_label
;
1708 if (get_min_precision (arg0
, SIGNED
)
1709 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1711 /* If both operands are sign extended from narrower types and
1712 the sum of the two precisions is smaller or equal to the
1713 result precision: if both arguments are at runtime
1714 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1715 and there will be no overflow, if both arguments are negative,
1716 maximum result will be 0x40..00 and there will be no overflow
1717 either, if one argument is positive and the other argument
1718 negative, the result when treated as signed will be negative
1719 and there will be always overflow, and if one argument is
1720 zero and the other negative the result will be zero and no
1723 ovf = (S) res < 0 */
1724 struct separate_ops ops
;
1725 ops
.code
= MULT_EXPR
;
1727 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1729 ops
.op0
= make_tree (ops
.type
, op0
);
1730 ops
.op1
= make_tree (ops
.type
, op1
);
1731 ops
.op2
= NULL_TREE
;
1733 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1734 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1735 mode
, NULL_RTX
, NULL
, done_label
,
1736 profile_probability::very_likely ());
1737 goto do_error_label
;
1739 /* The general case, do all the needed comparisons at runtime. */
1740 rtx_code_label
*do_main_label
, *after_negate_label
;
1742 rop0
= gen_reg_rtx (mode
);
1743 rop1
= gen_reg_rtx (mode
);
1744 emit_move_insn (rop0
, op0
);
1745 emit_move_insn (rop1
, op1
);
1748 do_main_label
= gen_label_rtx ();
1749 after_negate_label
= gen_label_rtx ();
1750 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1752 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1753 NULL
, after_negate_label
, profile_probability::very_likely ());
1754 /* Both arguments negative here, negate them and continue with
1755 normal unsigned overflow checking multiplication. */
1756 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1758 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1760 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1762 arg0
= error_mark_node
;
1763 arg1
= error_mark_node
;
1764 emit_jump (do_main_label
);
1765 emit_label (after_negate_label
);
1766 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1768 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1769 NULL
, do_main_label
,
1770 profile_probability::very_likely ());
1771 /* One argument is negative here, the other positive. This
1772 overflows always, unless one of the arguments is 0. But
1773 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1774 is, thus we can keep do_main code oring in overflow as is. */
1776 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1777 NULL
, do_main_label
,
1778 profile_probability::very_unlikely ());
1780 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1781 NULL
, do_main_label
,
1782 profile_probability::very_unlikely ());
1783 expand_arith_set_overflow (lhs
, target
);
1784 emit_label (do_main_label
);
1792 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1793 sign
= uns
? UNSIGNED
: SIGNED
;
1794 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1796 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1797 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1799 /* Optimize unsigned multiplication by power of 2 constant
1800 using 2 shifts, one for result, one to extract the shifted
1801 out bits to see if they are all zero.
1802 Don't do this if optimizing for size and we have umulv4_optab,
1803 in that case assume multiplication will be shorter.
1804 This is heuristics based on the single target that provides
1805 umulv4 right now (i?86/x86_64), if further targets add it, this
1806 might need to be revisited.
1807 Cases where both operands are constant should be folded already
1808 during GIMPLE, and cases where one operand is constant but not
1809 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1810 below can be done without multiplication, just by shifts and adds,
1811 or we'd need to divide the result (and hope it actually doesn't
1812 really divide nor multiply) and compare the result of the division
1813 with the original operand. */
1818 if (integer_pow2p (arg0
))
1820 std::swap (opn0
, opn1
);
1821 std::swap (argn0
, argn1
);
1823 int cnt
= tree_log2 (argn1
);
1824 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1826 rtx upper
= const0_rtx
;
1827 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1829 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1830 GET_MODE_PRECISION (mode
) - cnt
,
1832 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1833 NULL_RTX
, NULL
, done_label
,
1834 profile_probability::very_likely ());
1835 goto do_error_label
;
1838 if (icode
!= CODE_FOR_nothing
)
1840 class expand_operand ops
[4];
1841 rtx_insn
*last
= get_last_insn ();
1843 res
= gen_reg_rtx (mode
);
1844 create_output_operand (&ops
[0], res
, mode
);
1845 create_input_operand (&ops
[1], op0
, mode
);
1846 create_input_operand (&ops
[2], op1
, mode
);
1847 create_fixed_operand (&ops
[3], do_error
);
1848 if (maybe_expand_insn (icode
, 4, ops
))
1850 last
= get_last_insn ();
1851 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1853 && any_condjump_p (last
)
1854 && !find_reg_note (last
, REG_BR_PROB
, 0))
1855 add_reg_br_prob_note (last
,
1856 profile_probability::very_unlikely ());
1857 emit_jump (done_label
);
1861 delete_insns_since (last
);
1862 icode
= CODE_FOR_nothing
;
1866 if (icode
== CODE_FOR_nothing
)
1868 struct separate_ops ops
;
1869 int prec
= GET_MODE_PRECISION (mode
);
1870 scalar_int_mode hmode
, wmode
;
1871 ops
.op0
= make_tree (type
, op0
);
1872 ops
.op1
= make_tree (type
, op1
);
1873 ops
.op2
= NULL_TREE
;
1876 /* Optimize unsigned overflow check where we don't use the
1877 multiplication result, just whether overflow happened.
1878 If we can do MULT_HIGHPART_EXPR, that followed by
1879 comparison of the result against zero is cheapest.
1880 We'll still compute res, but it should be DCEd later. */
1886 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1887 && can_mult_highpart_p (mode
, uns
) == 1
1888 && single_imm_use (lhs
, &use
, &use_stmt
)
1889 && is_gimple_assign (use_stmt
)
1890 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1893 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1894 && targetm
.scalar_mode_supported_p (wmode
)
1895 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1898 ops
.code
= WIDEN_MULT_EXPR
;
1900 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1902 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1903 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1905 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1906 res
= convert_modes (mode
, wmode
, res
, uns
);
1908 /* For the unsigned multiplication, there was overflow if
1909 HIPART is non-zero. */
1910 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1911 NULL_RTX
, NULL
, done_label
,
1912 profile_probability::very_likely ());
1915 /* RES is used more than once, place it in a pseudo. */
1916 res
= force_reg (mode
, res
);
1918 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1920 /* RES is low half of the double width result, HIPART
1921 the high half. There was overflow if
1922 HIPART is different from RES < 0 ? -1 : 0. */
1923 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1924 NULL_RTX
, NULL
, done_label
,
1925 profile_probability::very_likely ());
1928 else if (can_mult_highpart_p (mode
, uns
) == 1)
1931 ops
.code
= MULT_HIGHPART_EXPR
;
1934 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1936 ops
.code
= MULT_EXPR
;
1937 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1939 /* For the unsigned multiplication, there was overflow if
1940 HIPART is non-zero. */
1941 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1942 NULL_RTX
, NULL
, done_label
,
1943 profile_probability::very_likely ());
1946 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1948 /* RES is low half of the double width result, HIPART
1949 the high half. There was overflow if
1950 HIPART is different from RES < 0 ? -1 : 0. */
1951 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1952 NULL_RTX
, NULL
, done_label
,
1953 profile_probability::very_likely ());
1957 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1958 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1960 rtx_code_label
*large_op0
= gen_label_rtx ();
1961 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1962 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1963 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1964 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1965 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1966 rtx_code_label
*do_overflow
= gen_label_rtx ();
1967 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1969 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1970 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1972 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1973 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1974 rtx signbit0
= const0_rtx
;
1976 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1978 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1980 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1981 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1982 rtx signbit1
= const0_rtx
;
1984 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1987 res
= gen_reg_rtx (mode
);
1989 /* True if op0 resp. op1 are known to be in the range of
1991 bool op0_small_p
= false;
1992 bool op1_small_p
= false;
1993 /* True if op0 resp. op1 are known to have all zeros or all ones
1994 in the upper half of bits, but are not known to be
1996 bool op0_medium_p
= false;
1997 bool op1_medium_p
= false;
1998 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1999 nonnegative, 1 if unknown. */
2005 else if (pos_neg0
== 2)
2009 else if (pos_neg1
== 2)
2012 unsigned int mprec0
= prec
;
2013 if (arg0
!= error_mark_node
)
2014 mprec0
= get_min_precision (arg0
, sign
);
2015 if (mprec0
<= hprec
)
2017 else if (!uns
&& mprec0
<= hprec
+ 1)
2018 op0_medium_p
= true;
2019 unsigned int mprec1
= prec
;
2020 if (arg1
!= error_mark_node
)
2021 mprec1
= get_min_precision (arg1
, sign
);
2022 if (mprec1
<= hprec
)
2024 else if (!uns
&& mprec1
<= hprec
+ 1)
2025 op1_medium_p
= true;
2027 int smaller_sign
= 1;
2028 int larger_sign
= 1;
2031 smaller_sign
= op0_sign
;
2032 larger_sign
= op1_sign
;
2034 else if (op1_small_p
)
2036 smaller_sign
= op1_sign
;
2037 larger_sign
= op0_sign
;
2039 else if (op0_sign
== op1_sign
)
2041 smaller_sign
= op0_sign
;
2042 larger_sign
= op0_sign
;
2046 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2047 NULL_RTX
, NULL
, large_op0
,
2048 profile_probability::unlikely ());
2051 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2052 NULL_RTX
, NULL
, small_op0_large_op1
,
2053 profile_probability::unlikely ());
2055 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2056 hmode to mode, the multiplication will never overflow. We can
2057 do just one hmode x hmode => mode widening multiplication. */
2058 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2059 ops
.op0
= make_tree (halfstype
, lopart0
);
2060 ops
.op1
= make_tree (halfstype
, lopart1
);
2061 ops
.code
= WIDEN_MULT_EXPR
;
2064 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2065 emit_move_insn (res
, thisres
);
2066 emit_jump (done_label
);
2068 emit_label (small_op0_large_op1
);
2070 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2071 but op1 is not, just swap the arguments and handle it as op1
2072 sign/zero extended, op0 not. */
2073 rtx larger
= gen_reg_rtx (mode
);
2074 rtx hipart
= gen_reg_rtx (hmode
);
2075 rtx lopart
= gen_reg_rtx (hmode
);
2076 emit_move_insn (larger
, op1
);
2077 emit_move_insn (hipart
, hipart1
);
2078 emit_move_insn (lopart
, lopart0
);
2079 emit_jump (one_small_one_large
);
2081 emit_label (large_op0
);
2084 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2085 NULL_RTX
, NULL
, both_ops_large
,
2086 profile_probability::unlikely ());
2088 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2089 but op0 is not, prepare larger, hipart and lopart pseudos and
2090 handle it together with small_op0_large_op1. */
2091 emit_move_insn (larger
, op0
);
2092 emit_move_insn (hipart
, hipart0
);
2093 emit_move_insn (lopart
, lopart1
);
2095 emit_label (one_small_one_large
);
2097 /* lopart is the low part of the operand that is sign extended
2098 to mode, larger is the other operand, hipart is the
2099 high part of larger and lopart0 and lopart1 are the low parts
2101 We perform lopart0 * lopart1 and lopart * hipart widening
2103 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2104 ops
.op0
= make_tree (halfutype
, lopart0
);
2105 ops
.op1
= make_tree (halfutype
, lopart1
);
2107 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2109 ops
.op0
= make_tree (halfutype
, lopart
);
2110 ops
.op1
= make_tree (halfutype
, hipart
);
2111 rtx loxhi
= gen_reg_rtx (mode
);
2112 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2113 emit_move_insn (loxhi
, tem
);
2117 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2118 if (larger_sign
== 0)
2119 emit_jump (after_hipart_neg
);
2120 else if (larger_sign
!= -1)
2121 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2122 NULL_RTX
, NULL
, after_hipart_neg
,
2123 profile_probability::even ());
2125 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2126 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2127 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2129 emit_move_insn (loxhi
, tem
);
2131 emit_label (after_hipart_neg
);
2133 /* if (lopart < 0) loxhi -= larger; */
2134 if (smaller_sign
== 0)
2135 emit_jump (after_lopart_neg
);
2136 else if (smaller_sign
!= -1)
2137 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2138 NULL_RTX
, NULL
, after_lopart_neg
,
2139 profile_probability::even ());
2141 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2143 emit_move_insn (loxhi
, tem
);
2145 emit_label (after_lopart_neg
);
2148 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2149 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2150 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2152 emit_move_insn (loxhi
, tem
);
2154 /* if (loxhi >> (bitsize / 2)
2155 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2156 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2157 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2159 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2160 rtx signbitloxhi
= const0_rtx
;
2162 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2163 convert_modes (hmode
, mode
,
2165 hprec
- 1, NULL_RTX
, 0);
2167 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2168 NULL_RTX
, NULL
, do_overflow
,
2169 profile_probability::very_unlikely ());
2171 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2172 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2174 tem
= convert_modes (mode
, hmode
,
2175 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2177 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2180 emit_move_insn (res
, tem
);
2181 emit_jump (done_label
);
2183 emit_label (both_ops_large
);
2185 /* If both operands are large (not sign (!uns) or zero (uns)
2186 extended from hmode), then perform the full multiplication
2187 which will be the result of the operation.
2188 The only cases which don't overflow are for signed multiplication
2189 some cases where both hipart0 and highpart1 are 0 or -1.
2190 For unsigned multiplication when high parts are both non-zero
2191 this overflows always. */
2192 ops
.code
= MULT_EXPR
;
2193 ops
.op0
= make_tree (type
, op0
);
2194 ops
.op1
= make_tree (type
, op1
);
2195 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2196 emit_move_insn (res
, tem
);
2202 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2203 NULL_RTX
, 1, OPTAB_WIDEN
);
2204 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2205 NULL_RTX
, NULL
, do_error
,
2206 profile_probability::very_unlikely ());
2211 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2212 NULL_RTX
, 1, OPTAB_WIDEN
);
2213 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2214 NULL_RTX
, NULL
, do_error
,
2215 profile_probability::very_unlikely ());
2218 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2219 the same, overflow happened if res is non-positive, if they
2220 are different, overflow happened if res is positive. */
2221 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2222 emit_jump (hipart_different
);
2223 else if (op0_sign
== 1 || op1_sign
== 1)
2224 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2225 NULL_RTX
, NULL
, hipart_different
,
2226 profile_probability::even ());
2228 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2229 NULL_RTX
, NULL
, do_error
,
2230 profile_probability::very_unlikely ());
2231 emit_jump (done_label
);
2233 emit_label (hipart_different
);
2235 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2236 NULL_RTX
, NULL
, do_error
,
2237 profile_probability::very_unlikely ());
2238 emit_jump (done_label
);
2241 emit_label (do_overflow
);
2243 /* Overflow, do full multiplication and fallthru into do_error. */
2244 ops
.op0
= make_tree (type
, op0
);
2245 ops
.op1
= make_tree (type
, op1
);
2246 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2247 emit_move_insn (res
, tem
);
2249 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2250 && targetm
.scalar_mode_supported_p (wmode
))
2251 /* Even emitting a libcall is better than not detecting overflow
2256 gcc_assert (!is_ubsan
);
2257 ops
.code
= MULT_EXPR
;
2259 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2260 emit_jump (done_label
);
2265 emit_label (do_error
);
2268 /* Expand the ubsan builtin call. */
2270 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2274 do_pending_stack_adjust ();
2277 expand_arith_set_overflow (lhs
, target
);
2280 emit_label (done_label
);
2283 if (uns0_p
&& uns1_p
&& !unsr_p
)
2285 rtx_code_label
*all_done_label
= gen_label_rtx ();
2286 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2287 NULL
, all_done_label
, profile_probability::very_likely ());
2288 expand_arith_set_overflow (lhs
, target
);
2289 emit_label (all_done_label
);
2293 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2295 rtx_code_label
*all_done_label
= gen_label_rtx ();
2296 rtx_code_label
*set_noovf
= gen_label_rtx ();
2297 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2298 NULL
, all_done_label
, profile_probability::very_likely ());
2299 expand_arith_set_overflow (lhs
, target
);
2300 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2301 NULL
, set_noovf
, profile_probability::very_likely ());
2302 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2303 NULL
, all_done_label
, profile_probability::very_unlikely ());
2304 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2305 all_done_label
, profile_probability::very_unlikely ());
2306 emit_label (set_noovf
);
2307 write_complex_part (target
, const0_rtx
, true);
2308 emit_label (all_done_label
);
2314 expand_ubsan_result_store (target
, res
);
2316 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2320 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2323 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2324 tree arg0
, tree arg1
)
2326 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2327 rtx_code_label
*loop_lab
= NULL
;
2328 rtx cntvar
= NULL_RTX
;
2329 tree cntv
= NULL_TREE
;
2330 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2331 tree sz
= TYPE_SIZE (eltype
);
2332 tree data
= NULL_TREE
;
2333 tree resv
= NULL_TREE
;
2334 rtx lhsr
= NULL_RTX
;
2335 rtx resvr
= NULL_RTX
;
2336 unsigned HOST_WIDE_INT const_cnt
= 0;
2337 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2342 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2343 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2344 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2345 optab_default
)) == unknown_optab
2346 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2347 == CODE_FOR_nothing
))
2350 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2353 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2354 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2360 do_pending_stack_adjust ();
2361 loop_lab
= gen_label_rtx ();
2362 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2363 cntv
= make_tree (sizetype
, cntvar
);
2364 emit_move_insn (cntvar
, const0_rtx
);
2365 emit_label (loop_lab
);
2367 if (TREE_CODE (arg0
) != VECTOR_CST
)
2369 rtx arg0r
= expand_normal (arg0
);
2370 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2372 if (TREE_CODE (arg1
) != VECTOR_CST
)
2374 rtx arg1r
= expand_normal (arg1
);
2375 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2377 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2379 tree op0
, op1
, res
= NULL_TREE
;
2382 tree atype
= build_array_type_nelts (eltype
, cnt
);
2383 op0
= uniform_vector_p (arg0
);
2384 if (op0
== NULL_TREE
)
2386 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2387 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2388 NULL_TREE
, NULL_TREE
);
2390 op1
= uniform_vector_p (arg1
);
2391 if (op1
== NULL_TREE
)
2393 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2394 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2395 NULL_TREE
, NULL_TREE
);
2399 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2400 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2401 NULL_TREE
, NULL_TREE
);
2406 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2407 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2408 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2410 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2416 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2417 false, false, false, true, &data
);
2420 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2421 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2423 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2424 false, false, false, true, &data
);
2427 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2436 struct separate_ops ops
;
2437 ops
.code
= PLUS_EXPR
;
2438 ops
.type
= TREE_TYPE (cntv
);
2440 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2441 ops
.op2
= NULL_TREE
;
2443 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2446 emit_move_insn (cntvar
, ret
);
2447 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2448 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2449 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2450 profile_probability::very_likely ());
2452 if (lhs
&& resv
== NULL_TREE
)
2454 struct separate_ops ops
;
2456 ops
.type
= TREE_TYPE (arg0
);
2459 ops
.op2
= NULL_TREE
;
2461 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2464 emit_move_insn (lhsr
, ret
);
2467 emit_move_insn (lhsr
, resvr
);
2470 /* Expand UBSAN_CHECK_ADD call STMT. */
2473 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2475 location_t loc
= gimple_location (stmt
);
2476 tree lhs
= gimple_call_lhs (stmt
);
2477 tree arg0
= gimple_call_arg (stmt
, 0);
2478 tree arg1
= gimple_call_arg (stmt
, 1);
2479 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2480 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2482 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2483 false, false, false, true, NULL
);
2486 /* Expand UBSAN_CHECK_SUB call STMT. */
2489 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2491 location_t loc
= gimple_location (stmt
);
2492 tree lhs
= gimple_call_lhs (stmt
);
2493 tree arg0
= gimple_call_arg (stmt
, 0);
2494 tree arg1
= gimple_call_arg (stmt
, 1);
2495 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2496 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2497 else if (integer_zerop (arg0
))
2498 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2500 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2501 false, false, false, true, NULL
);
2504 /* Expand UBSAN_CHECK_MUL call STMT. */
2507 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2509 location_t loc
= gimple_location (stmt
);
2510 tree lhs
= gimple_call_lhs (stmt
);
2511 tree arg0
= gimple_call_arg (stmt
, 0);
2512 tree arg1
= gimple_call_arg (stmt
, 1);
2513 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2514 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2516 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2520 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2523 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2525 tree lhs
= gimple_call_lhs (stmt
);
2526 if (lhs
== NULL_TREE
)
2528 tree arg0
= gimple_call_arg (stmt
, 0);
2529 tree arg1
= gimple_call_arg (stmt
, 1);
2530 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2531 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2532 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2533 int unsr_p
= TYPE_UNSIGNED (type
);
2534 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2535 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2536 int precres
= TYPE_PRECISION (type
);
2537 location_t loc
= gimple_location (stmt
);
2538 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2540 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2542 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2543 prec0
= MIN (prec0
, pr
);
2544 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2545 prec1
= MIN (prec1
, pr
);
2547 /* If uns0_p && uns1_p, precop is minimum needed precision
2548 of unsigned type to hold the exact result, otherwise
2549 precop is minimum needed precision of signed type to
2550 hold the exact result. */
2552 if (code
== MULT_EXPR
)
2553 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2556 if (uns0_p
== uns1_p
)
2557 precop
= MAX (prec0
, prec1
) + 1;
2559 precop
= MAX (prec0
+ 1, prec1
) + 1;
2561 precop
= MAX (prec0
, prec1
+ 1) + 1;
2563 int orig_precres
= precres
;
2567 if ((uns0_p
&& uns1_p
)
2568 ? ((precop
+ !unsr_p
) <= precres
2569 /* u1 - u2 -> ur can overflow, no matter what precision
2571 && (code
!= MINUS_EXPR
|| !unsr_p
))
2572 : (!unsr_p
&& precop
<= precres
))
2574 /* The infinity precision result will always fit into result. */
2575 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2576 write_complex_part (target
, const0_rtx
, true);
2577 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2578 struct separate_ops ops
;
2581 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2582 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2583 ops
.op2
= NULL_TREE
;
2585 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2586 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2590 /* For operations with low precision, if target doesn't have them, start
2591 with precres widening right away, otherwise do it only if the most
2592 simple cases can't be used. */
2593 const int min_precision
= targetm
.min_arithmetic_precision ();
2594 if (orig_precres
== precres
&& precres
< min_precision
)
2596 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2597 && prec1
<= precres
)
2598 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2599 && prec0
+ uns0_p
<= precres
2600 && prec1
+ uns1_p
<= precres
))
2602 arg0
= fold_convert_loc (loc
, type
, arg0
);
2603 arg1
= fold_convert_loc (loc
, type
, arg1
);
2607 if (integer_zerop (arg0
) && !unsr_p
)
2609 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2614 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2615 unsr_p
, unsr_p
, false, NULL
);
2618 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2619 unsr_p
, unsr_p
, false, NULL
);
2626 /* For sub-word operations, retry with a wider type first. */
2627 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2629 int p
= MAX (min_precision
, precop
);
2630 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2631 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2634 p
= TYPE_PRECISION (optype
);
2638 unsr_p
= TYPE_UNSIGNED (optype
);
2644 if (prec0
<= precres
&& prec1
<= precres
)
2649 types
[0] = build_nonstandard_integer_type (precres
, 0);
2655 types
[1] = build_nonstandard_integer_type (precres
, 1);
2657 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2658 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2659 if (code
!= MULT_EXPR
)
2660 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2661 uns0_p
, uns1_p
, false, NULL
);
2663 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2664 uns0_p
, uns1_p
, false, NULL
);
2668 /* Retry with a wider type. */
2669 if (orig_precres
== precres
)
2671 int p
= MAX (prec0
, prec1
);
2672 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2673 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2676 p
= TYPE_PRECISION (optype
);
2680 unsr_p
= TYPE_UNSIGNED (optype
);
2691 /* Expand ADD_OVERFLOW STMT. */
2694 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2696 expand_arith_overflow (PLUS_EXPR
, stmt
);
2699 /* Expand SUB_OVERFLOW STMT. */
2702 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2704 expand_arith_overflow (MINUS_EXPR
, stmt
);
2707 /* Expand MUL_OVERFLOW STMT. */
2710 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2712 expand_arith_overflow (MULT_EXPR
, stmt
);
2715 /* This should get folded in tree-vectorizer.cc. */
2718 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2723 /* This should get folded in tree-vectorizer.cc. */
2726 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2731 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2732 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2735 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2737 tree addr
= gimple_call_arg (stmt
, index
);
2738 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2739 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2740 if (TYPE_ALIGN (type
) != align
)
2741 type
= build_aligned_type (type
, align
);
2744 if (TREE_CODE (tmp
) == SSA_NAME
)
2746 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2747 if (gimple_assign_single_p (def
))
2748 tmp
= gimple_assign_rhs1 (def
);
2751 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2753 tree mem
= TREE_OPERAND (tmp
, 0);
2754 if (TREE_CODE (mem
) == TARGET_MEM_REF
2755 && types_compatible_p (TREE_TYPE (mem
), type
))
2757 tree offset
= TMR_OFFSET (mem
);
2758 if (type
!= TREE_TYPE (mem
)
2759 || alias_ptr_type
!= TREE_TYPE (offset
)
2760 || !integer_zerop (offset
))
2762 mem
= copy_node (mem
);
2763 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2764 wi::to_poly_wide (offset
));
2765 TREE_TYPE (mem
) = type
;
2771 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2774 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB. */
2777 expand_partial_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2779 class expand_operand ops
[4];
2780 tree type
, lhs
, rhs
, maskt
, biast
;
2781 rtx mem
, target
, mask
, bias
;
2784 maskt
= gimple_call_arg (stmt
, 2);
2785 lhs
= gimple_call_lhs (stmt
);
2786 if (lhs
== NULL_TREE
)
2788 type
= TREE_TYPE (lhs
);
2789 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2791 if (optab
== vec_mask_load_lanes_optab
)
2792 icode
= get_multi_vector_move (type
, optab
);
2793 else if (optab
== len_load_optab
)
2794 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2796 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2797 TYPE_MODE (TREE_TYPE (maskt
)));
2799 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2800 gcc_assert (MEM_P (mem
));
2801 mask
= expand_normal (maskt
);
2802 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2803 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2804 create_fixed_operand (&ops
[1], mem
);
2805 if (optab
== len_load_optab
)
2807 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2808 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2809 biast
= gimple_call_arg (stmt
, 3);
2810 bias
= expand_normal (biast
);
2811 create_input_operand (&ops
[3], bias
, QImode
);
2812 expand_insn (icode
, 4, ops
);
2816 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2817 expand_insn (icode
, 3, ops
);
2820 if (!rtx_equal_p (target
, ops
[0].value
))
2821 emit_move_insn (target
, ops
[0].value
);
2824 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2825 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2826 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2828 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB. */
2831 expand_partial_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2833 class expand_operand ops
[4];
2834 tree type
, lhs
, rhs
, maskt
, biast
;
2835 rtx mem
, reg
, mask
, bias
;
2838 maskt
= gimple_call_arg (stmt
, 2);
2839 rhs
= gimple_call_arg (stmt
, 3);
2840 type
= TREE_TYPE (rhs
);
2841 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2843 if (optab
== vec_mask_store_lanes_optab
)
2844 icode
= get_multi_vector_move (type
, optab
);
2845 else if (optab
== len_store_optab
)
2846 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2848 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2849 TYPE_MODE (TREE_TYPE (maskt
)));
2851 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2852 gcc_assert (MEM_P (mem
));
2853 mask
= expand_normal (maskt
);
2854 reg
= expand_normal (rhs
);
2855 create_fixed_operand (&ops
[0], mem
);
2856 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2857 if (optab
== len_store_optab
)
2859 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2860 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2861 biast
= gimple_call_arg (stmt
, 4);
2862 bias
= expand_normal (biast
);
2863 create_input_operand (&ops
[3], bias
, QImode
);
2864 expand_insn (icode
, 4, ops
);
2868 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2869 expand_insn (icode
, 3, ops
);
2873 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2874 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2875 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2877 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2878 The expansion of STMT happens based on OPTAB table associated. */
2881 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2883 class expand_operand ops
[6];
2885 tree lhs
= gimple_call_lhs (stmt
);
2886 tree op0a
= gimple_call_arg (stmt
, 0);
2887 tree op0b
= gimple_call_arg (stmt
, 1);
2888 tree op1
= gimple_call_arg (stmt
, 2);
2889 tree op2
= gimple_call_arg (stmt
, 3);
2890 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
2892 tree vec_cond_type
= TREE_TYPE (lhs
);
2893 tree op_mode
= TREE_TYPE (op0a
);
2894 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
2896 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2897 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
2899 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
2901 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
2902 rtx rtx_op1
= expand_normal (op1
);
2903 rtx rtx_op2
= expand_normal (op2
);
2905 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2906 create_output_operand (&ops
[0], target
, mode
);
2907 create_input_operand (&ops
[1], rtx_op1
, mode
);
2908 create_input_operand (&ops
[2], rtx_op2
, mode
);
2909 create_fixed_operand (&ops
[3], comparison
);
2910 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
2911 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
2912 expand_insn (icode
, 6, ops
);
2913 if (!rtx_equal_p (ops
[0].value
, target
))
2914 emit_move_insn (target
, ops
[0].value
);
2917 /* Expand VCOND_MASK optab internal function.
2918 The expansion of STMT happens based on OPTAB table associated. */
2921 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2923 class expand_operand ops
[4];
2925 tree lhs
= gimple_call_lhs (stmt
);
2926 tree op0
= gimple_call_arg (stmt
, 0);
2927 tree op1
= gimple_call_arg (stmt
, 1);
2928 tree op2
= gimple_call_arg (stmt
, 2);
2929 tree vec_cond_type
= TREE_TYPE (lhs
);
2931 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2932 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
2933 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
2934 rtx mask
, rtx_op1
, rtx_op2
;
2936 gcc_assert (icode
!= CODE_FOR_nothing
);
2938 mask
= expand_normal (op0
);
2939 rtx_op1
= expand_normal (op1
);
2940 rtx_op2
= expand_normal (op2
);
2942 mask
= force_reg (mask_mode
, mask
);
2943 rtx_op1
= force_reg (mode
, rtx_op1
);
2945 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2946 create_output_operand (&ops
[0], target
, mode
);
2947 create_input_operand (&ops
[1], rtx_op1
, mode
);
2948 create_input_operand (&ops
[2], rtx_op2
, mode
);
2949 create_input_operand (&ops
[3], mask
, mask_mode
);
2950 expand_insn (icode
, 4, ops
);
2951 if (!rtx_equal_p (ops
[0].value
, target
))
2952 emit_move_insn (target
, ops
[0].value
);
2955 /* Expand VEC_SET internal functions. */
2958 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2960 tree lhs
= gimple_call_lhs (stmt
);
2961 tree op0
= gimple_call_arg (stmt
, 0);
2962 tree op1
= gimple_call_arg (stmt
, 1);
2963 tree op2
= gimple_call_arg (stmt
, 2);
2964 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2965 rtx src
= expand_normal (op0
);
2967 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
2968 scalar_mode innermode
= GET_MODE_INNER (outermode
);
2970 rtx value
= expand_normal (op1
);
2971 rtx pos
= expand_normal (op2
);
2973 class expand_operand ops
[3];
2974 enum insn_code icode
= optab_handler (optab
, outermode
);
2976 if (icode
!= CODE_FOR_nothing
)
2978 rtx temp
= gen_reg_rtx (outermode
);
2979 emit_move_insn (temp
, src
);
2981 create_fixed_operand (&ops
[0], temp
);
2982 create_input_operand (&ops
[1], value
, innermode
);
2983 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
2985 if (maybe_expand_insn (icode
, 3, ops
))
2987 emit_move_insn (target
, temp
);
2995 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3000 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3002 /* When guessing was done, the hints should be already stripped away. */
3003 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3006 tree lhs
= gimple_call_lhs (stmt
);
3008 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3010 target
= const0_rtx
;
3011 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3012 if (lhs
&& val
!= target
)
3013 emit_move_insn (target
, val
);
3016 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3017 should never be called. */
3020 expand_VA_ARG (internal_fn
, gcall
*)
3025 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3026 dummy function should never be called. */
3029 expand_VEC_CONVERT (internal_fn
, gcall
*)
3034 /* Expand IFN_RAWMEMCHAR internal function. */
3037 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3039 expand_operand ops
[3];
3041 tree lhs
= gimple_call_lhs (stmt
);
3044 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3045 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3046 create_output_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3048 tree mem
= gimple_call_arg (stmt
, 0);
3049 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3050 create_fixed_operand (&ops
[1], mem_rtx
);
3052 tree pattern
= gimple_call_arg (stmt
, 1);
3053 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3054 rtx pattern_rtx
= expand_normal (pattern
);
3055 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3057 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3059 expand_insn (icode
, 3, ops
);
3060 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3061 emit_move_insn (lhs_rtx
, ops
[0].value
);
3064 /* Expand the IFN_UNIQUE function according to its first argument. */
3067 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3069 rtx pattern
= NULL_RTX
;
3070 enum ifn_unique_kind kind
3071 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3078 case IFN_UNIQUE_UNSPEC
:
3079 if (targetm
.have_unique ())
3080 pattern
= targetm
.gen_unique ();
3083 case IFN_UNIQUE_OACC_FORK
:
3084 case IFN_UNIQUE_OACC_JOIN
:
3085 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3087 tree lhs
= gimple_call_lhs (stmt
);
3088 rtx target
= const0_rtx
;
3091 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3093 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3094 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3096 if (kind
== IFN_UNIQUE_OACC_FORK
)
3097 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3099 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3107 emit_insn (pattern
);
3110 /* Expand the IFN_DEFERRED_INIT function:
3111 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3113 Initialize the LHS with zero/pattern according to its second argument
3115 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3116 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3118 The LHS variable is initialized including paddings.
3119 The reasons to choose 0xFE for pattern initialization are:
3120 1. It is a non-canonical virtual address on x86_64, and at the
3121 high end of the i386 kernel address space.
3122 2. It is a very large float value (-1.694739530317379e+38).
3123 3. It is also an unusual number for integers. */
3124 #define INIT_PATTERN_VALUE 0xFE
3126 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3128 tree lhs
= gimple_call_lhs (stmt
);
3129 tree var_size
= gimple_call_arg (stmt
, 0);
3130 enum auto_init_type init_type
3131 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3132 bool reg_lhs
= true;
3134 tree var_type
= TREE_TYPE (lhs
);
3135 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3137 if (TREE_CODE (lhs
) == SSA_NAME
)
3141 tree lhs_base
= lhs
;
3142 while (handled_component_p (lhs_base
))
3143 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3144 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3145 || non_mem_decl_p (lhs_base
));
3146 /* If this expands to a register and the underlying decl is wrapped in
3147 a MEM_REF that just serves as an access type change expose the decl
3148 if it is of correct size. This avoids a situation as in PR103271
3149 if the target does not support a direct move to the registers mode. */
3151 && TREE_CODE (lhs_base
) == MEM_REF
3152 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3153 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3154 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3155 && tree_fits_uhwi_p (var_size
)
3156 && tree_int_cst_equal
3158 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3160 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3161 var_type
= TREE_TYPE (lhs
);
3167 /* If the variable is not in register, expand to a memset
3168 to initialize it. */
3169 mark_addressable (lhs
);
3170 tree var_addr
= build_fold_addr_expr (lhs
);
3172 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3173 ? build_int_cst (integer_type_node
,
3175 : integer_zero_node
;
3176 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3177 3, var_addr
, value
, var_size
);
3178 /* Expand this memset call. */
3179 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3183 /* If this variable is in a register use expand_assignment.
3184 For boolean scalars force zero-init. */
3186 scalar_int_mode var_mode
;
3187 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3188 && tree_fits_uhwi_p (var_size
)
3189 && (init_type
== AUTO_INIT_PATTERN
3190 || !is_gimple_reg_type (var_type
))
3191 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3192 0).exists (&var_mode
)
3193 && have_insn_for (SET
, var_mode
))
3195 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3196 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3197 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3198 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3199 tree itype
= build_nonstandard_integer_type
3200 (total_bytes
* BITS_PER_UNIT
, 1);
3201 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3202 init
= wide_int_to_tree (itype
, w
);
3203 /* Pun the LHS to make sure its type has constant size
3204 unless it is an SSA name where that's already known. */
3205 if (TREE_CODE (lhs
) != SSA_NAME
)
3206 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3208 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3211 /* Use zero-init also for variable-length sizes. */
3212 init
= build_zero_cst (var_type
);
3214 expand_assignment (lhs
, init
, false);
3218 /* The size of an OpenACC compute dimension. */
3221 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3223 tree lhs
= gimple_call_lhs (stmt
);
3228 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3229 if (targetm
.have_oacc_dim_size ())
3231 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3232 VOIDmode
, EXPAND_NORMAL
);
3233 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3236 emit_move_insn (target
, GEN_INT (1));
3239 /* The position of an OpenACC execution engine along one compute axis. */
3242 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3244 tree lhs
= gimple_call_lhs (stmt
);
3249 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3250 if (targetm
.have_oacc_dim_pos ())
3252 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3253 VOIDmode
, EXPAND_NORMAL
);
3254 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3257 emit_move_insn (target
, const0_rtx
);
3260 /* This is expanded by oacc_device_lower pass. */
3263 expand_GOACC_LOOP (internal_fn
, gcall
*)
3268 /* This is expanded by oacc_device_lower pass. */
3271 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3276 /* This is expanded by oacc_device_lower pass. */
3279 expand_GOACC_TILE (internal_fn
, gcall
*)
3284 /* Set errno to EDOM. */
3287 expand_SET_EDOM (internal_fn
, gcall
*)
3290 #ifdef GEN_ERRNO_RTX
3291 rtx errno_rtx
= GEN_ERRNO_RTX
;
3293 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3295 emit_move_insn (errno_rtx
,
3296 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3302 /* Expand atomic bit test and set. */
3305 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3307 expand_ifn_atomic_bit_test_and (call
);
3310 /* Expand atomic bit test and complement. */
3313 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3315 expand_ifn_atomic_bit_test_and (call
);
3318 /* Expand atomic bit test and reset. */
3321 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3323 expand_ifn_atomic_bit_test_and (call
);
3326 /* Expand atomic bit test and set. */
3329 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3331 expand_ifn_atomic_compare_exchange (call
);
3334 /* Expand atomic add fetch and cmp with 0. */
3337 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3339 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3342 /* Expand atomic sub fetch and cmp with 0. */
3345 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3347 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3350 /* Expand atomic and fetch and cmp with 0. */
3353 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3355 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3358 /* Expand atomic or fetch and cmp with 0. */
3361 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3363 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3366 /* Expand atomic xor fetch and cmp with 0. */
3369 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3371 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3374 /* Expand LAUNDER to assignment, lhs = arg0. */
3377 expand_LAUNDER (internal_fn
, gcall
*call
)
3379 tree lhs
= gimple_call_lhs (call
);
3384 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3387 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3390 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3392 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3393 int rhs_index
= internal_fn_stored_value_index (ifn
);
3394 int mask_index
= internal_fn_mask_index (ifn
);
3395 tree base
= gimple_call_arg (stmt
, 0);
3396 tree offset
= gimple_call_arg (stmt
, 1);
3397 tree scale
= gimple_call_arg (stmt
, 2);
3398 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3400 rtx base_rtx
= expand_normal (base
);
3401 rtx offset_rtx
= expand_normal (offset
);
3402 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3403 rtx rhs_rtx
= expand_normal (rhs
);
3405 class expand_operand ops
[6];
3407 create_address_operand (&ops
[i
++], base_rtx
);
3408 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3409 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3410 create_integer_operand (&ops
[i
++], scale_int
);
3411 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3412 if (mask_index
>= 0)
3414 tree mask
= gimple_call_arg (stmt
, mask_index
);
3415 rtx mask_rtx
= expand_normal (mask
);
3416 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3419 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3420 TYPE_MODE (TREE_TYPE (offset
)));
3421 expand_insn (icode
, i
, ops
);
3424 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3427 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3429 tree lhs
= gimple_call_lhs (stmt
);
3430 tree base
= gimple_call_arg (stmt
, 0);
3431 tree offset
= gimple_call_arg (stmt
, 1);
3432 tree scale
= gimple_call_arg (stmt
, 2);
3434 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3435 rtx base_rtx
= expand_normal (base
);
3436 rtx offset_rtx
= expand_normal (offset
);
3437 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3440 class expand_operand ops
[6];
3441 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3442 create_address_operand (&ops
[i
++], base_rtx
);
3443 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3444 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3445 create_integer_operand (&ops
[i
++], scale_int
);
3446 if (optab
== mask_gather_load_optab
)
3448 tree mask
= gimple_call_arg (stmt
, 4);
3449 rtx mask_rtx
= expand_normal (mask
);
3450 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3452 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3453 TYPE_MODE (TREE_TYPE (offset
)));
3454 expand_insn (icode
, i
, ops
);
3455 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3456 emit_move_insn (lhs_rtx
, ops
[0].value
);
3459 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3460 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3463 contains_call_div_mod (rtx_insn
*insn
)
3465 subrtx_iterator::array_type array
;
3466 for (; insn
; insn
= NEXT_INSN (insn
))
3469 else if (INSN_P (insn
))
3470 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3471 switch (GET_CODE (*iter
))
3485 /* Expand DIVMOD() using:
3486 a) optab handler for udivmod/sdivmod if it is available.
3487 b) If optab_handler doesn't exist, generate call to
3488 target-specific divmod libfunc. */
3491 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3493 tree lhs
= gimple_call_lhs (call_stmt
);
3494 tree arg0
= gimple_call_arg (call_stmt
, 0);
3495 tree arg1
= gimple_call_arg (call_stmt
, 1);
3497 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3498 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3499 machine_mode mode
= TYPE_MODE (type
);
3500 bool unsignedp
= TYPE_UNSIGNED (type
);
3501 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3503 rtx op0
= expand_normal (arg0
);
3504 rtx op1
= expand_normal (arg1
);
3505 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3507 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3508 rtx_insn
*insns
= NULL
;
3510 if (TREE_CODE (arg1
) == INTEGER_CST
)
3512 /* For DIVMOD by integral constants, there could be efficient code
3513 expanded inline e.g. using shifts and plus/minus. Try to expand
3514 the division and modulo and if it emits any library calls or any
3515 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3517 scalar_int_mode int_mode
;
3518 if (remainder
== NULL_RTX
3520 && CONST_INT_P (op1
)
3521 && !pow2p_hwi (INTVAL (op1
))
3522 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3523 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3524 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3525 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3526 && optimize_insn_for_speed_p ())
3528 rtx_insn
*last
= get_last_insn ();
3529 remainder
= NULL_RTX
;
3530 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3531 TYPE_UNSIGNED (type
));
3532 if (quotient
!= NULL_RTX
)
3534 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3536 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3537 set_dst_reg_note (move
, REG_EQUAL
,
3538 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3539 ? UDIV
: DIV
, int_mode
,
3540 copy_rtx (op0
), op1
),
3542 move
= emit_move_insn (remainder
, remainder
);
3543 set_dst_reg_note (move
, REG_EQUAL
,
3544 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3545 ? UMOD
: MOD
, int_mode
,
3546 copy_rtx (op0
), op1
),
3551 delete_insns_since (last
);
3554 if (remainder
== NULL_RTX
)
3556 struct separate_ops ops
;
3557 ops
.code
= TRUNC_DIV_EXPR
;
3559 ops
.op0
= make_tree (ops
.type
, op0
);
3561 ops
.op2
= NULL_TREE
;
3562 ops
.location
= gimple_location (call_stmt
);
3564 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3565 if (contains_call_div_mod (get_insns ()))
3566 quotient
= NULL_RTX
;
3569 ops
.code
= TRUNC_MOD_EXPR
;
3570 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3572 if (contains_call_div_mod (get_insns ()))
3573 remainder
= NULL_RTX
;
3576 insns
= get_insns ();
3584 /* Check if optab_handler exists for divmod_optab for given mode. */
3585 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3587 quotient
= gen_reg_rtx (mode
);
3588 remainder
= gen_reg_rtx (mode
);
3589 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3592 /* Generate call to divmod libfunc if it exists. */
3593 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3594 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3595 "ient
, &remainder
);
3600 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3601 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3602 make_tree (TREE_TYPE (arg0
), quotient
),
3603 make_tree (TREE_TYPE (arg1
), remainder
)),
3604 target
, VOIDmode
, EXPAND_NORMAL
);
3610 expand_NOP (internal_fn
, gcall
*)
3612 /* Nothing. But it shouldn't really prevail. */
3615 /* Coroutines, all should have been processed at this stage. */
3618 expand_CO_FRAME (internal_fn
, gcall
*)
3624 expand_CO_YIELD (internal_fn
, gcall
*)
3630 expand_CO_SUSPN (internal_fn
, gcall
*)
3636 expand_CO_ACTOR (internal_fn
, gcall
*)
3641 /* Expand a call to FN using the operands in STMT. FN has a single
3642 output operand and NARGS input operands. */
3645 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3648 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3649 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3650 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3653 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3656 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3658 expand_operand ops
[3];
3661 tree lhs
= gimple_call_lhs (stmt
);
3662 tree lhs_type
= TREE_TYPE (lhs
);
3663 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3664 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3666 for (unsigned int i
= 0; i
< 2; ++i
)
3668 tree rhs
= gimple_call_arg (stmt
, i
);
3669 rhs_type
[i
] = TREE_TYPE (rhs
);
3670 rtx rhs_rtx
= expand_normal (rhs
);
3671 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3674 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3675 TYPE_MODE (lhs_type
));
3677 expand_insn (icode
, 3, ops
);
3678 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3679 emit_move_insn (lhs_rtx
, ops
[0].value
);
3682 /* Expanders for optabs that can use expand_direct_optab_fn. */
3684 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3685 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3687 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3688 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3690 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3691 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3693 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3694 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3696 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3697 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3699 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3700 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3702 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3703 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3705 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3706 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3708 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3709 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3711 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3712 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3714 /* RETURN_TYPE and ARGS are a return type and argument list that are
3715 in principle compatible with FN (which satisfies direct_internal_fn_p).
3716 Return the types that should be used to determine whether the
3717 target supports FN. */
3720 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3722 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3723 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3724 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3725 return tree_pair (type0
, type1
);
3728 /* CALL is a call whose return type and arguments are in principle
3729 compatible with FN (which satisfies direct_internal_fn_p). Return the
3730 types that should be used to determine whether the target supports FN. */
3733 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3735 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3736 tree op0
= (info
.type0
< 0
3737 ? gimple_call_lhs (call
)
3738 : gimple_call_arg (call
, info
.type0
));
3739 tree op1
= (info
.type1
< 0
3740 ? gimple_call_lhs (call
)
3741 : gimple_call_arg (call
, info
.type1
));
3742 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3745 /* Return true if OPTAB is supported for TYPES (whose modes should be
3746 the same) when the optimization type is OPT_TYPE. Used for simple
3750 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3751 optimization_type opt_type
)
3753 machine_mode mode
= TYPE_MODE (types
.first
);
3754 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3755 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3758 /* Return true if OPTAB is supported for TYPES, where the first type
3759 is the destination and the second type is the source. Used for
3763 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3764 optimization_type opt_type
)
3766 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3767 TYPE_MODE (types
.second
), opt_type
)
3768 != CODE_FOR_nothing
);
3771 /* Return true if load/store lanes optab OPTAB is supported for
3772 array type TYPES.first when the optimization type is OPT_TYPE. */
3775 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3776 optimization_type opt_type
)
3778 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3779 machine_mode imode
= TYPE_MODE (types
.first
);
3780 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3781 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3782 != CODE_FOR_nothing
);
3785 #define direct_unary_optab_supported_p direct_optab_supported_p
3786 #define direct_binary_optab_supported_p direct_optab_supported_p
3787 #define direct_ternary_optab_supported_p direct_optab_supported_p
3788 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3789 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3790 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3791 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3792 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3793 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3794 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3795 #define direct_len_load_optab_supported_p direct_optab_supported_p
3796 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3797 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3798 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3799 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3800 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3801 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3802 #define direct_len_store_optab_supported_p direct_optab_supported_p
3803 #define direct_while_optab_supported_p convert_optab_supported_p
3804 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3805 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3806 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3807 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3808 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3810 /* Return the optab used by internal function FN. */
3813 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3817 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3818 case IFN_##CODE: break;
3819 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3820 case IFN_##CODE: return OPTAB##_optab;
3821 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3822 UNSIGNED_OPTAB, TYPE) \
3823 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3824 ? UNSIGNED_OPTAB ## _optab \
3825 : SIGNED_OPTAB ## _optab);
3826 #include "internal-fn.def"
3834 /* Return the optab used by internal function FN. */
3837 direct_internal_fn_optab (internal_fn fn
)
3841 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3842 case IFN_##CODE: break;
3843 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3844 case IFN_##CODE: return OPTAB##_optab;
3845 #include "internal-fn.def"
3853 /* Return true if FN is supported for the types in TYPES when the
3854 optimization type is OPT_TYPE. The types are those associated with
3855 the "type0" and "type1" fields of FN's direct_internal_fn_info
3859 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3860 optimization_type opt_type
)
3864 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3865 case IFN_##CODE: break;
3866 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3868 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3870 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3871 UNSIGNED_OPTAB, TYPE) \
3874 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3875 ? UNSIGNED_OPTAB ## _optab \
3876 : SIGNED_OPTAB ## _optab); \
3877 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3880 #include "internal-fn.def"
3888 /* Return true if FN is supported for type TYPE when the optimization
3889 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3890 fields of FN's direct_internal_fn_info structure are the same. */
3893 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3894 optimization_type opt_type
)
3896 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3897 gcc_checking_assert (info
.type0
== info
.type1
);
3898 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3901 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3902 given that STMT is a call to a direct internal function. */
3905 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
3907 internal_fn fn
= gimple_call_internal_fn (stmt
);
3908 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3909 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
3912 /* Return true if FN is a binary operation and if FN is commutative. */
3915 commutative_binary_fn_p (internal_fn fn
)
3926 case IFN_COMPLEX_MUL
:
3927 case IFN_UBSAN_CHECK_ADD
:
3928 case IFN_UBSAN_CHECK_MUL
:
3929 case IFN_ADD_OVERFLOW
:
3930 case IFN_MUL_OVERFLOW
:
3938 /* Return true if FN is a ternary operation and if its first two arguments
3942 commutative_ternary_fn_p (internal_fn fn
)
3957 /* Return true if FN is an associative binary operation. */
3960 associative_binary_fn_p (internal_fn fn
)
3973 /* If FN is commutative in two consecutive arguments, return the
3974 index of the first, otherwise return -1. */
3977 first_commutative_argument (internal_fn fn
)
3997 if (commutative_binary_fn_p (fn
)
3998 || commutative_ternary_fn_p (fn
))
4004 /* Return true if IFN_SET_EDOM is supported. */
4007 set_edom_supported_p (void)
4016 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4018 expand_##CODE (internal_fn fn, gcall *stmt) \
4020 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4022 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4023 UNSIGNED_OPTAB, TYPE) \
4025 expand_##CODE (internal_fn fn, gcall *stmt) \
4027 tree_pair types = direct_internal_fn_types (fn, stmt); \
4028 optab which_optab = direct_internal_fn_optab (fn, types); \
4029 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4031 #include "internal-fn.def"
4033 /* Routines to expand each internal function, indexed by function number.
4034 Each routine has the prototype:
4036 expand_<NAME> (gcall *stmt)
4038 where STMT is the statement that performs the call. */
4039 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4040 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4041 #include "internal-fn.def"
4045 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
4047 #define FOR_EACH_CODE_MAPPING(T) \
4048 T (PLUS_EXPR, IFN_COND_ADD) \
4049 T (MINUS_EXPR, IFN_COND_SUB) \
4050 T (MULT_EXPR, IFN_COND_MUL) \
4051 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
4052 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
4053 T (RDIV_EXPR, IFN_COND_RDIV) \
4054 T (MIN_EXPR, IFN_COND_MIN) \
4055 T (MAX_EXPR, IFN_COND_MAX) \
4056 T (BIT_AND_EXPR, IFN_COND_AND) \
4057 T (BIT_IOR_EXPR, IFN_COND_IOR) \
4058 T (BIT_XOR_EXPR, IFN_COND_XOR) \
4059 T (LSHIFT_EXPR, IFN_COND_SHL) \
4060 T (RSHIFT_EXPR, IFN_COND_SHR) \
4061 T (NEGATE_EXPR, IFN_COND_NEG)
4063 /* Return a function that only performs CODE when a certain condition is met
4064 and that uses a given fallback value otherwise. For example, if CODE is
4065 a binary operation associated with conditional function FN:
4067 LHS = FN (COND, A, B, ELSE)
4069 is equivalent to the C expression:
4071 LHS = COND ? A CODE B : ELSE;
4073 operating elementwise if the operands are vectors.
4075 Return IFN_LAST if no such function exists. */
4078 get_conditional_internal_fn (tree_code code
)
4082 #define CASE(CODE, IFN) case CODE: return IFN;
4083 FOR_EACH_CODE_MAPPING(CASE
)
4090 /* If IFN implements the conditional form of a tree code, return that
4091 tree code, otherwise return ERROR_MARK. */
4094 conditional_internal_fn_code (internal_fn ifn
)
4098 #define CASE(CODE, IFN) case IFN: return CODE;
4099 FOR_EACH_CODE_MAPPING(CASE
)
4106 /* Invoke T(IFN) for each internal function IFN that also has an
4108 #define FOR_EACH_COND_FN_PAIR(T) \
4116 /* Return a function that only performs internal function FN when a
4117 certain condition is met and that uses a given fallback value otherwise.
4118 In other words, the returned function FN' is such that:
4120 LHS = FN' (COND, A1, ... An, ELSE)
4122 is equivalent to the C expression:
4124 LHS = COND ? FN (A1, ..., An) : ELSE;
4126 operating elementwise if the operands are vectors.
4128 Return IFN_LAST if no such function exists. */
4131 get_conditional_internal_fn (internal_fn fn
)
4135 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4136 FOR_EACH_COND_FN_PAIR(CASE
)
4143 /* If IFN implements the conditional form of an unconditional internal
4144 function, return that unconditional function, otherwise return IFN_LAST. */
4147 get_unconditional_internal_fn (internal_fn ifn
)
4151 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
4152 FOR_EACH_COND_FN_PAIR(CASE
)
4159 /* Return true if STMT can be interpreted as a conditional tree code
4160 operation of the form:
4162 LHS = COND ? OP (RHS1, ...) : ELSE;
4164 operating elementwise if the operands are vectors. This includes
4165 the case of an all-true COND, so that the operation always happens.
4167 When returning true, set:
4169 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4170 is known to be all-true
4171 - *CODE_OUT to the tree code
4172 - OPS[I] to operand I of *CODE_OUT
4173 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4174 condition is known to be all true. */
4177 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4178 tree_code
*code_out
,
4179 tree (&ops
)[3], tree
*else_out
)
4181 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4183 *cond_out
= NULL_TREE
;
4184 *code_out
= gimple_assign_rhs_code (assign
);
4185 ops
[0] = gimple_assign_rhs1 (assign
);
4186 ops
[1] = gimple_assign_rhs2 (assign
);
4187 ops
[2] = gimple_assign_rhs3 (assign
);
4188 *else_out
= NULL_TREE
;
4191 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4192 if (gimple_call_internal_p (call
))
4194 internal_fn ifn
= gimple_call_internal_fn (call
);
4195 tree_code code
= conditional_internal_fn_code (ifn
);
4196 if (code
!= ERROR_MARK
)
4198 *cond_out
= gimple_call_arg (call
, 0);
4200 unsigned int nops
= gimple_call_num_args (call
) - 2;
4201 for (unsigned int i
= 0; i
< 3; ++i
)
4202 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4203 *else_out
= gimple_call_arg (call
, nops
+ 1);
4204 if (integer_truep (*cond_out
))
4206 *cond_out
= NULL_TREE
;
4207 *else_out
= NULL_TREE
;
4215 /* Return true if IFN is some form of load from memory. */
4218 internal_load_fn_p (internal_fn fn
)
4223 case IFN_LOAD_LANES
:
4224 case IFN_MASK_LOAD_LANES
:
4225 case IFN_GATHER_LOAD
:
4226 case IFN_MASK_GATHER_LOAD
:
4235 /* Return true if IFN is some form of store to memory. */
4238 internal_store_fn_p (internal_fn fn
)
4242 case IFN_MASK_STORE
:
4243 case IFN_STORE_LANES
:
4244 case IFN_MASK_STORE_LANES
:
4245 case IFN_SCATTER_STORE
:
4246 case IFN_MASK_SCATTER_STORE
:
4255 /* Return true if IFN is some form of gather load or scatter store. */
4258 internal_gather_scatter_fn_p (internal_fn fn
)
4262 case IFN_GATHER_LOAD
:
4263 case IFN_MASK_GATHER_LOAD
:
4264 case IFN_SCATTER_STORE
:
4265 case IFN_MASK_SCATTER_STORE
:
4273 /* If FN takes a vector mask argument, return the index of that argument,
4274 otherwise return -1. */
4277 internal_fn_mask_index (internal_fn fn
)
4282 case IFN_MASK_LOAD_LANES
:
4283 case IFN_MASK_STORE
:
4284 case IFN_MASK_STORE_LANES
:
4287 case IFN_MASK_GATHER_LOAD
:
4288 case IFN_MASK_SCATTER_STORE
:
4292 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4293 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4297 /* If FN takes a value that should be stored to memory, return the index
4298 of that argument, otherwise return -1. */
4301 internal_fn_stored_value_index (internal_fn fn
)
4305 case IFN_MASK_STORE
:
4306 case IFN_MASK_STORE_LANES
:
4307 case IFN_SCATTER_STORE
:
4308 case IFN_MASK_SCATTER_STORE
:
4317 /* Return true if the target supports gather load or scatter store function
4318 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4319 while for stores it is the vector type of the stored data argument.
4320 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4321 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4322 offset from the shared base address of each loaded or stored element.
4323 SCALE is the amount by which these offsets should be multiplied
4324 *after* they have been extended to address width. */
4327 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4328 tree memory_element_type
,
4329 tree offset_vector_type
, int scale
)
4331 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4332 TYPE_SIZE (memory_element_type
)))
4334 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4335 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4337 optab optab
= direct_internal_fn_optab (ifn
);
4338 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4339 TYPE_MODE (offset_vector_type
));
4340 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4341 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4342 return (icode
!= CODE_FOR_nothing
4343 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4344 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4347 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4348 for pointers of type TYPE when the accesses have LENGTH bytes and their
4349 common byte alignment is ALIGN. */
4352 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4353 poly_uint64 length
, unsigned int align
)
4355 machine_mode mode
= TYPE_MODE (type
);
4356 optab optab
= direct_internal_fn_optab (ifn
);
4357 insn_code icode
= direct_optab_handler (optab
, mode
);
4358 if (icode
== CODE_FOR_nothing
)
4360 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4361 return (insn_operand_matches (icode
, 3, length_rtx
)
4362 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4365 /* Return the supported bias for IFN which is either IFN_LEN_LOAD
4366 or IFN_LEN_STORE. For now we only support the biases of 0 and -1
4367 (in case 0 is not an allowable length for len_load or len_store).
4368 If none of the biases match what the backend provides, return
4369 VECT_PARTIAL_BIAS_UNSUPPORTED. */
4372 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
4374 optab optab
= direct_internal_fn_optab (ifn
);
4375 insn_code icode
= direct_optab_handler (optab
, mode
);
4377 if (icode
!= CODE_FOR_nothing
)
4379 /* For now we only support biases of 0 or -1. Try both of them. */
4380 if (insn_operand_matches (icode
, 3, GEN_INT (0)))
4382 if (insn_operand_matches (icode
, 3, GEN_INT (-1)))
4386 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4389 /* Expand STMT as though it were a call to internal function FN. */
4392 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4394 internal_fn_expanders
[fn
] (fn
, stmt
);
4397 /* Expand STMT, which is a call to internal function FN. */
4400 expand_internal_call (gcall
*stmt
)
4402 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4405 /* If TYPE is a vector type, return true if IFN is a direct internal
4406 function that is supported for that type. If TYPE is a scalar type,
4407 return true if IFN is a direct internal function that is supported for
4408 the target's preferred vector version of TYPE. */
4411 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4413 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4414 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4417 if (VECTOR_TYPE_P (type
)
4418 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4421 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4422 if (VECTOR_MODE_P (vmode
))
4424 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4425 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4429 auto_vector_modes vector_modes
;
4430 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4431 for (machine_mode base_mode
: vector_modes
)
4432 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
4434 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4435 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4443 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
4449 expand_PHI (internal_fn
, gcall
*)
4455 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
4457 tree lhs
= gimple_call_lhs (stmt
);
4458 tree rhs1
= gimple_call_arg (stmt
, 0);
4459 tree rhs2
= gimple_call_arg (stmt
, 1);
4460 tree type
= TREE_TYPE (rhs1
);
4462 do_pending_stack_adjust ();
4464 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
4465 rtx op1
= expand_normal (rhs1
);
4466 rtx op2
= expand_normal (rhs2
);
4468 class expand_operand ops
[3];
4469 create_output_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
4470 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
4471 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
4472 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
4473 expand_insn (icode
, 3, ops
);
4474 if (!rtx_equal_p (target
, ops
[0].value
))
4475 emit_move_insn (target
, ops
[0].value
);