2 Copyright (C) 2011-2018 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
67 /* Fnspec of each internal function, indexed by function number. */
68 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
73 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
74 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
75 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
76 #include "internal-fn.def"
77 internal_fn_fnspec_array
[IFN_LAST
] = 0;
80 /* Create static initializers for the information returned by
81 direct_internal_fn. */
82 #define not_direct { -2, -2, false }
83 #define mask_load_direct { -1, 2, false }
84 #define load_lanes_direct { -1, -1, false }
85 #define mask_load_lanes_direct { -1, -1, false }
86 #define mask_store_direct { 3, 2, false }
87 #define store_lanes_direct { 0, 0, false }
88 #define mask_store_lanes_direct { 0, 0, false }
89 #define unary_direct { 0, 0, true }
90 #define binary_direct { 0, 0, true }
91 #define cond_unary_direct { 1, 1, true }
92 #define cond_binary_direct { 1, 1, true }
93 #define while_direct { 0, 2, false }
94 #define fold_extract_direct { 2, 2, false }
95 #define fold_left_direct { 1, 1, false }
97 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
98 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
99 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
100 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
101 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
102 #include "internal-fn.def"
106 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
107 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
109 static enum insn_code
110 get_multi_vector_move (tree array_type
, convert_optab optab
)
115 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
116 imode
= TYPE_MODE (array_type
);
117 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
119 return convert_optab_handler (optab
, imode
, vmode
);
122 /* Expand LOAD_LANES call STMT using optab OPTAB. */
125 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
127 struct expand_operand ops
[2];
131 lhs
= gimple_call_lhs (stmt
);
132 rhs
= gimple_call_arg (stmt
, 0);
133 type
= TREE_TYPE (lhs
);
135 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
136 mem
= expand_normal (rhs
);
138 gcc_assert (MEM_P (mem
));
139 PUT_MODE (mem
, TYPE_MODE (type
));
141 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
142 create_fixed_operand (&ops
[1], mem
);
143 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
146 /* Expand STORE_LANES call STMT using optab OPTAB. */
149 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
151 struct expand_operand ops
[2];
155 lhs
= gimple_call_lhs (stmt
);
156 rhs
= gimple_call_arg (stmt
, 0);
157 type
= TREE_TYPE (rhs
);
159 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
160 reg
= expand_normal (rhs
);
162 gcc_assert (MEM_P (target
));
163 PUT_MODE (target
, TYPE_MODE (type
));
165 create_fixed_operand (&ops
[0], target
);
166 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
167 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
171 expand_ANNOTATE (internal_fn
, gcall
*)
176 /* This should get expanded in omp_device_lower pass. */
179 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
184 /* This should get expanded in omp_device_lower pass. */
187 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
192 /* Allocate per-lane storage and begin non-uniform execution region. */
195 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
198 tree lhs
= gimple_call_lhs (stmt
);
200 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
202 target
= gen_reg_rtx (Pmode
);
203 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
204 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
205 struct expand_operand ops
[3];
206 create_output_operand (&ops
[0], target
, Pmode
);
207 create_input_operand (&ops
[1], size
, Pmode
);
208 create_input_operand (&ops
[2], align
, Pmode
);
209 gcc_assert (targetm
.have_omp_simt_enter ());
210 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
213 /* Deallocate per-lane storage and leave non-uniform execution region. */
216 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
218 gcc_checking_assert (!gimple_call_lhs (stmt
));
219 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
220 struct expand_operand ops
[1];
221 create_input_operand (&ops
[0], arg
, Pmode
);
222 gcc_assert (targetm
.have_omp_simt_exit ());
223 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
226 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
227 without SIMT execution this should be expanded in omp_device_lower pass. */
230 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
232 tree lhs
= gimple_call_lhs (stmt
);
236 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
237 gcc_assert (targetm
.have_omp_simt_lane ());
238 emit_insn (targetm
.gen_omp_simt_lane (target
));
241 /* This should get expanded in omp_device_lower pass. */
244 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
249 /* Lane index of the first SIMT lane that supplies a non-zero argument.
250 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
251 lane that executed the last iteration for handling OpenMP lastprivate. */
254 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
256 tree lhs
= gimple_call_lhs (stmt
);
260 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
261 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
262 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
263 struct expand_operand ops
[2];
264 create_output_operand (&ops
[0], target
, mode
);
265 create_input_operand (&ops
[1], cond
, mode
);
266 gcc_assert (targetm
.have_omp_simt_last_lane ());
267 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
270 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
273 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
275 tree lhs
= gimple_call_lhs (stmt
);
279 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
280 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
281 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
282 struct expand_operand ops
[2];
283 create_output_operand (&ops
[0], target
, mode
);
284 create_input_operand (&ops
[1], ctr
, mode
);
285 gcc_assert (targetm
.have_omp_simt_ordered ());
286 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
289 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
290 any lane supplies a non-zero argument. */
293 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
295 tree lhs
= gimple_call_lhs (stmt
);
299 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
300 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
301 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
302 struct expand_operand ops
[2];
303 create_output_operand (&ops
[0], target
, mode
);
304 create_input_operand (&ops
[1], cond
, mode
);
305 gcc_assert (targetm
.have_omp_simt_vote_any ());
306 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
309 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
310 is destination lane index XOR given offset. */
313 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
315 tree lhs
= gimple_call_lhs (stmt
);
319 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
320 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
321 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
322 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
323 struct expand_operand ops
[3];
324 create_output_operand (&ops
[0], target
, mode
);
325 create_input_operand (&ops
[1], src
, mode
);
326 create_input_operand (&ops
[2], idx
, SImode
);
327 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
328 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
331 /* Exchange between SIMT lanes according to given source lane index. */
334 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
336 tree lhs
= gimple_call_lhs (stmt
);
340 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
341 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
342 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
343 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
344 struct expand_operand ops
[3];
345 create_output_operand (&ops
[0], target
, mode
);
346 create_input_operand (&ops
[1], src
, mode
);
347 create_input_operand (&ops
[2], idx
, SImode
);
348 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
349 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
352 /* This should get expanded in adjust_simduid_builtins. */
355 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
360 /* This should get expanded in adjust_simduid_builtins. */
363 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
368 /* This should get expanded in adjust_simduid_builtins. */
371 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
376 /* This should get expanded in adjust_simduid_builtins. */
379 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
384 /* This should get expanded in adjust_simduid_builtins. */
387 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
392 /* This should get expanded in the sanopt pass. */
395 expand_UBSAN_NULL (internal_fn
, gcall
*)
400 /* This should get expanded in the sanopt pass. */
403 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
408 /* This should get expanded in the sanopt pass. */
411 expand_UBSAN_VPTR (internal_fn
, gcall
*)
416 /* This should get expanded in the sanopt pass. */
419 expand_UBSAN_PTR (internal_fn
, gcall
*)
424 /* This should get expanded in the sanopt pass. */
427 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
432 /* This should get expanded in the sanopt pass. */
435 expand_ASAN_CHECK (internal_fn
, gcall
*)
440 /* This should get expanded in the sanopt pass. */
443 expand_ASAN_MARK (internal_fn
, gcall
*)
448 /* This should get expanded in the sanopt pass. */
451 expand_ASAN_POISON (internal_fn
, gcall
*)
456 /* This should get expanded in the sanopt pass. */
459 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
464 /* This should get expanded in the tsan pass. */
467 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
472 /* This should get expanded in the lower pass. */
475 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
477 error_at (gimple_location (call
),
478 "invalid use of attribute %<fallthrough%>");
481 /* Return minimum precision needed to represent all values
482 of ARG in SIGNed integral type. */
485 get_min_precision (tree arg
, signop sign
)
487 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
489 signop orig_sign
= sign
;
490 if (TREE_CODE (arg
) == INTEGER_CST
)
493 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
495 widest_int w
= wi::to_widest (arg
);
496 w
= wi::ext (w
, prec
, sign
);
497 p
= wi::min_precision (w
, sign
);
500 p
= wi::min_precision (wi::to_wide (arg
), sign
);
501 return MIN (p
, prec
);
503 while (CONVERT_EXPR_P (arg
)
504 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
505 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
507 arg
= TREE_OPERAND (arg
, 0);
508 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
510 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
512 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
513 return prec
+ (orig_sign
!= sign
);
514 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
517 return prec
+ (orig_sign
!= sign
);
519 if (TREE_CODE (arg
) != SSA_NAME
)
520 return prec
+ (orig_sign
!= sign
);
521 wide_int arg_min
, arg_max
;
522 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
524 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
525 if (is_gimple_assign (g
)
526 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
528 tree t
= gimple_assign_rhs1 (g
);
529 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
530 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
533 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
535 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
537 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
538 return prec
+ (orig_sign
!= sign
);
539 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
542 return prec
+ (orig_sign
!= sign
);
546 return prec
+ (orig_sign
!= sign
);
548 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
550 int p1
= wi::min_precision (arg_min
, sign
);
551 int p2
= wi::min_precision (arg_max
, sign
);
553 prec
= MIN (prec
, p1
);
555 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
557 int p
= wi::min_precision (arg_max
, UNSIGNED
);
558 prec
= MIN (prec
, p
);
560 return prec
+ (orig_sign
!= sign
);
563 /* Helper for expand_*_overflow. Set the __imag__ part to true
564 (1 except for signed:1 type, in which case store -1). */
567 expand_arith_set_overflow (tree lhs
, rtx target
)
569 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
570 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
571 write_complex_part (target
, constm1_rtx
, true);
573 write_complex_part (target
, const1_rtx
, true);
576 /* Helper for expand_*_overflow. Store RES into the __real__ part
577 of TARGET. If RES has larger MODE than __real__ part of TARGET,
578 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
579 if LHS has smaller precision than its mode. */
582 expand_arith_overflow_result_store (tree lhs
, rtx target
,
583 scalar_int_mode mode
, rtx res
)
585 scalar_int_mode tgtmode
586 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
590 rtx_code_label
*done_label
= gen_label_rtx ();
591 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
592 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
593 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
594 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
595 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
596 profile_probability::very_likely ());
597 expand_arith_set_overflow (lhs
, target
);
598 emit_label (done_label
);
600 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
601 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
604 rtx_code_label
*done_label
= gen_label_rtx ();
605 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
610 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
612 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
613 true, OPTAB_LIB_WIDEN
);
617 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
619 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
622 do_compare_rtx_and_jump (res
, lres
,
623 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
624 profile_probability::very_likely ());
625 expand_arith_set_overflow (lhs
, target
);
626 emit_label (done_label
);
628 write_complex_part (target
, lres
, false);
631 /* Helper for expand_*_overflow. Store RES into TARGET. */
634 expand_ubsan_result_store (rtx target
, rtx res
)
636 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
637 /* If this is a scalar in a register that is stored in a wider mode
638 than the declared mode, compute the result into its declared mode
639 and then convert to the wider mode. Our value is the computed
641 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
643 emit_move_insn (target
, res
);
646 /* Add sub/add overflow checking to the statement STMT.
647 CODE says whether the operation is +, or -. */
650 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
651 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
652 bool uns1_p
, bool is_ubsan
, tree
*datap
)
654 rtx res
, target
= NULL_RTX
;
656 rtx_code_label
*done_label
= gen_label_rtx ();
657 rtx_code_label
*do_error
= gen_label_rtx ();
658 do_pending_stack_adjust ();
659 rtx op0
= expand_normal (arg0
);
660 rtx op1
= expand_normal (arg1
);
661 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
662 int prec
= GET_MODE_PRECISION (mode
);
663 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
667 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
671 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
673 write_complex_part (target
, const0_rtx
, true);
676 /* We assume both operands and result have the same precision
677 here (GET_MODE_BITSIZE (mode)), S stands for signed type
678 with that precision, U for unsigned type with that precision,
679 sgn for unsigned most significant bit in that precision.
680 s1 is signed first operand, u1 is unsigned first operand,
681 s2 is signed second operand, u2 is unsigned second operand,
682 sr is signed result, ur is unsigned result and the following
683 rules say how to compute result (which is always result of
684 the operands as if both were unsigned, cast to the right
685 signedness) and how to compute whether operation overflowed.
688 res = (S) ((U) s1 + (U) s2)
689 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
691 res = (S) ((U) s1 - (U) s2)
692 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
695 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
698 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
700 res = (S) ((U) s1 + u2)
701 ovf = ((U) res ^ sgn) < u2
706 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
708 res = (S) ((U) s1 - u2)
709 ovf = u2 > ((U) s1 ^ sgn)
712 ovf = s1 < 0 || u2 > (U) s1
715 ovf = u1 >= ((U) s2 ^ sgn)
720 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
722 res = (U) s1 + (U) s2
723 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
726 ovf = (U) res < u2 || res < 0
729 ovf = u1 >= u2 ? res < 0 : res >= 0
731 res = (U) s1 - (U) s2
732 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
734 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
736 /* PLUS_EXPR is commutative, if operand signedness differs,
737 canonicalize to the first operand being signed and second
738 unsigned to simplify following code. */
739 std::swap (op0
, op1
);
740 std::swap (arg0
, arg1
);
746 if (uns0_p
&& uns1_p
&& unsr_p
)
748 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
749 : usubv4_optab
, mode
);
750 if (icode
!= CODE_FOR_nothing
)
752 struct expand_operand ops
[4];
753 rtx_insn
*last
= get_last_insn ();
755 res
= gen_reg_rtx (mode
);
756 create_output_operand (&ops
[0], res
, mode
);
757 create_input_operand (&ops
[1], op0
, mode
);
758 create_input_operand (&ops
[2], op1
, mode
);
759 create_fixed_operand (&ops
[3], do_error
);
760 if (maybe_expand_insn (icode
, 4, ops
))
762 last
= get_last_insn ();
763 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
765 && any_condjump_p (last
)
766 && !find_reg_note (last
, REG_BR_PROB
, 0))
767 add_reg_br_prob_note (last
,
768 profile_probability::very_unlikely ());
769 emit_jump (done_label
);
773 delete_insns_since (last
);
776 /* Compute the operation. On RTL level, the addition is always
778 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
779 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
781 /* For PLUS_EXPR, the operation is commutative, so we can pick
782 operand to compare against. For prec <= BITS_PER_WORD, I think
783 preferring REG operand is better over CONST_INT, because
784 the CONST_INT might enlarge the instruction or CSE would need
785 to figure out we'd already loaded it into a register before.
786 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
787 as then the multi-word comparison can be perhaps simplified. */
788 if (code
== PLUS_EXPR
789 && (prec
<= BITS_PER_WORD
790 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
791 : CONST_SCALAR_INT_P (op1
)))
793 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
794 true, mode
, NULL_RTX
, NULL
, done_label
,
795 profile_probability::very_likely ());
800 if (!uns0_p
&& uns1_p
&& !unsr_p
)
802 /* Compute the operation. On RTL level, the addition is always
804 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
805 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
806 rtx tem
= expand_binop (mode
, add_optab
,
807 code
== PLUS_EXPR
? res
: op0
, sgn
,
808 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
809 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
810 done_label
, profile_probability::very_likely ());
815 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
817 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
819 /* As we've changed op1, we have to avoid using the value range
820 for the original argument. */
821 arg1
= error_mark_node
;
827 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
829 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
831 /* As we've changed op0, we have to avoid using the value range
832 for the original argument. */
833 arg0
= error_mark_node
;
839 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
841 /* Compute the operation. On RTL level, the addition is always
843 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
845 int pos_neg
= get_range_pos_neg (arg0
);
847 /* If ARG0 is known to be always negative, this is always overflow. */
848 emit_jump (do_error
);
849 else if (pos_neg
== 3)
850 /* If ARG0 is not known to be always positive, check at runtime. */
851 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
852 NULL
, do_error
, profile_probability::very_unlikely ());
853 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
854 done_label
, profile_probability::very_likely ());
859 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
861 /* Compute the operation. On RTL level, the addition is always
863 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
865 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
867 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
868 done_label
, profile_probability::very_likely ());
873 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
875 /* Compute the operation. On RTL level, the addition is always
877 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
879 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
880 NULL
, do_error
, profile_probability::very_unlikely ());
882 /* The operation is commutative, so we can pick operand to compare
883 against. For prec <= BITS_PER_WORD, I think preferring REG operand
884 is better over CONST_INT, because the CONST_INT might enlarge the
885 instruction or CSE would need to figure out we'd already loaded it
886 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
887 might be more beneficial, as then the multi-word comparison can be
888 perhaps simplified. */
889 if (prec
<= BITS_PER_WORD
890 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
891 : CONST_SCALAR_INT_P (op0
))
893 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
894 done_label
, profile_probability::very_likely ());
899 if (!uns0_p
&& !uns1_p
&& unsr_p
)
901 /* Compute the operation. On RTL level, the addition is always
903 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
904 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
905 int pos_neg
= get_range_pos_neg (arg1
);
906 if (code
== PLUS_EXPR
)
908 int pos_neg0
= get_range_pos_neg (arg0
);
909 if (pos_neg0
!= 3 && pos_neg
== 3)
911 std::swap (op0
, op1
);
918 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
919 ? and_optab
: ior_optab
,
920 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
921 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
922 NULL
, done_label
, profile_probability::very_likely ());
926 rtx_code_label
*do_ior_label
= gen_label_rtx ();
927 do_compare_rtx_and_jump (op1
, const0_rtx
,
928 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
929 NULL_RTX
, NULL
, do_ior_label
,
930 profile_probability::even ());
931 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
933 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
934 NULL
, done_label
, profile_probability::very_likely ());
935 emit_jump (do_error
);
936 emit_label (do_ior_label
);
937 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
939 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
940 NULL
, done_label
, profile_probability::very_likely ());
946 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
948 /* Compute the operation. On RTL level, the addition is always
950 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
952 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
953 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
954 op0_geu_op1
, profile_probability::even ());
955 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
956 NULL
, done_label
, profile_probability::very_likely ());
957 emit_jump (do_error
);
958 emit_label (op0_geu_op1
);
959 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
960 NULL
, done_label
, profile_probability::very_likely ());
964 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
969 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
970 : subv4_optab
, mode
);
971 if (icode
!= CODE_FOR_nothing
)
973 struct expand_operand ops
[4];
974 rtx_insn
*last
= get_last_insn ();
976 res
= gen_reg_rtx (mode
);
977 create_output_operand (&ops
[0], res
, mode
);
978 create_input_operand (&ops
[1], op0
, mode
);
979 create_input_operand (&ops
[2], op1
, mode
);
980 create_fixed_operand (&ops
[3], do_error
);
981 if (maybe_expand_insn (icode
, 4, ops
))
983 last
= get_last_insn ();
984 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
986 && any_condjump_p (last
)
987 && !find_reg_note (last
, REG_BR_PROB
, 0))
988 add_reg_br_prob_note (last
,
989 profile_probability::very_unlikely ());
990 emit_jump (done_label
);
994 delete_insns_since (last
);
997 /* Compute the operation. On RTL level, the addition is always
999 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1000 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1002 /* If we can prove that one of the arguments (for MINUS_EXPR only
1003 the second operand, as subtraction is not commutative) is always
1004 non-negative or always negative, we can do just one comparison
1005 and conditional jump. */
1006 int pos_neg
= get_range_pos_neg (arg1
);
1007 if (code
== PLUS_EXPR
)
1009 int pos_neg0
= get_range_pos_neg (arg0
);
1010 if (pos_neg0
!= 3 && pos_neg
== 3)
1012 std::swap (op0
, op1
);
1017 /* Addition overflows if and only if the two operands have the same sign,
1018 and the result has the opposite sign. Subtraction overflows if and
1019 only if the two operands have opposite sign, and the subtrahend has
1020 the same sign as the result. Here 0 is counted as positive. */
1023 /* Compute op0 ^ op1 (operands have opposite sign). */
1024 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1027 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1028 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1032 if (code
== PLUS_EXPR
)
1034 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1035 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1036 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1041 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1042 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1043 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1047 /* No overflow if the result has bit sign cleared. */
1048 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1049 NULL
, done_label
, profile_probability::very_likely ());
1052 /* Compare the result of the operation with the first operand.
1053 No overflow for addition if second operand is positive and result
1054 is larger or second operand is negative and result is smaller.
1055 Likewise for subtraction with sign of second operand flipped. */
1057 do_compare_rtx_and_jump (res
, op0
,
1058 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1059 false, mode
, NULL_RTX
, NULL
, done_label
,
1060 profile_probability::very_likely ());
1064 emit_label (do_error
);
1067 /* Expand the ubsan builtin call. */
1069 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1073 do_pending_stack_adjust ();
1076 expand_arith_set_overflow (lhs
, target
);
1079 emit_label (done_label
);
1084 expand_ubsan_result_store (target
, res
);
1088 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1091 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1096 /* Add negate overflow checking to the statement STMT. */
1099 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1104 rtx_code_label
*done_label
, *do_error
;
1105 rtx target
= NULL_RTX
;
1107 done_label
= gen_label_rtx ();
1108 do_error
= gen_label_rtx ();
1110 do_pending_stack_adjust ();
1111 op1
= expand_normal (arg1
);
1113 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1116 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1118 write_complex_part (target
, const0_rtx
, true);
1121 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1122 if (icode
!= CODE_FOR_nothing
)
1124 struct expand_operand ops
[3];
1125 rtx_insn
*last
= get_last_insn ();
1127 res
= gen_reg_rtx (mode
);
1128 create_output_operand (&ops
[0], res
, mode
);
1129 create_input_operand (&ops
[1], op1
, mode
);
1130 create_fixed_operand (&ops
[2], do_error
);
1131 if (maybe_expand_insn (icode
, 3, ops
))
1133 last
= get_last_insn ();
1134 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1136 && any_condjump_p (last
)
1137 && !find_reg_note (last
, REG_BR_PROB
, 0))
1138 add_reg_br_prob_note (last
,
1139 profile_probability::very_unlikely ());
1140 emit_jump (done_label
);
1144 delete_insns_since (last
);
1145 icode
= CODE_FOR_nothing
;
1149 if (icode
== CODE_FOR_nothing
)
1151 /* Compute the operation. On RTL level, the addition is always
1153 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1155 /* Compare the operand with the most negative value. */
1156 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1157 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1158 done_label
, profile_probability::very_likely ());
1161 emit_label (do_error
);
1164 /* Expand the ubsan builtin call. */
1166 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1167 arg1
, NULL_TREE
, datap
);
1170 do_pending_stack_adjust ();
1173 expand_arith_set_overflow (lhs
, target
);
1176 emit_label (done_label
);
1181 expand_ubsan_result_store (target
, res
);
1183 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1187 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1188 mode MODE can be expanded without using a libcall. */
1191 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1192 rtx op0
, rtx op1
, bool uns
)
1194 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1195 != CODE_FOR_nothing
)
1198 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1199 != CODE_FOR_nothing
)
1202 rtx_insn
*last
= get_last_insn ();
1203 if (CONSTANT_P (op0
))
1204 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1206 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1207 if (CONSTANT_P (op1
))
1208 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1210 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1211 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1212 delete_insns_since (last
);
1213 return ret
!= NULL_RTX
;
1216 /* Add mul overflow checking to the statement STMT. */
1219 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1220 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1225 rtx_code_label
*done_label
, *do_error
;
1226 rtx target
= NULL_RTX
;
1228 enum insn_code icode
;
1230 done_label
= gen_label_rtx ();
1231 do_error
= gen_label_rtx ();
1233 do_pending_stack_adjust ();
1234 op0
= expand_normal (arg0
);
1235 op1
= expand_normal (arg1
);
1237 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1241 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1243 write_complex_part (target
, const0_rtx
, true);
1247 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1249 /* We assume both operands and result have the same precision
1250 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1251 with that precision, U for unsigned type with that precision,
1252 sgn for unsigned most significant bit in that precision.
1253 s1 is signed first operand, u1 is unsigned first operand,
1254 s2 is signed second operand, u2 is unsigned second operand,
1255 sr is signed result, ur is unsigned result and the following
1256 rules say how to compute result (which is always result of
1257 the operands as if both were unsigned, cast to the right
1258 signedness) and how to compute whether operation overflowed.
1259 main_ovf (false) stands for jump on signed multiplication
1260 overflow or the main algorithm with uns == false.
1261 main_ovf (true) stands for jump on unsigned multiplication
1262 overflow or the main algorithm with uns == true.
1265 res = (S) ((U) s1 * (U) s2)
1266 ovf = main_ovf (false)
1269 ovf = main_ovf (true)
1272 ovf = (s1 < 0 && u2) || main_ovf (true)
1275 ovf = res < 0 || main_ovf (true)
1277 res = (S) ((U) s1 * u2)
1278 ovf = (S) u2 >= 0 ? main_ovf (false)
1279 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1281 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1282 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1284 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1286 if (uns0_p
&& !uns1_p
)
1288 /* Multiplication is commutative, if operand signedness differs,
1289 canonicalize to the first operand being signed and second
1290 unsigned to simplify following code. */
1291 std::swap (op0
, op1
);
1292 std::swap (arg0
, arg1
);
1297 int pos_neg0
= get_range_pos_neg (arg0
);
1298 int pos_neg1
= get_range_pos_neg (arg1
);
1301 if (!uns0_p
&& uns1_p
&& unsr_p
)
1306 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1309 /* If s1 is negative, avoid the main code, just multiply and
1310 signal overflow if op1 is not 0. */
1311 struct separate_ops ops
;
1312 ops
.code
= MULT_EXPR
;
1313 ops
.type
= TREE_TYPE (arg1
);
1314 ops
.op0
= make_tree (ops
.type
, op0
);
1315 ops
.op1
= make_tree (ops
.type
, op1
);
1316 ops
.op2
= NULL_TREE
;
1318 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1319 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1320 NULL
, done_label
, profile_probability::very_likely ());
1321 goto do_error_label
;
1323 rtx_code_label
*do_main_label
;
1324 do_main_label
= gen_label_rtx ();
1325 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1326 NULL
, do_main_label
, profile_probability::very_likely ());
1327 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1328 NULL
, do_main_label
, profile_probability::very_likely ());
1329 expand_arith_set_overflow (lhs
, target
);
1330 emit_label (do_main_label
);
1338 if (uns0_p
&& uns1_p
&& !unsr_p
)
1341 /* Rest of handling of this case after res is computed. */
1346 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1353 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1354 avoid the main code, just multiply and signal overflow
1355 unless 0 * u2 or -1 * ((U) Smin). */
1356 struct separate_ops ops
;
1357 ops
.code
= MULT_EXPR
;
1358 ops
.type
= TREE_TYPE (arg1
);
1359 ops
.op0
= make_tree (ops
.type
, op0
);
1360 ops
.op1
= make_tree (ops
.type
, op1
);
1361 ops
.op2
= NULL_TREE
;
1363 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1364 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1365 NULL
, done_label
, profile_probability::very_likely ());
1366 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1367 NULL
, do_error
, profile_probability::very_unlikely ());
1369 prec
= GET_MODE_PRECISION (mode
);
1371 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1372 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1373 NULL
, done_label
, profile_probability::very_likely ());
1374 goto do_error_label
;
1376 /* Rest of handling of this case after res is computed. */
1384 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1387 switch (pos_neg0
| pos_neg1
)
1389 case 1: /* Both operands known to be non-negative. */
1391 case 2: /* Both operands known to be negative. */
1392 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1393 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1394 /* Avoid looking at arg0/arg1 ranges, as we've changed
1396 arg0
= error_mark_node
;
1397 arg1
= error_mark_node
;
1400 if ((pos_neg0
^ pos_neg1
) == 3)
1402 /* If one operand is known to be negative and the other
1403 non-negative, this overflows always, unless the non-negative
1404 one is 0. Just do normal multiply and set overflow
1405 unless one of the operands is 0. */
1406 struct separate_ops ops
;
1407 ops
.code
= MULT_EXPR
;
1409 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1411 ops
.op0
= make_tree (ops
.type
, op0
);
1412 ops
.op1
= make_tree (ops
.type
, op1
);
1413 ops
.op2
= NULL_TREE
;
1415 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1416 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1418 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1419 NULL_RTX
, NULL
, done_label
,
1420 profile_probability::very_likely ());
1421 goto do_error_label
;
1423 /* The general case, do all the needed comparisons at runtime. */
1424 rtx_code_label
*do_main_label
, *after_negate_label
;
1426 rop0
= gen_reg_rtx (mode
);
1427 rop1
= gen_reg_rtx (mode
);
1428 emit_move_insn (rop0
, op0
);
1429 emit_move_insn (rop1
, op1
);
1432 do_main_label
= gen_label_rtx ();
1433 after_negate_label
= gen_label_rtx ();
1434 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1436 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1437 NULL
, after_negate_label
, profile_probability::very_likely ());
1438 /* Both arguments negative here, negate them and continue with
1439 normal unsigned overflow checking multiplication. */
1440 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1442 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1444 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1446 arg0
= error_mark_node
;
1447 arg1
= error_mark_node
;
1448 emit_jump (do_main_label
);
1449 emit_label (after_negate_label
);
1450 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1452 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1453 NULL
, do_main_label
, profile_probability::very_likely ());
1454 /* One argument is negative here, the other positive. This
1455 overflows always, unless one of the arguments is 0. But
1456 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1457 is, thus we can keep do_main code oring in overflow as is. */
1458 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1459 NULL
, do_main_label
, profile_probability::very_likely ());
1460 expand_arith_set_overflow (lhs
, target
);
1461 emit_label (do_main_label
);
1469 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1470 sign
= uns
? UNSIGNED
: SIGNED
;
1471 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1473 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1474 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1476 /* Optimize unsigned multiplication by power of 2 constant
1477 using 2 shifts, one for result, one to extract the shifted
1478 out bits to see if they are all zero.
1479 Don't do this if optimizing for size and we have umulv4_optab,
1480 in that case assume multiplication will be shorter.
1481 This is heuristics based on the single target that provides
1482 umulv4 right now (i?86/x86_64), if further targets add it, this
1483 might need to be revisited.
1484 Cases where both operands are constant should be folded already
1485 during GIMPLE, and cases where one operand is constant but not
1486 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1487 below can be done without multiplication, just by shifts and adds,
1488 or we'd need to divide the result (and hope it actually doesn't
1489 really divide nor multiply) and compare the result of the division
1490 with the original operand. */
1495 if (integer_pow2p (arg0
))
1497 std::swap (opn0
, opn1
);
1498 std::swap (argn0
, argn1
);
1500 int cnt
= tree_log2 (argn1
);
1501 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1503 rtx upper
= const0_rtx
;
1504 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1506 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1507 GET_MODE_PRECISION (mode
) - cnt
,
1509 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1510 NULL_RTX
, NULL
, done_label
,
1511 profile_probability::very_likely ());
1512 goto do_error_label
;
1515 if (icode
!= CODE_FOR_nothing
)
1517 struct expand_operand ops
[4];
1518 rtx_insn
*last
= get_last_insn ();
1520 res
= gen_reg_rtx (mode
);
1521 create_output_operand (&ops
[0], res
, mode
);
1522 create_input_operand (&ops
[1], op0
, mode
);
1523 create_input_operand (&ops
[2], op1
, mode
);
1524 create_fixed_operand (&ops
[3], do_error
);
1525 if (maybe_expand_insn (icode
, 4, ops
))
1527 last
= get_last_insn ();
1528 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1530 && any_condjump_p (last
)
1531 && !find_reg_note (last
, REG_BR_PROB
, 0))
1532 add_reg_br_prob_note (last
,
1533 profile_probability::very_unlikely ());
1534 emit_jump (done_label
);
1538 delete_insns_since (last
);
1539 icode
= CODE_FOR_nothing
;
1543 if (icode
== CODE_FOR_nothing
)
1545 struct separate_ops ops
;
1546 int prec
= GET_MODE_PRECISION (mode
);
1547 scalar_int_mode hmode
, wmode
;
1548 ops
.op0
= make_tree (type
, op0
);
1549 ops
.op1
= make_tree (type
, op1
);
1550 ops
.op2
= NULL_TREE
;
1553 /* Optimize unsigned overflow check where we don't use the
1554 multiplication result, just whether overflow happened.
1555 If we can do MULT_HIGHPART_EXPR, that followed by
1556 comparison of the result against zero is cheapest.
1557 We'll still compute res, but it should be DCEd later. */
1563 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1564 && can_mult_highpart_p (mode
, uns
) == 1
1565 && single_imm_use (lhs
, &use
, &use_stmt
)
1566 && is_gimple_assign (use_stmt
)
1567 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1570 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1571 && targetm
.scalar_mode_supported_p (wmode
)
1572 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1575 ops
.code
= WIDEN_MULT_EXPR
;
1577 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1579 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1580 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1582 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1583 res
= convert_modes (mode
, wmode
, res
, uns
);
1585 /* For the unsigned multiplication, there was overflow if
1586 HIPART is non-zero. */
1587 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1588 NULL_RTX
, NULL
, done_label
,
1589 profile_probability::very_likely ());
1592 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1594 /* RES is low half of the double width result, HIPART
1595 the high half. There was overflow if
1596 HIPART is different from RES < 0 ? -1 : 0. */
1597 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1598 NULL_RTX
, NULL
, done_label
,
1599 profile_probability::very_likely ());
1602 else if (can_mult_highpart_p (mode
, uns
) == 1)
1605 ops
.code
= MULT_HIGHPART_EXPR
;
1608 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1610 ops
.code
= MULT_EXPR
;
1611 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1613 /* For the unsigned multiplication, there was overflow if
1614 HIPART is non-zero. */
1615 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1616 NULL_RTX
, NULL
, done_label
,
1617 profile_probability::very_likely ());
1620 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1622 /* RES is low half of the double width result, HIPART
1623 the high half. There was overflow if
1624 HIPART is different from RES < 0 ? -1 : 0. */
1625 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1626 NULL_RTX
, NULL
, done_label
,
1627 profile_probability::very_likely ());
1631 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1632 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1634 rtx_code_label
*large_op0
= gen_label_rtx ();
1635 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1636 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1637 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1638 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1639 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1640 rtx_code_label
*do_overflow
= gen_label_rtx ();
1641 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1643 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1644 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1646 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1647 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1648 rtx signbit0
= const0_rtx
;
1650 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1652 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1654 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1655 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1656 rtx signbit1
= const0_rtx
;
1658 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1661 res
= gen_reg_rtx (mode
);
1663 /* True if op0 resp. op1 are known to be in the range of
1665 bool op0_small_p
= false;
1666 bool op1_small_p
= false;
1667 /* True if op0 resp. op1 are known to have all zeros or all ones
1668 in the upper half of bits, but are not known to be
1670 bool op0_medium_p
= false;
1671 bool op1_medium_p
= false;
1672 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1673 nonnegative, 1 if unknown. */
1679 else if (pos_neg0
== 2)
1683 else if (pos_neg1
== 2)
1686 unsigned int mprec0
= prec
;
1687 if (arg0
!= error_mark_node
)
1688 mprec0
= get_min_precision (arg0
, sign
);
1689 if (mprec0
<= hprec
)
1691 else if (!uns
&& mprec0
<= hprec
+ 1)
1692 op0_medium_p
= true;
1693 unsigned int mprec1
= prec
;
1694 if (arg1
!= error_mark_node
)
1695 mprec1
= get_min_precision (arg1
, sign
);
1696 if (mprec1
<= hprec
)
1698 else if (!uns
&& mprec1
<= hprec
+ 1)
1699 op1_medium_p
= true;
1701 int smaller_sign
= 1;
1702 int larger_sign
= 1;
1705 smaller_sign
= op0_sign
;
1706 larger_sign
= op1_sign
;
1708 else if (op1_small_p
)
1710 smaller_sign
= op1_sign
;
1711 larger_sign
= op0_sign
;
1713 else if (op0_sign
== op1_sign
)
1715 smaller_sign
= op0_sign
;
1716 larger_sign
= op0_sign
;
1720 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1721 NULL_RTX
, NULL
, large_op0
,
1722 profile_probability::unlikely ());
1725 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1726 NULL_RTX
, NULL
, small_op0_large_op1
,
1727 profile_probability::unlikely ());
1729 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1730 hmode to mode, the multiplication will never overflow. We can
1731 do just one hmode x hmode => mode widening multiplication. */
1732 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1733 if (GET_CODE (lopart0
) == SUBREG
)
1735 lopart0s
= shallow_copy_rtx (lopart0
);
1736 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1737 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1739 if (GET_CODE (lopart1
) == SUBREG
)
1741 lopart1s
= shallow_copy_rtx (lopart1
);
1742 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1743 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1745 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1746 ops
.op0
= make_tree (halfstype
, lopart0s
);
1747 ops
.op1
= make_tree (halfstype
, lopart1s
);
1748 ops
.code
= WIDEN_MULT_EXPR
;
1751 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1752 emit_move_insn (res
, thisres
);
1753 emit_jump (done_label
);
1755 emit_label (small_op0_large_op1
);
1757 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1758 but op1 is not, just swap the arguments and handle it as op1
1759 sign/zero extended, op0 not. */
1760 rtx larger
= gen_reg_rtx (mode
);
1761 rtx hipart
= gen_reg_rtx (hmode
);
1762 rtx lopart
= gen_reg_rtx (hmode
);
1763 emit_move_insn (larger
, op1
);
1764 emit_move_insn (hipart
, hipart1
);
1765 emit_move_insn (lopart
, lopart0
);
1766 emit_jump (one_small_one_large
);
1768 emit_label (large_op0
);
1771 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1772 NULL_RTX
, NULL
, both_ops_large
,
1773 profile_probability::unlikely ());
1775 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1776 but op0 is not, prepare larger, hipart and lopart pseudos and
1777 handle it together with small_op0_large_op1. */
1778 emit_move_insn (larger
, op0
);
1779 emit_move_insn (hipart
, hipart0
);
1780 emit_move_insn (lopart
, lopart1
);
1782 emit_label (one_small_one_large
);
1784 /* lopart is the low part of the operand that is sign extended
1785 to mode, larger is the other operand, hipart is the
1786 high part of larger and lopart0 and lopart1 are the low parts
1788 We perform lopart0 * lopart1 and lopart * hipart widening
1790 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1791 ops
.op0
= make_tree (halfutype
, lopart0
);
1792 ops
.op1
= make_tree (halfutype
, lopart1
);
1794 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1796 ops
.op0
= make_tree (halfutype
, lopart
);
1797 ops
.op1
= make_tree (halfutype
, hipart
);
1798 rtx loxhi
= gen_reg_rtx (mode
);
1799 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1800 emit_move_insn (loxhi
, tem
);
1804 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1805 if (larger_sign
== 0)
1806 emit_jump (after_hipart_neg
);
1807 else if (larger_sign
!= -1)
1808 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1809 NULL_RTX
, NULL
, after_hipart_neg
,
1810 profile_probability::even ());
1812 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1813 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1814 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1816 emit_move_insn (loxhi
, tem
);
1818 emit_label (after_hipart_neg
);
1820 /* if (lopart < 0) loxhi -= larger; */
1821 if (smaller_sign
== 0)
1822 emit_jump (after_lopart_neg
);
1823 else if (smaller_sign
!= -1)
1824 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1825 NULL_RTX
, NULL
, after_lopart_neg
,
1826 profile_probability::even ());
1828 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1830 emit_move_insn (loxhi
, tem
);
1832 emit_label (after_lopart_neg
);
1835 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1836 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1837 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1839 emit_move_insn (loxhi
, tem
);
1841 /* if (loxhi >> (bitsize / 2)
1842 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1843 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1844 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1846 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1847 rtx signbitloxhi
= const0_rtx
;
1849 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1850 convert_modes (hmode
, mode
,
1852 hprec
- 1, NULL_RTX
, 0);
1854 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1855 NULL_RTX
, NULL
, do_overflow
,
1856 profile_probability::very_unlikely ());
1858 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1859 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1861 tem
= convert_modes (mode
, hmode
,
1862 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1864 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1867 emit_move_insn (res
, tem
);
1868 emit_jump (done_label
);
1870 emit_label (both_ops_large
);
1872 /* If both operands are large (not sign (!uns) or zero (uns)
1873 extended from hmode), then perform the full multiplication
1874 which will be the result of the operation.
1875 The only cases which don't overflow are for signed multiplication
1876 some cases where both hipart0 and highpart1 are 0 or -1.
1877 For unsigned multiplication when high parts are both non-zero
1878 this overflows always. */
1879 ops
.code
= MULT_EXPR
;
1880 ops
.op0
= make_tree (type
, op0
);
1881 ops
.op1
= make_tree (type
, op1
);
1882 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1883 emit_move_insn (res
, tem
);
1889 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1890 NULL_RTX
, 1, OPTAB_WIDEN
);
1891 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1892 NULL_RTX
, NULL
, do_error
,
1893 profile_probability::very_unlikely ());
1898 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1899 NULL_RTX
, 1, OPTAB_WIDEN
);
1900 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1901 NULL_RTX
, NULL
, do_error
,
1902 profile_probability::very_unlikely ());
1905 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1906 the same, overflow happened if res is non-positive, if they
1907 are different, overflow happened if res is positive. */
1908 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1909 emit_jump (hipart_different
);
1910 else if (op0_sign
== 1 || op1_sign
== 1)
1911 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1912 NULL_RTX
, NULL
, hipart_different
,
1913 profile_probability::even ());
1915 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1916 NULL_RTX
, NULL
, do_error
,
1917 profile_probability::very_unlikely ());
1918 emit_jump (done_label
);
1920 emit_label (hipart_different
);
1922 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1923 NULL_RTX
, NULL
, do_error
,
1924 profile_probability::very_unlikely ());
1925 emit_jump (done_label
);
1928 emit_label (do_overflow
);
1930 /* Overflow, do full multiplication and fallthru into do_error. */
1931 ops
.op0
= make_tree (type
, op0
);
1932 ops
.op1
= make_tree (type
, op1
);
1933 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1934 emit_move_insn (res
, tem
);
1936 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1937 && targetm
.scalar_mode_supported_p (wmode
))
1938 /* Even emitting a libcall is better than not detecting overflow
1943 gcc_assert (!is_ubsan
);
1944 ops
.code
= MULT_EXPR
;
1946 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1947 emit_jump (done_label
);
1952 emit_label (do_error
);
1955 /* Expand the ubsan builtin call. */
1957 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1961 do_pending_stack_adjust ();
1964 expand_arith_set_overflow (lhs
, target
);
1967 emit_label (done_label
);
1970 if (uns0_p
&& uns1_p
&& !unsr_p
)
1972 rtx_code_label
*all_done_label
= gen_label_rtx ();
1973 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1974 NULL
, all_done_label
, profile_probability::very_likely ());
1975 expand_arith_set_overflow (lhs
, target
);
1976 emit_label (all_done_label
);
1980 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1982 rtx_code_label
*all_done_label
= gen_label_rtx ();
1983 rtx_code_label
*set_noovf
= gen_label_rtx ();
1984 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1985 NULL
, all_done_label
, profile_probability::very_likely ());
1986 expand_arith_set_overflow (lhs
, target
);
1987 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1988 NULL
, set_noovf
, profile_probability::very_likely ());
1989 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1990 NULL
, all_done_label
, profile_probability::very_unlikely ());
1991 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1992 all_done_label
, profile_probability::very_unlikely ());
1993 emit_label (set_noovf
);
1994 write_complex_part (target
, const0_rtx
, true);
1995 emit_label (all_done_label
);
2001 expand_ubsan_result_store (target
, res
);
2003 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2007 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2010 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2011 tree arg0
, tree arg1
)
2013 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2014 rtx_code_label
*loop_lab
= NULL
;
2015 rtx cntvar
= NULL_RTX
;
2016 tree cntv
= NULL_TREE
;
2017 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2018 tree sz
= TYPE_SIZE (eltype
);
2019 tree data
= NULL_TREE
;
2020 tree resv
= NULL_TREE
;
2021 rtx lhsr
= NULL_RTX
;
2022 rtx resvr
= NULL_RTX
;
2023 unsigned HOST_WIDE_INT const_cnt
= 0;
2024 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2029 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2030 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2031 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2032 optab_default
)) == unknown_optab
2033 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2034 == CODE_FOR_nothing
))
2037 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2040 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2041 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2047 do_pending_stack_adjust ();
2048 loop_lab
= gen_label_rtx ();
2049 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2050 cntv
= make_tree (sizetype
, cntvar
);
2051 emit_move_insn (cntvar
, const0_rtx
);
2052 emit_label (loop_lab
);
2054 if (TREE_CODE (arg0
) != VECTOR_CST
)
2056 rtx arg0r
= expand_normal (arg0
);
2057 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2059 if (TREE_CODE (arg1
) != VECTOR_CST
)
2061 rtx arg1r
= expand_normal (arg1
);
2062 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2064 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2066 tree op0
, op1
, res
= NULL_TREE
;
2069 tree atype
= build_array_type_nelts (eltype
, cnt
);
2070 op0
= uniform_vector_p (arg0
);
2071 if (op0
== NULL_TREE
)
2073 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2074 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2075 NULL_TREE
, NULL_TREE
);
2077 op1
= uniform_vector_p (arg1
);
2078 if (op1
== NULL_TREE
)
2080 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2081 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2082 NULL_TREE
, NULL_TREE
);
2086 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2087 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2088 NULL_TREE
, NULL_TREE
);
2093 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2094 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2095 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2097 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2103 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2104 false, false, false, true, &data
);
2107 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2108 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2110 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2111 false, false, false, true, &data
);
2114 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2123 struct separate_ops ops
;
2124 ops
.code
= PLUS_EXPR
;
2125 ops
.type
= TREE_TYPE (cntv
);
2127 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2128 ops
.op2
= NULL_TREE
;
2130 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2133 emit_move_insn (cntvar
, ret
);
2134 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2135 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2136 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2137 profile_probability::very_likely ());
2139 if (lhs
&& resv
== NULL_TREE
)
2141 struct separate_ops ops
;
2143 ops
.type
= TREE_TYPE (arg0
);
2146 ops
.op2
= NULL_TREE
;
2148 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2151 emit_move_insn (lhsr
, ret
);
2154 emit_move_insn (lhsr
, resvr
);
2157 /* Expand UBSAN_CHECK_ADD call STMT. */
2160 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2162 location_t loc
= gimple_location (stmt
);
2163 tree lhs
= gimple_call_lhs (stmt
);
2164 tree arg0
= gimple_call_arg (stmt
, 0);
2165 tree arg1
= gimple_call_arg (stmt
, 1);
2166 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2167 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2169 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2170 false, false, false, true, NULL
);
2173 /* Expand UBSAN_CHECK_SUB call STMT. */
2176 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2178 location_t loc
= gimple_location (stmt
);
2179 tree lhs
= gimple_call_lhs (stmt
);
2180 tree arg0
= gimple_call_arg (stmt
, 0);
2181 tree arg1
= gimple_call_arg (stmt
, 1);
2182 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2183 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2184 else if (integer_zerop (arg0
))
2185 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2187 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2188 false, false, false, true, NULL
);
2191 /* Expand UBSAN_CHECK_MUL call STMT. */
2194 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2196 location_t loc
= gimple_location (stmt
);
2197 tree lhs
= gimple_call_lhs (stmt
);
2198 tree arg0
= gimple_call_arg (stmt
, 0);
2199 tree arg1
= gimple_call_arg (stmt
, 1);
2200 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2201 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2203 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2207 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2210 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2212 tree lhs
= gimple_call_lhs (stmt
);
2213 if (lhs
== NULL_TREE
)
2215 tree arg0
= gimple_call_arg (stmt
, 0);
2216 tree arg1
= gimple_call_arg (stmt
, 1);
2217 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2218 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2219 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2220 int unsr_p
= TYPE_UNSIGNED (type
);
2221 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2222 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2223 int precres
= TYPE_PRECISION (type
);
2224 location_t loc
= gimple_location (stmt
);
2225 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2227 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2229 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2230 prec0
= MIN (prec0
, pr
);
2231 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2232 prec1
= MIN (prec1
, pr
);
2234 /* If uns0_p && uns1_p, precop is minimum needed precision
2235 of unsigned type to hold the exact result, otherwise
2236 precop is minimum needed precision of signed type to
2237 hold the exact result. */
2239 if (code
== MULT_EXPR
)
2240 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2243 if (uns0_p
== uns1_p
)
2244 precop
= MAX (prec0
, prec1
) + 1;
2246 precop
= MAX (prec0
+ 1, prec1
) + 1;
2248 precop
= MAX (prec0
, prec1
+ 1) + 1;
2250 int orig_precres
= precres
;
2254 if ((uns0_p
&& uns1_p
)
2255 ? ((precop
+ !unsr_p
) <= precres
2256 /* u1 - u2 -> ur can overflow, no matter what precision
2258 && (code
!= MINUS_EXPR
|| !unsr_p
))
2259 : (!unsr_p
&& precop
<= precres
))
2261 /* The infinity precision result will always fit into result. */
2262 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2263 write_complex_part (target
, const0_rtx
, true);
2264 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2265 struct separate_ops ops
;
2268 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2269 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2270 ops
.op2
= NULL_TREE
;
2272 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2273 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2277 /* For operations with low precision, if target doesn't have them, start
2278 with precres widening right away, otherwise do it only if the most
2279 simple cases can't be used. */
2280 const int min_precision
= targetm
.min_arithmetic_precision ();
2281 if (orig_precres
== precres
&& precres
< min_precision
)
2283 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2284 && prec1
<= precres
)
2285 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2286 && prec0
+ uns0_p
<= precres
2287 && prec1
+ uns1_p
<= precres
))
2289 arg0
= fold_convert_loc (loc
, type
, arg0
);
2290 arg1
= fold_convert_loc (loc
, type
, arg1
);
2294 if (integer_zerop (arg0
) && !unsr_p
)
2296 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2301 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2302 unsr_p
, unsr_p
, false, NULL
);
2305 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2306 unsr_p
, unsr_p
, false, NULL
);
2313 /* For sub-word operations, retry with a wider type first. */
2314 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2316 int p
= MAX (min_precision
, precop
);
2317 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2318 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2321 p
= TYPE_PRECISION (optype
);
2325 unsr_p
= TYPE_UNSIGNED (optype
);
2331 if (prec0
<= precres
&& prec1
<= precres
)
2336 types
[0] = build_nonstandard_integer_type (precres
, 0);
2342 types
[1] = build_nonstandard_integer_type (precres
, 1);
2344 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2345 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2346 if (code
!= MULT_EXPR
)
2347 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2348 uns0_p
, uns1_p
, false, NULL
);
2350 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2351 uns0_p
, uns1_p
, false, NULL
);
2355 /* Retry with a wider type. */
2356 if (orig_precres
== precres
)
2358 int p
= MAX (prec0
, prec1
);
2359 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2360 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2363 p
= TYPE_PRECISION (optype
);
2367 unsr_p
= TYPE_UNSIGNED (optype
);
2378 /* Expand ADD_OVERFLOW STMT. */
2381 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2383 expand_arith_overflow (PLUS_EXPR
, stmt
);
2386 /* Expand SUB_OVERFLOW STMT. */
2389 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2391 expand_arith_overflow (MINUS_EXPR
, stmt
);
2394 /* Expand MUL_OVERFLOW STMT. */
2397 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2399 expand_arith_overflow (MULT_EXPR
, stmt
);
2402 /* This should get folded in tree-vectorizer.c. */
2405 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2410 /* This should get folded in tree-vectorizer.c. */
2413 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2418 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2419 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2422 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2424 tree addr
= gimple_call_arg (stmt
, index
);
2425 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2426 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2427 if (TYPE_ALIGN (type
) != align
)
2428 type
= build_aligned_type (type
, align
);
2431 if (TREE_CODE (tmp
) == SSA_NAME
)
2433 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2434 if (gimple_assign_single_p (def
))
2435 tmp
= gimple_assign_rhs1 (def
);
2438 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2440 tree mem
= TREE_OPERAND (tmp
, 0);
2441 if (TREE_CODE (mem
) == TARGET_MEM_REF
2442 && types_compatible_p (TREE_TYPE (mem
), type
))
2444 tree offset
= TMR_OFFSET (mem
);
2445 if (alias_ptr_type
!= TREE_TYPE (offset
) || !integer_zerop (offset
))
2447 mem
= copy_node (mem
);
2448 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2449 wi::to_poly_wide (offset
));
2455 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2458 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2461 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2463 struct expand_operand ops
[3];
2464 tree type
, lhs
, rhs
, maskt
;
2465 rtx mem
, target
, mask
;
2468 maskt
= gimple_call_arg (stmt
, 2);
2469 lhs
= gimple_call_lhs (stmt
);
2470 if (lhs
== NULL_TREE
)
2472 type
= TREE_TYPE (lhs
);
2473 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2475 if (optab
== vec_mask_load_lanes_optab
)
2476 icode
= get_multi_vector_move (type
, optab
);
2478 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2479 TYPE_MODE (TREE_TYPE (maskt
)));
2481 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2482 gcc_assert (MEM_P (mem
));
2483 mask
= expand_normal (maskt
);
2484 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2485 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2486 create_fixed_operand (&ops
[1], mem
);
2487 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2488 expand_insn (icode
, 3, ops
);
2491 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2493 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2496 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2498 struct expand_operand ops
[3];
2499 tree type
, lhs
, rhs
, maskt
;
2503 maskt
= gimple_call_arg (stmt
, 2);
2504 rhs
= gimple_call_arg (stmt
, 3);
2505 type
= TREE_TYPE (rhs
);
2506 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2508 if (optab
== vec_mask_store_lanes_optab
)
2509 icode
= get_multi_vector_move (type
, optab
);
2511 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2512 TYPE_MODE (TREE_TYPE (maskt
)));
2514 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2515 gcc_assert (MEM_P (mem
));
2516 mask
= expand_normal (maskt
);
2517 reg
= expand_normal (rhs
);
2518 create_fixed_operand (&ops
[0], mem
);
2519 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2520 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2521 expand_insn (icode
, 3, ops
);
2524 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2527 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2532 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2534 /* When guessing was done, the hints should be already stripped away. */
2535 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2538 tree lhs
= gimple_call_lhs (stmt
);
2540 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2542 target
= const0_rtx
;
2543 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2544 if (lhs
&& val
!= target
)
2545 emit_move_insn (target
, val
);
2548 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2549 should never be called. */
2552 expand_VA_ARG (internal_fn
, gcall
*)
2557 /* Expand the IFN_UNIQUE function according to its first argument. */
2560 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2562 rtx pattern
= NULL_RTX
;
2563 enum ifn_unique_kind kind
2564 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2571 case IFN_UNIQUE_UNSPEC
:
2572 if (targetm
.have_unique ())
2573 pattern
= targetm
.gen_unique ();
2576 case IFN_UNIQUE_OACC_FORK
:
2577 case IFN_UNIQUE_OACC_JOIN
:
2578 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2580 tree lhs
= gimple_call_lhs (stmt
);
2581 rtx target
= const0_rtx
;
2584 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2586 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2587 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2589 if (kind
== IFN_UNIQUE_OACC_FORK
)
2590 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2592 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2600 emit_insn (pattern
);
2603 /* The size of an OpenACC compute dimension. */
2606 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2608 tree lhs
= gimple_call_lhs (stmt
);
2613 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2614 if (targetm
.have_oacc_dim_size ())
2616 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2617 VOIDmode
, EXPAND_NORMAL
);
2618 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2621 emit_move_insn (target
, GEN_INT (1));
2624 /* The position of an OpenACC execution engine along one compute axis. */
2627 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2629 tree lhs
= gimple_call_lhs (stmt
);
2634 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2635 if (targetm
.have_oacc_dim_pos ())
2637 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2638 VOIDmode
, EXPAND_NORMAL
);
2639 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2642 emit_move_insn (target
, const0_rtx
);
2645 /* This is expanded by oacc_device_lower pass. */
2648 expand_GOACC_LOOP (internal_fn
, gcall
*)
2653 /* This is expanded by oacc_device_lower pass. */
2656 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2661 /* This is expanded by oacc_device_lower pass. */
2664 expand_GOACC_TILE (internal_fn
, gcall
*)
2669 /* Set errno to EDOM. */
2672 expand_SET_EDOM (internal_fn
, gcall
*)
2675 #ifdef GEN_ERRNO_RTX
2676 rtx errno_rtx
= GEN_ERRNO_RTX
;
2678 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2680 emit_move_insn (errno_rtx
,
2681 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2687 /* Expand atomic bit test and set. */
2690 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2692 expand_ifn_atomic_bit_test_and (call
);
2695 /* Expand atomic bit test and complement. */
2698 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2700 expand_ifn_atomic_bit_test_and (call
);
2703 /* Expand atomic bit test and reset. */
2706 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2708 expand_ifn_atomic_bit_test_and (call
);
2711 /* Expand atomic bit test and set. */
2714 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2716 expand_ifn_atomic_compare_exchange (call
);
2719 /* Expand LAUNDER to assignment, lhs = arg0. */
2722 expand_LAUNDER (internal_fn
, gcall
*call
)
2724 tree lhs
= gimple_call_lhs (call
);
2729 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2732 /* Expand DIVMOD() using:
2733 a) optab handler for udivmod/sdivmod if it is available.
2734 b) If optab_handler doesn't exist, generate call to
2735 target-specific divmod libfunc. */
2738 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2740 tree lhs
= gimple_call_lhs (call_stmt
);
2741 tree arg0
= gimple_call_arg (call_stmt
, 0);
2742 tree arg1
= gimple_call_arg (call_stmt
, 1);
2744 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2745 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2746 machine_mode mode
= TYPE_MODE (type
);
2747 bool unsignedp
= TYPE_UNSIGNED (type
);
2748 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2750 rtx op0
= expand_normal (arg0
);
2751 rtx op1
= expand_normal (arg1
);
2752 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2754 rtx quotient
, remainder
, libfunc
;
2756 /* Check if optab_handler exists for divmod_optab for given mode. */
2757 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2759 quotient
= gen_reg_rtx (mode
);
2760 remainder
= gen_reg_rtx (mode
);
2761 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2764 /* Generate call to divmod libfunc if it exists. */
2765 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2766 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2767 "ient
, &remainder
);
2772 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2773 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2774 make_tree (TREE_TYPE (arg0
), quotient
),
2775 make_tree (TREE_TYPE (arg1
), remainder
)),
2776 target
, VOIDmode
, EXPAND_NORMAL
);
2782 expand_NOP (internal_fn
, gcall
*)
2784 /* Nothing. But it shouldn't really prevail. */
2787 /* Expand a call to FN using the operands in STMT. FN has a single
2788 output operand and NARGS input operands. */
2791 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2794 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2796 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2797 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2799 tree lhs
= gimple_call_lhs (stmt
);
2800 tree lhs_type
= TREE_TYPE (lhs
);
2801 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2803 /* Do not assign directly to a promoted subreg, since there is no
2804 guarantee that the instruction will leave the upper bits of the
2805 register in the state required by SUBREG_PROMOTED_SIGN. */
2807 if (GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
2810 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
2812 for (unsigned int i
= 0; i
< nargs
; ++i
)
2814 tree rhs
= gimple_call_arg (stmt
, i
);
2815 tree rhs_type
= TREE_TYPE (rhs
);
2816 rtx rhs_rtx
= expand_normal (rhs
);
2817 if (INTEGRAL_TYPE_P (rhs_type
))
2818 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2819 TYPE_MODE (rhs_type
),
2820 TYPE_UNSIGNED (rhs_type
));
2822 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2825 expand_insn (icode
, nargs
+ 1, ops
);
2826 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2828 /* If the return value has an integral type, convert the instruction
2829 result to that type. This is useful for things that return an
2830 int regardless of the size of the input. If the instruction result
2831 is smaller than required, assume that it is signed.
2833 If the return value has a nonintegral type, its mode must match
2834 the instruction result. */
2835 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2837 /* If this is a scalar in a register that is stored in a wider
2838 mode than the declared mode, compute the result into its
2839 declared mode and then convert to the wider mode. */
2840 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2841 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2842 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2843 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2845 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2846 emit_move_insn (lhs_rtx
, ops
[0].value
);
2849 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2850 convert_move (lhs_rtx
, ops
[0].value
, 0);
2855 /* Expand WHILE_ULT call STMT using optab OPTAB. */
2858 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2860 expand_operand ops
[3];
2863 tree lhs
= gimple_call_lhs (stmt
);
2864 tree lhs_type
= TREE_TYPE (lhs
);
2865 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2866 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
2868 for (unsigned int i
= 0; i
< 2; ++i
)
2870 tree rhs
= gimple_call_arg (stmt
, i
);
2871 rhs_type
[i
] = TREE_TYPE (rhs
);
2872 rtx rhs_rtx
= expand_normal (rhs
);
2873 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
2876 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
2877 TYPE_MODE (lhs_type
));
2879 expand_insn (icode
, 3, ops
);
2880 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2881 emit_move_insn (lhs_rtx
, ops
[0].value
);
2884 /* Expanders for optabs that can use expand_direct_optab_fn. */
2886 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2887 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2889 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2890 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2892 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
2893 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2895 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
2896 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2898 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
2899 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2901 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
2902 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2904 /* RETURN_TYPE and ARGS are a return type and argument list that are
2905 in principle compatible with FN (which satisfies direct_internal_fn_p).
2906 Return the types that should be used to determine whether the
2907 target supports FN. */
2910 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2912 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2913 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2914 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2915 return tree_pair (type0
, type1
);
2918 /* CALL is a call whose return type and arguments are in principle
2919 compatible with FN (which satisfies direct_internal_fn_p). Return the
2920 types that should be used to determine whether the target supports FN. */
2923 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2925 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2926 tree op0
= (info
.type0
< 0
2927 ? gimple_call_lhs (call
)
2928 : gimple_call_arg (call
, info
.type0
));
2929 tree op1
= (info
.type1
< 0
2930 ? gimple_call_lhs (call
)
2931 : gimple_call_arg (call
, info
.type1
));
2932 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2935 /* Return true if OPTAB is supported for TYPES (whose modes should be
2936 the same) when the optimization type is OPT_TYPE. Used for simple
2940 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2941 optimization_type opt_type
)
2943 machine_mode mode
= TYPE_MODE (types
.first
);
2944 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2945 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2948 /* Return true if OPTAB is supported for TYPES, where the first type
2949 is the destination and the second type is the source. Used for
2953 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
2954 optimization_type opt_type
)
2956 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
2957 TYPE_MODE (types
.second
), opt_type
)
2958 != CODE_FOR_nothing
);
2961 /* Return true if load/store lanes optab OPTAB is supported for
2962 array type TYPES.first when the optimization type is OPT_TYPE. */
2965 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2966 optimization_type opt_type
)
2968 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2969 machine_mode imode
= TYPE_MODE (types
.first
);
2970 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2971 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2972 != CODE_FOR_nothing
);
2975 #define direct_unary_optab_supported_p direct_optab_supported_p
2976 #define direct_binary_optab_supported_p direct_optab_supported_p
2977 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
2978 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
2979 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2980 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2981 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
2982 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2983 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2984 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
2985 #define direct_while_optab_supported_p convert_optab_supported_p
2986 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
2987 #define direct_fold_left_optab_supported_p direct_optab_supported_p
2989 /* Return the optab used by internal function FN. */
2992 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
2996 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2997 case IFN_##CODE: break;
2998 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2999 case IFN_##CODE: return OPTAB##_optab;
3000 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3001 UNSIGNED_OPTAB, TYPE) \
3002 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3003 ? UNSIGNED_OPTAB ## _optab \
3004 : SIGNED_OPTAB ## _optab);
3005 #include "internal-fn.def"
3013 /* Return true if FN is supported for the types in TYPES when the
3014 optimization type is OPT_TYPE. The types are those associated with
3015 the "type0" and "type1" fields of FN's direct_internal_fn_info
3019 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3020 optimization_type opt_type
)
3024 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3025 case IFN_##CODE: break;
3026 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3028 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3030 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3031 UNSIGNED_OPTAB, TYPE) \
3034 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3035 ? UNSIGNED_OPTAB ## _optab \
3036 : SIGNED_OPTAB ## _optab); \
3037 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3040 #include "internal-fn.def"
3048 /* Return true if FN is supported for type TYPE when the optimization
3049 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3050 fields of FN's direct_internal_fn_info structure are the same. */
3053 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3054 optimization_type opt_type
)
3056 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3057 gcc_checking_assert (info
.type0
== info
.type1
);
3058 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3061 /* Return true if IFN_SET_EDOM is supported. */
3064 set_edom_supported_p (void)
3073 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3075 expand_##CODE (internal_fn fn, gcall *stmt) \
3077 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3079 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3080 UNSIGNED_OPTAB, TYPE) \
3082 expand_##CODE (internal_fn fn, gcall *stmt) \
3084 tree_pair types = direct_internal_fn_types (fn, stmt); \
3085 optab which_optab = direct_internal_fn_optab (fn, types); \
3086 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3088 #include "internal-fn.def"
3090 /* Routines to expand each internal function, indexed by function number.
3091 Each routine has the prototype:
3093 expand_<NAME> (gcall *stmt)
3095 where STMT is the statement that performs the call. */
3096 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3097 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3098 #include "internal-fn.def"
3102 /* Return a function that performs the conditional form of CODE, i.e.:
3104 LHS = RHS1 ? RHS2 CODE RHS3 : RHS2
3106 (operating elementwise if the operands are vectors). Return IFN_LAST
3107 if no such function exists. */
3110 get_conditional_internal_fn (tree_code code
)
3115 return IFN_COND_ADD
;
3117 return IFN_COND_SUB
;
3119 return IFN_COND_MIN
;
3121 return IFN_COND_MAX
;
3123 return IFN_COND_AND
;
3125 return IFN_COND_IOR
;
3127 return IFN_COND_XOR
;
3133 /* Expand STMT as though it were a call to internal function FN. */
3136 expand_internal_call (internal_fn fn
, gcall
*stmt
)
3138 internal_fn_expanders
[fn
] (fn
, stmt
);
3141 /* Expand STMT, which is a call to internal function FN. */
3144 expand_internal_call (gcall
*stmt
)
3146 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
3150 expand_PHI (internal_fn
, gcall
*)