2 Copyright (C) 2011-2018 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
67 /* Fnspec of each internal function, indexed by function number. */
68 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
73 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
74 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
75 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
76 #include "internal-fn.def"
77 internal_fn_fnspec_array
[IFN_LAST
] = 0;
80 /* Create static initializers for the information returned by
81 direct_internal_fn. */
82 #define not_direct { -2, -2, false }
83 #define mask_load_direct { -1, 2, false }
84 #define load_lanes_direct { -1, -1, false }
85 #define mask_load_lanes_direct { -1, -1, false }
86 #define mask_store_direct { 3, 2, false }
87 #define store_lanes_direct { 0, 0, false }
88 #define mask_store_lanes_direct { 0, 0, false }
89 #define unary_direct { 0, 0, true }
90 #define binary_direct { 0, 0, true }
91 #define cond_binary_direct { 1, 1, true }
92 #define while_direct { 0, 2, false }
94 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
95 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
96 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
97 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
98 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
99 #include "internal-fn.def"
103 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
104 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
106 static enum insn_code
107 get_multi_vector_move (tree array_type
, convert_optab optab
)
112 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
113 imode
= TYPE_MODE (array_type
);
114 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
116 return convert_optab_handler (optab
, imode
, vmode
);
119 /* Expand LOAD_LANES call STMT using optab OPTAB. */
122 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
124 struct expand_operand ops
[2];
128 lhs
= gimple_call_lhs (stmt
);
129 rhs
= gimple_call_arg (stmt
, 0);
130 type
= TREE_TYPE (lhs
);
132 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
133 mem
= expand_normal (rhs
);
135 gcc_assert (MEM_P (mem
));
136 PUT_MODE (mem
, TYPE_MODE (type
));
138 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
139 create_fixed_operand (&ops
[1], mem
);
140 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
143 /* Expand STORE_LANES call STMT using optab OPTAB. */
146 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
148 struct expand_operand ops
[2];
152 lhs
= gimple_call_lhs (stmt
);
153 rhs
= gimple_call_arg (stmt
, 0);
154 type
= TREE_TYPE (rhs
);
156 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
157 reg
= expand_normal (rhs
);
159 gcc_assert (MEM_P (target
));
160 PUT_MODE (target
, TYPE_MODE (type
));
162 create_fixed_operand (&ops
[0], target
);
163 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
164 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
168 expand_ANNOTATE (internal_fn
, gcall
*)
173 /* This should get expanded in omp_device_lower pass. */
176 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
181 /* This should get expanded in omp_device_lower pass. */
184 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
189 /* Allocate per-lane storage and begin non-uniform execution region. */
192 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
195 tree lhs
= gimple_call_lhs (stmt
);
197 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
199 target
= gen_reg_rtx (Pmode
);
200 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
201 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
202 struct expand_operand ops
[3];
203 create_output_operand (&ops
[0], target
, Pmode
);
204 create_input_operand (&ops
[1], size
, Pmode
);
205 create_input_operand (&ops
[2], align
, Pmode
);
206 gcc_assert (targetm
.have_omp_simt_enter ());
207 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
210 /* Deallocate per-lane storage and leave non-uniform execution region. */
213 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
215 gcc_checking_assert (!gimple_call_lhs (stmt
));
216 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
217 struct expand_operand ops
[1];
218 create_input_operand (&ops
[0], arg
, Pmode
);
219 gcc_assert (targetm
.have_omp_simt_exit ());
220 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
223 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
224 without SIMT execution this should be expanded in omp_device_lower pass. */
227 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
229 tree lhs
= gimple_call_lhs (stmt
);
233 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
234 gcc_assert (targetm
.have_omp_simt_lane ());
235 emit_insn (targetm
.gen_omp_simt_lane (target
));
238 /* This should get expanded in omp_device_lower pass. */
241 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
246 /* Lane index of the first SIMT lane that supplies a non-zero argument.
247 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
248 lane that executed the last iteration for handling OpenMP lastprivate. */
251 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
253 tree lhs
= gimple_call_lhs (stmt
);
257 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
258 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
259 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
260 struct expand_operand ops
[2];
261 create_output_operand (&ops
[0], target
, mode
);
262 create_input_operand (&ops
[1], cond
, mode
);
263 gcc_assert (targetm
.have_omp_simt_last_lane ());
264 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
267 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
270 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
272 tree lhs
= gimple_call_lhs (stmt
);
276 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
277 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
278 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
279 struct expand_operand ops
[2];
280 create_output_operand (&ops
[0], target
, mode
);
281 create_input_operand (&ops
[1], ctr
, mode
);
282 gcc_assert (targetm
.have_omp_simt_ordered ());
283 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
286 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
287 any lane supplies a non-zero argument. */
290 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
292 tree lhs
= gimple_call_lhs (stmt
);
296 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
297 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
298 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
299 struct expand_operand ops
[2];
300 create_output_operand (&ops
[0], target
, mode
);
301 create_input_operand (&ops
[1], cond
, mode
);
302 gcc_assert (targetm
.have_omp_simt_vote_any ());
303 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
306 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
307 is destination lane index XOR given offset. */
310 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
312 tree lhs
= gimple_call_lhs (stmt
);
316 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
317 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
318 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
319 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
320 struct expand_operand ops
[3];
321 create_output_operand (&ops
[0], target
, mode
);
322 create_input_operand (&ops
[1], src
, mode
);
323 create_input_operand (&ops
[2], idx
, SImode
);
324 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
325 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
328 /* Exchange between SIMT lanes according to given source lane index. */
331 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
333 tree lhs
= gimple_call_lhs (stmt
);
337 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
338 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
339 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
340 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
341 struct expand_operand ops
[3];
342 create_output_operand (&ops
[0], target
, mode
);
343 create_input_operand (&ops
[1], src
, mode
);
344 create_input_operand (&ops
[2], idx
, SImode
);
345 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
346 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
349 /* This should get expanded in adjust_simduid_builtins. */
352 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
357 /* This should get expanded in adjust_simduid_builtins. */
360 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
365 /* This should get expanded in adjust_simduid_builtins. */
368 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
373 /* This should get expanded in adjust_simduid_builtins. */
376 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
381 /* This should get expanded in adjust_simduid_builtins. */
384 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
389 /* This should get expanded in the sanopt pass. */
392 expand_UBSAN_NULL (internal_fn
, gcall
*)
397 /* This should get expanded in the sanopt pass. */
400 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
405 /* This should get expanded in the sanopt pass. */
408 expand_UBSAN_VPTR (internal_fn
, gcall
*)
413 /* This should get expanded in the sanopt pass. */
416 expand_UBSAN_PTR (internal_fn
, gcall
*)
421 /* This should get expanded in the sanopt pass. */
424 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
429 /* This should get expanded in the sanopt pass. */
432 expand_ASAN_CHECK (internal_fn
, gcall
*)
437 /* This should get expanded in the sanopt pass. */
440 expand_ASAN_MARK (internal_fn
, gcall
*)
445 /* This should get expanded in the sanopt pass. */
448 expand_ASAN_POISON (internal_fn
, gcall
*)
453 /* This should get expanded in the sanopt pass. */
456 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
461 /* This should get expanded in the tsan pass. */
464 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
469 /* This should get expanded in the lower pass. */
472 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
474 error_at (gimple_location (call
),
475 "invalid use of attribute %<fallthrough%>");
478 /* Return minimum precision needed to represent all values
479 of ARG in SIGNed integral type. */
482 get_min_precision (tree arg
, signop sign
)
484 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
486 signop orig_sign
= sign
;
487 if (TREE_CODE (arg
) == INTEGER_CST
)
490 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
492 widest_int w
= wi::to_widest (arg
);
493 w
= wi::ext (w
, prec
, sign
);
494 p
= wi::min_precision (w
, sign
);
497 p
= wi::min_precision (wi::to_wide (arg
), sign
);
498 return MIN (p
, prec
);
500 while (CONVERT_EXPR_P (arg
)
501 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
502 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
504 arg
= TREE_OPERAND (arg
, 0);
505 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
507 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
509 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
510 return prec
+ (orig_sign
!= sign
);
511 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
514 return prec
+ (orig_sign
!= sign
);
516 if (TREE_CODE (arg
) != SSA_NAME
)
517 return prec
+ (orig_sign
!= sign
);
518 wide_int arg_min
, arg_max
;
519 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
521 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
522 if (is_gimple_assign (g
)
523 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
525 tree t
= gimple_assign_rhs1 (g
);
526 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
527 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
530 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
532 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
534 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
535 return prec
+ (orig_sign
!= sign
);
536 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
539 return prec
+ (orig_sign
!= sign
);
543 return prec
+ (orig_sign
!= sign
);
545 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
547 int p1
= wi::min_precision (arg_min
, sign
);
548 int p2
= wi::min_precision (arg_max
, sign
);
550 prec
= MIN (prec
, p1
);
552 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
554 int p
= wi::min_precision (arg_max
, UNSIGNED
);
555 prec
= MIN (prec
, p
);
557 return prec
+ (orig_sign
!= sign
);
560 /* Helper for expand_*_overflow. Set the __imag__ part to true
561 (1 except for signed:1 type, in which case store -1). */
564 expand_arith_set_overflow (tree lhs
, rtx target
)
566 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
567 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
568 write_complex_part (target
, constm1_rtx
, true);
570 write_complex_part (target
, const1_rtx
, true);
573 /* Helper for expand_*_overflow. Store RES into the __real__ part
574 of TARGET. If RES has larger MODE than __real__ part of TARGET,
575 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
576 if LHS has smaller precision than its mode. */
579 expand_arith_overflow_result_store (tree lhs
, rtx target
,
580 scalar_int_mode mode
, rtx res
)
582 scalar_int_mode tgtmode
583 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
587 rtx_code_label
*done_label
= gen_label_rtx ();
588 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
589 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
590 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
591 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
592 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
593 profile_probability::very_likely ());
594 expand_arith_set_overflow (lhs
, target
);
595 emit_label (done_label
);
597 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
598 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
601 rtx_code_label
*done_label
= gen_label_rtx ();
602 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
607 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
609 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
610 true, OPTAB_LIB_WIDEN
);
614 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
616 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
619 do_compare_rtx_and_jump (res
, lres
,
620 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
621 profile_probability::very_likely ());
622 expand_arith_set_overflow (lhs
, target
);
623 emit_label (done_label
);
625 write_complex_part (target
, lres
, false);
628 /* Helper for expand_*_overflow. Store RES into TARGET. */
631 expand_ubsan_result_store (rtx target
, rtx res
)
633 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
634 /* If this is a scalar in a register that is stored in a wider mode
635 than the declared mode, compute the result into its declared mode
636 and then convert to the wider mode. Our value is the computed
638 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
640 emit_move_insn (target
, res
);
643 /* Add sub/add overflow checking to the statement STMT.
644 CODE says whether the operation is +, or -. */
647 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
648 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
649 bool uns1_p
, bool is_ubsan
, tree
*datap
)
651 rtx res
, target
= NULL_RTX
;
653 rtx_code_label
*done_label
= gen_label_rtx ();
654 rtx_code_label
*do_error
= gen_label_rtx ();
655 do_pending_stack_adjust ();
656 rtx op0
= expand_normal (arg0
);
657 rtx op1
= expand_normal (arg1
);
658 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
659 int prec
= GET_MODE_PRECISION (mode
);
660 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
664 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
668 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
670 write_complex_part (target
, const0_rtx
, true);
673 /* We assume both operands and result have the same precision
674 here (GET_MODE_BITSIZE (mode)), S stands for signed type
675 with that precision, U for unsigned type with that precision,
676 sgn for unsigned most significant bit in that precision.
677 s1 is signed first operand, u1 is unsigned first operand,
678 s2 is signed second operand, u2 is unsigned second operand,
679 sr is signed result, ur is unsigned result and the following
680 rules say how to compute result (which is always result of
681 the operands as if both were unsigned, cast to the right
682 signedness) and how to compute whether operation overflowed.
685 res = (S) ((U) s1 + (U) s2)
686 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
688 res = (S) ((U) s1 - (U) s2)
689 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
692 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
695 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
697 res = (S) ((U) s1 + u2)
698 ovf = ((U) res ^ sgn) < u2
703 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
705 res = (S) ((U) s1 - u2)
706 ovf = u2 > ((U) s1 ^ sgn)
709 ovf = s1 < 0 || u2 > (U) s1
712 ovf = u1 >= ((U) s2 ^ sgn)
717 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
719 res = (U) s1 + (U) s2
720 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
723 ovf = (U) res < u2 || res < 0
726 ovf = u1 >= u2 ? res < 0 : res >= 0
728 res = (U) s1 - (U) s2
729 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
731 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
733 /* PLUS_EXPR is commutative, if operand signedness differs,
734 canonicalize to the first operand being signed and second
735 unsigned to simplify following code. */
736 std::swap (op0
, op1
);
737 std::swap (arg0
, arg1
);
743 if (uns0_p
&& uns1_p
&& unsr_p
)
745 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
746 : usubv4_optab
, mode
);
747 if (icode
!= CODE_FOR_nothing
)
749 struct expand_operand ops
[4];
750 rtx_insn
*last
= get_last_insn ();
752 res
= gen_reg_rtx (mode
);
753 create_output_operand (&ops
[0], res
, mode
);
754 create_input_operand (&ops
[1], op0
, mode
);
755 create_input_operand (&ops
[2], op1
, mode
);
756 create_fixed_operand (&ops
[3], do_error
);
757 if (maybe_expand_insn (icode
, 4, ops
))
759 last
= get_last_insn ();
760 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
762 && any_condjump_p (last
)
763 && !find_reg_note (last
, REG_BR_PROB
, 0))
764 add_reg_br_prob_note (last
,
765 profile_probability::very_unlikely ());
766 emit_jump (done_label
);
770 delete_insns_since (last
);
773 /* Compute the operation. On RTL level, the addition is always
775 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
776 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
778 /* For PLUS_EXPR, the operation is commutative, so we can pick
779 operand to compare against. For prec <= BITS_PER_WORD, I think
780 preferring REG operand is better over CONST_INT, because
781 the CONST_INT might enlarge the instruction or CSE would need
782 to figure out we'd already loaded it into a register before.
783 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
784 as then the multi-word comparison can be perhaps simplified. */
785 if (code
== PLUS_EXPR
786 && (prec
<= BITS_PER_WORD
787 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
788 : CONST_SCALAR_INT_P (op1
)))
790 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
791 true, mode
, NULL_RTX
, NULL
, done_label
,
792 profile_probability::very_likely ());
797 if (!uns0_p
&& uns1_p
&& !unsr_p
)
799 /* Compute the operation. On RTL level, the addition is always
801 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
802 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
803 rtx tem
= expand_binop (mode
, add_optab
,
804 code
== PLUS_EXPR
? res
: op0
, sgn
,
805 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
806 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
807 done_label
, profile_probability::very_likely ());
812 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
814 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
816 /* As we've changed op1, we have to avoid using the value range
817 for the original argument. */
818 arg1
= error_mark_node
;
824 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
826 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
828 /* As we've changed op0, we have to avoid using the value range
829 for the original argument. */
830 arg0
= error_mark_node
;
836 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
838 /* Compute the operation. On RTL level, the addition is always
840 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
842 int pos_neg
= get_range_pos_neg (arg0
);
844 /* If ARG0 is known to be always negative, this is always overflow. */
845 emit_jump (do_error
);
846 else if (pos_neg
== 3)
847 /* If ARG0 is not known to be always positive, check at runtime. */
848 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
849 NULL
, do_error
, profile_probability::very_unlikely ());
850 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
851 done_label
, profile_probability::very_likely ());
856 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
858 /* Compute the operation. On RTL level, the addition is always
860 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
862 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
864 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
865 done_label
, profile_probability::very_likely ());
870 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
872 /* Compute the operation. On RTL level, the addition is always
874 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
876 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
877 NULL
, do_error
, profile_probability::very_unlikely ());
879 /* The operation is commutative, so we can pick operand to compare
880 against. For prec <= BITS_PER_WORD, I think preferring REG operand
881 is better over CONST_INT, because the CONST_INT might enlarge the
882 instruction or CSE would need to figure out we'd already loaded it
883 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
884 might be more beneficial, as then the multi-word comparison can be
885 perhaps simplified. */
886 if (prec
<= BITS_PER_WORD
887 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
888 : CONST_SCALAR_INT_P (op0
))
890 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
891 done_label
, profile_probability::very_likely ());
896 if (!uns0_p
&& !uns1_p
&& unsr_p
)
898 /* Compute the operation. On RTL level, the addition is always
900 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
901 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
902 int pos_neg
= get_range_pos_neg (arg1
);
903 if (code
== PLUS_EXPR
)
905 int pos_neg0
= get_range_pos_neg (arg0
);
906 if (pos_neg0
!= 3 && pos_neg
== 3)
908 std::swap (op0
, op1
);
915 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
916 ? and_optab
: ior_optab
,
917 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
918 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
919 NULL
, done_label
, profile_probability::very_likely ());
923 rtx_code_label
*do_ior_label
= gen_label_rtx ();
924 do_compare_rtx_and_jump (op1
, const0_rtx
,
925 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
926 NULL_RTX
, NULL
, do_ior_label
,
927 profile_probability::even ());
928 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
930 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
931 NULL
, done_label
, profile_probability::very_likely ());
932 emit_jump (do_error
);
933 emit_label (do_ior_label
);
934 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
936 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
937 NULL
, done_label
, profile_probability::very_likely ());
943 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
945 /* Compute the operation. On RTL level, the addition is always
947 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
949 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
950 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
951 op0_geu_op1
, profile_probability::even ());
952 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
953 NULL
, done_label
, profile_probability::very_likely ());
954 emit_jump (do_error
);
955 emit_label (op0_geu_op1
);
956 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
957 NULL
, done_label
, profile_probability::very_likely ());
961 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
966 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
967 : subv4_optab
, mode
);
968 if (icode
!= CODE_FOR_nothing
)
970 struct expand_operand ops
[4];
971 rtx_insn
*last
= get_last_insn ();
973 res
= gen_reg_rtx (mode
);
974 create_output_operand (&ops
[0], res
, mode
);
975 create_input_operand (&ops
[1], op0
, mode
);
976 create_input_operand (&ops
[2], op1
, mode
);
977 create_fixed_operand (&ops
[3], do_error
);
978 if (maybe_expand_insn (icode
, 4, ops
))
980 last
= get_last_insn ();
981 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
983 && any_condjump_p (last
)
984 && !find_reg_note (last
, REG_BR_PROB
, 0))
985 add_reg_br_prob_note (last
,
986 profile_probability::very_unlikely ());
987 emit_jump (done_label
);
991 delete_insns_since (last
);
994 /* Compute the operation. On RTL level, the addition is always
996 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
997 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
999 /* If we can prove that one of the arguments (for MINUS_EXPR only
1000 the second operand, as subtraction is not commutative) is always
1001 non-negative or always negative, we can do just one comparison
1002 and conditional jump. */
1003 int pos_neg
= get_range_pos_neg (arg1
);
1004 if (code
== PLUS_EXPR
)
1006 int pos_neg0
= get_range_pos_neg (arg0
);
1007 if (pos_neg0
!= 3 && pos_neg
== 3)
1009 std::swap (op0
, op1
);
1014 /* Addition overflows if and only if the two operands have the same sign,
1015 and the result has the opposite sign. Subtraction overflows if and
1016 only if the two operands have opposite sign, and the subtrahend has
1017 the same sign as the result. Here 0 is counted as positive. */
1020 /* Compute op0 ^ op1 (operands have opposite sign). */
1021 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1024 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1025 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1029 if (code
== PLUS_EXPR
)
1031 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1032 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1033 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1038 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1039 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1040 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1044 /* No overflow if the result has bit sign cleared. */
1045 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1046 NULL
, done_label
, profile_probability::very_likely ());
1049 /* Compare the result of the operation with the first operand.
1050 No overflow for addition if second operand is positive and result
1051 is larger or second operand is negative and result is smaller.
1052 Likewise for subtraction with sign of second operand flipped. */
1054 do_compare_rtx_and_jump (res
, op0
,
1055 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1056 false, mode
, NULL_RTX
, NULL
, done_label
,
1057 profile_probability::very_likely ());
1061 emit_label (do_error
);
1064 /* Expand the ubsan builtin call. */
1066 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1070 do_pending_stack_adjust ();
1073 expand_arith_set_overflow (lhs
, target
);
1076 emit_label (done_label
);
1081 expand_ubsan_result_store (target
, res
);
1085 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1088 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1093 /* Add negate overflow checking to the statement STMT. */
1096 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1101 rtx_code_label
*done_label
, *do_error
;
1102 rtx target
= NULL_RTX
;
1104 done_label
= gen_label_rtx ();
1105 do_error
= gen_label_rtx ();
1107 do_pending_stack_adjust ();
1108 op1
= expand_normal (arg1
);
1110 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1113 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1115 write_complex_part (target
, const0_rtx
, true);
1118 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1119 if (icode
!= CODE_FOR_nothing
)
1121 struct expand_operand ops
[3];
1122 rtx_insn
*last
= get_last_insn ();
1124 res
= gen_reg_rtx (mode
);
1125 create_output_operand (&ops
[0], res
, mode
);
1126 create_input_operand (&ops
[1], op1
, mode
);
1127 create_fixed_operand (&ops
[2], do_error
);
1128 if (maybe_expand_insn (icode
, 3, ops
))
1130 last
= get_last_insn ();
1131 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1133 && any_condjump_p (last
)
1134 && !find_reg_note (last
, REG_BR_PROB
, 0))
1135 add_reg_br_prob_note (last
,
1136 profile_probability::very_unlikely ());
1137 emit_jump (done_label
);
1141 delete_insns_since (last
);
1142 icode
= CODE_FOR_nothing
;
1146 if (icode
== CODE_FOR_nothing
)
1148 /* Compute the operation. On RTL level, the addition is always
1150 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1152 /* Compare the operand with the most negative value. */
1153 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1154 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1155 done_label
, profile_probability::very_likely ());
1158 emit_label (do_error
);
1161 /* Expand the ubsan builtin call. */
1163 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1164 arg1
, NULL_TREE
, datap
);
1167 do_pending_stack_adjust ();
1170 expand_arith_set_overflow (lhs
, target
);
1173 emit_label (done_label
);
1178 expand_ubsan_result_store (target
, res
);
1180 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1184 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1185 mode MODE can be expanded without using a libcall. */
1188 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1189 rtx op0
, rtx op1
, bool uns
)
1191 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1192 != CODE_FOR_nothing
)
1195 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1196 != CODE_FOR_nothing
)
1199 rtx_insn
*last
= get_last_insn ();
1200 if (CONSTANT_P (op0
))
1201 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1203 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1204 if (CONSTANT_P (op1
))
1205 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1207 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1208 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1209 delete_insns_since (last
);
1210 return ret
!= NULL_RTX
;
1213 /* Add mul overflow checking to the statement STMT. */
1216 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1217 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1222 rtx_code_label
*done_label
, *do_error
;
1223 rtx target
= NULL_RTX
;
1225 enum insn_code icode
;
1227 done_label
= gen_label_rtx ();
1228 do_error
= gen_label_rtx ();
1230 do_pending_stack_adjust ();
1231 op0
= expand_normal (arg0
);
1232 op1
= expand_normal (arg1
);
1234 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1238 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1240 write_complex_part (target
, const0_rtx
, true);
1244 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1246 /* We assume both operands and result have the same precision
1247 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1248 with that precision, U for unsigned type with that precision,
1249 sgn for unsigned most significant bit in that precision.
1250 s1 is signed first operand, u1 is unsigned first operand,
1251 s2 is signed second operand, u2 is unsigned second operand,
1252 sr is signed result, ur is unsigned result and the following
1253 rules say how to compute result (which is always result of
1254 the operands as if both were unsigned, cast to the right
1255 signedness) and how to compute whether operation overflowed.
1256 main_ovf (false) stands for jump on signed multiplication
1257 overflow or the main algorithm with uns == false.
1258 main_ovf (true) stands for jump on unsigned multiplication
1259 overflow or the main algorithm with uns == true.
1262 res = (S) ((U) s1 * (U) s2)
1263 ovf = main_ovf (false)
1266 ovf = main_ovf (true)
1269 ovf = (s1 < 0 && u2) || main_ovf (true)
1272 ovf = res < 0 || main_ovf (true)
1274 res = (S) ((U) s1 * u2)
1275 ovf = (S) u2 >= 0 ? main_ovf (false)
1276 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1278 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1279 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1281 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1283 if (uns0_p
&& !uns1_p
)
1285 /* Multiplication is commutative, if operand signedness differs,
1286 canonicalize to the first operand being signed and second
1287 unsigned to simplify following code. */
1288 std::swap (op0
, op1
);
1289 std::swap (arg0
, arg1
);
1294 int pos_neg0
= get_range_pos_neg (arg0
);
1295 int pos_neg1
= get_range_pos_neg (arg1
);
1298 if (!uns0_p
&& uns1_p
&& unsr_p
)
1303 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1306 /* If s1 is negative, avoid the main code, just multiply and
1307 signal overflow if op1 is not 0. */
1308 struct separate_ops ops
;
1309 ops
.code
= MULT_EXPR
;
1310 ops
.type
= TREE_TYPE (arg1
);
1311 ops
.op0
= make_tree (ops
.type
, op0
);
1312 ops
.op1
= make_tree (ops
.type
, op1
);
1313 ops
.op2
= NULL_TREE
;
1315 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1316 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1317 NULL
, done_label
, profile_probability::very_likely ());
1318 goto do_error_label
;
1320 rtx_code_label
*do_main_label
;
1321 do_main_label
= gen_label_rtx ();
1322 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1323 NULL
, do_main_label
, profile_probability::very_likely ());
1324 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1325 NULL
, do_main_label
, profile_probability::very_likely ());
1326 expand_arith_set_overflow (lhs
, target
);
1327 emit_label (do_main_label
);
1335 if (uns0_p
&& uns1_p
&& !unsr_p
)
1338 /* Rest of handling of this case after res is computed. */
1343 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1350 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1351 avoid the main code, just multiply and signal overflow
1352 unless 0 * u2 or -1 * ((U) Smin). */
1353 struct separate_ops ops
;
1354 ops
.code
= MULT_EXPR
;
1355 ops
.type
= TREE_TYPE (arg1
);
1356 ops
.op0
= make_tree (ops
.type
, op0
);
1357 ops
.op1
= make_tree (ops
.type
, op1
);
1358 ops
.op2
= NULL_TREE
;
1360 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1361 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1362 NULL
, done_label
, profile_probability::very_likely ());
1363 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1364 NULL
, do_error
, profile_probability::very_unlikely ());
1366 prec
= GET_MODE_PRECISION (mode
);
1368 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1369 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1370 NULL
, done_label
, profile_probability::very_likely ());
1371 goto do_error_label
;
1373 /* Rest of handling of this case after res is computed. */
1381 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1384 switch (pos_neg0
| pos_neg1
)
1386 case 1: /* Both operands known to be non-negative. */
1388 case 2: /* Both operands known to be negative. */
1389 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1390 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1391 /* Avoid looking at arg0/arg1 ranges, as we've changed
1393 arg0
= error_mark_node
;
1394 arg1
= error_mark_node
;
1397 if ((pos_neg0
^ pos_neg1
) == 3)
1399 /* If one operand is known to be negative and the other
1400 non-negative, this overflows always, unless the non-negative
1401 one is 0. Just do normal multiply and set overflow
1402 unless one of the operands is 0. */
1403 struct separate_ops ops
;
1404 ops
.code
= MULT_EXPR
;
1406 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1408 ops
.op0
= make_tree (ops
.type
, op0
);
1409 ops
.op1
= make_tree (ops
.type
, op1
);
1410 ops
.op2
= NULL_TREE
;
1412 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1413 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1415 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1416 NULL_RTX
, NULL
, done_label
,
1417 profile_probability::very_likely ());
1418 goto do_error_label
;
1420 /* The general case, do all the needed comparisons at runtime. */
1421 rtx_code_label
*do_main_label
, *after_negate_label
;
1423 rop0
= gen_reg_rtx (mode
);
1424 rop1
= gen_reg_rtx (mode
);
1425 emit_move_insn (rop0
, op0
);
1426 emit_move_insn (rop1
, op1
);
1429 do_main_label
= gen_label_rtx ();
1430 after_negate_label
= gen_label_rtx ();
1431 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1433 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1434 NULL
, after_negate_label
, profile_probability::very_likely ());
1435 /* Both arguments negative here, negate them and continue with
1436 normal unsigned overflow checking multiplication. */
1437 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1439 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1441 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1443 arg0
= error_mark_node
;
1444 arg1
= error_mark_node
;
1445 emit_jump (do_main_label
);
1446 emit_label (after_negate_label
);
1447 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1449 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1450 NULL
, do_main_label
, profile_probability::very_likely ());
1451 /* One argument is negative here, the other positive. This
1452 overflows always, unless one of the arguments is 0. But
1453 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1454 is, thus we can keep do_main code oring in overflow as is. */
1455 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1456 NULL
, do_main_label
, profile_probability::very_likely ());
1457 expand_arith_set_overflow (lhs
, target
);
1458 emit_label (do_main_label
);
1466 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1467 sign
= uns
? UNSIGNED
: SIGNED
;
1468 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1470 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1471 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1473 /* Optimize unsigned multiplication by power of 2 constant
1474 using 2 shifts, one for result, one to extract the shifted
1475 out bits to see if they are all zero.
1476 Don't do this if optimizing for size and we have umulv4_optab,
1477 in that case assume multiplication will be shorter.
1478 This is heuristics based on the single target that provides
1479 umulv4 right now (i?86/x86_64), if further targets add it, this
1480 might need to be revisited.
1481 Cases where both operands are constant should be folded already
1482 during GIMPLE, and cases where one operand is constant but not
1483 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1484 below can be done without multiplication, just by shifts and adds,
1485 or we'd need to divide the result (and hope it actually doesn't
1486 really divide nor multiply) and compare the result of the division
1487 with the original operand. */
1492 if (integer_pow2p (arg0
))
1494 std::swap (opn0
, opn1
);
1495 std::swap (argn0
, argn1
);
1497 int cnt
= tree_log2 (argn1
);
1498 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1500 rtx upper
= const0_rtx
;
1501 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1503 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1504 GET_MODE_PRECISION (mode
) - cnt
,
1506 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1507 NULL_RTX
, NULL
, done_label
,
1508 profile_probability::very_likely ());
1509 goto do_error_label
;
1512 if (icode
!= CODE_FOR_nothing
)
1514 struct expand_operand ops
[4];
1515 rtx_insn
*last
= get_last_insn ();
1517 res
= gen_reg_rtx (mode
);
1518 create_output_operand (&ops
[0], res
, mode
);
1519 create_input_operand (&ops
[1], op0
, mode
);
1520 create_input_operand (&ops
[2], op1
, mode
);
1521 create_fixed_operand (&ops
[3], do_error
);
1522 if (maybe_expand_insn (icode
, 4, ops
))
1524 last
= get_last_insn ();
1525 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1527 && any_condjump_p (last
)
1528 && !find_reg_note (last
, REG_BR_PROB
, 0))
1529 add_reg_br_prob_note (last
,
1530 profile_probability::very_unlikely ());
1531 emit_jump (done_label
);
1535 delete_insns_since (last
);
1536 icode
= CODE_FOR_nothing
;
1540 if (icode
== CODE_FOR_nothing
)
1542 struct separate_ops ops
;
1543 int prec
= GET_MODE_PRECISION (mode
);
1544 scalar_int_mode hmode
, wmode
;
1545 ops
.op0
= make_tree (type
, op0
);
1546 ops
.op1
= make_tree (type
, op1
);
1547 ops
.op2
= NULL_TREE
;
1550 /* Optimize unsigned overflow check where we don't use the
1551 multiplication result, just whether overflow happened.
1552 If we can do MULT_HIGHPART_EXPR, that followed by
1553 comparison of the result against zero is cheapest.
1554 We'll still compute res, but it should be DCEd later. */
1560 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1561 && can_mult_highpart_p (mode
, uns
) == 1
1562 && single_imm_use (lhs
, &use
, &use_stmt
)
1563 && is_gimple_assign (use_stmt
)
1564 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1567 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1568 && targetm
.scalar_mode_supported_p (wmode
)
1569 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1572 ops
.code
= WIDEN_MULT_EXPR
;
1574 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1576 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1577 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1579 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1580 res
= convert_modes (mode
, wmode
, res
, uns
);
1582 /* For the unsigned multiplication, there was overflow if
1583 HIPART is non-zero. */
1584 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1585 NULL_RTX
, NULL
, done_label
,
1586 profile_probability::very_likely ());
1589 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1591 /* RES is low half of the double width result, HIPART
1592 the high half. There was overflow if
1593 HIPART is different from RES < 0 ? -1 : 0. */
1594 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1595 NULL_RTX
, NULL
, done_label
,
1596 profile_probability::very_likely ());
1599 else if (can_mult_highpart_p (mode
, uns
) == 1)
1602 ops
.code
= MULT_HIGHPART_EXPR
;
1605 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1607 ops
.code
= MULT_EXPR
;
1608 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1610 /* For the unsigned multiplication, there was overflow if
1611 HIPART is non-zero. */
1612 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1613 NULL_RTX
, NULL
, done_label
,
1614 profile_probability::very_likely ());
1617 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1619 /* RES is low half of the double width result, HIPART
1620 the high half. There was overflow if
1621 HIPART is different from RES < 0 ? -1 : 0. */
1622 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1623 NULL_RTX
, NULL
, done_label
,
1624 profile_probability::very_likely ());
1628 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1629 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1631 rtx_code_label
*large_op0
= gen_label_rtx ();
1632 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1633 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1634 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1635 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1636 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1637 rtx_code_label
*do_overflow
= gen_label_rtx ();
1638 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1640 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1641 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1643 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1644 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1645 rtx signbit0
= const0_rtx
;
1647 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1649 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1651 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1652 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1653 rtx signbit1
= const0_rtx
;
1655 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1658 res
= gen_reg_rtx (mode
);
1660 /* True if op0 resp. op1 are known to be in the range of
1662 bool op0_small_p
= false;
1663 bool op1_small_p
= false;
1664 /* True if op0 resp. op1 are known to have all zeros or all ones
1665 in the upper half of bits, but are not known to be
1667 bool op0_medium_p
= false;
1668 bool op1_medium_p
= false;
1669 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1670 nonnegative, 1 if unknown. */
1676 else if (pos_neg0
== 2)
1680 else if (pos_neg1
== 2)
1683 unsigned int mprec0
= prec
;
1684 if (arg0
!= error_mark_node
)
1685 mprec0
= get_min_precision (arg0
, sign
);
1686 if (mprec0
<= hprec
)
1688 else if (!uns
&& mprec0
<= hprec
+ 1)
1689 op0_medium_p
= true;
1690 unsigned int mprec1
= prec
;
1691 if (arg1
!= error_mark_node
)
1692 mprec1
= get_min_precision (arg1
, sign
);
1693 if (mprec1
<= hprec
)
1695 else if (!uns
&& mprec1
<= hprec
+ 1)
1696 op1_medium_p
= true;
1698 int smaller_sign
= 1;
1699 int larger_sign
= 1;
1702 smaller_sign
= op0_sign
;
1703 larger_sign
= op1_sign
;
1705 else if (op1_small_p
)
1707 smaller_sign
= op1_sign
;
1708 larger_sign
= op0_sign
;
1710 else if (op0_sign
== op1_sign
)
1712 smaller_sign
= op0_sign
;
1713 larger_sign
= op0_sign
;
1717 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1718 NULL_RTX
, NULL
, large_op0
,
1719 profile_probability::unlikely ());
1722 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1723 NULL_RTX
, NULL
, small_op0_large_op1
,
1724 profile_probability::unlikely ());
1726 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1727 hmode to mode, the multiplication will never overflow. We can
1728 do just one hmode x hmode => mode widening multiplication. */
1729 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1730 if (GET_CODE (lopart0
) == SUBREG
)
1732 lopart0s
= shallow_copy_rtx (lopart0
);
1733 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1734 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1736 if (GET_CODE (lopart1
) == SUBREG
)
1738 lopart1s
= shallow_copy_rtx (lopart1
);
1739 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1740 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1742 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1743 ops
.op0
= make_tree (halfstype
, lopart0s
);
1744 ops
.op1
= make_tree (halfstype
, lopart1s
);
1745 ops
.code
= WIDEN_MULT_EXPR
;
1748 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1749 emit_move_insn (res
, thisres
);
1750 emit_jump (done_label
);
1752 emit_label (small_op0_large_op1
);
1754 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1755 but op1 is not, just swap the arguments and handle it as op1
1756 sign/zero extended, op0 not. */
1757 rtx larger
= gen_reg_rtx (mode
);
1758 rtx hipart
= gen_reg_rtx (hmode
);
1759 rtx lopart
= gen_reg_rtx (hmode
);
1760 emit_move_insn (larger
, op1
);
1761 emit_move_insn (hipart
, hipart1
);
1762 emit_move_insn (lopart
, lopart0
);
1763 emit_jump (one_small_one_large
);
1765 emit_label (large_op0
);
1768 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1769 NULL_RTX
, NULL
, both_ops_large
,
1770 profile_probability::unlikely ());
1772 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1773 but op0 is not, prepare larger, hipart and lopart pseudos and
1774 handle it together with small_op0_large_op1. */
1775 emit_move_insn (larger
, op0
);
1776 emit_move_insn (hipart
, hipart0
);
1777 emit_move_insn (lopart
, lopart1
);
1779 emit_label (one_small_one_large
);
1781 /* lopart is the low part of the operand that is sign extended
1782 to mode, larger is the other operand, hipart is the
1783 high part of larger and lopart0 and lopart1 are the low parts
1785 We perform lopart0 * lopart1 and lopart * hipart widening
1787 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1788 ops
.op0
= make_tree (halfutype
, lopart0
);
1789 ops
.op1
= make_tree (halfutype
, lopart1
);
1791 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1793 ops
.op0
= make_tree (halfutype
, lopart
);
1794 ops
.op1
= make_tree (halfutype
, hipart
);
1795 rtx loxhi
= gen_reg_rtx (mode
);
1796 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1797 emit_move_insn (loxhi
, tem
);
1801 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1802 if (larger_sign
== 0)
1803 emit_jump (after_hipart_neg
);
1804 else if (larger_sign
!= -1)
1805 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1806 NULL_RTX
, NULL
, after_hipart_neg
,
1807 profile_probability::even ());
1809 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1810 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1811 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1813 emit_move_insn (loxhi
, tem
);
1815 emit_label (after_hipart_neg
);
1817 /* if (lopart < 0) loxhi -= larger; */
1818 if (smaller_sign
== 0)
1819 emit_jump (after_lopart_neg
);
1820 else if (smaller_sign
!= -1)
1821 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1822 NULL_RTX
, NULL
, after_lopart_neg
,
1823 profile_probability::even ());
1825 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1827 emit_move_insn (loxhi
, tem
);
1829 emit_label (after_lopart_neg
);
1832 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1833 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1834 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1836 emit_move_insn (loxhi
, tem
);
1838 /* if (loxhi >> (bitsize / 2)
1839 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1840 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1841 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1843 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1844 rtx signbitloxhi
= const0_rtx
;
1846 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1847 convert_modes (hmode
, mode
,
1849 hprec
- 1, NULL_RTX
, 0);
1851 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1852 NULL_RTX
, NULL
, do_overflow
,
1853 profile_probability::very_unlikely ());
1855 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1856 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1858 tem
= convert_modes (mode
, hmode
,
1859 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1861 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1864 emit_move_insn (res
, tem
);
1865 emit_jump (done_label
);
1867 emit_label (both_ops_large
);
1869 /* If both operands are large (not sign (!uns) or zero (uns)
1870 extended from hmode), then perform the full multiplication
1871 which will be the result of the operation.
1872 The only cases which don't overflow are for signed multiplication
1873 some cases where both hipart0 and highpart1 are 0 or -1.
1874 For unsigned multiplication when high parts are both non-zero
1875 this overflows always. */
1876 ops
.code
= MULT_EXPR
;
1877 ops
.op0
= make_tree (type
, op0
);
1878 ops
.op1
= make_tree (type
, op1
);
1879 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1880 emit_move_insn (res
, tem
);
1886 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1887 NULL_RTX
, 1, OPTAB_WIDEN
);
1888 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1889 NULL_RTX
, NULL
, do_error
,
1890 profile_probability::very_unlikely ());
1895 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1896 NULL_RTX
, 1, OPTAB_WIDEN
);
1897 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1898 NULL_RTX
, NULL
, do_error
,
1899 profile_probability::very_unlikely ());
1902 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1903 the same, overflow happened if res is non-positive, if they
1904 are different, overflow happened if res is positive. */
1905 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1906 emit_jump (hipart_different
);
1907 else if (op0_sign
== 1 || op1_sign
== 1)
1908 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1909 NULL_RTX
, NULL
, hipart_different
,
1910 profile_probability::even ());
1912 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1913 NULL_RTX
, NULL
, do_error
,
1914 profile_probability::very_unlikely ());
1915 emit_jump (done_label
);
1917 emit_label (hipart_different
);
1919 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1920 NULL_RTX
, NULL
, do_error
,
1921 profile_probability::very_unlikely ());
1922 emit_jump (done_label
);
1925 emit_label (do_overflow
);
1927 /* Overflow, do full multiplication and fallthru into do_error. */
1928 ops
.op0
= make_tree (type
, op0
);
1929 ops
.op1
= make_tree (type
, op1
);
1930 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1931 emit_move_insn (res
, tem
);
1933 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1934 && targetm
.scalar_mode_supported_p (wmode
))
1935 /* Even emitting a libcall is better than not detecting overflow
1940 gcc_assert (!is_ubsan
);
1941 ops
.code
= MULT_EXPR
;
1943 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1944 emit_jump (done_label
);
1949 emit_label (do_error
);
1952 /* Expand the ubsan builtin call. */
1954 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1958 do_pending_stack_adjust ();
1961 expand_arith_set_overflow (lhs
, target
);
1964 emit_label (done_label
);
1967 if (uns0_p
&& uns1_p
&& !unsr_p
)
1969 rtx_code_label
*all_done_label
= gen_label_rtx ();
1970 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1971 NULL
, all_done_label
, profile_probability::very_likely ());
1972 expand_arith_set_overflow (lhs
, target
);
1973 emit_label (all_done_label
);
1977 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1979 rtx_code_label
*all_done_label
= gen_label_rtx ();
1980 rtx_code_label
*set_noovf
= gen_label_rtx ();
1981 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1982 NULL
, all_done_label
, profile_probability::very_likely ());
1983 expand_arith_set_overflow (lhs
, target
);
1984 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1985 NULL
, set_noovf
, profile_probability::very_likely ());
1986 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1987 NULL
, all_done_label
, profile_probability::very_unlikely ());
1988 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1989 all_done_label
, profile_probability::very_unlikely ());
1990 emit_label (set_noovf
);
1991 write_complex_part (target
, const0_rtx
, true);
1992 emit_label (all_done_label
);
1998 expand_ubsan_result_store (target
, res
);
2000 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2004 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2007 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2008 tree arg0
, tree arg1
)
2010 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2011 rtx_code_label
*loop_lab
= NULL
;
2012 rtx cntvar
= NULL_RTX
;
2013 tree cntv
= NULL_TREE
;
2014 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2015 tree sz
= TYPE_SIZE (eltype
);
2016 tree data
= NULL_TREE
;
2017 tree resv
= NULL_TREE
;
2018 rtx lhsr
= NULL_RTX
;
2019 rtx resvr
= NULL_RTX
;
2020 unsigned HOST_WIDE_INT const_cnt
= 0;
2021 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2026 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2027 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2028 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2029 optab_default
)) == unknown_optab
2030 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2031 == CODE_FOR_nothing
))
2034 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2037 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2038 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2044 do_pending_stack_adjust ();
2045 loop_lab
= gen_label_rtx ();
2046 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2047 cntv
= make_tree (sizetype
, cntvar
);
2048 emit_move_insn (cntvar
, const0_rtx
);
2049 emit_label (loop_lab
);
2051 if (TREE_CODE (arg0
) != VECTOR_CST
)
2053 rtx arg0r
= expand_normal (arg0
);
2054 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2056 if (TREE_CODE (arg1
) != VECTOR_CST
)
2058 rtx arg1r
= expand_normal (arg1
);
2059 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2061 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2063 tree op0
, op1
, res
= NULL_TREE
;
2066 tree atype
= build_array_type_nelts (eltype
, cnt
);
2067 op0
= uniform_vector_p (arg0
);
2068 if (op0
== NULL_TREE
)
2070 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2071 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2072 NULL_TREE
, NULL_TREE
);
2074 op1
= uniform_vector_p (arg1
);
2075 if (op1
== NULL_TREE
)
2077 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2078 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2079 NULL_TREE
, NULL_TREE
);
2083 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2084 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2085 NULL_TREE
, NULL_TREE
);
2090 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2091 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2092 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2094 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2100 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2101 false, false, false, true, &data
);
2104 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2105 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2107 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2108 false, false, false, true, &data
);
2111 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2120 struct separate_ops ops
;
2121 ops
.code
= PLUS_EXPR
;
2122 ops
.type
= TREE_TYPE (cntv
);
2124 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2125 ops
.op2
= NULL_TREE
;
2127 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2130 emit_move_insn (cntvar
, ret
);
2131 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2132 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2133 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2134 profile_probability::very_likely ());
2136 if (lhs
&& resv
== NULL_TREE
)
2138 struct separate_ops ops
;
2140 ops
.type
= TREE_TYPE (arg0
);
2143 ops
.op2
= NULL_TREE
;
2145 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2148 emit_move_insn (lhsr
, ret
);
2151 emit_move_insn (lhsr
, resvr
);
2154 /* Expand UBSAN_CHECK_ADD call STMT. */
2157 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2159 location_t loc
= gimple_location (stmt
);
2160 tree lhs
= gimple_call_lhs (stmt
);
2161 tree arg0
= gimple_call_arg (stmt
, 0);
2162 tree arg1
= gimple_call_arg (stmt
, 1);
2163 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2164 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2166 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2167 false, false, false, true, NULL
);
2170 /* Expand UBSAN_CHECK_SUB call STMT. */
2173 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2175 location_t loc
= gimple_location (stmt
);
2176 tree lhs
= gimple_call_lhs (stmt
);
2177 tree arg0
= gimple_call_arg (stmt
, 0);
2178 tree arg1
= gimple_call_arg (stmt
, 1);
2179 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2180 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2181 else if (integer_zerop (arg0
))
2182 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2184 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2185 false, false, false, true, NULL
);
2188 /* Expand UBSAN_CHECK_MUL call STMT. */
2191 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2193 location_t loc
= gimple_location (stmt
);
2194 tree lhs
= gimple_call_lhs (stmt
);
2195 tree arg0
= gimple_call_arg (stmt
, 0);
2196 tree arg1
= gimple_call_arg (stmt
, 1);
2197 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2198 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2200 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2204 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2207 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2209 tree lhs
= gimple_call_lhs (stmt
);
2210 if (lhs
== NULL_TREE
)
2212 tree arg0
= gimple_call_arg (stmt
, 0);
2213 tree arg1
= gimple_call_arg (stmt
, 1);
2214 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2215 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2216 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2217 int unsr_p
= TYPE_UNSIGNED (type
);
2218 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2219 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2220 int precres
= TYPE_PRECISION (type
);
2221 location_t loc
= gimple_location (stmt
);
2222 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2224 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2226 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2227 prec0
= MIN (prec0
, pr
);
2228 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2229 prec1
= MIN (prec1
, pr
);
2231 /* If uns0_p && uns1_p, precop is minimum needed precision
2232 of unsigned type to hold the exact result, otherwise
2233 precop is minimum needed precision of signed type to
2234 hold the exact result. */
2236 if (code
== MULT_EXPR
)
2237 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2240 if (uns0_p
== uns1_p
)
2241 precop
= MAX (prec0
, prec1
) + 1;
2243 precop
= MAX (prec0
+ 1, prec1
) + 1;
2245 precop
= MAX (prec0
, prec1
+ 1) + 1;
2247 int orig_precres
= precres
;
2251 if ((uns0_p
&& uns1_p
)
2252 ? ((precop
+ !unsr_p
) <= precres
2253 /* u1 - u2 -> ur can overflow, no matter what precision
2255 && (code
!= MINUS_EXPR
|| !unsr_p
))
2256 : (!unsr_p
&& precop
<= precres
))
2258 /* The infinity precision result will always fit into result. */
2259 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2260 write_complex_part (target
, const0_rtx
, true);
2261 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2262 struct separate_ops ops
;
2265 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2266 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2267 ops
.op2
= NULL_TREE
;
2269 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2270 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2274 /* For operations with low precision, if target doesn't have them, start
2275 with precres widening right away, otherwise do it only if the most
2276 simple cases can't be used. */
2277 const int min_precision
= targetm
.min_arithmetic_precision ();
2278 if (orig_precres
== precres
&& precres
< min_precision
)
2280 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2281 && prec1
<= precres
)
2282 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2283 && prec0
+ uns0_p
<= precres
2284 && prec1
+ uns1_p
<= precres
))
2286 arg0
= fold_convert_loc (loc
, type
, arg0
);
2287 arg1
= fold_convert_loc (loc
, type
, arg1
);
2291 if (integer_zerop (arg0
) && !unsr_p
)
2293 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2298 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2299 unsr_p
, unsr_p
, false, NULL
);
2302 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2303 unsr_p
, unsr_p
, false, NULL
);
2310 /* For sub-word operations, retry with a wider type first. */
2311 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2313 int p
= MAX (min_precision
, precop
);
2314 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2315 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2318 p
= TYPE_PRECISION (optype
);
2322 unsr_p
= TYPE_UNSIGNED (optype
);
2328 if (prec0
<= precres
&& prec1
<= precres
)
2333 types
[0] = build_nonstandard_integer_type (precres
, 0);
2339 types
[1] = build_nonstandard_integer_type (precres
, 1);
2341 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2342 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2343 if (code
!= MULT_EXPR
)
2344 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2345 uns0_p
, uns1_p
, false, NULL
);
2347 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2348 uns0_p
, uns1_p
, false, NULL
);
2352 /* Retry with a wider type. */
2353 if (orig_precres
== precres
)
2355 int p
= MAX (prec0
, prec1
);
2356 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2357 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2360 p
= TYPE_PRECISION (optype
);
2364 unsr_p
= TYPE_UNSIGNED (optype
);
2375 /* Expand ADD_OVERFLOW STMT. */
2378 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2380 expand_arith_overflow (PLUS_EXPR
, stmt
);
2383 /* Expand SUB_OVERFLOW STMT. */
2386 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2388 expand_arith_overflow (MINUS_EXPR
, stmt
);
2391 /* Expand MUL_OVERFLOW STMT. */
2394 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2396 expand_arith_overflow (MULT_EXPR
, stmt
);
2399 /* This should get folded in tree-vectorizer.c. */
2402 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2407 /* This should get folded in tree-vectorizer.c. */
2410 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2415 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2418 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2420 struct expand_operand ops
[3];
2421 tree type
, lhs
, rhs
, maskt
, ptr
;
2422 rtx mem
, target
, mask
;
2426 maskt
= gimple_call_arg (stmt
, 2);
2427 lhs
= gimple_call_lhs (stmt
);
2428 if (lhs
== NULL_TREE
)
2430 type
= TREE_TYPE (lhs
);
2431 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2432 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2433 if (TYPE_ALIGN (type
) != align
)
2434 type
= build_aligned_type (type
, align
);
2435 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2437 if (optab
== vec_mask_load_lanes_optab
)
2438 icode
= get_multi_vector_move (type
, optab
);
2440 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2441 TYPE_MODE (TREE_TYPE (maskt
)));
2443 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2444 gcc_assert (MEM_P (mem
));
2445 mask
= expand_normal (maskt
);
2446 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2447 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2448 create_fixed_operand (&ops
[1], mem
);
2449 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2450 expand_insn (icode
, 3, ops
);
2453 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2455 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2458 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2460 struct expand_operand ops
[3];
2461 tree type
, lhs
, rhs
, maskt
, ptr
;
2466 maskt
= gimple_call_arg (stmt
, 2);
2467 rhs
= gimple_call_arg (stmt
, 3);
2468 type
= TREE_TYPE (rhs
);
2469 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2470 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2471 if (TYPE_ALIGN (type
) != align
)
2472 type
= build_aligned_type (type
, align
);
2473 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2475 if (optab
== vec_mask_store_lanes_optab
)
2476 icode
= get_multi_vector_move (type
, optab
);
2478 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2479 TYPE_MODE (TREE_TYPE (maskt
)));
2481 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2482 gcc_assert (MEM_P (mem
));
2483 mask
= expand_normal (maskt
);
2484 reg
= expand_normal (rhs
);
2485 create_fixed_operand (&ops
[0], mem
);
2486 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2487 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2488 expand_insn (icode
, 3, ops
);
2491 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2494 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2499 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2501 /* When guessing was done, the hints should be already stripped away. */
2502 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2505 tree lhs
= gimple_call_lhs (stmt
);
2507 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2509 target
= const0_rtx
;
2510 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2511 if (lhs
&& val
!= target
)
2512 emit_move_insn (target
, val
);
2515 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2516 should never be called. */
2519 expand_VA_ARG (internal_fn
, gcall
*)
2524 /* Expand the IFN_UNIQUE function according to its first argument. */
2527 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2529 rtx pattern
= NULL_RTX
;
2530 enum ifn_unique_kind kind
2531 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2538 case IFN_UNIQUE_UNSPEC
:
2539 if (targetm
.have_unique ())
2540 pattern
= targetm
.gen_unique ();
2543 case IFN_UNIQUE_OACC_FORK
:
2544 case IFN_UNIQUE_OACC_JOIN
:
2545 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2547 tree lhs
= gimple_call_lhs (stmt
);
2548 rtx target
= const0_rtx
;
2551 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2553 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2554 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2556 if (kind
== IFN_UNIQUE_OACC_FORK
)
2557 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2559 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2567 emit_insn (pattern
);
2570 /* The size of an OpenACC compute dimension. */
2573 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2575 tree lhs
= gimple_call_lhs (stmt
);
2580 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2581 if (targetm
.have_oacc_dim_size ())
2583 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2584 VOIDmode
, EXPAND_NORMAL
);
2585 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2588 emit_move_insn (target
, GEN_INT (1));
2591 /* The position of an OpenACC execution engine along one compute axis. */
2594 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2596 tree lhs
= gimple_call_lhs (stmt
);
2601 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2602 if (targetm
.have_oacc_dim_pos ())
2604 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2605 VOIDmode
, EXPAND_NORMAL
);
2606 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2609 emit_move_insn (target
, const0_rtx
);
2612 /* This is expanded by oacc_device_lower pass. */
2615 expand_GOACC_LOOP (internal_fn
, gcall
*)
2620 /* This is expanded by oacc_device_lower pass. */
2623 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2628 /* This is expanded by oacc_device_lower pass. */
2631 expand_GOACC_TILE (internal_fn
, gcall
*)
2636 /* Set errno to EDOM. */
2639 expand_SET_EDOM (internal_fn
, gcall
*)
2642 #ifdef GEN_ERRNO_RTX
2643 rtx errno_rtx
= GEN_ERRNO_RTX
;
2645 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2647 emit_move_insn (errno_rtx
,
2648 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2654 /* Expand atomic bit test and set. */
2657 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2659 expand_ifn_atomic_bit_test_and (call
);
2662 /* Expand atomic bit test and complement. */
2665 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2667 expand_ifn_atomic_bit_test_and (call
);
2670 /* Expand atomic bit test and reset. */
2673 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2675 expand_ifn_atomic_bit_test_and (call
);
2678 /* Expand atomic bit test and set. */
2681 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2683 expand_ifn_atomic_compare_exchange (call
);
2686 /* Expand LAUNDER to assignment, lhs = arg0. */
2689 expand_LAUNDER (internal_fn
, gcall
*call
)
2691 tree lhs
= gimple_call_lhs (call
);
2696 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2699 /* Expand DIVMOD() using:
2700 a) optab handler for udivmod/sdivmod if it is available.
2701 b) If optab_handler doesn't exist, generate call to
2702 target-specific divmod libfunc. */
2705 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2707 tree lhs
= gimple_call_lhs (call_stmt
);
2708 tree arg0
= gimple_call_arg (call_stmt
, 0);
2709 tree arg1
= gimple_call_arg (call_stmt
, 1);
2711 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2712 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2713 machine_mode mode
= TYPE_MODE (type
);
2714 bool unsignedp
= TYPE_UNSIGNED (type
);
2715 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2717 rtx op0
= expand_normal (arg0
);
2718 rtx op1
= expand_normal (arg1
);
2719 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2721 rtx quotient
, remainder
, libfunc
;
2723 /* Check if optab_handler exists for divmod_optab for given mode. */
2724 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2726 quotient
= gen_reg_rtx (mode
);
2727 remainder
= gen_reg_rtx (mode
);
2728 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2731 /* Generate call to divmod libfunc if it exists. */
2732 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2733 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2734 "ient
, &remainder
);
2739 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2740 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2741 make_tree (TREE_TYPE (arg0
), quotient
),
2742 make_tree (TREE_TYPE (arg1
), remainder
)),
2743 target
, VOIDmode
, EXPAND_NORMAL
);
2749 expand_NOP (internal_fn
, gcall
*)
2751 /* Nothing. But it shouldn't really prevail. */
2754 /* Expand a call to FN using the operands in STMT. FN has a single
2755 output operand and NARGS input operands. */
2758 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2761 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2763 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2764 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2766 tree lhs
= gimple_call_lhs (stmt
);
2767 tree lhs_type
= TREE_TYPE (lhs
);
2768 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2770 /* Do not assign directly to a promoted subreg, since there is no
2771 guarantee that the instruction will leave the upper bits of the
2772 register in the state required by SUBREG_PROMOTED_SIGN. */
2774 if (GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
2777 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
2779 for (unsigned int i
= 0; i
< nargs
; ++i
)
2781 tree rhs
= gimple_call_arg (stmt
, i
);
2782 tree rhs_type
= TREE_TYPE (rhs
);
2783 rtx rhs_rtx
= expand_normal (rhs
);
2784 if (INTEGRAL_TYPE_P (rhs_type
))
2785 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2786 TYPE_MODE (rhs_type
),
2787 TYPE_UNSIGNED (rhs_type
));
2789 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2792 expand_insn (icode
, nargs
+ 1, ops
);
2793 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2795 /* If the return value has an integral type, convert the instruction
2796 result to that type. This is useful for things that return an
2797 int regardless of the size of the input. If the instruction result
2798 is smaller than required, assume that it is signed.
2800 If the return value has a nonintegral type, its mode must match
2801 the instruction result. */
2802 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2804 /* If this is a scalar in a register that is stored in a wider
2805 mode than the declared mode, compute the result into its
2806 declared mode and then convert to the wider mode. */
2807 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2808 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2809 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2810 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2812 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2813 emit_move_insn (lhs_rtx
, ops
[0].value
);
2816 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2817 convert_move (lhs_rtx
, ops
[0].value
, 0);
2822 /* Expand WHILE_ULT call STMT using optab OPTAB. */
2825 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2827 expand_operand ops
[3];
2830 tree lhs
= gimple_call_lhs (stmt
);
2831 tree lhs_type
= TREE_TYPE (lhs
);
2832 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2833 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
2835 for (unsigned int i
= 0; i
< 2; ++i
)
2837 tree rhs
= gimple_call_arg (stmt
, i
);
2838 rhs_type
[i
] = TREE_TYPE (rhs
);
2839 rtx rhs_rtx
= expand_normal (rhs
);
2840 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
2843 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
2844 TYPE_MODE (lhs_type
));
2846 expand_insn (icode
, 3, ops
);
2847 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2848 emit_move_insn (lhs_rtx
, ops
[0].value
);
2851 /* Expanders for optabs that can use expand_direct_optab_fn. */
2853 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2854 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2856 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2857 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2859 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
2860 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2862 /* RETURN_TYPE and ARGS are a return type and argument list that are
2863 in principle compatible with FN (which satisfies direct_internal_fn_p).
2864 Return the types that should be used to determine whether the
2865 target supports FN. */
2868 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2870 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2871 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2872 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2873 return tree_pair (type0
, type1
);
2876 /* CALL is a call whose return type and arguments are in principle
2877 compatible with FN (which satisfies direct_internal_fn_p). Return the
2878 types that should be used to determine whether the target supports FN. */
2881 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2883 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2884 tree op0
= (info
.type0
< 0
2885 ? gimple_call_lhs (call
)
2886 : gimple_call_arg (call
, info
.type0
));
2887 tree op1
= (info
.type1
< 0
2888 ? gimple_call_lhs (call
)
2889 : gimple_call_arg (call
, info
.type1
));
2890 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2893 /* Return true if OPTAB is supported for TYPES (whose modes should be
2894 the same) when the optimization type is OPT_TYPE. Used for simple
2898 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2899 optimization_type opt_type
)
2901 machine_mode mode
= TYPE_MODE (types
.first
);
2902 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2903 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2906 /* Return true if OPTAB is supported for TYPES, where the first type
2907 is the destination and the second type is the source. Used for
2911 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
2912 optimization_type opt_type
)
2914 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
2915 TYPE_MODE (types
.second
), opt_type
)
2916 != CODE_FOR_nothing
);
2919 /* Return true if load/store lanes optab OPTAB is supported for
2920 array type TYPES.first when the optimization type is OPT_TYPE. */
2923 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2924 optimization_type opt_type
)
2926 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2927 machine_mode imode
= TYPE_MODE (types
.first
);
2928 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2929 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2930 != CODE_FOR_nothing
);
2933 #define direct_unary_optab_supported_p direct_optab_supported_p
2934 #define direct_binary_optab_supported_p direct_optab_supported_p
2935 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
2936 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2937 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2938 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
2939 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2940 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2941 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
2942 #define direct_while_optab_supported_p convert_optab_supported_p
2944 /* Return the optab used by internal function FN. */
2947 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
2951 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2952 case IFN_##CODE: break;
2953 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2954 case IFN_##CODE: return OPTAB##_optab;
2955 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
2956 UNSIGNED_OPTAB, TYPE) \
2957 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
2958 ? UNSIGNED_OPTAB ## _optab \
2959 : SIGNED_OPTAB ## _optab);
2960 #include "internal-fn.def"
2968 /* Return true if FN is supported for the types in TYPES when the
2969 optimization type is OPT_TYPE. The types are those associated with
2970 the "type0" and "type1" fields of FN's direct_internal_fn_info
2974 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2975 optimization_type opt_type
)
2979 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2980 case IFN_##CODE: break;
2981 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2983 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2985 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
2986 UNSIGNED_OPTAB, TYPE) \
2989 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
2990 ? UNSIGNED_OPTAB ## _optab \
2991 : SIGNED_OPTAB ## _optab); \
2992 return direct_##TYPE##_optab_supported_p (which_optab, types, \
2995 #include "internal-fn.def"
3003 /* Return true if FN is supported for type TYPE when the optimization
3004 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3005 fields of FN's direct_internal_fn_info structure are the same. */
3008 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3009 optimization_type opt_type
)
3011 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3012 gcc_checking_assert (info
.type0
== info
.type1
);
3013 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3016 /* Return true if IFN_SET_EDOM is supported. */
3019 set_edom_supported_p (void)
3028 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3030 expand_##CODE (internal_fn fn, gcall *stmt) \
3032 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3034 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3035 UNSIGNED_OPTAB, TYPE) \
3037 expand_##CODE (internal_fn fn, gcall *stmt) \
3039 tree_pair types = direct_internal_fn_types (fn, stmt); \
3040 optab which_optab = direct_internal_fn_optab (fn, types); \
3041 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3043 #include "internal-fn.def"
3045 /* Routines to expand each internal function, indexed by function number.
3046 Each routine has the prototype:
3048 expand_<NAME> (gcall *stmt)
3050 where STMT is the statement that performs the call. */
3051 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3052 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3053 #include "internal-fn.def"
3057 /* Return a function that performs the conditional form of CODE, i.e.:
3059 LHS = RHS1 ? RHS2 CODE RHS3 : RHS2
3061 (operating elementwise if the operands are vectors). Return IFN_LAST
3062 if no such function exists. */
3065 get_conditional_internal_fn (tree_code code
)
3070 return IFN_COND_ADD
;
3072 return IFN_COND_SUB
;
3074 return IFN_COND_MIN
;
3076 return IFN_COND_MAX
;
3078 return IFN_COND_AND
;
3080 return IFN_COND_IOR
;
3082 return IFN_COND_XOR
;
3088 /* Expand STMT as though it were a call to internal function FN. */
3091 expand_internal_call (internal_fn fn
, gcall
*stmt
)
3093 internal_fn_expanders
[fn
] (fn
, stmt
);
3096 /* Expand STMT, which is a call to internal function FN. */
3099 expand_internal_call (gcall
*stmt
)
3101 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
3105 expand_PHI (internal_fn
, gcall
*)