2 Copyright (C) 2011-2014 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
24 #include "internal-fn.h"
25 #include "stor-layout.h"
27 #include "insn-codes.h"
35 #include "hard-reg-set.h"
38 #include "dominance.h"
40 #include "basic-block.h"
41 #include "tree-ssa-alias.h"
42 #include "internal-fn.h"
43 #include "gimple-expr.h"
48 #include "stringpool.h"
49 #include "tree-ssanames.h"
50 #include "diagnostic-core.h"
52 /* The names of each internal function, indexed by function number. */
53 const char *const internal_fn_name_array
[] = {
54 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
55 #include "internal-fn.def"
56 #undef DEF_INTERNAL_FN
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
64 #undef DEF_INTERNAL_FN
68 /* Fnspec of each internal function, indexed by function number. */
69 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
74 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
75 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
76 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
77 #include "internal-fn.def"
78 #undef DEF_INTERNAL_FN
79 internal_fn_fnspec_array
[IFN_LAST
] = 0;
82 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
83 for load-lanes-style optab OPTAB. The insn must exist. */
86 get_multi_vector_move (tree array_type
, convert_optab optab
)
92 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
93 imode
= TYPE_MODE (array_type
);
94 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
96 icode
= convert_optab_handler (optab
, imode
, vmode
);
97 gcc_assert (icode
!= CODE_FOR_nothing
);
101 /* Expand LOAD_LANES call STMT. */
104 expand_LOAD_LANES (gcall
*stmt
)
106 struct expand_operand ops
[2];
110 lhs
= gimple_call_lhs (stmt
);
111 rhs
= gimple_call_arg (stmt
, 0);
112 type
= TREE_TYPE (lhs
);
114 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
115 mem
= expand_normal (rhs
);
117 gcc_assert (MEM_P (mem
));
118 PUT_MODE (mem
, TYPE_MODE (type
));
120 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
121 create_fixed_operand (&ops
[1], mem
);
122 expand_insn (get_multi_vector_move (type
, vec_load_lanes_optab
), 2, ops
);
125 /* Expand STORE_LANES call STMT. */
128 expand_STORE_LANES (gcall
*stmt
)
130 struct expand_operand ops
[2];
134 lhs
= gimple_call_lhs (stmt
);
135 rhs
= gimple_call_arg (stmt
, 0);
136 type
= TREE_TYPE (rhs
);
138 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
139 reg
= expand_normal (rhs
);
141 gcc_assert (MEM_P (target
));
142 PUT_MODE (target
, TYPE_MODE (type
));
144 create_fixed_operand (&ops
[0], target
);
145 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
146 expand_insn (get_multi_vector_move (type
, vec_store_lanes_optab
), 2, ops
);
150 expand_ANNOTATE (gcall
*stmt ATTRIBUTE_UNUSED
)
155 /* This should get expanded in adjust_simduid_builtins. */
158 expand_GOMP_SIMD_LANE (gcall
*stmt ATTRIBUTE_UNUSED
)
163 /* This should get expanded in adjust_simduid_builtins. */
166 expand_GOMP_SIMD_VF (gcall
*stmt ATTRIBUTE_UNUSED
)
171 /* This should get expanded in adjust_simduid_builtins. */
174 expand_GOMP_SIMD_LAST_LANE (gcall
*stmt ATTRIBUTE_UNUSED
)
179 /* This should get expanded in the sanopt pass. */
182 expand_UBSAN_NULL (gcall
*stmt ATTRIBUTE_UNUSED
)
187 /* This should get expanded in the sanopt pass. */
190 expand_UBSAN_BOUNDS (gcall
*stmt ATTRIBUTE_UNUSED
)
195 /* This should get expanded in the sanopt pass. */
198 expand_UBSAN_OBJECT_SIZE (gcall
*stmt ATTRIBUTE_UNUSED
)
203 /* This should get expanded in the sanopt pass. */
206 expand_ASAN_CHECK (gcall
*stmt ATTRIBUTE_UNUSED
)
211 /* Helper function for expand_addsub_overflow. Return 1
212 if ARG interpreted as signed in its precision is known to be always
213 positive or 2 if ARG is known to be always negative, or 3 if ARG may
214 be positive or negative. */
217 get_range_pos_neg (tree arg
)
219 if (arg
== error_mark_node
)
222 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
224 if (TREE_CODE (arg
) == INTEGER_CST
)
226 wide_int w
= wi::sext (arg
, prec
);
232 while (CONVERT_EXPR_P (arg
)
233 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
234 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
236 arg
= TREE_OPERAND (arg
, 0);
237 /* Narrower value zero extended into wider type
238 will always result in positive values. */
239 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
240 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
242 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
247 if (TREE_CODE (arg
) != SSA_NAME
)
249 wide_int arg_min
, arg_max
;
250 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
252 gimple g
= SSA_NAME_DEF_STMT (arg
);
253 if (is_gimple_assign (g
)
254 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
256 tree t
= gimple_assign_rhs1 (g
);
257 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
258 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
260 if (TYPE_UNSIGNED (TREE_TYPE (t
))
261 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
263 prec
= TYPE_PRECISION (TREE_TYPE (t
));
272 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
274 /* For unsigned values, the "positive" range comes
275 below the "negative" range. */
276 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
278 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
283 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
285 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
291 /* Return minimum precision needed to represent all values
292 of ARG in SIGNed integral type. */
295 get_min_precision (tree arg
, signop sign
)
297 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
299 signop orig_sign
= sign
;
300 if (TREE_CODE (arg
) == INTEGER_CST
)
303 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
305 widest_int w
= wi::to_widest (arg
);
306 w
= wi::ext (w
, prec
, sign
);
307 p
= wi::min_precision (w
, sign
);
310 p
= wi::min_precision (arg
, sign
);
311 return MIN (p
, prec
);
313 while (CONVERT_EXPR_P (arg
)
314 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
315 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
317 arg
= TREE_OPERAND (arg
, 0);
318 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
320 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
322 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
323 return prec
+ (orig_sign
!= sign
);
324 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
327 return prec
+ (orig_sign
!= sign
);
329 if (TREE_CODE (arg
) != SSA_NAME
)
330 return prec
+ (orig_sign
!= sign
);
331 wide_int arg_min
, arg_max
;
332 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
334 gimple g
= SSA_NAME_DEF_STMT (arg
);
335 if (is_gimple_assign (g
)
336 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
338 tree t
= gimple_assign_rhs1 (g
);
339 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
340 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
343 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
345 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
347 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
348 return prec
+ (orig_sign
!= sign
);
349 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
352 return prec
+ (orig_sign
!= sign
);
356 return prec
+ (orig_sign
!= sign
);
358 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
360 int p1
= wi::min_precision (arg_min
, sign
);
361 int p2
= wi::min_precision (arg_max
, sign
);
363 prec
= MIN (prec
, p1
);
365 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
367 int p
= wi::min_precision (arg_max
, SIGNED
);
368 prec
= MIN (prec
, p
);
370 return prec
+ (orig_sign
!= sign
);
373 /* Helper for expand_*_overflow. Store RES into the __real__ part
374 of TARGET. If RES has larger MODE than __real__ part of TARGET,
375 set the __imag__ part to 1 if RES doesn't fit into it. */
378 expand_arith_overflow_result_store (tree lhs
, rtx target
,
379 machine_mode mode
, rtx res
)
381 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
385 rtx_code_label
*done_label
= gen_label_rtx ();
386 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
387 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
388 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
389 emit_cmp_and_jump_insns (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
390 EQ
, NULL_RTX
, mode
, false, done_label
,
392 write_complex_part (target
, const1_rtx
, true);
393 emit_label (done_label
);
395 write_complex_part (target
, lres
, false);
398 /* Helper for expand_*_overflow. Store RES into TARGET. */
401 expand_ubsan_result_store (rtx target
, rtx res
)
403 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
404 /* If this is a scalar in a register that is stored in a wider mode
405 than the declared mode, compute the result into its declared mode
406 and then convert to the wider mode. Our value is the computed
408 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
410 emit_move_insn (target
, res
);
413 /* Add sub/add overflow checking to the statement STMT.
414 CODE says whether the operation is +, or -. */
417 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
418 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
419 bool uns1_p
, bool is_ubsan
)
421 rtx res
, target
= NULL_RTX
;
423 rtx_code_label
*done_label
= gen_label_rtx ();
424 rtx_code_label
*do_error
= gen_label_rtx ();
425 do_pending_stack_adjust ();
426 rtx op0
= expand_normal (arg0
);
427 rtx op1
= expand_normal (arg1
);
428 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
429 int prec
= GET_MODE_PRECISION (mode
);
430 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
434 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
438 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
440 write_complex_part (target
, const0_rtx
, true);
443 /* We assume both operands and result have the same precision
444 here (GET_MODE_BITSIZE (mode)), S stands for signed type
445 with that precision, U for unsigned type with that precision,
446 sgn for unsigned most significant bit in that precision.
447 s1 is signed first operand, u1 is unsigned first operand,
448 s2 is signed second operand, u2 is unsigned second operand,
449 sr is signed result, ur is unsigned result and the following
450 rules say how to compute result (which is always result of
451 the operands as if both were unsigned, cast to the right
452 signedness) and how to compute whether operation overflowed.
455 res = (S) ((U) s1 + (U) s2)
456 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
458 res = (S) ((U) s1 - (U) s2)
459 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
462 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
465 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
467 res = (S) ((U) s1 + u2)
468 ovf = ((U) res ^ sgn) < u2
473 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
475 res = (S) ((U) s1 - u2)
476 ovf = u2 > ((U) s1 ^ sgn)
479 ovf = s1 < 0 || u2 > (U) s1
482 ovf = u1 >= ((U) s2 ^ sgn)
487 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
489 res = (U) s1 + (U) s2
490 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
493 ovf = (U) res < u2 || res < 0
496 ovf = u1 >= u2 ? res < 0 : res >= 0
498 res = (U) s1 - (U) s2
499 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
501 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
503 /* PLUS_EXPR is commutative, if operand signedness differs,
504 canonicalize to the first operand being signed and second
505 unsigned to simplify following code. */
517 if (uns0_p
&& uns1_p
&& unsr_p
)
519 /* Compute the operation. On RTL level, the addition is always
521 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
522 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
524 /* For PLUS_EXPR, the operation is commutative, so we can pick
525 operand to compare against. For prec <= BITS_PER_WORD, I think
526 preferring REG operand is better over CONST_INT, because
527 the CONST_INT might enlarge the instruction or CSE would need
528 to figure out we'd already loaded it into a register before.
529 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
530 as then the multi-word comparison can be perhaps simplified. */
531 if (code
== PLUS_EXPR
532 && (prec
<= BITS_PER_WORD
533 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
534 : CONST_SCALAR_INT_P (op1
)))
536 emit_cmp_and_jump_insns (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
537 NULL_RTX
, mode
, false, done_label
,
543 if (!uns0_p
&& uns1_p
&& !unsr_p
)
545 /* Compute the operation. On RTL level, the addition is always
547 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
548 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
549 rtx tem
= expand_binop (mode
, add_optab
,
550 code
== PLUS_EXPR
? res
: op0
, sgn
,
551 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
552 emit_cmp_and_jump_insns (tem
, op1
, GEU
, NULL_RTX
, mode
, false,
553 done_label
, PROB_VERY_LIKELY
);
558 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
560 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
562 /* As we've changed op1, we have to avoid using the value range
563 for the original argument. */
564 arg1
= error_mark_node
;
570 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
572 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
574 /* As we've changed op0, we have to avoid using the value range
575 for the original argument. */
576 arg0
= error_mark_node
;
582 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
584 /* Compute the operation. On RTL level, the addition is always
586 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
588 int pos_neg
= get_range_pos_neg (arg0
);
590 /* If ARG0 is known to be always negative, this is always overflow. */
591 emit_jump (do_error
);
592 else if (pos_neg
== 3)
593 /* If ARG0 is not known to be always positive, check at runtime. */
594 emit_cmp_and_jump_insns (op0
, const0_rtx
, LT
, NULL_RTX
, mode
, false,
595 do_error
, PROB_VERY_UNLIKELY
);
596 emit_cmp_and_jump_insns (op1
, op0
, LEU
, NULL_RTX
, mode
, false,
597 done_label
, PROB_VERY_LIKELY
);
602 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
604 /* Compute the operation. On RTL level, the addition is always
606 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
608 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
610 emit_cmp_and_jump_insns (op0
, tem
, LTU
, NULL_RTX
, mode
, false,
611 done_label
, PROB_VERY_LIKELY
);
616 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
618 /* Compute the operation. On RTL level, the addition is always
620 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
622 emit_cmp_and_jump_insns (res
, const0_rtx
, LT
, NULL_RTX
, mode
, false,
623 do_error
, PROB_VERY_UNLIKELY
);
625 /* The operation is commutative, so we can pick operand to compare
626 against. For prec <= BITS_PER_WORD, I think preferring REG operand
627 is better over CONST_INT, because the CONST_INT might enlarge the
628 instruction or CSE would need to figure out we'd already loaded it
629 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
630 might be more beneficial, as then the multi-word comparison can be
631 perhaps simplified. */
632 if (prec
<= BITS_PER_WORD
633 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
634 : CONST_SCALAR_INT_P (op0
))
636 emit_cmp_and_jump_insns (res
, tem
, GEU
, NULL_RTX
, mode
, false,
637 done_label
, PROB_VERY_LIKELY
);
642 if (!uns0_p
&& !uns1_p
&& unsr_p
)
644 /* Compute the operation. On RTL level, the addition is always
646 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
647 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
648 int pos_neg
= get_range_pos_neg (arg1
);
649 if (code
== PLUS_EXPR
)
651 int pos_neg0
= get_range_pos_neg (arg0
);
652 if (pos_neg0
!= 3 && pos_neg
== 3)
663 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
664 ? and_optab
: ior_optab
,
665 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
666 emit_cmp_and_jump_insns (tem
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
667 done_label
, PROB_VERY_LIKELY
);
671 rtx_code_label
*do_ior_label
= gen_label_rtx ();
672 emit_cmp_and_jump_insns (op1
, const0_rtx
,
673 code
== MINUS_EXPR
? GE
: LT
, NULL_RTX
,
674 mode
, false, do_ior_label
, PROB_EVEN
);
675 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
677 emit_cmp_and_jump_insns (tem
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
678 done_label
, PROB_VERY_LIKELY
);
679 emit_jump (do_error
);
680 emit_label (do_ior_label
);
681 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
683 emit_cmp_and_jump_insns (tem
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
684 done_label
, PROB_VERY_LIKELY
);
690 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
692 /* Compute the operation. On RTL level, the addition is always
694 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
696 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
697 emit_cmp_and_jump_insns (op0
, op1
, GEU
, NULL_RTX
, mode
, false,
698 op0_geu_op1
, PROB_EVEN
);
699 emit_cmp_and_jump_insns (res
, const0_rtx
, LT
, NULL_RTX
, mode
, false,
700 done_label
, PROB_VERY_LIKELY
);
701 emit_jump (do_error
);
702 emit_label (op0_geu_op1
);
703 emit_cmp_and_jump_insns (res
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
704 done_label
, PROB_VERY_LIKELY
);
708 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
712 enum insn_code icode
;
713 icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
: subv4_optab
, mode
);
714 if (icode
!= CODE_FOR_nothing
)
716 struct expand_operand ops
[4];
717 rtx_insn
*last
= get_last_insn ();
719 res
= gen_reg_rtx (mode
);
720 create_output_operand (&ops
[0], res
, mode
);
721 create_input_operand (&ops
[1], op0
, mode
);
722 create_input_operand (&ops
[2], op1
, mode
);
723 create_fixed_operand (&ops
[3], do_error
);
724 if (maybe_expand_insn (icode
, 4, ops
))
726 last
= get_last_insn ();
727 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
729 && any_condjump_p (last
)
730 && !find_reg_note (last
, REG_BR_PROB
, 0))
731 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
732 emit_jump (done_label
);
736 delete_insns_since (last
);
737 icode
= CODE_FOR_nothing
;
741 if (icode
== CODE_FOR_nothing
)
743 rtx_code_label
*sub_check
= gen_label_rtx ();
746 /* Compute the operation. On RTL level, the addition is always
748 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
749 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
751 /* If we can prove one of the arguments (for MINUS_EXPR only
752 the second operand, as subtraction is not commutative) is always
753 non-negative or always negative, we can do just one comparison
754 and conditional jump instead of 2 at runtime, 3 present in the
755 emitted code. If one of the arguments is CONST_INT, all we
756 need is to make sure it is op1, then the first
757 emit_cmp_and_jump_insns will be just folded. Otherwise try
758 to use range info if available. */
759 if (code
== PLUS_EXPR
&& CONST_INT_P (op0
))
765 else if (CONST_INT_P (op1
))
767 else if (code
== PLUS_EXPR
&& TREE_CODE (arg0
) == SSA_NAME
)
769 pos_neg
= get_range_pos_neg (arg0
);
777 if (pos_neg
== 3 && !CONST_INT_P (op1
) && TREE_CODE (arg1
) == SSA_NAME
)
778 pos_neg
= get_range_pos_neg (arg1
);
780 /* If the op1 is negative, we have to use a different check. */
782 emit_cmp_and_jump_insns (op1
, const0_rtx
, LT
, NULL_RTX
, mode
,
783 false, sub_check
, PROB_EVEN
);
785 /* Compare the result of the operation with one of the operands. */
787 emit_cmp_and_jump_insns (res
, op0
, code
== PLUS_EXPR
? GE
: LE
,
788 NULL_RTX
, mode
, false, done_label
,
791 /* If we get here, we have to print the error. */
794 emit_jump (do_error
);
796 emit_label (sub_check
);
799 /* We have k = a + b for b < 0 here. k <= a must hold. */
801 emit_cmp_and_jump_insns (res
, op0
, code
== PLUS_EXPR
? LE
: GE
,
802 NULL_RTX
, mode
, false, done_label
,
807 emit_label (do_error
);
810 /* Expand the ubsan builtin call. */
812 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
816 do_pending_stack_adjust ();
819 write_complex_part (target
, const1_rtx
, true);
822 emit_label (done_label
);
827 expand_ubsan_result_store (target
, res
);
831 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
834 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
839 /* Add negate overflow checking to the statement STMT. */
842 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
)
846 rtx_code_label
*done_label
, *do_error
;
847 rtx target
= NULL_RTX
;
849 done_label
= gen_label_rtx ();
850 do_error
= gen_label_rtx ();
852 do_pending_stack_adjust ();
853 op1
= expand_normal (arg1
);
855 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
858 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
860 write_complex_part (target
, const0_rtx
, true);
863 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
864 if (icode
!= CODE_FOR_nothing
)
866 struct expand_operand ops
[3];
867 rtx_insn
*last
= get_last_insn ();
869 res
= gen_reg_rtx (mode
);
870 create_output_operand (&ops
[0], res
, mode
);
871 create_input_operand (&ops
[1], op1
, mode
);
872 create_fixed_operand (&ops
[2], do_error
);
873 if (maybe_expand_insn (icode
, 3, ops
))
875 last
= get_last_insn ();
876 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
878 && any_condjump_p (last
)
879 && !find_reg_note (last
, REG_BR_PROB
, 0))
880 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
881 emit_jump (done_label
);
885 delete_insns_since (last
);
886 icode
= CODE_FOR_nothing
;
890 if (icode
== CODE_FOR_nothing
)
892 /* Compute the operation. On RTL level, the addition is always
894 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
896 /* Compare the operand with the most negative value. */
897 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
898 emit_cmp_and_jump_insns (op1
, minv
, NE
, NULL_RTX
, mode
, false,
899 done_label
, PROB_VERY_LIKELY
);
902 emit_label (do_error
);
905 /* Expand the ubsan builtin call. */
907 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
911 do_pending_stack_adjust ();
914 write_complex_part (target
, const1_rtx
, true);
917 emit_label (done_label
);
922 expand_ubsan_result_store (target
, res
);
924 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
928 /* Add mul overflow checking to the statement STMT. */
931 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
932 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
)
936 rtx_code_label
*done_label
, *do_error
;
937 rtx target
= NULL_RTX
;
939 enum insn_code icode
;
941 done_label
= gen_label_rtx ();
942 do_error
= gen_label_rtx ();
944 do_pending_stack_adjust ();
945 op0
= expand_normal (arg0
);
946 op1
= expand_normal (arg1
);
948 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
952 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
954 write_complex_part (target
, const0_rtx
, true);
958 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
960 /* We assume both operands and result have the same precision
961 here (GET_MODE_BITSIZE (mode)), S stands for signed type
962 with that precision, U for unsigned type with that precision,
963 sgn for unsigned most significant bit in that precision.
964 s1 is signed first operand, u1 is unsigned first operand,
965 s2 is signed second operand, u2 is unsigned second operand,
966 sr is signed result, ur is unsigned result and the following
967 rules say how to compute result (which is always result of
968 the operands as if both were unsigned, cast to the right
969 signedness) and how to compute whether operation overflowed.
970 main_ovf (false) stands for jump on signed multiplication
971 overflow or the main algorithm with uns == false.
972 main_ovf (true) stands for jump on unsigned multiplication
973 overflow or the main algorithm with uns == true.
976 res = (S) ((U) s1 * (U) s2)
977 ovf = main_ovf (false)
980 ovf = main_ovf (true)
983 ovf = (s1 < 0 && u2) || main_ovf (true)
986 ovf = res < 0 || main_ovf (true)
988 res = (S) ((U) s1 * u2)
989 ovf = (S) u2 >= 0 ? main_ovf (false)
990 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
992 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
993 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
995 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
997 if (uns0_p
&& !uns1_p
)
999 /* Multiplication is commutative, if operand signedness differs,
1000 canonicalize to the first operand being signed and second
1001 unsigned to simplify following code. */
1012 int pos_neg0
= get_range_pos_neg (arg0
);
1013 int pos_neg1
= get_range_pos_neg (arg1
);
1016 if (!uns0_p
&& uns1_p
&& unsr_p
)
1021 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1024 /* If s1 is negative, avoid the main code, just multiply and
1025 signal overflow if op1 is not 0. */
1026 struct separate_ops ops
;
1027 ops
.code
= MULT_EXPR
;
1028 ops
.type
= TREE_TYPE (arg1
);
1029 ops
.op0
= make_tree (ops
.type
, op0
);
1030 ops
.op1
= make_tree (ops
.type
, op1
);
1031 ops
.op2
= NULL_TREE
;
1033 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1034 emit_cmp_and_jump_insns (op1
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1035 false, done_label
, PROB_VERY_LIKELY
);
1036 goto do_error_label
;
1038 rtx_code_label
*do_main_label
;
1039 do_main_label
= gen_label_rtx ();
1040 emit_cmp_and_jump_insns (op0
, const0_rtx
, GE
, NULL_RTX
, mode
,
1041 false, do_main_label
, PROB_VERY_LIKELY
);
1042 emit_cmp_and_jump_insns (op1
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1043 false, do_main_label
, PROB_VERY_LIKELY
);
1044 write_complex_part (target
, const1_rtx
, true);
1045 emit_label (do_main_label
);
1053 if (uns0_p
&& uns1_p
&& !unsr_p
)
1056 /* Rest of handling of this case after res is computed. */
1061 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1068 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1069 avoid the main code, just multiply and signal overflow
1070 unless 0 * u2 or -1 * ((U) Smin). */
1071 struct separate_ops ops
;
1072 ops
.code
= MULT_EXPR
;
1073 ops
.type
= TREE_TYPE (arg1
);
1074 ops
.op0
= make_tree (ops
.type
, op0
);
1075 ops
.op1
= make_tree (ops
.type
, op1
);
1076 ops
.op2
= NULL_TREE
;
1078 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1079 emit_cmp_and_jump_insns (op0
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1080 false, done_label
, PROB_VERY_LIKELY
);
1081 emit_cmp_and_jump_insns (op0
, constm1_rtx
, NE
, NULL_RTX
, mode
,
1082 false, do_error
, PROB_VERY_UNLIKELY
);
1084 prec
= GET_MODE_PRECISION (mode
);
1086 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1087 emit_cmp_and_jump_insns (op1
, sgn
, EQ
, NULL_RTX
, mode
,
1088 false, done_label
, PROB_VERY_LIKELY
);
1089 goto do_error_label
;
1091 /* Rest of handling of this case after res is computed. */
1099 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1102 switch (pos_neg0
| pos_neg1
)
1104 case 1: /* Both operands known to be non-negative. */
1106 case 2: /* Both operands known to be negative. */
1107 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1108 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1109 /* Avoid looking at arg0/arg1 ranges, as we've changed
1111 arg0
= error_mark_node
;
1112 arg1
= error_mark_node
;
1115 if ((pos_neg0
^ pos_neg1
) == 3)
1117 /* If one operand is known to be negative and the other
1118 non-negative, this overflows always, unless the non-negative
1119 one is 0. Just do normal multiply and set overflow
1120 unless one of the operands is 0. */
1121 struct separate_ops ops
;
1122 ops
.code
= MULT_EXPR
;
1124 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1126 ops
.op0
= make_tree (ops
.type
, op0
);
1127 ops
.op1
= make_tree (ops
.type
, op1
);
1128 ops
.op2
= NULL_TREE
;
1130 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1131 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1133 emit_cmp_and_jump_insns (tem
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1134 false, done_label
, PROB_VERY_LIKELY
);
1135 goto do_error_label
;
1137 /* The general case, do all the needed comparisons at runtime. */
1138 rtx_code_label
*do_main_label
, *after_negate_label
;
1140 rop0
= gen_reg_rtx (mode
);
1141 rop1
= gen_reg_rtx (mode
);
1142 emit_move_insn (rop0
, op0
);
1143 emit_move_insn (rop1
, op1
);
1146 do_main_label
= gen_label_rtx ();
1147 after_negate_label
= gen_label_rtx ();
1148 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1150 emit_cmp_and_jump_insns (tem
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
1151 after_negate_label
, PROB_VERY_LIKELY
);
1152 /* Both arguments negative here, negate them and continue with
1153 normal unsigned overflow checking multiplication. */
1154 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1156 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1158 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1160 arg0
= error_mark_node
;
1161 arg1
= error_mark_node
;
1162 emit_jump (do_main_label
);
1163 emit_label (after_negate_label
);
1164 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1166 emit_cmp_and_jump_insns (tem2
, const0_rtx
, GE
, NULL_RTX
, mode
, false,
1167 do_main_label
, PROB_VERY_LIKELY
);
1168 /* One argument is negative here, the other positive. This
1169 overflows always, unless one of the arguments is 0. But
1170 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1171 is, thus we can keep do_main code oring in overflow as is. */
1172 emit_cmp_and_jump_insns (tem
, const0_rtx
, EQ
, NULL_RTX
, mode
, false,
1173 do_main_label
, PROB_VERY_LIKELY
);
1174 write_complex_part (target
, const1_rtx
, true);
1175 emit_label (do_main_label
);
1183 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1184 sign
= uns
? UNSIGNED
: SIGNED
;
1185 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1186 if (icode
!= CODE_FOR_nothing
)
1188 struct expand_operand ops
[4];
1189 rtx_insn
*last
= get_last_insn ();
1191 res
= gen_reg_rtx (mode
);
1192 create_output_operand (&ops
[0], res
, mode
);
1193 create_input_operand (&ops
[1], op0
, mode
);
1194 create_input_operand (&ops
[2], op1
, mode
);
1195 create_fixed_operand (&ops
[3], do_error
);
1196 if (maybe_expand_insn (icode
, 4, ops
))
1198 last
= get_last_insn ();
1199 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1201 && any_condjump_p (last
)
1202 && !find_reg_note (last
, REG_BR_PROB
, 0))
1203 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1204 emit_jump (done_label
);
1208 delete_insns_since (last
);
1209 icode
= CODE_FOR_nothing
;
1213 if (icode
== CODE_FOR_nothing
)
1215 struct separate_ops ops
;
1216 int prec
= GET_MODE_PRECISION (mode
);
1217 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1218 ops
.op0
= make_tree (type
, op0
);
1219 ops
.op1
= make_tree (type
, op1
);
1220 ops
.op2
= NULL_TREE
;
1222 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1223 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1225 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1226 ops
.code
= WIDEN_MULT_EXPR
;
1228 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1230 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1231 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1233 hipart
= gen_lowpart (mode
, hipart
);
1234 res
= gen_lowpart (mode
, res
);
1236 /* For the unsigned multiplication, there was overflow if
1237 HIPART is non-zero. */
1238 emit_cmp_and_jump_insns (hipart
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1239 false, done_label
, PROB_VERY_LIKELY
);
1242 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1244 /* RES is low half of the double width result, HIPART
1245 the high half. There was overflow if
1246 HIPART is different from RES < 0 ? -1 : 0. */
1247 emit_cmp_and_jump_insns (signbit
, hipart
, EQ
, NULL_RTX
, mode
,
1248 false, done_label
, PROB_VERY_LIKELY
);
1251 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1253 rtx_code_label
*large_op0
= gen_label_rtx ();
1254 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1255 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1256 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1257 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1258 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1259 rtx_code_label
*do_overflow
= gen_label_rtx ();
1260 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1262 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1263 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1265 hipart0
= gen_lowpart (hmode
, hipart0
);
1266 rtx lopart0
= gen_lowpart (hmode
, op0
);
1267 rtx signbit0
= const0_rtx
;
1269 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1271 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1273 hipart1
= gen_lowpart (hmode
, hipart1
);
1274 rtx lopart1
= gen_lowpart (hmode
, op1
);
1275 rtx signbit1
= const0_rtx
;
1277 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1280 res
= gen_reg_rtx (mode
);
1282 /* True if op0 resp. op1 are known to be in the range of
1284 bool op0_small_p
= false;
1285 bool op1_small_p
= false;
1286 /* True if op0 resp. op1 are known to have all zeros or all ones
1287 in the upper half of bits, but are not known to be
1289 bool op0_medium_p
= false;
1290 bool op1_medium_p
= false;
1291 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1292 nonnegative, 1 if unknown. */
1298 else if (pos_neg0
== 2)
1302 else if (pos_neg1
== 2)
1305 unsigned int mprec0
= prec
;
1306 if (arg0
!= error_mark_node
)
1307 mprec0
= get_min_precision (arg0
, sign
);
1308 if (mprec0
<= hprec
)
1310 else if (!uns
&& mprec0
<= hprec
+ 1)
1311 op0_medium_p
= true;
1312 unsigned int mprec1
= prec
;
1313 if (arg1
!= error_mark_node
)
1314 mprec1
= get_min_precision (arg1
, sign
);
1315 if (mprec1
<= hprec
)
1317 else if (!uns
&& mprec1
<= hprec
+ 1)
1318 op1_medium_p
= true;
1320 int smaller_sign
= 1;
1321 int larger_sign
= 1;
1324 smaller_sign
= op0_sign
;
1325 larger_sign
= op1_sign
;
1327 else if (op1_small_p
)
1329 smaller_sign
= op1_sign
;
1330 larger_sign
= op0_sign
;
1332 else if (op0_sign
== op1_sign
)
1334 smaller_sign
= op0_sign
;
1335 larger_sign
= op0_sign
;
1339 emit_cmp_and_jump_insns (signbit0
, hipart0
, NE
, NULL_RTX
, hmode
,
1340 false, large_op0
, PROB_UNLIKELY
);
1343 emit_cmp_and_jump_insns (signbit1
, hipart1
, NE
, NULL_RTX
, hmode
,
1344 false, small_op0_large_op1
,
1347 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1348 hmode to mode, the multiplication will never overflow. We can
1349 do just one hmode x hmode => mode widening multiplication. */
1350 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1351 if (GET_CODE (lopart0
) == SUBREG
)
1353 lopart0s
= shallow_copy_rtx (lopart0
);
1354 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1355 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1357 if (GET_CODE (lopart1
) == SUBREG
)
1359 lopart1s
= shallow_copy_rtx (lopart1
);
1360 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1361 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1363 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1364 ops
.op0
= make_tree (halfstype
, lopart0s
);
1365 ops
.op1
= make_tree (halfstype
, lopart1s
);
1366 ops
.code
= WIDEN_MULT_EXPR
;
1369 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1370 emit_move_insn (res
, thisres
);
1371 emit_jump (done_label
);
1373 emit_label (small_op0_large_op1
);
1375 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1376 but op1 is not, just swap the arguments and handle it as op1
1377 sign/zero extended, op0 not. */
1378 rtx larger
= gen_reg_rtx (mode
);
1379 rtx hipart
= gen_reg_rtx (hmode
);
1380 rtx lopart
= gen_reg_rtx (hmode
);
1381 emit_move_insn (larger
, op1
);
1382 emit_move_insn (hipart
, hipart1
);
1383 emit_move_insn (lopart
, lopart0
);
1384 emit_jump (one_small_one_large
);
1386 emit_label (large_op0
);
1389 emit_cmp_and_jump_insns (signbit1
, hipart1
, NE
, NULL_RTX
, hmode
,
1390 false, both_ops_large
, PROB_UNLIKELY
);
1392 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1393 but op0 is not, prepare larger, hipart and lopart pseudos and
1394 handle it together with small_op0_large_op1. */
1395 emit_move_insn (larger
, op0
);
1396 emit_move_insn (hipart
, hipart0
);
1397 emit_move_insn (lopart
, lopart1
);
1399 emit_label (one_small_one_large
);
1401 /* lopart is the low part of the operand that is sign extended
1402 to mode, larger is the the other operand, hipart is the
1403 high part of larger and lopart0 and lopart1 are the low parts
1405 We perform lopart0 * lopart1 and lopart * hipart widening
1407 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1408 ops
.op0
= make_tree (halfutype
, lopart0
);
1409 ops
.op1
= make_tree (halfutype
, lopart1
);
1411 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1413 ops
.op0
= make_tree (halfutype
, lopart
);
1414 ops
.op1
= make_tree (halfutype
, hipart
);
1415 rtx loxhi
= gen_reg_rtx (mode
);
1416 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1417 emit_move_insn (loxhi
, tem
);
1421 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1422 if (larger_sign
== 0)
1423 emit_jump (after_hipart_neg
);
1424 else if (larger_sign
!= -1)
1425 emit_cmp_and_jump_insns (hipart
, const0_rtx
, GE
, NULL_RTX
,
1426 hmode
, false, after_hipart_neg
,
1429 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1430 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1431 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1433 emit_move_insn (loxhi
, tem
);
1435 emit_label (after_hipart_neg
);
1437 /* if (lopart < 0) loxhi -= larger; */
1438 if (smaller_sign
== 0)
1439 emit_jump (after_lopart_neg
);
1440 else if (smaller_sign
!= -1)
1441 emit_cmp_and_jump_insns (lopart
, const0_rtx
, GE
, NULL_RTX
,
1442 hmode
, false, after_lopart_neg
,
1445 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1447 emit_move_insn (loxhi
, tem
);
1449 emit_label (after_lopart_neg
);
1452 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1453 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1454 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1456 emit_move_insn (loxhi
, tem
);
1458 /* if (loxhi >> (bitsize / 2)
1459 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1460 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1461 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1463 hipartloxhi
= gen_lowpart (hmode
, hipartloxhi
);
1464 rtx signbitloxhi
= const0_rtx
;
1466 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1467 gen_lowpart (hmode
, loxhi
),
1468 hprec
- 1, NULL_RTX
, 0);
1470 emit_cmp_and_jump_insns (signbitloxhi
, hipartloxhi
, NE
, NULL_RTX
,
1471 hmode
, false, do_overflow
,
1472 PROB_VERY_UNLIKELY
);
1474 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1475 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1477 tem
= convert_modes (mode
, hmode
, gen_lowpart (hmode
, lo0xlo1
), 1);
1479 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1482 emit_move_insn (res
, tem
);
1483 emit_jump (done_label
);
1485 emit_label (both_ops_large
);
1487 /* If both operands are large (not sign (!uns) or zero (uns)
1488 extended from hmode), then perform the full multiplication
1489 which will be the result of the operation.
1490 The only cases which don't overflow are for signed multiplication
1491 some cases where both hipart0 and highpart1 are 0 or -1.
1492 For unsigned multiplication when high parts are both non-zero
1493 this overflows always. */
1494 ops
.code
= MULT_EXPR
;
1495 ops
.op0
= make_tree (type
, op0
);
1496 ops
.op1
= make_tree (type
, op1
);
1497 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1498 emit_move_insn (res
, tem
);
1504 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1505 NULL_RTX
, 1, OPTAB_DIRECT
);
1506 emit_cmp_and_jump_insns (tem
, const1_rtx
, GTU
, NULL_RTX
,
1507 hmode
, true, do_error
,
1508 PROB_VERY_UNLIKELY
);
1513 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1514 NULL_RTX
, 1, OPTAB_DIRECT
);
1515 emit_cmp_and_jump_insns (tem
, const1_rtx
, GTU
, NULL_RTX
,
1516 hmode
, true, do_error
,
1517 PROB_VERY_UNLIKELY
);
1520 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1521 the same, overflow happened if res is negative, if they are
1522 different, overflow happened if res is positive. */
1523 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1524 emit_jump (hipart_different
);
1525 else if (op0_sign
== 1 || op1_sign
== 1)
1526 emit_cmp_and_jump_insns (hipart0
, hipart1
, NE
, NULL_RTX
, hmode
,
1527 true, hipart_different
, PROB_EVEN
);
1529 emit_cmp_and_jump_insns (res
, const0_rtx
, LT
, NULL_RTX
, mode
,
1530 false, do_error
, PROB_VERY_UNLIKELY
);
1531 emit_jump (done_label
);
1533 emit_label (hipart_different
);
1535 emit_cmp_and_jump_insns (res
, const0_rtx
, GE
, NULL_RTX
, mode
,
1536 false, do_error
, PROB_VERY_UNLIKELY
);
1537 emit_jump (done_label
);
1540 emit_label (do_overflow
);
1542 /* Overflow, do full multiplication and fallthru into do_error. */
1543 ops
.op0
= make_tree (type
, op0
);
1544 ops
.op1
= make_tree (type
, op1
);
1545 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1546 emit_move_insn (res
, tem
);
1550 gcc_assert (!is_ubsan
);
1551 ops
.code
= MULT_EXPR
;
1553 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1554 emit_jump (done_label
);
1559 emit_label (do_error
);
1562 /* Expand the ubsan builtin call. */
1564 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1568 do_pending_stack_adjust ();
1571 write_complex_part (target
, const1_rtx
, true);
1574 emit_label (done_label
);
1577 if (uns0_p
&& uns1_p
&& !unsr_p
)
1579 rtx_code_label
*all_done_label
= gen_label_rtx ();
1580 emit_cmp_and_jump_insns (res
, const0_rtx
, GE
, NULL_RTX
, mode
,
1581 false, all_done_label
, PROB_VERY_LIKELY
);
1582 write_complex_part (target
, const1_rtx
, true);
1583 emit_label (all_done_label
);
1587 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1589 rtx_code_label
*all_done_label
= gen_label_rtx ();
1590 rtx_code_label
*set_noovf
= gen_label_rtx ();
1591 emit_cmp_and_jump_insns (op1
, const0_rtx
, GE
, NULL_RTX
, mode
,
1592 false, all_done_label
, PROB_VERY_LIKELY
);
1593 write_complex_part (target
, const1_rtx
, true);
1594 emit_cmp_and_jump_insns (op0
, const0_rtx
, EQ
, NULL_RTX
, mode
,
1595 false, set_noovf
, PROB_VERY_LIKELY
);
1596 emit_cmp_and_jump_insns (op0
, constm1_rtx
, NE
, NULL_RTX
, mode
,
1597 false, all_done_label
, PROB_VERY_UNLIKELY
);
1598 emit_cmp_and_jump_insns (op1
, res
, NE
, NULL_RTX
, mode
,
1599 false, all_done_label
, PROB_VERY_UNLIKELY
);
1600 emit_label (set_noovf
);
1601 write_complex_part (target
, const0_rtx
, true);
1602 emit_label (all_done_label
);
1608 expand_ubsan_result_store (target
, res
);
1610 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1614 /* Expand UBSAN_CHECK_ADD call STMT. */
1617 expand_UBSAN_CHECK_ADD (gcall
*stmt
)
1619 location_t loc
= gimple_location (stmt
);
1620 tree lhs
= gimple_call_lhs (stmt
);
1621 tree arg0
= gimple_call_arg (stmt
, 0);
1622 tree arg1
= gimple_call_arg (stmt
, 1);
1623 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
1624 false, false, false, true);
1627 /* Expand UBSAN_CHECK_SUB call STMT. */
1630 expand_UBSAN_CHECK_SUB (gcall
*stmt
)
1632 location_t loc
= gimple_location (stmt
);
1633 tree lhs
= gimple_call_lhs (stmt
);
1634 tree arg0
= gimple_call_arg (stmt
, 0);
1635 tree arg1
= gimple_call_arg (stmt
, 1);
1636 if (integer_zerop (arg0
))
1637 expand_neg_overflow (loc
, lhs
, arg1
, true);
1639 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
1640 false, false, false, true);
1643 /* Expand UBSAN_CHECK_MUL call STMT. */
1646 expand_UBSAN_CHECK_MUL (gcall
*stmt
)
1648 location_t loc
= gimple_location (stmt
);
1649 tree lhs
= gimple_call_lhs (stmt
);
1650 tree arg0
= gimple_call_arg (stmt
, 0);
1651 tree arg1
= gimple_call_arg (stmt
, 1);
1652 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true);
1655 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1658 expand_arith_overflow (enum tree_code code
, gimple stmt
)
1660 tree lhs
= gimple_call_lhs (stmt
);
1661 if (lhs
== NULL_TREE
)
1663 tree arg0
= gimple_call_arg (stmt
, 0);
1664 tree arg1
= gimple_call_arg (stmt
, 1);
1665 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
1666 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
1667 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
1668 int unsr_p
= TYPE_UNSIGNED (type
);
1669 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
1670 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
1671 int precres
= TYPE_PRECISION (type
);
1672 location_t loc
= gimple_location (stmt
);
1673 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
1675 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
1677 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
1678 prec0
= MIN (prec0
, pr
);
1679 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
1680 prec1
= MIN (prec1
, pr
);
1682 /* If uns0_p && uns1_p, precop is minimum needed precision
1683 of unsigned type to hold the exact result, otherwise
1684 precop is minimum needed precision of signed type to
1685 hold the exact result. */
1687 if (code
== MULT_EXPR
)
1688 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
1691 if (uns0_p
== uns1_p
)
1692 precop
= MAX (prec0
, prec1
) + 1;
1694 precop
= MAX (prec0
+ 1, prec1
) + 1;
1696 precop
= MAX (prec0
, prec1
+ 1) + 1;
1698 int orig_precres
= precres
;
1702 if ((uns0_p
&& uns1_p
)
1703 ? ((precop
+ !unsr_p
) <= precres
1704 /* u1 - u2 -> ur can overflow, no matter what precision
1706 && (code
!= MINUS_EXPR
|| !unsr_p
))
1707 : (!unsr_p
&& precop
<= precres
))
1709 /* The infinity precision result will always fit into result. */
1710 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1711 write_complex_part (target
, const0_rtx
, true);
1712 enum machine_mode mode
= TYPE_MODE (type
);
1713 struct separate_ops ops
;
1716 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
1717 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
1718 ops
.op2
= NULL_TREE
;
1720 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1721 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
1725 #ifdef WORD_REGISTER_OPERATIONS
1726 /* For sub-word operations, if target doesn't have them, start
1727 with precres widening right away, otherwise do it only
1728 if the most simple cases can't be used. */
1729 if (orig_precres
== precres
&& precres
< BITS_PER_WORD
)
1733 if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
&& prec1
<= precres
)
1734 || ((!uns0_p
|| !uns1_p
) && !unsr_p
1735 && prec0
+ uns0_p
<= precres
1736 && prec1
+ uns1_p
<= precres
))
1738 arg0
= fold_convert_loc (loc
, type
, arg0
);
1739 arg1
= fold_convert_loc (loc
, type
, arg1
);
1743 if (integer_zerop (arg0
) && !unsr_p
)
1744 expand_neg_overflow (loc
, lhs
, arg1
, false);
1747 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
,
1748 unsr_p
, unsr_p
, unsr_p
, false);
1751 expand_mul_overflow (loc
, lhs
, arg0
, arg1
,
1752 unsr_p
, unsr_p
, unsr_p
, false);
1759 /* For sub-word operations, retry with a wider type first. */
1760 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
1762 #ifdef WORD_REGISTER_OPERATIONS
1763 int p
= BITS_PER_WORD
;
1767 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1768 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1771 p
= TYPE_PRECISION (optype
);
1775 unsr_p
= TYPE_UNSIGNED (optype
);
1781 if (prec0
<= precres
&& prec1
<= precres
)
1786 types
[0] = build_nonstandard_integer_type (precres
, 0);
1792 types
[1] = build_nonstandard_integer_type (precres
, 1);
1794 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
1795 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
1796 if (code
!= MULT_EXPR
)
1797 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
1798 uns0_p
, uns1_p
, false);
1800 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
1801 uns0_p
, uns1_p
, false);
1805 /* Retry with a wider type. */
1806 if (orig_precres
== precres
)
1808 int p
= MAX (prec0
, prec1
);
1809 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1810 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1813 p
= TYPE_PRECISION (optype
);
1817 unsr_p
= TYPE_UNSIGNED (optype
);
1828 /* Expand ADD_OVERFLOW STMT. */
1831 expand_ADD_OVERFLOW (gcall
*stmt
)
1833 expand_arith_overflow (PLUS_EXPR
, stmt
);
1836 /* Expand SUB_OVERFLOW STMT. */
1839 expand_SUB_OVERFLOW (gcall
*stmt
)
1841 expand_arith_overflow (MINUS_EXPR
, stmt
);
1844 /* Expand MUL_OVERFLOW STMT. */
1847 expand_MUL_OVERFLOW (gcall
*stmt
)
1849 expand_arith_overflow (MULT_EXPR
, stmt
);
1852 /* This should get folded in tree-vectorizer.c. */
1855 expand_LOOP_VECTORIZED (gcall
*stmt ATTRIBUTE_UNUSED
)
1861 expand_MASK_LOAD (gcall
*stmt
)
1863 struct expand_operand ops
[3];
1864 tree type
, lhs
, rhs
, maskt
;
1865 rtx mem
, target
, mask
;
1867 maskt
= gimple_call_arg (stmt
, 2);
1868 lhs
= gimple_call_lhs (stmt
);
1869 if (lhs
== NULL_TREE
)
1871 type
= TREE_TYPE (lhs
);
1872 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
1873 gimple_call_arg (stmt
, 1));
1875 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1876 gcc_assert (MEM_P (mem
));
1877 mask
= expand_normal (maskt
);
1878 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1879 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
1880 create_fixed_operand (&ops
[1], mem
);
1881 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1882 expand_insn (optab_handler (maskload_optab
, TYPE_MODE (type
)), 3, ops
);
1886 expand_MASK_STORE (gcall
*stmt
)
1888 struct expand_operand ops
[3];
1889 tree type
, lhs
, rhs
, maskt
;
1892 maskt
= gimple_call_arg (stmt
, 2);
1893 rhs
= gimple_call_arg (stmt
, 3);
1894 type
= TREE_TYPE (rhs
);
1895 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
1896 gimple_call_arg (stmt
, 1));
1898 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1899 gcc_assert (MEM_P (mem
));
1900 mask
= expand_normal (maskt
);
1901 reg
= expand_normal (rhs
);
1902 create_fixed_operand (&ops
[0], mem
);
1903 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
1904 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1905 expand_insn (optab_handler (maskstore_optab
, TYPE_MODE (type
)), 3, ops
);
1909 expand_ABNORMAL_DISPATCHER (gcall
*)
1914 expand_BUILTIN_EXPECT (gcall
*stmt
)
1916 /* When guessing was done, the hints should be already stripped away. */
1917 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
1920 tree lhs
= gimple_call_lhs (stmt
);
1922 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1924 target
= const0_rtx
;
1925 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
1926 if (lhs
&& val
!= target
)
1927 emit_move_insn (target
, val
);
1930 /* Routines to expand each internal function, indexed by function number.
1931 Each routine has the prototype:
1933 expand_<NAME> (gcall *stmt)
1935 where STMT is the statement that performs the call. */
1936 static void (*const internal_fn_expanders
[]) (gcall
*) = {
1937 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
1938 #include "internal-fn.def"
1939 #undef DEF_INTERNAL_FN
1943 /* Expand STMT, which is a call to internal function FN. */
1946 expand_internal_call (gcall
*stmt
)
1948 internal_fn_expanders
[(int) gimple_call_internal_fn (stmt
)] (stmt
);