2 Copyright (C) 2011-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
26 #include "double-int.h"
34 #include "fold-const.h"
35 #include "internal-fn.h"
36 #include "stor-layout.h"
39 #include "hard-reg-set.h"
43 #include "statistics.h"
45 #include "fixed-value.h"
46 #include "insn-config.h"
55 #include "insn-codes.h"
58 #include "dominance.h"
60 #include "basic-block.h"
61 #include "tree-ssa-alias.h"
62 #include "gimple-expr.h"
67 #include "stringpool.h"
68 #include "tree-ssanames.h"
69 #include "diagnostic-core.h"
71 /* The names of each internal function, indexed by function number. */
72 const char *const internal_fn_name_array
[] = {
73 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
74 #include "internal-fn.def"
75 #undef DEF_INTERNAL_FN
79 /* The ECF_* flags of each internal function, indexed by function number. */
80 const int internal_fn_flags_array
[] = {
81 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
82 #include "internal-fn.def"
83 #undef DEF_INTERNAL_FN
87 /* Fnspec of each internal function, indexed by function number. */
88 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
93 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
94 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
95 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
96 #include "internal-fn.def"
97 #undef DEF_INTERNAL_FN
98 internal_fn_fnspec_array
[IFN_LAST
] = 0;
101 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
102 for load-lanes-style optab OPTAB. The insn must exist. */
104 static enum insn_code
105 get_multi_vector_move (tree array_type
, convert_optab optab
)
107 enum insn_code icode
;
111 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
112 imode
= TYPE_MODE (array_type
);
113 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
115 icode
= convert_optab_handler (optab
, imode
, vmode
);
116 gcc_assert (icode
!= CODE_FOR_nothing
);
120 /* Expand LOAD_LANES call STMT. */
123 expand_LOAD_LANES (gcall
*stmt
)
125 struct expand_operand ops
[2];
129 lhs
= gimple_call_lhs (stmt
);
130 rhs
= gimple_call_arg (stmt
, 0);
131 type
= TREE_TYPE (lhs
);
133 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
134 mem
= expand_normal (rhs
);
136 gcc_assert (MEM_P (mem
));
137 PUT_MODE (mem
, TYPE_MODE (type
));
139 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
140 create_fixed_operand (&ops
[1], mem
);
141 expand_insn (get_multi_vector_move (type
, vec_load_lanes_optab
), 2, ops
);
144 /* Expand STORE_LANES call STMT. */
147 expand_STORE_LANES (gcall
*stmt
)
149 struct expand_operand ops
[2];
153 lhs
= gimple_call_lhs (stmt
);
154 rhs
= gimple_call_arg (stmt
, 0);
155 type
= TREE_TYPE (rhs
);
157 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
158 reg
= expand_normal (rhs
);
160 gcc_assert (MEM_P (target
));
161 PUT_MODE (target
, TYPE_MODE (type
));
163 create_fixed_operand (&ops
[0], target
);
164 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
165 expand_insn (get_multi_vector_move (type
, vec_store_lanes_optab
), 2, ops
);
169 expand_ANNOTATE (gcall
*)
174 /* This should get expanded in adjust_simduid_builtins. */
177 expand_GOMP_SIMD_LANE (gcall
*)
182 /* This should get expanded in adjust_simduid_builtins. */
185 expand_GOMP_SIMD_VF (gcall
*)
190 /* This should get expanded in adjust_simduid_builtins. */
193 expand_GOMP_SIMD_LAST_LANE (gcall
*)
198 /* This should get expanded in the sanopt pass. */
201 expand_UBSAN_NULL (gcall
*)
206 /* This should get expanded in the sanopt pass. */
209 expand_UBSAN_BOUNDS (gcall
*)
214 /* This should get expanded in the sanopt pass. */
217 expand_UBSAN_VPTR (gcall
*)
222 /* This should get expanded in the sanopt pass. */
225 expand_UBSAN_OBJECT_SIZE (gcall
*)
230 /* This should get expanded in the sanopt pass. */
233 expand_ASAN_CHECK (gcall
*)
238 /* This should get expanded in the tsan pass. */
241 expand_TSAN_FUNC_EXIT (gcall
*)
246 /* Helper function for expand_addsub_overflow. Return 1
247 if ARG interpreted as signed in its precision is known to be always
248 positive or 2 if ARG is known to be always negative, or 3 if ARG may
249 be positive or negative. */
252 get_range_pos_neg (tree arg
)
254 if (arg
== error_mark_node
)
257 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
259 if (TREE_CODE (arg
) == INTEGER_CST
)
261 wide_int w
= wi::sext (arg
, prec
);
267 while (CONVERT_EXPR_P (arg
)
268 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
269 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
271 arg
= TREE_OPERAND (arg
, 0);
272 /* Narrower value zero extended into wider type
273 will always result in positive values. */
274 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
275 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
277 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
282 if (TREE_CODE (arg
) != SSA_NAME
)
284 wide_int arg_min
, arg_max
;
285 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
287 gimple g
= SSA_NAME_DEF_STMT (arg
);
288 if (is_gimple_assign (g
)
289 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
291 tree t
= gimple_assign_rhs1 (g
);
292 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
293 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
295 if (TYPE_UNSIGNED (TREE_TYPE (t
))
296 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
298 prec
= TYPE_PRECISION (TREE_TYPE (t
));
307 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
309 /* For unsigned values, the "positive" range comes
310 below the "negative" range. */
311 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
313 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
318 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
320 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
326 /* Return minimum precision needed to represent all values
327 of ARG in SIGNed integral type. */
330 get_min_precision (tree arg
, signop sign
)
332 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
334 signop orig_sign
= sign
;
335 if (TREE_CODE (arg
) == INTEGER_CST
)
338 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
340 widest_int w
= wi::to_widest (arg
);
341 w
= wi::ext (w
, prec
, sign
);
342 p
= wi::min_precision (w
, sign
);
345 p
= wi::min_precision (arg
, sign
);
346 return MIN (p
, prec
);
348 while (CONVERT_EXPR_P (arg
)
349 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
350 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
352 arg
= TREE_OPERAND (arg
, 0);
353 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
355 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
357 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
358 return prec
+ (orig_sign
!= sign
);
359 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
362 return prec
+ (orig_sign
!= sign
);
364 if (TREE_CODE (arg
) != SSA_NAME
)
365 return prec
+ (orig_sign
!= sign
);
366 wide_int arg_min
, arg_max
;
367 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
369 gimple g
= SSA_NAME_DEF_STMT (arg
);
370 if (is_gimple_assign (g
)
371 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
373 tree t
= gimple_assign_rhs1 (g
);
374 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
375 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
378 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
380 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
382 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
383 return prec
+ (orig_sign
!= sign
);
384 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
387 return prec
+ (orig_sign
!= sign
);
391 return prec
+ (orig_sign
!= sign
);
393 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
395 int p1
= wi::min_precision (arg_min
, sign
);
396 int p2
= wi::min_precision (arg_max
, sign
);
398 prec
= MIN (prec
, p1
);
400 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
402 int p
= wi::min_precision (arg_max
, SIGNED
);
403 prec
= MIN (prec
, p
);
405 return prec
+ (orig_sign
!= sign
);
408 /* Helper for expand_*_overflow. Store RES into the __real__ part
409 of TARGET. If RES has larger MODE than __real__ part of TARGET,
410 set the __imag__ part to 1 if RES doesn't fit into it. */
413 expand_arith_overflow_result_store (tree lhs
, rtx target
,
414 machine_mode mode
, rtx res
)
416 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
420 rtx_code_label
*done_label
= gen_label_rtx ();
421 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
422 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
423 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
424 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
425 EQ
, true, mode
, NULL_RTX
, NULL_RTX
, done_label
,
427 write_complex_part (target
, const1_rtx
, true);
428 emit_label (done_label
);
430 write_complex_part (target
, lres
, false);
433 /* Helper for expand_*_overflow. Store RES into TARGET. */
436 expand_ubsan_result_store (rtx target
, rtx res
)
438 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
439 /* If this is a scalar in a register that is stored in a wider mode
440 than the declared mode, compute the result into its declared mode
441 and then convert to the wider mode. Our value is the computed
443 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
445 emit_move_insn (target
, res
);
448 /* Add sub/add overflow checking to the statement STMT.
449 CODE says whether the operation is +, or -. */
452 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
453 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
454 bool uns1_p
, bool is_ubsan
)
456 rtx res
, target
= NULL_RTX
;
458 rtx_code_label
*done_label
= gen_label_rtx ();
459 rtx_code_label
*do_error
= gen_label_rtx ();
460 do_pending_stack_adjust ();
461 rtx op0
= expand_normal (arg0
);
462 rtx op1
= expand_normal (arg1
);
463 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
464 int prec
= GET_MODE_PRECISION (mode
);
465 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
469 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
473 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
475 write_complex_part (target
, const0_rtx
, true);
478 /* We assume both operands and result have the same precision
479 here (GET_MODE_BITSIZE (mode)), S stands for signed type
480 with that precision, U for unsigned type with that precision,
481 sgn for unsigned most significant bit in that precision.
482 s1 is signed first operand, u1 is unsigned first operand,
483 s2 is signed second operand, u2 is unsigned second operand,
484 sr is signed result, ur is unsigned result and the following
485 rules say how to compute result (which is always result of
486 the operands as if both were unsigned, cast to the right
487 signedness) and how to compute whether operation overflowed.
490 res = (S) ((U) s1 + (U) s2)
491 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
493 res = (S) ((U) s1 - (U) s2)
494 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
497 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
500 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
502 res = (S) ((U) s1 + u2)
503 ovf = ((U) res ^ sgn) < u2
508 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
510 res = (S) ((U) s1 - u2)
511 ovf = u2 > ((U) s1 ^ sgn)
514 ovf = s1 < 0 || u2 > (U) s1
517 ovf = u1 >= ((U) s2 ^ sgn)
522 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
524 res = (U) s1 + (U) s2
525 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
528 ovf = (U) res < u2 || res < 0
531 ovf = u1 >= u2 ? res < 0 : res >= 0
533 res = (U) s1 - (U) s2
534 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
536 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
538 /* PLUS_EXPR is commutative, if operand signedness differs,
539 canonicalize to the first operand being signed and second
540 unsigned to simplify following code. */
552 if (uns0_p
&& uns1_p
&& unsr_p
)
554 /* Compute the operation. On RTL level, the addition is always
556 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
557 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
559 /* For PLUS_EXPR, the operation is commutative, so we can pick
560 operand to compare against. For prec <= BITS_PER_WORD, I think
561 preferring REG operand is better over CONST_INT, because
562 the CONST_INT might enlarge the instruction or CSE would need
563 to figure out we'd already loaded it into a register before.
564 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
565 as then the multi-word comparison can be perhaps simplified. */
566 if (code
== PLUS_EXPR
567 && (prec
<= BITS_PER_WORD
568 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
569 : CONST_SCALAR_INT_P (op1
)))
571 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
572 true, mode
, NULL_RTX
, NULL_RTX
, done_label
,
578 if (!uns0_p
&& uns1_p
&& !unsr_p
)
580 /* Compute the operation. On RTL level, the addition is always
582 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
583 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
584 rtx tem
= expand_binop (mode
, add_optab
,
585 code
== PLUS_EXPR
? res
: op0
, sgn
,
586 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
587 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL_RTX
,
588 done_label
, PROB_VERY_LIKELY
);
593 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
595 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
597 /* As we've changed op1, we have to avoid using the value range
598 for the original argument. */
599 arg1
= error_mark_node
;
605 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
607 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
609 /* As we've changed op0, we have to avoid using the value range
610 for the original argument. */
611 arg0
= error_mark_node
;
617 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
619 /* Compute the operation. On RTL level, the addition is always
621 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
623 int pos_neg
= get_range_pos_neg (arg0
);
625 /* If ARG0 is known to be always negative, this is always overflow. */
626 emit_jump (do_error
);
627 else if (pos_neg
== 3)
628 /* If ARG0 is not known to be always positive, check at runtime. */
629 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
630 NULL_RTX
, do_error
, PROB_VERY_UNLIKELY
);
631 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL_RTX
,
632 done_label
, PROB_VERY_LIKELY
);
637 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
639 /* Compute the operation. On RTL level, the addition is always
641 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
643 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
645 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL_RTX
,
646 done_label
, PROB_VERY_LIKELY
);
651 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
653 /* Compute the operation. On RTL level, the addition is always
655 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
657 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
658 NULL_RTX
, do_error
, PROB_VERY_UNLIKELY
);
660 /* The operation is commutative, so we can pick operand to compare
661 against. For prec <= BITS_PER_WORD, I think preferring REG operand
662 is better over CONST_INT, because the CONST_INT might enlarge the
663 instruction or CSE would need to figure out we'd already loaded it
664 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
665 might be more beneficial, as then the multi-word comparison can be
666 perhaps simplified. */
667 if (prec
<= BITS_PER_WORD
668 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
669 : CONST_SCALAR_INT_P (op0
))
671 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL_RTX
,
672 done_label
, PROB_VERY_LIKELY
);
677 if (!uns0_p
&& !uns1_p
&& unsr_p
)
679 /* Compute the operation. On RTL level, the addition is always
681 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
682 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
683 int pos_neg
= get_range_pos_neg (arg1
);
684 if (code
== PLUS_EXPR
)
686 int pos_neg0
= get_range_pos_neg (arg0
);
687 if (pos_neg0
!= 3 && pos_neg
== 3)
698 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
699 ? and_optab
: ior_optab
,
700 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
701 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
702 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
706 rtx_code_label
*do_ior_label
= gen_label_rtx ();
707 do_compare_rtx_and_jump (op1
, const0_rtx
,
708 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
709 NULL_RTX
, NULL_RTX
, do_ior_label
,
711 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
713 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
714 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
715 emit_jump (do_error
);
716 emit_label (do_ior_label
);
717 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
719 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
720 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
726 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
728 /* Compute the operation. On RTL level, the addition is always
730 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
732 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
733 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL_RTX
,
734 op0_geu_op1
, PROB_EVEN
);
735 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
736 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
737 emit_jump (do_error
);
738 emit_label (op0_geu_op1
);
739 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
740 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
744 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
748 enum insn_code icode
;
749 icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
: subv4_optab
, mode
);
750 if (icode
!= CODE_FOR_nothing
)
752 struct expand_operand ops
[4];
753 rtx_insn
*last
= get_last_insn ();
755 res
= gen_reg_rtx (mode
);
756 create_output_operand (&ops
[0], res
, mode
);
757 create_input_operand (&ops
[1], op0
, mode
);
758 create_input_operand (&ops
[2], op1
, mode
);
759 create_fixed_operand (&ops
[3], do_error
);
760 if (maybe_expand_insn (icode
, 4, ops
))
762 last
= get_last_insn ();
763 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
765 && any_condjump_p (last
)
766 && !find_reg_note (last
, REG_BR_PROB
, 0))
767 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
768 emit_jump (done_label
);
772 delete_insns_since (last
);
773 icode
= CODE_FOR_nothing
;
777 if (icode
== CODE_FOR_nothing
)
779 rtx_code_label
*sub_check
= gen_label_rtx ();
782 /* Compute the operation. On RTL level, the addition is always
784 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
785 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
787 /* If we can prove one of the arguments (for MINUS_EXPR only
788 the second operand, as subtraction is not commutative) is always
789 non-negative or always negative, we can do just one comparison
790 and conditional jump instead of 2 at runtime, 3 present in the
791 emitted code. If one of the arguments is CONST_INT, all we
792 need is to make sure it is op1, then the first
793 do_compare_rtx_and_jump will be just folded. Otherwise try
794 to use range info if available. */
795 if (code
== PLUS_EXPR
&& CONST_INT_P (op0
))
801 else if (CONST_INT_P (op1
))
803 else if (code
== PLUS_EXPR
&& TREE_CODE (arg0
) == SSA_NAME
)
805 pos_neg
= get_range_pos_neg (arg0
);
813 if (pos_neg
== 3 && !CONST_INT_P (op1
) && TREE_CODE (arg1
) == SSA_NAME
)
814 pos_neg
= get_range_pos_neg (arg1
);
816 /* If the op1 is negative, we have to use a different check. */
818 do_compare_rtx_and_jump (op1
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
819 NULL_RTX
, sub_check
, PROB_EVEN
);
821 /* Compare the result of the operation with one of the operands. */
823 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? GE
: LE
,
824 false, mode
, NULL_RTX
, NULL_RTX
, done_label
,
827 /* If we get here, we have to print the error. */
830 emit_jump (do_error
);
832 emit_label (sub_check
);
835 /* We have k = a + b for b < 0 here. k <= a must hold. */
837 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? LE
: GE
,
838 false, mode
, NULL_RTX
, NULL_RTX
, done_label
,
843 emit_label (do_error
);
846 /* Expand the ubsan builtin call. */
848 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
852 do_pending_stack_adjust ();
855 write_complex_part (target
, const1_rtx
, true);
858 emit_label (done_label
);
863 expand_ubsan_result_store (target
, res
);
867 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
870 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
875 /* Add negate overflow checking to the statement STMT. */
878 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
)
882 rtx_code_label
*done_label
, *do_error
;
883 rtx target
= NULL_RTX
;
885 done_label
= gen_label_rtx ();
886 do_error
= gen_label_rtx ();
888 do_pending_stack_adjust ();
889 op1
= expand_normal (arg1
);
891 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
894 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
896 write_complex_part (target
, const0_rtx
, true);
899 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
900 if (icode
!= CODE_FOR_nothing
)
902 struct expand_operand ops
[3];
903 rtx_insn
*last
= get_last_insn ();
905 res
= gen_reg_rtx (mode
);
906 create_output_operand (&ops
[0], res
, mode
);
907 create_input_operand (&ops
[1], op1
, mode
);
908 create_fixed_operand (&ops
[2], do_error
);
909 if (maybe_expand_insn (icode
, 3, ops
))
911 last
= get_last_insn ();
912 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
914 && any_condjump_p (last
)
915 && !find_reg_note (last
, REG_BR_PROB
, 0))
916 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
917 emit_jump (done_label
);
921 delete_insns_since (last
);
922 icode
= CODE_FOR_nothing
;
926 if (icode
== CODE_FOR_nothing
)
928 /* Compute the operation. On RTL level, the addition is always
930 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
932 /* Compare the operand with the most negative value. */
933 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
934 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL_RTX
,
935 done_label
, PROB_VERY_LIKELY
);
938 emit_label (do_error
);
941 /* Expand the ubsan builtin call. */
943 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
947 do_pending_stack_adjust ();
950 write_complex_part (target
, const1_rtx
, true);
953 emit_label (done_label
);
958 expand_ubsan_result_store (target
, res
);
960 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
964 /* Add mul overflow checking to the statement STMT. */
967 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
968 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
)
972 rtx_code_label
*done_label
, *do_error
;
973 rtx target
= NULL_RTX
;
975 enum insn_code icode
;
977 done_label
= gen_label_rtx ();
978 do_error
= gen_label_rtx ();
980 do_pending_stack_adjust ();
981 op0
= expand_normal (arg0
);
982 op1
= expand_normal (arg1
);
984 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
988 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
990 write_complex_part (target
, const0_rtx
, true);
994 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
996 /* We assume both operands and result have the same precision
997 here (GET_MODE_BITSIZE (mode)), S stands for signed type
998 with that precision, U for unsigned type with that precision,
999 sgn for unsigned most significant bit in that precision.
1000 s1 is signed first operand, u1 is unsigned first operand,
1001 s2 is signed second operand, u2 is unsigned second operand,
1002 sr is signed result, ur is unsigned result and the following
1003 rules say how to compute result (which is always result of
1004 the operands as if both were unsigned, cast to the right
1005 signedness) and how to compute whether operation overflowed.
1006 main_ovf (false) stands for jump on signed multiplication
1007 overflow or the main algorithm with uns == false.
1008 main_ovf (true) stands for jump on unsigned multiplication
1009 overflow or the main algorithm with uns == true.
1012 res = (S) ((U) s1 * (U) s2)
1013 ovf = main_ovf (false)
1016 ovf = main_ovf (true)
1019 ovf = (s1 < 0 && u2) || main_ovf (true)
1022 ovf = res < 0 || main_ovf (true)
1024 res = (S) ((U) s1 * u2)
1025 ovf = (S) u2 >= 0 ? main_ovf (false)
1026 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1028 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1029 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1031 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1033 if (uns0_p
&& !uns1_p
)
1035 /* Multiplication is commutative, if operand signedness differs,
1036 canonicalize to the first operand being signed and second
1037 unsigned to simplify following code. */
1048 int pos_neg0
= get_range_pos_neg (arg0
);
1049 int pos_neg1
= get_range_pos_neg (arg1
);
1052 if (!uns0_p
&& uns1_p
&& unsr_p
)
1057 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1060 /* If s1 is negative, avoid the main code, just multiply and
1061 signal overflow if op1 is not 0. */
1062 struct separate_ops ops
;
1063 ops
.code
= MULT_EXPR
;
1064 ops
.type
= TREE_TYPE (arg1
);
1065 ops
.op0
= make_tree (ops
.type
, op0
);
1066 ops
.op1
= make_tree (ops
.type
, op1
);
1067 ops
.op2
= NULL_TREE
;
1069 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1070 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1071 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
1072 goto do_error_label
;
1074 rtx_code_label
*do_main_label
;
1075 do_main_label
= gen_label_rtx ();
1076 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1077 NULL_RTX
, do_main_label
, PROB_VERY_LIKELY
);
1078 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1079 NULL_RTX
, do_main_label
, PROB_VERY_LIKELY
);
1080 write_complex_part (target
, const1_rtx
, true);
1081 emit_label (do_main_label
);
1089 if (uns0_p
&& uns1_p
&& !unsr_p
)
1092 /* Rest of handling of this case after res is computed. */
1097 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1104 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1105 avoid the main code, just multiply and signal overflow
1106 unless 0 * u2 or -1 * ((U) Smin). */
1107 struct separate_ops ops
;
1108 ops
.code
= MULT_EXPR
;
1109 ops
.type
= TREE_TYPE (arg1
);
1110 ops
.op0
= make_tree (ops
.type
, op0
);
1111 ops
.op1
= make_tree (ops
.type
, op1
);
1112 ops
.op2
= NULL_TREE
;
1114 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1115 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1116 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
1117 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1118 NULL_RTX
, do_error
, PROB_VERY_UNLIKELY
);
1120 prec
= GET_MODE_PRECISION (mode
);
1122 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1123 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1124 NULL_RTX
, done_label
, PROB_VERY_LIKELY
);
1125 goto do_error_label
;
1127 /* Rest of handling of this case after res is computed. */
1135 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1138 switch (pos_neg0
| pos_neg1
)
1140 case 1: /* Both operands known to be non-negative. */
1142 case 2: /* Both operands known to be negative. */
1143 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1144 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1145 /* Avoid looking at arg0/arg1 ranges, as we've changed
1147 arg0
= error_mark_node
;
1148 arg1
= error_mark_node
;
1151 if ((pos_neg0
^ pos_neg1
) == 3)
1153 /* If one operand is known to be negative and the other
1154 non-negative, this overflows always, unless the non-negative
1155 one is 0. Just do normal multiply and set overflow
1156 unless one of the operands is 0. */
1157 struct separate_ops ops
;
1158 ops
.code
= MULT_EXPR
;
1160 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1162 ops
.op0
= make_tree (ops
.type
, op0
);
1163 ops
.op1
= make_tree (ops
.type
, op1
);
1164 ops
.op2
= NULL_TREE
;
1166 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1167 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1169 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1170 NULL_RTX
, NULL_RTX
, done_label
,
1172 goto do_error_label
;
1174 /* The general case, do all the needed comparisons at runtime. */
1175 rtx_code_label
*do_main_label
, *after_negate_label
;
1177 rop0
= gen_reg_rtx (mode
);
1178 rop1
= gen_reg_rtx (mode
);
1179 emit_move_insn (rop0
, op0
);
1180 emit_move_insn (rop1
, op1
);
1183 do_main_label
= gen_label_rtx ();
1184 after_negate_label
= gen_label_rtx ();
1185 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1187 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1188 NULL_RTX
, after_negate_label
,
1190 /* Both arguments negative here, negate them and continue with
1191 normal unsigned overflow checking multiplication. */
1192 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1194 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1196 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1198 arg0
= error_mark_node
;
1199 arg1
= error_mark_node
;
1200 emit_jump (do_main_label
);
1201 emit_label (after_negate_label
);
1202 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1204 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1205 NULL_RTX
, do_main_label
, PROB_VERY_LIKELY
);
1206 /* One argument is negative here, the other positive. This
1207 overflows always, unless one of the arguments is 0. But
1208 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1209 is, thus we can keep do_main code oring in overflow as is. */
1210 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1211 NULL_RTX
, do_main_label
, PROB_VERY_LIKELY
);
1212 write_complex_part (target
, const1_rtx
, true);
1213 emit_label (do_main_label
);
1221 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1222 sign
= uns
? UNSIGNED
: SIGNED
;
1223 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1224 if (icode
!= CODE_FOR_nothing
)
1226 struct expand_operand ops
[4];
1227 rtx_insn
*last
= get_last_insn ();
1229 res
= gen_reg_rtx (mode
);
1230 create_output_operand (&ops
[0], res
, mode
);
1231 create_input_operand (&ops
[1], op0
, mode
);
1232 create_input_operand (&ops
[2], op1
, mode
);
1233 create_fixed_operand (&ops
[3], do_error
);
1234 if (maybe_expand_insn (icode
, 4, ops
))
1236 last
= get_last_insn ();
1237 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1239 && any_condjump_p (last
)
1240 && !find_reg_note (last
, REG_BR_PROB
, 0))
1241 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1242 emit_jump (done_label
);
1246 delete_insns_since (last
);
1247 icode
= CODE_FOR_nothing
;
1251 if (icode
== CODE_FOR_nothing
)
1253 struct separate_ops ops
;
1254 int prec
= GET_MODE_PRECISION (mode
);
1255 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1256 ops
.op0
= make_tree (type
, op0
);
1257 ops
.op1
= make_tree (type
, op1
);
1258 ops
.op2
= NULL_TREE
;
1260 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1261 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1263 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1264 ops
.code
= WIDEN_MULT_EXPR
;
1266 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1268 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1269 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1271 hipart
= gen_lowpart (mode
, hipart
);
1272 res
= gen_lowpart (mode
, res
);
1274 /* For the unsigned multiplication, there was overflow if
1275 HIPART is non-zero. */
1276 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1277 NULL_RTX
, NULL_RTX
, done_label
,
1281 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1283 /* RES is low half of the double width result, HIPART
1284 the high half. There was overflow if
1285 HIPART is different from RES < 0 ? -1 : 0. */
1286 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1287 NULL_RTX
, NULL_RTX
, done_label
,
1291 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1293 rtx_code_label
*large_op0
= gen_label_rtx ();
1294 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1295 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1296 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1297 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1298 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1299 rtx_code_label
*do_overflow
= gen_label_rtx ();
1300 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1302 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1303 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1305 hipart0
= gen_lowpart (hmode
, hipart0
);
1306 rtx lopart0
= gen_lowpart (hmode
, op0
);
1307 rtx signbit0
= const0_rtx
;
1309 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1311 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1313 hipart1
= gen_lowpart (hmode
, hipart1
);
1314 rtx lopart1
= gen_lowpart (hmode
, op1
);
1315 rtx signbit1
= const0_rtx
;
1317 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1320 res
= gen_reg_rtx (mode
);
1322 /* True if op0 resp. op1 are known to be in the range of
1324 bool op0_small_p
= false;
1325 bool op1_small_p
= false;
1326 /* True if op0 resp. op1 are known to have all zeros or all ones
1327 in the upper half of bits, but are not known to be
1329 bool op0_medium_p
= false;
1330 bool op1_medium_p
= false;
1331 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1332 nonnegative, 1 if unknown. */
1338 else if (pos_neg0
== 2)
1342 else if (pos_neg1
== 2)
1345 unsigned int mprec0
= prec
;
1346 if (arg0
!= error_mark_node
)
1347 mprec0
= get_min_precision (arg0
, sign
);
1348 if (mprec0
<= hprec
)
1350 else if (!uns
&& mprec0
<= hprec
+ 1)
1351 op0_medium_p
= true;
1352 unsigned int mprec1
= prec
;
1353 if (arg1
!= error_mark_node
)
1354 mprec1
= get_min_precision (arg1
, sign
);
1355 if (mprec1
<= hprec
)
1357 else if (!uns
&& mprec1
<= hprec
+ 1)
1358 op1_medium_p
= true;
1360 int smaller_sign
= 1;
1361 int larger_sign
= 1;
1364 smaller_sign
= op0_sign
;
1365 larger_sign
= op1_sign
;
1367 else if (op1_small_p
)
1369 smaller_sign
= op1_sign
;
1370 larger_sign
= op0_sign
;
1372 else if (op0_sign
== op1_sign
)
1374 smaller_sign
= op0_sign
;
1375 larger_sign
= op0_sign
;
1379 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1380 NULL_RTX
, NULL_RTX
, large_op0
,
1384 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1385 NULL_RTX
, NULL_RTX
, small_op0_large_op1
,
1388 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1389 hmode to mode, the multiplication will never overflow. We can
1390 do just one hmode x hmode => mode widening multiplication. */
1391 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1392 if (GET_CODE (lopart0
) == SUBREG
)
1394 lopart0s
= shallow_copy_rtx (lopart0
);
1395 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1396 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1398 if (GET_CODE (lopart1
) == SUBREG
)
1400 lopart1s
= shallow_copy_rtx (lopart1
);
1401 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1402 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1404 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1405 ops
.op0
= make_tree (halfstype
, lopart0s
);
1406 ops
.op1
= make_tree (halfstype
, lopart1s
);
1407 ops
.code
= WIDEN_MULT_EXPR
;
1410 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1411 emit_move_insn (res
, thisres
);
1412 emit_jump (done_label
);
1414 emit_label (small_op0_large_op1
);
1416 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1417 but op1 is not, just swap the arguments and handle it as op1
1418 sign/zero extended, op0 not. */
1419 rtx larger
= gen_reg_rtx (mode
);
1420 rtx hipart
= gen_reg_rtx (hmode
);
1421 rtx lopart
= gen_reg_rtx (hmode
);
1422 emit_move_insn (larger
, op1
);
1423 emit_move_insn (hipart
, hipart1
);
1424 emit_move_insn (lopart
, lopart0
);
1425 emit_jump (one_small_one_large
);
1427 emit_label (large_op0
);
1430 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1431 NULL_RTX
, NULL_RTX
, both_ops_large
,
1434 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1435 but op0 is not, prepare larger, hipart and lopart pseudos and
1436 handle it together with small_op0_large_op1. */
1437 emit_move_insn (larger
, op0
);
1438 emit_move_insn (hipart
, hipart0
);
1439 emit_move_insn (lopart
, lopart1
);
1441 emit_label (one_small_one_large
);
1443 /* lopart is the low part of the operand that is sign extended
1444 to mode, larger is the the other operand, hipart is the
1445 high part of larger and lopart0 and lopart1 are the low parts
1447 We perform lopart0 * lopart1 and lopart * hipart widening
1449 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1450 ops
.op0
= make_tree (halfutype
, lopart0
);
1451 ops
.op1
= make_tree (halfutype
, lopart1
);
1453 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1455 ops
.op0
= make_tree (halfutype
, lopart
);
1456 ops
.op1
= make_tree (halfutype
, hipart
);
1457 rtx loxhi
= gen_reg_rtx (mode
);
1458 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1459 emit_move_insn (loxhi
, tem
);
1463 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1464 if (larger_sign
== 0)
1465 emit_jump (after_hipart_neg
);
1466 else if (larger_sign
!= -1)
1467 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1468 NULL_RTX
, NULL_RTX
, after_hipart_neg
,
1471 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1472 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1473 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1475 emit_move_insn (loxhi
, tem
);
1477 emit_label (after_hipart_neg
);
1479 /* if (lopart < 0) loxhi -= larger; */
1480 if (smaller_sign
== 0)
1481 emit_jump (after_lopart_neg
);
1482 else if (smaller_sign
!= -1)
1483 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1484 NULL_RTX
, NULL_RTX
, after_lopart_neg
,
1487 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1489 emit_move_insn (loxhi
, tem
);
1491 emit_label (after_lopart_neg
);
1494 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1495 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1496 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1498 emit_move_insn (loxhi
, tem
);
1500 /* if (loxhi >> (bitsize / 2)
1501 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1502 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1503 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1505 hipartloxhi
= gen_lowpart (hmode
, hipartloxhi
);
1506 rtx signbitloxhi
= const0_rtx
;
1508 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1509 gen_lowpart (hmode
, loxhi
),
1510 hprec
- 1, NULL_RTX
, 0);
1512 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1513 NULL_RTX
, NULL_RTX
, do_overflow
,
1514 PROB_VERY_UNLIKELY
);
1516 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1517 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1519 tem
= convert_modes (mode
, hmode
, gen_lowpart (hmode
, lo0xlo1
), 1);
1521 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1524 emit_move_insn (res
, tem
);
1525 emit_jump (done_label
);
1527 emit_label (both_ops_large
);
1529 /* If both operands are large (not sign (!uns) or zero (uns)
1530 extended from hmode), then perform the full multiplication
1531 which will be the result of the operation.
1532 The only cases which don't overflow are for signed multiplication
1533 some cases where both hipart0 and highpart1 are 0 or -1.
1534 For unsigned multiplication when high parts are both non-zero
1535 this overflows always. */
1536 ops
.code
= MULT_EXPR
;
1537 ops
.op0
= make_tree (type
, op0
);
1538 ops
.op1
= make_tree (type
, op1
);
1539 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1540 emit_move_insn (res
, tem
);
1546 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1547 NULL_RTX
, 1, OPTAB_DIRECT
);
1548 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1549 NULL_RTX
, NULL_RTX
, do_error
,
1550 PROB_VERY_UNLIKELY
);
1555 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1556 NULL_RTX
, 1, OPTAB_DIRECT
);
1557 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1558 NULL_RTX
, NULL_RTX
, do_error
,
1559 PROB_VERY_UNLIKELY
);
1562 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1563 the same, overflow happened if res is negative, if they are
1564 different, overflow happened if res is positive. */
1565 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1566 emit_jump (hipart_different
);
1567 else if (op0_sign
== 1 || op1_sign
== 1)
1568 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1569 NULL_RTX
, NULL_RTX
, hipart_different
,
1572 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1573 NULL_RTX
, NULL_RTX
, do_error
,
1574 PROB_VERY_UNLIKELY
);
1575 emit_jump (done_label
);
1577 emit_label (hipart_different
);
1579 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1580 NULL_RTX
, NULL_RTX
, do_error
,
1581 PROB_VERY_UNLIKELY
);
1582 emit_jump (done_label
);
1585 emit_label (do_overflow
);
1587 /* Overflow, do full multiplication and fallthru into do_error. */
1588 ops
.op0
= make_tree (type
, op0
);
1589 ops
.op1
= make_tree (type
, op1
);
1590 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1591 emit_move_insn (res
, tem
);
1595 gcc_assert (!is_ubsan
);
1596 ops
.code
= MULT_EXPR
;
1598 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1599 emit_jump (done_label
);
1604 emit_label (do_error
);
1607 /* Expand the ubsan builtin call. */
1609 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1613 do_pending_stack_adjust ();
1616 write_complex_part (target
, const1_rtx
, true);
1619 emit_label (done_label
);
1622 if (uns0_p
&& uns1_p
&& !unsr_p
)
1624 rtx_code_label
*all_done_label
= gen_label_rtx ();
1625 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1626 NULL_RTX
, all_done_label
, PROB_VERY_LIKELY
);
1627 write_complex_part (target
, const1_rtx
, true);
1628 emit_label (all_done_label
);
1632 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1634 rtx_code_label
*all_done_label
= gen_label_rtx ();
1635 rtx_code_label
*set_noovf
= gen_label_rtx ();
1636 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1637 NULL_RTX
, all_done_label
, PROB_VERY_LIKELY
);
1638 write_complex_part (target
, const1_rtx
, true);
1639 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1640 NULL_RTX
, set_noovf
, PROB_VERY_LIKELY
);
1641 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1642 NULL_RTX
, all_done_label
, PROB_VERY_UNLIKELY
);
1643 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL_RTX
,
1644 all_done_label
, PROB_VERY_UNLIKELY
);
1645 emit_label (set_noovf
);
1646 write_complex_part (target
, const0_rtx
, true);
1647 emit_label (all_done_label
);
1653 expand_ubsan_result_store (target
, res
);
1655 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1659 /* Expand UBSAN_CHECK_ADD call STMT. */
1662 expand_UBSAN_CHECK_ADD (gcall
*stmt
)
1664 location_t loc
= gimple_location (stmt
);
1665 tree lhs
= gimple_call_lhs (stmt
);
1666 tree arg0
= gimple_call_arg (stmt
, 0);
1667 tree arg1
= gimple_call_arg (stmt
, 1);
1668 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
1669 false, false, false, true);
1672 /* Expand UBSAN_CHECK_SUB call STMT. */
1675 expand_UBSAN_CHECK_SUB (gcall
*stmt
)
1677 location_t loc
= gimple_location (stmt
);
1678 tree lhs
= gimple_call_lhs (stmt
);
1679 tree arg0
= gimple_call_arg (stmt
, 0);
1680 tree arg1
= gimple_call_arg (stmt
, 1);
1681 if (integer_zerop (arg0
))
1682 expand_neg_overflow (loc
, lhs
, arg1
, true);
1684 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
1685 false, false, false, true);
1688 /* Expand UBSAN_CHECK_MUL call STMT. */
1691 expand_UBSAN_CHECK_MUL (gcall
*stmt
)
1693 location_t loc
= gimple_location (stmt
);
1694 tree lhs
= gimple_call_lhs (stmt
);
1695 tree arg0
= gimple_call_arg (stmt
, 0);
1696 tree arg1
= gimple_call_arg (stmt
, 1);
1697 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true);
1700 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1703 expand_arith_overflow (enum tree_code code
, gimple stmt
)
1705 tree lhs
= gimple_call_lhs (stmt
);
1706 if (lhs
== NULL_TREE
)
1708 tree arg0
= gimple_call_arg (stmt
, 0);
1709 tree arg1
= gimple_call_arg (stmt
, 1);
1710 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
1711 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
1712 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
1713 int unsr_p
= TYPE_UNSIGNED (type
);
1714 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
1715 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
1716 int precres
= TYPE_PRECISION (type
);
1717 location_t loc
= gimple_location (stmt
);
1718 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
1720 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
1722 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
1723 prec0
= MIN (prec0
, pr
);
1724 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
1725 prec1
= MIN (prec1
, pr
);
1727 /* If uns0_p && uns1_p, precop is minimum needed precision
1728 of unsigned type to hold the exact result, otherwise
1729 precop is minimum needed precision of signed type to
1730 hold the exact result. */
1732 if (code
== MULT_EXPR
)
1733 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
1736 if (uns0_p
== uns1_p
)
1737 precop
= MAX (prec0
, prec1
) + 1;
1739 precop
= MAX (prec0
+ 1, prec1
) + 1;
1741 precop
= MAX (prec0
, prec1
+ 1) + 1;
1743 int orig_precres
= precres
;
1747 if ((uns0_p
&& uns1_p
)
1748 ? ((precop
+ !unsr_p
) <= precres
1749 /* u1 - u2 -> ur can overflow, no matter what precision
1751 && (code
!= MINUS_EXPR
|| !unsr_p
))
1752 : (!unsr_p
&& precop
<= precres
))
1754 /* The infinity precision result will always fit into result. */
1755 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1756 write_complex_part (target
, const0_rtx
, true);
1757 enum machine_mode mode
= TYPE_MODE (type
);
1758 struct separate_ops ops
;
1761 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
1762 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
1763 ops
.op2
= NULL_TREE
;
1765 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1766 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
1770 #ifdef WORD_REGISTER_OPERATIONS
1771 /* For sub-word operations, if target doesn't have them, start
1772 with precres widening right away, otherwise do it only
1773 if the most simple cases can't be used. */
1774 if (orig_precres
== precres
&& precres
< BITS_PER_WORD
)
1778 if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
&& prec1
<= precres
)
1779 || ((!uns0_p
|| !uns1_p
) && !unsr_p
1780 && prec0
+ uns0_p
<= precres
1781 && prec1
+ uns1_p
<= precres
))
1783 arg0
= fold_convert_loc (loc
, type
, arg0
);
1784 arg1
= fold_convert_loc (loc
, type
, arg1
);
1788 if (integer_zerop (arg0
) && !unsr_p
)
1789 expand_neg_overflow (loc
, lhs
, arg1
, false);
1792 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
,
1793 unsr_p
, unsr_p
, unsr_p
, false);
1796 expand_mul_overflow (loc
, lhs
, arg0
, arg1
,
1797 unsr_p
, unsr_p
, unsr_p
, false);
1804 /* For sub-word operations, retry with a wider type first. */
1805 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
1807 #ifdef WORD_REGISTER_OPERATIONS
1808 int p
= BITS_PER_WORD
;
1812 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1813 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1816 p
= TYPE_PRECISION (optype
);
1820 unsr_p
= TYPE_UNSIGNED (optype
);
1826 if (prec0
<= precres
&& prec1
<= precres
)
1831 types
[0] = build_nonstandard_integer_type (precres
, 0);
1837 types
[1] = build_nonstandard_integer_type (precres
, 1);
1839 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
1840 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
1841 if (code
!= MULT_EXPR
)
1842 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
1843 uns0_p
, uns1_p
, false);
1845 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
1846 uns0_p
, uns1_p
, false);
1850 /* Retry with a wider type. */
1851 if (orig_precres
== precres
)
1853 int p
= MAX (prec0
, prec1
);
1854 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1855 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1858 p
= TYPE_PRECISION (optype
);
1862 unsr_p
= TYPE_UNSIGNED (optype
);
1873 /* Expand ADD_OVERFLOW STMT. */
1876 expand_ADD_OVERFLOW (gcall
*stmt
)
1878 expand_arith_overflow (PLUS_EXPR
, stmt
);
1881 /* Expand SUB_OVERFLOW STMT. */
1884 expand_SUB_OVERFLOW (gcall
*stmt
)
1886 expand_arith_overflow (MINUS_EXPR
, stmt
);
1889 /* Expand MUL_OVERFLOW STMT. */
1892 expand_MUL_OVERFLOW (gcall
*stmt
)
1894 expand_arith_overflow (MULT_EXPR
, stmt
);
1897 /* This should get folded in tree-vectorizer.c. */
1900 expand_LOOP_VECTORIZED (gcall
*)
1906 expand_MASK_LOAD (gcall
*stmt
)
1908 struct expand_operand ops
[3];
1909 tree type
, lhs
, rhs
, maskt
;
1910 rtx mem
, target
, mask
;
1912 maskt
= gimple_call_arg (stmt
, 2);
1913 lhs
= gimple_call_lhs (stmt
);
1914 if (lhs
== NULL_TREE
)
1916 type
= TREE_TYPE (lhs
);
1917 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
1918 gimple_call_arg (stmt
, 1));
1920 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1921 gcc_assert (MEM_P (mem
));
1922 mask
= expand_normal (maskt
);
1923 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1924 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
1925 create_fixed_operand (&ops
[1], mem
);
1926 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1927 expand_insn (optab_handler (maskload_optab
, TYPE_MODE (type
)), 3, ops
);
1931 expand_MASK_STORE (gcall
*stmt
)
1933 struct expand_operand ops
[3];
1934 tree type
, lhs
, rhs
, maskt
;
1937 maskt
= gimple_call_arg (stmt
, 2);
1938 rhs
= gimple_call_arg (stmt
, 3);
1939 type
= TREE_TYPE (rhs
);
1940 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
1941 gimple_call_arg (stmt
, 1));
1943 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1944 gcc_assert (MEM_P (mem
));
1945 mask
= expand_normal (maskt
);
1946 reg
= expand_normal (rhs
);
1947 create_fixed_operand (&ops
[0], mem
);
1948 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
1949 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1950 expand_insn (optab_handler (maskstore_optab
, TYPE_MODE (type
)), 3, ops
);
1954 expand_ABNORMAL_DISPATCHER (gcall
*)
1959 expand_BUILTIN_EXPECT (gcall
*stmt
)
1961 /* When guessing was done, the hints should be already stripped away. */
1962 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
1965 tree lhs
= gimple_call_lhs (stmt
);
1967 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1969 target
= const0_rtx
;
1970 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
1971 if (lhs
&& val
!= target
)
1972 emit_move_insn (target
, val
);
1975 /* Routines to expand each internal function, indexed by function number.
1976 Each routine has the prototype:
1978 expand_<NAME> (gcall *stmt)
1980 where STMT is the statement that performs the call. */
1981 static void (*const internal_fn_expanders
[]) (gcall
*) = {
1982 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
1983 #include "internal-fn.def"
1984 #undef DEF_INTERNAL_FN
1988 /* Expand STMT, which is a call to internal function FN. */
1991 expand_internal_call (gcall
*stmt
)
1993 internal_fn_expanders
[(int) gimple_call_internal_fn (stmt
)] (stmt
);