2 Copyright (C) 2011-2016 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
46 /* The names of each internal function, indexed by function number. */
47 const char *const internal_fn_name_array
[] = {
48 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
49 #include "internal-fn.def"
53 /* The ECF_* flags of each internal function, indexed by function number. */
54 const int internal_fn_flags_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
56 #include "internal-fn.def"
60 /* Fnspec of each internal function, indexed by function number. */
61 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
66 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
67 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
68 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
69 #include "internal-fn.def"
70 internal_fn_fnspec_array
[IFN_LAST
] = 0;
73 /* Create static initializers for the information returned by
74 direct_internal_fn. */
75 #define not_direct { -2, -2, false }
76 #define mask_load_direct { -1, 2, false }
77 #define load_lanes_direct { -1, -1, false }
78 #define mask_store_direct { 3, 2, false }
79 #define store_lanes_direct { 0, 0, false }
80 #define unary_direct { 0, 0, true }
81 #define binary_direct { 0, 0, true }
83 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
84 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
85 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
86 #include "internal-fn.def"
90 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
91 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
94 get_multi_vector_move (tree array_type
, convert_optab optab
)
99 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
100 imode
= TYPE_MODE (array_type
);
101 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
103 return convert_optab_handler (optab
, imode
, vmode
);
106 /* Expand LOAD_LANES call STMT using optab OPTAB. */
109 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
111 struct expand_operand ops
[2];
115 lhs
= gimple_call_lhs (stmt
);
116 rhs
= gimple_call_arg (stmt
, 0);
117 type
= TREE_TYPE (lhs
);
119 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
120 mem
= expand_normal (rhs
);
122 gcc_assert (MEM_P (mem
));
123 PUT_MODE (mem
, TYPE_MODE (type
));
125 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
126 create_fixed_operand (&ops
[1], mem
);
127 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
130 /* Expand STORE_LANES call STMT using optab OPTAB. */
133 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
135 struct expand_operand ops
[2];
139 lhs
= gimple_call_lhs (stmt
);
140 rhs
= gimple_call_arg (stmt
, 0);
141 type
= TREE_TYPE (rhs
);
143 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
144 reg
= expand_normal (rhs
);
146 gcc_assert (MEM_P (target
));
147 PUT_MODE (target
, TYPE_MODE (type
));
149 create_fixed_operand (&ops
[0], target
);
150 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
151 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
155 expand_ANNOTATE (internal_fn
, gcall
*)
160 /* This should get expanded in adjust_simduid_builtins. */
163 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
168 /* This should get expanded in adjust_simduid_builtins. */
171 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
176 /* This should get expanded in adjust_simduid_builtins. */
179 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
184 /* This should get expanded in adjust_simduid_builtins. */
187 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
192 /* This should get expanded in adjust_simduid_builtins. */
195 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
200 /* This should get expanded in the sanopt pass. */
203 expand_UBSAN_NULL (internal_fn
, gcall
*)
208 /* This should get expanded in the sanopt pass. */
211 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
216 /* This should get expanded in the sanopt pass. */
219 expand_UBSAN_VPTR (internal_fn
, gcall
*)
224 /* This should get expanded in the sanopt pass. */
227 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
232 /* This should get expanded in the sanopt pass. */
235 expand_ASAN_CHECK (internal_fn
, gcall
*)
240 /* This should get expanded in the tsan pass. */
243 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
248 /* This should get expanded in the lower pass. */
251 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
253 error_at (gimple_location (call
),
254 "invalid use of attribute %<fallthrough%>");
257 /* Helper function for expand_addsub_overflow. Return 1
258 if ARG interpreted as signed in its precision is known to be always
259 positive or 2 if ARG is known to be always negative, or 3 if ARG may
260 be positive or negative. */
263 get_range_pos_neg (tree arg
)
265 if (arg
== error_mark_node
)
268 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
270 if (TREE_CODE (arg
) == INTEGER_CST
)
272 wide_int w
= wi::sext (arg
, prec
);
278 while (CONVERT_EXPR_P (arg
)
279 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
280 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
282 arg
= TREE_OPERAND (arg
, 0);
283 /* Narrower value zero extended into wider type
284 will always result in positive values. */
285 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
286 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
288 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
293 if (TREE_CODE (arg
) != SSA_NAME
)
295 wide_int arg_min
, arg_max
;
296 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
298 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
299 if (is_gimple_assign (g
)
300 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
302 tree t
= gimple_assign_rhs1 (g
);
303 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
304 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
306 if (TYPE_UNSIGNED (TREE_TYPE (t
))
307 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
309 prec
= TYPE_PRECISION (TREE_TYPE (t
));
318 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
320 /* For unsigned values, the "positive" range comes
321 below the "negative" range. */
322 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
324 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
329 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
331 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
337 /* Return minimum precision needed to represent all values
338 of ARG in SIGNed integral type. */
341 get_min_precision (tree arg
, signop sign
)
343 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
345 signop orig_sign
= sign
;
346 if (TREE_CODE (arg
) == INTEGER_CST
)
349 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
351 widest_int w
= wi::to_widest (arg
);
352 w
= wi::ext (w
, prec
, sign
);
353 p
= wi::min_precision (w
, sign
);
356 p
= wi::min_precision (arg
, sign
);
357 return MIN (p
, prec
);
359 while (CONVERT_EXPR_P (arg
)
360 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
361 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
363 arg
= TREE_OPERAND (arg
, 0);
364 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
366 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
368 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
369 return prec
+ (orig_sign
!= sign
);
370 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
373 return prec
+ (orig_sign
!= sign
);
375 if (TREE_CODE (arg
) != SSA_NAME
)
376 return prec
+ (orig_sign
!= sign
);
377 wide_int arg_min
, arg_max
;
378 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
380 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
381 if (is_gimple_assign (g
)
382 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
384 tree t
= gimple_assign_rhs1 (g
);
385 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
386 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
389 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
391 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
393 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
394 return prec
+ (orig_sign
!= sign
);
395 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
398 return prec
+ (orig_sign
!= sign
);
402 return prec
+ (orig_sign
!= sign
);
404 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
406 int p1
= wi::min_precision (arg_min
, sign
);
407 int p2
= wi::min_precision (arg_max
, sign
);
409 prec
= MIN (prec
, p1
);
411 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
413 int p
= wi::min_precision (arg_max
, UNSIGNED
);
414 prec
= MIN (prec
, p
);
416 return prec
+ (orig_sign
!= sign
);
419 /* Helper for expand_*_overflow. Set the __imag__ part to true
420 (1 except for signed:1 type, in which case store -1). */
423 expand_arith_set_overflow (tree lhs
, rtx target
)
425 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
426 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
427 write_complex_part (target
, constm1_rtx
, true);
429 write_complex_part (target
, const1_rtx
, true);
432 /* Helper for expand_*_overflow. Store RES into the __real__ part
433 of TARGET. If RES has larger MODE than __real__ part of TARGET,
434 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
435 if LHS has smaller precision than its mode. */
438 expand_arith_overflow_result_store (tree lhs
, rtx target
,
439 machine_mode mode
, rtx res
)
441 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
445 rtx_code_label
*done_label
= gen_label_rtx ();
446 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
447 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
448 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
449 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
450 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
452 expand_arith_set_overflow (lhs
, target
);
453 emit_label (done_label
);
455 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
456 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
459 rtx_code_label
*done_label
= gen_label_rtx ();
460 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
465 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
467 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
468 true, OPTAB_LIB_WIDEN
);
472 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
474 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
477 do_compare_rtx_and_jump (res
, lres
,
478 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
480 expand_arith_set_overflow (lhs
, target
);
481 emit_label (done_label
);
483 write_complex_part (target
, lres
, false);
486 /* Helper for expand_*_overflow. Store RES into TARGET. */
489 expand_ubsan_result_store (rtx target
, rtx res
)
491 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
492 /* If this is a scalar in a register that is stored in a wider mode
493 than the declared mode, compute the result into its declared mode
494 and then convert to the wider mode. Our value is the computed
496 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
498 emit_move_insn (target
, res
);
501 /* Add sub/add overflow checking to the statement STMT.
502 CODE says whether the operation is +, or -. */
505 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
506 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
507 bool uns1_p
, bool is_ubsan
)
509 rtx res
, target
= NULL_RTX
;
511 rtx_code_label
*done_label
= gen_label_rtx ();
512 rtx_code_label
*do_error
= gen_label_rtx ();
513 do_pending_stack_adjust ();
514 rtx op0
= expand_normal (arg0
);
515 rtx op1
= expand_normal (arg1
);
516 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
517 int prec
= GET_MODE_PRECISION (mode
);
518 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
522 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
526 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
528 write_complex_part (target
, const0_rtx
, true);
531 /* We assume both operands and result have the same precision
532 here (GET_MODE_BITSIZE (mode)), S stands for signed type
533 with that precision, U for unsigned type with that precision,
534 sgn for unsigned most significant bit in that precision.
535 s1 is signed first operand, u1 is unsigned first operand,
536 s2 is signed second operand, u2 is unsigned second operand,
537 sr is signed result, ur is unsigned result and the following
538 rules say how to compute result (which is always result of
539 the operands as if both were unsigned, cast to the right
540 signedness) and how to compute whether operation overflowed.
543 res = (S) ((U) s1 + (U) s2)
544 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
546 res = (S) ((U) s1 - (U) s2)
547 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
550 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
553 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
555 res = (S) ((U) s1 + u2)
556 ovf = ((U) res ^ sgn) < u2
561 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
563 res = (S) ((U) s1 - u2)
564 ovf = u2 > ((U) s1 ^ sgn)
567 ovf = s1 < 0 || u2 > (U) s1
570 ovf = u1 >= ((U) s2 ^ sgn)
575 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
577 res = (U) s1 + (U) s2
578 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
581 ovf = (U) res < u2 || res < 0
584 ovf = u1 >= u2 ? res < 0 : res >= 0
586 res = (U) s1 - (U) s2
587 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
589 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
591 /* PLUS_EXPR is commutative, if operand signedness differs,
592 canonicalize to the first operand being signed and second
593 unsigned to simplify following code. */
594 std::swap (op0
, op1
);
595 std::swap (arg0
, arg1
);
601 if (uns0_p
&& uns1_p
&& unsr_p
)
603 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
604 : usubv4_optab
, mode
);
605 if (icode
!= CODE_FOR_nothing
)
607 struct expand_operand ops
[4];
608 rtx_insn
*last
= get_last_insn ();
610 res
= gen_reg_rtx (mode
);
611 create_output_operand (&ops
[0], res
, mode
);
612 create_input_operand (&ops
[1], op0
, mode
);
613 create_input_operand (&ops
[2], op1
, mode
);
614 create_fixed_operand (&ops
[3], do_error
);
615 if (maybe_expand_insn (icode
, 4, ops
))
617 last
= get_last_insn ();
618 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
620 && any_condjump_p (last
)
621 && !find_reg_note (last
, REG_BR_PROB
, 0))
622 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
623 emit_jump (done_label
);
627 delete_insns_since (last
);
630 /* Compute the operation. On RTL level, the addition is always
632 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
633 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
635 /* For PLUS_EXPR, the operation is commutative, so we can pick
636 operand to compare against. For prec <= BITS_PER_WORD, I think
637 preferring REG operand is better over CONST_INT, because
638 the CONST_INT might enlarge the instruction or CSE would need
639 to figure out we'd already loaded it into a register before.
640 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
641 as then the multi-word comparison can be perhaps simplified. */
642 if (code
== PLUS_EXPR
643 && (prec
<= BITS_PER_WORD
644 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
645 : CONST_SCALAR_INT_P (op1
)))
647 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
648 true, mode
, NULL_RTX
, NULL
, done_label
,
654 if (!uns0_p
&& uns1_p
&& !unsr_p
)
656 /* Compute the operation. On RTL level, the addition is always
658 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
659 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
660 rtx tem
= expand_binop (mode
, add_optab
,
661 code
== PLUS_EXPR
? res
: op0
, sgn
,
662 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
663 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
664 done_label
, PROB_VERY_LIKELY
);
669 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
671 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
673 /* As we've changed op1, we have to avoid using the value range
674 for the original argument. */
675 arg1
= error_mark_node
;
681 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
683 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
685 /* As we've changed op0, we have to avoid using the value range
686 for the original argument. */
687 arg0
= error_mark_node
;
693 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
695 /* Compute the operation. On RTL level, the addition is always
697 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
699 int pos_neg
= get_range_pos_neg (arg0
);
701 /* If ARG0 is known to be always negative, this is always overflow. */
702 emit_jump (do_error
);
703 else if (pos_neg
== 3)
704 /* If ARG0 is not known to be always positive, check at runtime. */
705 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
706 NULL
, do_error
, PROB_VERY_UNLIKELY
);
707 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
708 done_label
, PROB_VERY_LIKELY
);
713 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
715 /* Compute the operation. On RTL level, the addition is always
717 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
719 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
721 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
722 done_label
, PROB_VERY_LIKELY
);
727 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
729 /* Compute the operation. On RTL level, the addition is always
731 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
733 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
734 NULL
, do_error
, PROB_VERY_UNLIKELY
);
736 /* The operation is commutative, so we can pick operand to compare
737 against. For prec <= BITS_PER_WORD, I think preferring REG operand
738 is better over CONST_INT, because the CONST_INT might enlarge the
739 instruction or CSE would need to figure out we'd already loaded it
740 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
741 might be more beneficial, as then the multi-word comparison can be
742 perhaps simplified. */
743 if (prec
<= BITS_PER_WORD
744 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
745 : CONST_SCALAR_INT_P (op0
))
747 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
748 done_label
, PROB_VERY_LIKELY
);
753 if (!uns0_p
&& !uns1_p
&& unsr_p
)
755 /* Compute the operation. On RTL level, the addition is always
757 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
758 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
759 int pos_neg
= get_range_pos_neg (arg1
);
760 if (code
== PLUS_EXPR
)
762 int pos_neg0
= get_range_pos_neg (arg0
);
763 if (pos_neg0
!= 3 && pos_neg
== 3)
765 std::swap (op0
, op1
);
772 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
773 ? and_optab
: ior_optab
,
774 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
775 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
776 NULL
, done_label
, PROB_VERY_LIKELY
);
780 rtx_code_label
*do_ior_label
= gen_label_rtx ();
781 do_compare_rtx_and_jump (op1
, const0_rtx
,
782 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
783 NULL_RTX
, NULL
, do_ior_label
,
785 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
787 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
788 NULL
, done_label
, PROB_VERY_LIKELY
);
789 emit_jump (do_error
);
790 emit_label (do_ior_label
);
791 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
793 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
794 NULL
, done_label
, PROB_VERY_LIKELY
);
800 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
802 /* Compute the operation. On RTL level, the addition is always
804 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
806 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
807 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
808 op0_geu_op1
, PROB_EVEN
);
809 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
810 NULL
, done_label
, PROB_VERY_LIKELY
);
811 emit_jump (do_error
);
812 emit_label (op0_geu_op1
);
813 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
814 NULL
, done_label
, PROB_VERY_LIKELY
);
818 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
823 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
824 : subv4_optab
, mode
);
825 if (icode
!= CODE_FOR_nothing
)
827 struct expand_operand ops
[4];
828 rtx_insn
*last
= get_last_insn ();
830 res
= gen_reg_rtx (mode
);
831 create_output_operand (&ops
[0], res
, mode
);
832 create_input_operand (&ops
[1], op0
, mode
);
833 create_input_operand (&ops
[2], op1
, mode
);
834 create_fixed_operand (&ops
[3], do_error
);
835 if (maybe_expand_insn (icode
, 4, ops
))
837 last
= get_last_insn ();
838 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
840 && any_condjump_p (last
)
841 && !find_reg_note (last
, REG_BR_PROB
, 0))
842 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
843 emit_jump (done_label
);
847 delete_insns_since (last
);
850 /* Compute the operation. On RTL level, the addition is always
852 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
853 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
855 /* If we can prove that one of the arguments (for MINUS_EXPR only
856 the second operand, as subtraction is not commutative) is always
857 non-negative or always negative, we can do just one comparison
858 and conditional jump. */
859 int pos_neg
= get_range_pos_neg (arg1
);
860 if (code
== PLUS_EXPR
)
862 int pos_neg0
= get_range_pos_neg (arg0
);
863 if (pos_neg0
!= 3 && pos_neg
== 3)
865 std::swap (op0
, op1
);
870 /* Addition overflows if and only if the two operands have the same sign,
871 and the result has the opposite sign. Subtraction overflows if and
872 only if the two operands have opposite sign, and the subtrahend has
873 the same sign as the result. Here 0 is counted as positive. */
876 /* Compute op0 ^ op1 (operands have opposite sign). */
877 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
880 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
881 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
885 if (code
== PLUS_EXPR
)
887 /* Compute (res ^ op1) & ~(op0 ^ op1). */
888 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
889 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
894 /* Compute (op0 ^ op1) & ~(res ^ op1). */
895 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
896 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
900 /* No overflow if the result has bit sign cleared. */
901 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
902 NULL
, done_label
, PROB_VERY_LIKELY
);
905 /* Compare the result of the operation with the first operand.
906 No overflow for addition if second operand is positive and result
907 is larger or second operand is negative and result is smaller.
908 Likewise for subtraction with sign of second operand flipped. */
910 do_compare_rtx_and_jump (res
, op0
,
911 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
912 false, mode
, NULL_RTX
, NULL
, done_label
,
917 emit_label (do_error
);
920 /* Expand the ubsan builtin call. */
922 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
926 do_pending_stack_adjust ();
929 expand_arith_set_overflow (lhs
, target
);
932 emit_label (done_label
);
937 expand_ubsan_result_store (target
, res
);
941 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
944 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
949 /* Add negate overflow checking to the statement STMT. */
952 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
)
956 rtx_code_label
*done_label
, *do_error
;
957 rtx target
= NULL_RTX
;
959 done_label
= gen_label_rtx ();
960 do_error
= gen_label_rtx ();
962 do_pending_stack_adjust ();
963 op1
= expand_normal (arg1
);
965 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
968 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
970 write_complex_part (target
, const0_rtx
, true);
973 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
974 if (icode
!= CODE_FOR_nothing
)
976 struct expand_operand ops
[3];
977 rtx_insn
*last
= get_last_insn ();
979 res
= gen_reg_rtx (mode
);
980 create_output_operand (&ops
[0], res
, mode
);
981 create_input_operand (&ops
[1], op1
, mode
);
982 create_fixed_operand (&ops
[2], do_error
);
983 if (maybe_expand_insn (icode
, 3, ops
))
985 last
= get_last_insn ();
986 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
988 && any_condjump_p (last
)
989 && !find_reg_note (last
, REG_BR_PROB
, 0))
990 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
991 emit_jump (done_label
);
995 delete_insns_since (last
);
996 icode
= CODE_FOR_nothing
;
1000 if (icode
== CODE_FOR_nothing
)
1002 /* Compute the operation. On RTL level, the addition is always
1004 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1006 /* Compare the operand with the most negative value. */
1007 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1008 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1009 done_label
, PROB_VERY_LIKELY
);
1012 emit_label (do_error
);
1015 /* Expand the ubsan builtin call. */
1017 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1021 do_pending_stack_adjust ();
1024 expand_arith_set_overflow (lhs
, target
);
1027 emit_label (done_label
);
1032 expand_ubsan_result_store (target
, res
);
1034 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1038 /* Add mul overflow checking to the statement STMT. */
1041 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1042 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
)
1046 rtx_code_label
*done_label
, *do_error
;
1047 rtx target
= NULL_RTX
;
1049 enum insn_code icode
;
1051 done_label
= gen_label_rtx ();
1052 do_error
= gen_label_rtx ();
1054 do_pending_stack_adjust ();
1055 op0
= expand_normal (arg0
);
1056 op1
= expand_normal (arg1
);
1058 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
1062 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1064 write_complex_part (target
, const0_rtx
, true);
1068 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1070 /* We assume both operands and result have the same precision
1071 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1072 with that precision, U for unsigned type with that precision,
1073 sgn for unsigned most significant bit in that precision.
1074 s1 is signed first operand, u1 is unsigned first operand,
1075 s2 is signed second operand, u2 is unsigned second operand,
1076 sr is signed result, ur is unsigned result and the following
1077 rules say how to compute result (which is always result of
1078 the operands as if both were unsigned, cast to the right
1079 signedness) and how to compute whether operation overflowed.
1080 main_ovf (false) stands for jump on signed multiplication
1081 overflow or the main algorithm with uns == false.
1082 main_ovf (true) stands for jump on unsigned multiplication
1083 overflow or the main algorithm with uns == true.
1086 res = (S) ((U) s1 * (U) s2)
1087 ovf = main_ovf (false)
1090 ovf = main_ovf (true)
1093 ovf = (s1 < 0 && u2) || main_ovf (true)
1096 ovf = res < 0 || main_ovf (true)
1098 res = (S) ((U) s1 * u2)
1099 ovf = (S) u2 >= 0 ? main_ovf (false)
1100 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1102 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1103 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1105 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1107 if (uns0_p
&& !uns1_p
)
1109 /* Multiplication is commutative, if operand signedness differs,
1110 canonicalize to the first operand being signed and second
1111 unsigned to simplify following code. */
1112 std::swap (op0
, op1
);
1113 std::swap (arg0
, arg1
);
1118 int pos_neg0
= get_range_pos_neg (arg0
);
1119 int pos_neg1
= get_range_pos_neg (arg1
);
1122 if (!uns0_p
&& uns1_p
&& unsr_p
)
1127 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1130 /* If s1 is negative, avoid the main code, just multiply and
1131 signal overflow if op1 is not 0. */
1132 struct separate_ops ops
;
1133 ops
.code
= MULT_EXPR
;
1134 ops
.type
= TREE_TYPE (arg1
);
1135 ops
.op0
= make_tree (ops
.type
, op0
);
1136 ops
.op1
= make_tree (ops
.type
, op1
);
1137 ops
.op2
= NULL_TREE
;
1139 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1140 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1141 NULL
, done_label
, PROB_VERY_LIKELY
);
1142 goto do_error_label
;
1144 rtx_code_label
*do_main_label
;
1145 do_main_label
= gen_label_rtx ();
1146 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1147 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1148 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1149 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1150 expand_arith_set_overflow (lhs
, target
);
1151 emit_label (do_main_label
);
1159 if (uns0_p
&& uns1_p
&& !unsr_p
)
1162 /* Rest of handling of this case after res is computed. */
1167 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1174 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1175 avoid the main code, just multiply and signal overflow
1176 unless 0 * u2 or -1 * ((U) Smin). */
1177 struct separate_ops ops
;
1178 ops
.code
= MULT_EXPR
;
1179 ops
.type
= TREE_TYPE (arg1
);
1180 ops
.op0
= make_tree (ops
.type
, op0
);
1181 ops
.op1
= make_tree (ops
.type
, op1
);
1182 ops
.op2
= NULL_TREE
;
1184 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1185 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1186 NULL
, done_label
, PROB_VERY_LIKELY
);
1187 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1188 NULL
, do_error
, PROB_VERY_UNLIKELY
);
1190 prec
= GET_MODE_PRECISION (mode
);
1192 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1193 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1194 NULL
, done_label
, PROB_VERY_LIKELY
);
1195 goto do_error_label
;
1197 /* Rest of handling of this case after res is computed. */
1205 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1208 switch (pos_neg0
| pos_neg1
)
1210 case 1: /* Both operands known to be non-negative. */
1212 case 2: /* Both operands known to be negative. */
1213 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1214 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1215 /* Avoid looking at arg0/arg1 ranges, as we've changed
1217 arg0
= error_mark_node
;
1218 arg1
= error_mark_node
;
1221 if ((pos_neg0
^ pos_neg1
) == 3)
1223 /* If one operand is known to be negative and the other
1224 non-negative, this overflows always, unless the non-negative
1225 one is 0. Just do normal multiply and set overflow
1226 unless one of the operands is 0. */
1227 struct separate_ops ops
;
1228 ops
.code
= MULT_EXPR
;
1230 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1232 ops
.op0
= make_tree (ops
.type
, op0
);
1233 ops
.op1
= make_tree (ops
.type
, op1
);
1234 ops
.op2
= NULL_TREE
;
1236 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1237 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1239 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1240 NULL_RTX
, NULL
, done_label
,
1242 goto do_error_label
;
1244 /* The general case, do all the needed comparisons at runtime. */
1245 rtx_code_label
*do_main_label
, *after_negate_label
;
1247 rop0
= gen_reg_rtx (mode
);
1248 rop1
= gen_reg_rtx (mode
);
1249 emit_move_insn (rop0
, op0
);
1250 emit_move_insn (rop1
, op1
);
1253 do_main_label
= gen_label_rtx ();
1254 after_negate_label
= gen_label_rtx ();
1255 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1257 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1258 NULL
, after_negate_label
, PROB_VERY_LIKELY
);
1259 /* Both arguments negative here, negate them and continue with
1260 normal unsigned overflow checking multiplication. */
1261 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1263 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1265 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1267 arg0
= error_mark_node
;
1268 arg1
= error_mark_node
;
1269 emit_jump (do_main_label
);
1270 emit_label (after_negate_label
);
1271 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1273 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1274 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1275 /* One argument is negative here, the other positive. This
1276 overflows always, unless one of the arguments is 0. But
1277 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1278 is, thus we can keep do_main code oring in overflow as is. */
1279 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1280 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1281 expand_arith_set_overflow (lhs
, target
);
1282 emit_label (do_main_label
);
1290 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1291 sign
= uns
? UNSIGNED
: SIGNED
;
1292 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1293 if (icode
!= CODE_FOR_nothing
)
1295 struct expand_operand ops
[4];
1296 rtx_insn
*last
= get_last_insn ();
1298 res
= gen_reg_rtx (mode
);
1299 create_output_operand (&ops
[0], res
, mode
);
1300 create_input_operand (&ops
[1], op0
, mode
);
1301 create_input_operand (&ops
[2], op1
, mode
);
1302 create_fixed_operand (&ops
[3], do_error
);
1303 if (maybe_expand_insn (icode
, 4, ops
))
1305 last
= get_last_insn ();
1306 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1308 && any_condjump_p (last
)
1309 && !find_reg_note (last
, REG_BR_PROB
, 0))
1310 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1311 emit_jump (done_label
);
1315 delete_insns_since (last
);
1316 icode
= CODE_FOR_nothing
;
1320 if (icode
== CODE_FOR_nothing
)
1322 struct separate_ops ops
;
1323 int prec
= GET_MODE_PRECISION (mode
);
1324 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1325 ops
.op0
= make_tree (type
, op0
);
1326 ops
.op1
= make_tree (type
, op1
);
1327 ops
.op2
= NULL_TREE
;
1329 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1330 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1332 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1333 ops
.code
= WIDEN_MULT_EXPR
;
1335 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1337 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1338 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1340 hipart
= gen_lowpart (mode
, hipart
);
1341 res
= gen_lowpart (mode
, res
);
1343 /* For the unsigned multiplication, there was overflow if
1344 HIPART is non-zero. */
1345 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1346 NULL_RTX
, NULL
, done_label
,
1350 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1352 /* RES is low half of the double width result, HIPART
1353 the high half. There was overflow if
1354 HIPART is different from RES < 0 ? -1 : 0. */
1355 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1356 NULL_RTX
, NULL
, done_label
,
1360 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1362 rtx_code_label
*large_op0
= gen_label_rtx ();
1363 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1364 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1365 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1366 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1367 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1368 rtx_code_label
*do_overflow
= gen_label_rtx ();
1369 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1371 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1372 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1374 hipart0
= gen_lowpart (hmode
, hipart0
);
1375 rtx lopart0
= gen_lowpart (hmode
, op0
);
1376 rtx signbit0
= const0_rtx
;
1378 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1380 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1382 hipart1
= gen_lowpart (hmode
, hipart1
);
1383 rtx lopart1
= gen_lowpart (hmode
, op1
);
1384 rtx signbit1
= const0_rtx
;
1386 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1389 res
= gen_reg_rtx (mode
);
1391 /* True if op0 resp. op1 are known to be in the range of
1393 bool op0_small_p
= false;
1394 bool op1_small_p
= false;
1395 /* True if op0 resp. op1 are known to have all zeros or all ones
1396 in the upper half of bits, but are not known to be
1398 bool op0_medium_p
= false;
1399 bool op1_medium_p
= false;
1400 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1401 nonnegative, 1 if unknown. */
1407 else if (pos_neg0
== 2)
1411 else if (pos_neg1
== 2)
1414 unsigned int mprec0
= prec
;
1415 if (arg0
!= error_mark_node
)
1416 mprec0
= get_min_precision (arg0
, sign
);
1417 if (mprec0
<= hprec
)
1419 else if (!uns
&& mprec0
<= hprec
+ 1)
1420 op0_medium_p
= true;
1421 unsigned int mprec1
= prec
;
1422 if (arg1
!= error_mark_node
)
1423 mprec1
= get_min_precision (arg1
, sign
);
1424 if (mprec1
<= hprec
)
1426 else if (!uns
&& mprec1
<= hprec
+ 1)
1427 op1_medium_p
= true;
1429 int smaller_sign
= 1;
1430 int larger_sign
= 1;
1433 smaller_sign
= op0_sign
;
1434 larger_sign
= op1_sign
;
1436 else if (op1_small_p
)
1438 smaller_sign
= op1_sign
;
1439 larger_sign
= op0_sign
;
1441 else if (op0_sign
== op1_sign
)
1443 smaller_sign
= op0_sign
;
1444 larger_sign
= op0_sign
;
1448 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1449 NULL_RTX
, NULL
, large_op0
,
1453 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1454 NULL_RTX
, NULL
, small_op0_large_op1
,
1457 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1458 hmode to mode, the multiplication will never overflow. We can
1459 do just one hmode x hmode => mode widening multiplication. */
1460 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1461 if (GET_CODE (lopart0
) == SUBREG
)
1463 lopart0s
= shallow_copy_rtx (lopart0
);
1464 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1465 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1467 if (GET_CODE (lopart1
) == SUBREG
)
1469 lopart1s
= shallow_copy_rtx (lopart1
);
1470 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1471 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1473 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1474 ops
.op0
= make_tree (halfstype
, lopart0s
);
1475 ops
.op1
= make_tree (halfstype
, lopart1s
);
1476 ops
.code
= WIDEN_MULT_EXPR
;
1479 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1480 emit_move_insn (res
, thisres
);
1481 emit_jump (done_label
);
1483 emit_label (small_op0_large_op1
);
1485 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1486 but op1 is not, just swap the arguments and handle it as op1
1487 sign/zero extended, op0 not. */
1488 rtx larger
= gen_reg_rtx (mode
);
1489 rtx hipart
= gen_reg_rtx (hmode
);
1490 rtx lopart
= gen_reg_rtx (hmode
);
1491 emit_move_insn (larger
, op1
);
1492 emit_move_insn (hipart
, hipart1
);
1493 emit_move_insn (lopart
, lopart0
);
1494 emit_jump (one_small_one_large
);
1496 emit_label (large_op0
);
1499 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1500 NULL_RTX
, NULL
, both_ops_large
,
1503 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1504 but op0 is not, prepare larger, hipart and lopart pseudos and
1505 handle it together with small_op0_large_op1. */
1506 emit_move_insn (larger
, op0
);
1507 emit_move_insn (hipart
, hipart0
);
1508 emit_move_insn (lopart
, lopart1
);
1510 emit_label (one_small_one_large
);
1512 /* lopart is the low part of the operand that is sign extended
1513 to mode, larger is the other operand, hipart is the
1514 high part of larger and lopart0 and lopart1 are the low parts
1516 We perform lopart0 * lopart1 and lopart * hipart widening
1518 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1519 ops
.op0
= make_tree (halfutype
, lopart0
);
1520 ops
.op1
= make_tree (halfutype
, lopart1
);
1522 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1524 ops
.op0
= make_tree (halfutype
, lopart
);
1525 ops
.op1
= make_tree (halfutype
, hipart
);
1526 rtx loxhi
= gen_reg_rtx (mode
);
1527 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1528 emit_move_insn (loxhi
, tem
);
1532 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1533 if (larger_sign
== 0)
1534 emit_jump (after_hipart_neg
);
1535 else if (larger_sign
!= -1)
1536 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1537 NULL_RTX
, NULL
, after_hipart_neg
,
1540 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1541 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1542 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1544 emit_move_insn (loxhi
, tem
);
1546 emit_label (after_hipart_neg
);
1548 /* if (lopart < 0) loxhi -= larger; */
1549 if (smaller_sign
== 0)
1550 emit_jump (after_lopart_neg
);
1551 else if (smaller_sign
!= -1)
1552 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1553 NULL_RTX
, NULL
, after_lopart_neg
,
1556 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1558 emit_move_insn (loxhi
, tem
);
1560 emit_label (after_lopart_neg
);
1563 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1564 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1565 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1567 emit_move_insn (loxhi
, tem
);
1569 /* if (loxhi >> (bitsize / 2)
1570 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1571 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1572 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1574 hipartloxhi
= gen_lowpart (hmode
, hipartloxhi
);
1575 rtx signbitloxhi
= const0_rtx
;
1577 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1578 gen_lowpart (hmode
, loxhi
),
1579 hprec
- 1, NULL_RTX
, 0);
1581 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1582 NULL_RTX
, NULL
, do_overflow
,
1583 PROB_VERY_UNLIKELY
);
1585 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1586 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1588 tem
= convert_modes (mode
, hmode
, gen_lowpart (hmode
, lo0xlo1
), 1);
1590 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1593 emit_move_insn (res
, tem
);
1594 emit_jump (done_label
);
1596 emit_label (both_ops_large
);
1598 /* If both operands are large (not sign (!uns) or zero (uns)
1599 extended from hmode), then perform the full multiplication
1600 which will be the result of the operation.
1601 The only cases which don't overflow are for signed multiplication
1602 some cases where both hipart0 and highpart1 are 0 or -1.
1603 For unsigned multiplication when high parts are both non-zero
1604 this overflows always. */
1605 ops
.code
= MULT_EXPR
;
1606 ops
.op0
= make_tree (type
, op0
);
1607 ops
.op1
= make_tree (type
, op1
);
1608 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1609 emit_move_insn (res
, tem
);
1615 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1616 NULL_RTX
, 1, OPTAB_DIRECT
);
1617 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1618 NULL_RTX
, NULL
, do_error
,
1619 PROB_VERY_UNLIKELY
);
1624 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1625 NULL_RTX
, 1, OPTAB_DIRECT
);
1626 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1627 NULL_RTX
, NULL
, do_error
,
1628 PROB_VERY_UNLIKELY
);
1631 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1632 the same, overflow happened if res is negative, if they are
1633 different, overflow happened if res is positive. */
1634 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1635 emit_jump (hipart_different
);
1636 else if (op0_sign
== 1 || op1_sign
== 1)
1637 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1638 NULL_RTX
, NULL
, hipart_different
,
1641 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1642 NULL_RTX
, NULL
, do_error
,
1643 PROB_VERY_UNLIKELY
);
1644 emit_jump (done_label
);
1646 emit_label (hipart_different
);
1648 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1649 NULL_RTX
, NULL
, do_error
,
1650 PROB_VERY_UNLIKELY
);
1651 emit_jump (done_label
);
1654 emit_label (do_overflow
);
1656 /* Overflow, do full multiplication and fallthru into do_error. */
1657 ops
.op0
= make_tree (type
, op0
);
1658 ops
.op1
= make_tree (type
, op1
);
1659 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1660 emit_move_insn (res
, tem
);
1664 gcc_assert (!is_ubsan
);
1665 ops
.code
= MULT_EXPR
;
1667 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1668 emit_jump (done_label
);
1673 emit_label (do_error
);
1676 /* Expand the ubsan builtin call. */
1678 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1682 do_pending_stack_adjust ();
1685 expand_arith_set_overflow (lhs
, target
);
1688 emit_label (done_label
);
1691 if (uns0_p
&& uns1_p
&& !unsr_p
)
1693 rtx_code_label
*all_done_label
= gen_label_rtx ();
1694 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1695 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1696 expand_arith_set_overflow (lhs
, target
);
1697 emit_label (all_done_label
);
1701 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1703 rtx_code_label
*all_done_label
= gen_label_rtx ();
1704 rtx_code_label
*set_noovf
= gen_label_rtx ();
1705 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1706 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1707 expand_arith_set_overflow (lhs
, target
);
1708 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1709 NULL
, set_noovf
, PROB_VERY_LIKELY
);
1710 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1711 NULL
, all_done_label
, PROB_VERY_UNLIKELY
);
1712 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1713 all_done_label
, PROB_VERY_UNLIKELY
);
1714 emit_label (set_noovf
);
1715 write_complex_part (target
, const0_rtx
, true);
1716 emit_label (all_done_label
);
1722 expand_ubsan_result_store (target
, res
);
1724 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1728 /* Expand UBSAN_CHECK_ADD call STMT. */
1731 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
1733 location_t loc
= gimple_location (stmt
);
1734 tree lhs
= gimple_call_lhs (stmt
);
1735 tree arg0
= gimple_call_arg (stmt
, 0);
1736 tree arg1
= gimple_call_arg (stmt
, 1);
1737 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
1738 false, false, false, true);
1741 /* Expand UBSAN_CHECK_SUB call STMT. */
1744 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
1746 location_t loc
= gimple_location (stmt
);
1747 tree lhs
= gimple_call_lhs (stmt
);
1748 tree arg0
= gimple_call_arg (stmt
, 0);
1749 tree arg1
= gimple_call_arg (stmt
, 1);
1750 if (integer_zerop (arg0
))
1751 expand_neg_overflow (loc
, lhs
, arg1
, true);
1753 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
1754 false, false, false, true);
1757 /* Expand UBSAN_CHECK_MUL call STMT. */
1760 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
1762 location_t loc
= gimple_location (stmt
);
1763 tree lhs
= gimple_call_lhs (stmt
);
1764 tree arg0
= gimple_call_arg (stmt
, 0);
1765 tree arg1
= gimple_call_arg (stmt
, 1);
1766 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true);
1769 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1772 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
1774 tree lhs
= gimple_call_lhs (stmt
);
1775 if (lhs
== NULL_TREE
)
1777 tree arg0
= gimple_call_arg (stmt
, 0);
1778 tree arg1
= gimple_call_arg (stmt
, 1);
1779 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
1780 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
1781 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
1782 int unsr_p
= TYPE_UNSIGNED (type
);
1783 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
1784 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
1785 int precres
= TYPE_PRECISION (type
);
1786 location_t loc
= gimple_location (stmt
);
1787 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
1789 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
1791 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
1792 prec0
= MIN (prec0
, pr
);
1793 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
1794 prec1
= MIN (prec1
, pr
);
1796 /* If uns0_p && uns1_p, precop is minimum needed precision
1797 of unsigned type to hold the exact result, otherwise
1798 precop is minimum needed precision of signed type to
1799 hold the exact result. */
1801 if (code
== MULT_EXPR
)
1802 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
1805 if (uns0_p
== uns1_p
)
1806 precop
= MAX (prec0
, prec1
) + 1;
1808 precop
= MAX (prec0
+ 1, prec1
) + 1;
1810 precop
= MAX (prec0
, prec1
+ 1) + 1;
1812 int orig_precres
= precres
;
1816 if ((uns0_p
&& uns1_p
)
1817 ? ((precop
+ !unsr_p
) <= precres
1818 /* u1 - u2 -> ur can overflow, no matter what precision
1820 && (code
!= MINUS_EXPR
|| !unsr_p
))
1821 : (!unsr_p
&& precop
<= precres
))
1823 /* The infinity precision result will always fit into result. */
1824 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1825 write_complex_part (target
, const0_rtx
, true);
1826 enum machine_mode mode
= TYPE_MODE (type
);
1827 struct separate_ops ops
;
1830 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
1831 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
1832 ops
.op2
= NULL_TREE
;
1834 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1835 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
1839 /* For operations with low precision, if target doesn't have them, start
1840 with precres widening right away, otherwise do it only if the most
1841 simple cases can't be used. */
1842 const int min_precision
= targetm
.min_arithmetic_precision ();
1843 if (orig_precres
== precres
&& precres
< min_precision
)
1845 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
1846 && prec1
<= precres
)
1847 || ((!uns0_p
|| !uns1_p
) && !unsr_p
1848 && prec0
+ uns0_p
<= precres
1849 && prec1
+ uns1_p
<= precres
))
1851 arg0
= fold_convert_loc (loc
, type
, arg0
);
1852 arg1
= fold_convert_loc (loc
, type
, arg1
);
1856 if (integer_zerop (arg0
) && !unsr_p
)
1858 expand_neg_overflow (loc
, lhs
, arg1
, false);
1863 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
,
1864 unsr_p
, unsr_p
, unsr_p
, false);
1867 expand_mul_overflow (loc
, lhs
, arg0
, arg1
,
1868 unsr_p
, unsr_p
, unsr_p
, false);
1875 /* For sub-word operations, retry with a wider type first. */
1876 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
1878 int p
= MAX (min_precision
, precop
);
1879 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1880 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1883 p
= TYPE_PRECISION (optype
);
1887 unsr_p
= TYPE_UNSIGNED (optype
);
1893 if (prec0
<= precres
&& prec1
<= precres
)
1898 types
[0] = build_nonstandard_integer_type (precres
, 0);
1904 types
[1] = build_nonstandard_integer_type (precres
, 1);
1906 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
1907 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
1908 if (code
!= MULT_EXPR
)
1909 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
1910 uns0_p
, uns1_p
, false);
1912 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
1913 uns0_p
, uns1_p
, false);
1917 /* Retry with a wider type. */
1918 if (orig_precres
== precres
)
1920 int p
= MAX (prec0
, prec1
);
1921 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1922 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1925 p
= TYPE_PRECISION (optype
);
1929 unsr_p
= TYPE_UNSIGNED (optype
);
1940 /* Expand ADD_OVERFLOW STMT. */
1943 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
1945 expand_arith_overflow (PLUS_EXPR
, stmt
);
1948 /* Expand SUB_OVERFLOW STMT. */
1951 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
1953 expand_arith_overflow (MINUS_EXPR
, stmt
);
1956 /* Expand MUL_OVERFLOW STMT. */
1959 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
1961 expand_arith_overflow (MULT_EXPR
, stmt
);
1964 /* This should get folded in tree-vectorizer.c. */
1967 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
1972 /* Expand MASK_LOAD call STMT using optab OPTAB. */
1975 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
1977 struct expand_operand ops
[3];
1978 tree type
, lhs
, rhs
, maskt
, ptr
;
1979 rtx mem
, target
, mask
;
1982 maskt
= gimple_call_arg (stmt
, 2);
1983 lhs
= gimple_call_lhs (stmt
);
1984 if (lhs
== NULL_TREE
)
1986 type
= TREE_TYPE (lhs
);
1987 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
1988 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
1989 if (TYPE_ALIGN (type
) != align
)
1990 type
= build_aligned_type (type
, align
);
1991 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
1993 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1994 gcc_assert (MEM_P (mem
));
1995 mask
= expand_normal (maskt
);
1996 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1997 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
1998 create_fixed_operand (&ops
[1], mem
);
1999 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2000 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2001 TYPE_MODE (TREE_TYPE (maskt
))),
2005 /* Expand MASK_STORE call STMT using optab OPTAB. */
2008 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2010 struct expand_operand ops
[3];
2011 tree type
, lhs
, rhs
, maskt
, ptr
;
2015 maskt
= gimple_call_arg (stmt
, 2);
2016 rhs
= gimple_call_arg (stmt
, 3);
2017 type
= TREE_TYPE (rhs
);
2018 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2019 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2020 if (TYPE_ALIGN (type
) != align
)
2021 type
= build_aligned_type (type
, align
);
2022 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2024 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2025 gcc_assert (MEM_P (mem
));
2026 mask
= expand_normal (maskt
);
2027 reg
= expand_normal (rhs
);
2028 create_fixed_operand (&ops
[0], mem
);
2029 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2030 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2031 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2032 TYPE_MODE (TREE_TYPE (maskt
))),
2037 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2042 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2044 /* When guessing was done, the hints should be already stripped away. */
2045 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2048 tree lhs
= gimple_call_lhs (stmt
);
2050 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2052 target
= const0_rtx
;
2053 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2054 if (lhs
&& val
!= target
)
2055 emit_move_insn (target
, val
);
2058 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2059 should never be called. */
2062 expand_VA_ARG (internal_fn
, gcall
*)
2067 /* Expand the IFN_UNIQUE function according to its first argument. */
2070 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2072 rtx pattern
= NULL_RTX
;
2073 enum ifn_unique_kind kind
2074 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2081 case IFN_UNIQUE_UNSPEC
:
2082 if (targetm
.have_unique ())
2083 pattern
= targetm
.gen_unique ();
2086 case IFN_UNIQUE_OACC_FORK
:
2087 case IFN_UNIQUE_OACC_JOIN
:
2088 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2090 tree lhs
= gimple_call_lhs (stmt
);
2091 rtx target
= const0_rtx
;
2094 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2096 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2097 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2099 if (kind
== IFN_UNIQUE_OACC_FORK
)
2100 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2102 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2110 emit_insn (pattern
);
2113 /* The size of an OpenACC compute dimension. */
2116 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2118 tree lhs
= gimple_call_lhs (stmt
);
2123 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2124 if (targetm
.have_oacc_dim_size ())
2126 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2127 VOIDmode
, EXPAND_NORMAL
);
2128 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2131 emit_move_insn (target
, GEN_INT (1));
2134 /* The position of an OpenACC execution engine along one compute axis. */
2137 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2139 tree lhs
= gimple_call_lhs (stmt
);
2144 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2145 if (targetm
.have_oacc_dim_pos ())
2147 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2148 VOIDmode
, EXPAND_NORMAL
);
2149 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2152 emit_move_insn (target
, const0_rtx
);
2155 /* This is expanded by oacc_device_lower pass. */
2158 expand_GOACC_LOOP (internal_fn
, gcall
*)
2163 /* This is expanded by oacc_device_lower pass. */
2166 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2171 /* Set errno to EDOM. */
2174 expand_SET_EDOM (internal_fn
, gcall
*)
2177 #ifdef GEN_ERRNO_RTX
2178 rtx errno_rtx
= GEN_ERRNO_RTX
;
2180 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2182 emit_move_insn (errno_rtx
,
2183 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2189 /* Expand atomic bit test and set. */
2192 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2194 expand_ifn_atomic_bit_test_and (call
);
2197 /* Expand atomic bit test and complement. */
2200 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2202 expand_ifn_atomic_bit_test_and (call
);
2205 /* Expand atomic bit test and reset. */
2208 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2210 expand_ifn_atomic_bit_test_and (call
);
2213 /* Expand atomic bit test and set. */
2216 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2218 expand_ifn_atomic_compare_exchange (call
);
2221 /* Expand LAUNDER to assignment, lhs = arg0. */
2224 expand_LAUNDER (internal_fn
, gcall
*call
)
2226 tree lhs
= gimple_call_lhs (call
);
2231 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2234 /* Expand DIVMOD() using:
2235 a) optab handler for udivmod/sdivmod if it is available.
2236 b) If optab_handler doesn't exist, generate call to
2237 target-specific divmod libfunc. */
2240 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2242 tree lhs
= gimple_call_lhs (call_stmt
);
2243 tree arg0
= gimple_call_arg (call_stmt
, 0);
2244 tree arg1
= gimple_call_arg (call_stmt
, 1);
2246 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2247 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2248 machine_mode mode
= TYPE_MODE (type
);
2249 bool unsignedp
= TYPE_UNSIGNED (type
);
2250 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2252 rtx op0
= expand_normal (arg0
);
2253 rtx op1
= expand_normal (arg1
);
2254 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2256 rtx quotient
, remainder
, libfunc
;
2258 /* Check if optab_handler exists for divmod_optab for given mode. */
2259 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2261 quotient
= gen_reg_rtx (mode
);
2262 remainder
= gen_reg_rtx (mode
);
2263 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2266 /* Generate call to divmod libfunc if it exists. */
2267 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2268 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2269 "ient
, &remainder
);
2274 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2275 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2276 make_tree (TREE_TYPE (arg0
), quotient
),
2277 make_tree (TREE_TYPE (arg1
), remainder
)),
2278 target
, VOIDmode
, EXPAND_NORMAL
);
2281 /* Expand a call to FN using the operands in STMT. FN has a single
2282 output operand and NARGS input operands. */
2285 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2288 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2290 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2291 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2293 tree lhs
= gimple_call_lhs (stmt
);
2294 tree lhs_type
= TREE_TYPE (lhs
);
2295 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2296 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2298 for (unsigned int i
= 0; i
< nargs
; ++i
)
2300 tree rhs
= gimple_call_arg (stmt
, i
);
2301 tree rhs_type
= TREE_TYPE (rhs
);
2302 rtx rhs_rtx
= expand_normal (rhs
);
2303 if (INTEGRAL_TYPE_P (rhs_type
))
2304 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2305 TYPE_MODE (rhs_type
),
2306 TYPE_UNSIGNED (rhs_type
));
2308 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2311 expand_insn (icode
, nargs
+ 1, ops
);
2312 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2314 /* If the return value has an integral type, convert the instruction
2315 result to that type. This is useful for things that return an
2316 int regardless of the size of the input. If the instruction result
2317 is smaller than required, assume that it is signed.
2319 If the return value has a nonintegral type, its mode must match
2320 the instruction result. */
2321 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2323 /* If this is a scalar in a register that is stored in a wider
2324 mode than the declared mode, compute the result into its
2325 declared mode and then convert to the wider mode. */
2326 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2327 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2328 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2329 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2331 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2332 emit_move_insn (lhs_rtx
, ops
[0].value
);
2335 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2336 convert_move (lhs_rtx
, ops
[0].value
, 0);
2341 /* Expanders for optabs that can use expand_direct_optab_fn. */
2343 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2344 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2346 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2347 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2349 /* RETURN_TYPE and ARGS are a return type and argument list that are
2350 in principle compatible with FN (which satisfies direct_internal_fn_p).
2351 Return the types that should be used to determine whether the
2352 target supports FN. */
2355 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2357 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2358 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2359 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2360 return tree_pair (type0
, type1
);
2363 /* CALL is a call whose return type and arguments are in principle
2364 compatible with FN (which satisfies direct_internal_fn_p). Return the
2365 types that should be used to determine whether the target supports FN. */
2368 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2370 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2371 tree op0
= (info
.type0
< 0
2372 ? gimple_call_lhs (call
)
2373 : gimple_call_arg (call
, info
.type0
));
2374 tree op1
= (info
.type1
< 0
2375 ? gimple_call_lhs (call
)
2376 : gimple_call_arg (call
, info
.type1
));
2377 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2380 /* Return true if OPTAB is supported for TYPES (whose modes should be
2381 the same) when the optimization type is OPT_TYPE. Used for simple
2385 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2386 optimization_type opt_type
)
2388 machine_mode mode
= TYPE_MODE (types
.first
);
2389 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2390 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2393 /* Return true if load/store lanes optab OPTAB is supported for
2394 array type TYPES.first when the optimization type is OPT_TYPE. */
2397 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2398 optimization_type opt_type
)
2400 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2401 machine_mode imode
= TYPE_MODE (types
.first
);
2402 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2403 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2404 != CODE_FOR_nothing
);
2407 #define direct_unary_optab_supported_p direct_optab_supported_p
2408 #define direct_binary_optab_supported_p direct_optab_supported_p
2409 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2410 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2411 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2412 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2414 /* Return true if FN is supported for the types in TYPES when the
2415 optimization type is OPT_TYPE. The types are those associated with
2416 the "type0" and "type1" fields of FN's direct_internal_fn_info
2420 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2421 optimization_type opt_type
)
2425 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2426 case IFN_##CODE: break;
2427 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2429 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2431 #include "internal-fn.def"
2439 /* Return true if FN is supported for type TYPE when the optimization
2440 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2441 fields of FN's direct_internal_fn_info structure are the same. */
2444 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2445 optimization_type opt_type
)
2447 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2448 gcc_checking_assert (info
.type0
== info
.type1
);
2449 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2452 /* Return true if IFN_SET_EDOM is supported. */
2455 set_edom_supported_p (void)
2464 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2466 expand_##CODE (internal_fn fn, gcall *stmt) \
2468 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2470 #include "internal-fn.def"
2472 /* Routines to expand each internal function, indexed by function number.
2473 Each routine has the prototype:
2475 expand_<NAME> (gcall *stmt)
2477 where STMT is the statement that performs the call. */
2478 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2479 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2480 #include "internal-fn.def"
2484 /* Expand STMT as though it were a call to internal function FN. */
2487 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2489 internal_fn_expanders
[fn
] (fn
, stmt
);
2492 /* Expand STMT, which is a call to internal function FN. */
2495 expand_internal_call (gcall
*stmt
)
2497 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);