2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 /* The names of each internal function, indexed by function number. */
48 const char *const internal_fn_name_array
[] = {
49 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
50 #include "internal-fn.def"
54 /* The ECF_* flags of each internal function, indexed by function number. */
55 const int internal_fn_flags_array
[] = {
56 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
57 #include "internal-fn.def"
61 /* Fnspec of each internal function, indexed by function number. */
62 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
67 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
68 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
69 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
70 #include "internal-fn.def"
71 internal_fn_fnspec_array
[IFN_LAST
] = 0;
74 /* Create static initializers for the information returned by
75 direct_internal_fn. */
76 #define not_direct { -2, -2, false }
77 #define mask_load_direct { -1, 2, false }
78 #define load_lanes_direct { -1, -1, false }
79 #define mask_store_direct { 3, 2, false }
80 #define store_lanes_direct { 0, 0, false }
81 #define unary_direct { 0, 0, true }
82 #define binary_direct { 0, 0, true }
84 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
85 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
86 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
87 #include "internal-fn.def"
91 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
92 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
95 get_multi_vector_move (tree array_type
, convert_optab optab
)
100 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
101 imode
= TYPE_MODE (array_type
);
102 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
104 return convert_optab_handler (optab
, imode
, vmode
);
107 /* Expand LOAD_LANES call STMT using optab OPTAB. */
110 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
112 struct expand_operand ops
[2];
116 lhs
= gimple_call_lhs (stmt
);
117 rhs
= gimple_call_arg (stmt
, 0);
118 type
= TREE_TYPE (lhs
);
120 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
121 mem
= expand_normal (rhs
);
123 gcc_assert (MEM_P (mem
));
124 PUT_MODE (mem
, TYPE_MODE (type
));
126 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
127 create_fixed_operand (&ops
[1], mem
);
128 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
131 /* Expand STORE_LANES call STMT using optab OPTAB. */
134 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
136 struct expand_operand ops
[2];
140 lhs
= gimple_call_lhs (stmt
);
141 rhs
= gimple_call_arg (stmt
, 0);
142 type
= TREE_TYPE (rhs
);
144 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
145 reg
= expand_normal (rhs
);
147 gcc_assert (MEM_P (target
));
148 PUT_MODE (target
, TYPE_MODE (type
));
150 create_fixed_operand (&ops
[0], target
);
151 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
152 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
156 expand_ANNOTATE (internal_fn
, gcall
*)
161 /* This should get expanded in omp_device_lower pass. */
164 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
169 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
170 without SIMT execution this should be expanded in omp_device_lower pass. */
173 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
175 tree lhs
= gimple_call_lhs (stmt
);
179 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
180 gcc_assert (targetm
.have_omp_simt_lane ());
181 emit_insn (targetm
.gen_omp_simt_lane (target
));
184 /* This should get expanded in omp_device_lower pass. */
187 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
192 /* Lane index of the first SIMT lane that supplies a non-zero argument.
193 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
194 lane that executed the last iteration for handling OpenMP lastprivate. */
197 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
199 tree lhs
= gimple_call_lhs (stmt
);
203 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
204 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
205 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
206 struct expand_operand ops
[2];
207 create_output_operand (&ops
[0], target
, mode
);
208 create_input_operand (&ops
[1], cond
, mode
);
209 gcc_assert (targetm
.have_omp_simt_last_lane ());
210 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
213 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
216 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
218 tree lhs
= gimple_call_lhs (stmt
);
222 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
223 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
224 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
225 struct expand_operand ops
[2];
226 create_output_operand (&ops
[0], target
, mode
);
227 create_input_operand (&ops
[1], ctr
, mode
);
228 gcc_assert (targetm
.have_omp_simt_ordered ());
229 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
232 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
233 any lane supplies a non-zero argument. */
236 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
238 tree lhs
= gimple_call_lhs (stmt
);
242 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
243 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
244 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
245 struct expand_operand ops
[2];
246 create_output_operand (&ops
[0], target
, mode
);
247 create_input_operand (&ops
[1], cond
, mode
);
248 gcc_assert (targetm
.have_omp_simt_vote_any ());
249 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
252 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
253 is destination lane index XOR given offset. */
256 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
258 tree lhs
= gimple_call_lhs (stmt
);
262 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
263 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
264 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
265 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
266 struct expand_operand ops
[3];
267 create_output_operand (&ops
[0], target
, mode
);
268 create_input_operand (&ops
[1], src
, mode
);
269 create_input_operand (&ops
[2], idx
, SImode
);
270 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
271 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
274 /* Exchange between SIMT lanes according to given source lane index. */
277 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
279 tree lhs
= gimple_call_lhs (stmt
);
283 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
284 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
285 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
286 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
287 struct expand_operand ops
[3];
288 create_output_operand (&ops
[0], target
, mode
);
289 create_input_operand (&ops
[1], src
, mode
);
290 create_input_operand (&ops
[2], idx
, SImode
);
291 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
292 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
295 /* This should get expanded in adjust_simduid_builtins. */
298 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
303 /* This should get expanded in adjust_simduid_builtins. */
306 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
311 /* This should get expanded in adjust_simduid_builtins. */
314 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
319 /* This should get expanded in adjust_simduid_builtins. */
322 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
327 /* This should get expanded in adjust_simduid_builtins. */
330 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
335 /* This should get expanded in the sanopt pass. */
338 expand_UBSAN_NULL (internal_fn
, gcall
*)
343 /* This should get expanded in the sanopt pass. */
346 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
351 /* This should get expanded in the sanopt pass. */
354 expand_UBSAN_VPTR (internal_fn
, gcall
*)
359 /* This should get expanded in the sanopt pass. */
362 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
367 /* This should get expanded in the sanopt pass. */
370 expand_ASAN_CHECK (internal_fn
, gcall
*)
375 /* This should get expanded in the sanopt pass. */
378 expand_ASAN_MARK (internal_fn
, gcall
*)
384 /* This should get expanded in the tsan pass. */
387 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
392 /* This should get expanded in the lower pass. */
395 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
397 error_at (gimple_location (call
),
398 "invalid use of attribute %<fallthrough%>");
401 /* Helper function for expand_addsub_overflow. Return 1
402 if ARG interpreted as signed in its precision is known to be always
403 positive or 2 if ARG is known to be always negative, or 3 if ARG may
404 be positive or negative. */
407 get_range_pos_neg (tree arg
)
409 if (arg
== error_mark_node
)
412 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
414 if (TREE_CODE (arg
) == INTEGER_CST
)
416 wide_int w
= wi::sext (arg
, prec
);
422 while (CONVERT_EXPR_P (arg
)
423 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
424 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
426 arg
= TREE_OPERAND (arg
, 0);
427 /* Narrower value zero extended into wider type
428 will always result in positive values. */
429 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
430 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
432 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
437 if (TREE_CODE (arg
) != SSA_NAME
)
439 wide_int arg_min
, arg_max
;
440 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
442 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
443 if (is_gimple_assign (g
)
444 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
446 tree t
= gimple_assign_rhs1 (g
);
447 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
448 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
450 if (TYPE_UNSIGNED (TREE_TYPE (t
))
451 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
453 prec
= TYPE_PRECISION (TREE_TYPE (t
));
462 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
464 /* For unsigned values, the "positive" range comes
465 below the "negative" range. */
466 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
468 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
473 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
475 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
481 /* Return minimum precision needed to represent all values
482 of ARG in SIGNed integral type. */
485 get_min_precision (tree arg
, signop sign
)
487 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
489 signop orig_sign
= sign
;
490 if (TREE_CODE (arg
) == INTEGER_CST
)
493 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
495 widest_int w
= wi::to_widest (arg
);
496 w
= wi::ext (w
, prec
, sign
);
497 p
= wi::min_precision (w
, sign
);
500 p
= wi::min_precision (arg
, sign
);
501 return MIN (p
, prec
);
503 while (CONVERT_EXPR_P (arg
)
504 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
505 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
507 arg
= TREE_OPERAND (arg
, 0);
508 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
510 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
512 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
513 return prec
+ (orig_sign
!= sign
);
514 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
517 return prec
+ (orig_sign
!= sign
);
519 if (TREE_CODE (arg
) != SSA_NAME
)
520 return prec
+ (orig_sign
!= sign
);
521 wide_int arg_min
, arg_max
;
522 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
524 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
525 if (is_gimple_assign (g
)
526 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
528 tree t
= gimple_assign_rhs1 (g
);
529 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
530 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
533 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
535 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
537 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
538 return prec
+ (orig_sign
!= sign
);
539 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
542 return prec
+ (orig_sign
!= sign
);
546 return prec
+ (orig_sign
!= sign
);
548 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
550 int p1
= wi::min_precision (arg_min
, sign
);
551 int p2
= wi::min_precision (arg_max
, sign
);
553 prec
= MIN (prec
, p1
);
555 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
557 int p
= wi::min_precision (arg_max
, UNSIGNED
);
558 prec
= MIN (prec
, p
);
560 return prec
+ (orig_sign
!= sign
);
563 /* Helper for expand_*_overflow. Set the __imag__ part to true
564 (1 except for signed:1 type, in which case store -1). */
567 expand_arith_set_overflow (tree lhs
, rtx target
)
569 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
570 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
571 write_complex_part (target
, constm1_rtx
, true);
573 write_complex_part (target
, const1_rtx
, true);
576 /* Helper for expand_*_overflow. Store RES into the __real__ part
577 of TARGET. If RES has larger MODE than __real__ part of TARGET,
578 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
579 if LHS has smaller precision than its mode. */
582 expand_arith_overflow_result_store (tree lhs
, rtx target
,
583 machine_mode mode
, rtx res
)
585 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
589 rtx_code_label
*done_label
= gen_label_rtx ();
590 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
591 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
592 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
593 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
594 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
596 expand_arith_set_overflow (lhs
, target
);
597 emit_label (done_label
);
599 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
600 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
603 rtx_code_label
*done_label
= gen_label_rtx ();
604 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
609 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
611 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
612 true, OPTAB_LIB_WIDEN
);
616 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
618 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
621 do_compare_rtx_and_jump (res
, lres
,
622 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
624 expand_arith_set_overflow (lhs
, target
);
625 emit_label (done_label
);
627 write_complex_part (target
, lres
, false);
630 /* Helper for expand_*_overflow. Store RES into TARGET. */
633 expand_ubsan_result_store (rtx target
, rtx res
)
635 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
636 /* If this is a scalar in a register that is stored in a wider mode
637 than the declared mode, compute the result into its declared mode
638 and then convert to the wider mode. Our value is the computed
640 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
642 emit_move_insn (target
, res
);
645 /* Add sub/add overflow checking to the statement STMT.
646 CODE says whether the operation is +, or -. */
649 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
650 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
651 bool uns1_p
, bool is_ubsan
, tree
*datap
)
653 rtx res
, target
= NULL_RTX
;
655 rtx_code_label
*done_label
= gen_label_rtx ();
656 rtx_code_label
*do_error
= gen_label_rtx ();
657 do_pending_stack_adjust ();
658 rtx op0
= expand_normal (arg0
);
659 rtx op1
= expand_normal (arg1
);
660 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
661 int prec
= GET_MODE_PRECISION (mode
);
662 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
666 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
670 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
672 write_complex_part (target
, const0_rtx
, true);
675 /* We assume both operands and result have the same precision
676 here (GET_MODE_BITSIZE (mode)), S stands for signed type
677 with that precision, U for unsigned type with that precision,
678 sgn for unsigned most significant bit in that precision.
679 s1 is signed first operand, u1 is unsigned first operand,
680 s2 is signed second operand, u2 is unsigned second operand,
681 sr is signed result, ur is unsigned result and the following
682 rules say how to compute result (which is always result of
683 the operands as if both were unsigned, cast to the right
684 signedness) and how to compute whether operation overflowed.
687 res = (S) ((U) s1 + (U) s2)
688 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
690 res = (S) ((U) s1 - (U) s2)
691 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
694 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
697 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
699 res = (S) ((U) s1 + u2)
700 ovf = ((U) res ^ sgn) < u2
705 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
707 res = (S) ((U) s1 - u2)
708 ovf = u2 > ((U) s1 ^ sgn)
711 ovf = s1 < 0 || u2 > (U) s1
714 ovf = u1 >= ((U) s2 ^ sgn)
719 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
721 res = (U) s1 + (U) s2
722 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
725 ovf = (U) res < u2 || res < 0
728 ovf = u1 >= u2 ? res < 0 : res >= 0
730 res = (U) s1 - (U) s2
731 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
733 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
735 /* PLUS_EXPR is commutative, if operand signedness differs,
736 canonicalize to the first operand being signed and second
737 unsigned to simplify following code. */
738 std::swap (op0
, op1
);
739 std::swap (arg0
, arg1
);
745 if (uns0_p
&& uns1_p
&& unsr_p
)
747 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
748 : usubv4_optab
, mode
);
749 if (icode
!= CODE_FOR_nothing
)
751 struct expand_operand ops
[4];
752 rtx_insn
*last
= get_last_insn ();
754 res
= gen_reg_rtx (mode
);
755 create_output_operand (&ops
[0], res
, mode
);
756 create_input_operand (&ops
[1], op0
, mode
);
757 create_input_operand (&ops
[2], op1
, mode
);
758 create_fixed_operand (&ops
[3], do_error
);
759 if (maybe_expand_insn (icode
, 4, ops
))
761 last
= get_last_insn ();
762 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
764 && any_condjump_p (last
)
765 && !find_reg_note (last
, REG_BR_PROB
, 0))
766 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
767 emit_jump (done_label
);
771 delete_insns_since (last
);
774 /* Compute the operation. On RTL level, the addition is always
776 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
777 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
779 /* For PLUS_EXPR, the operation is commutative, so we can pick
780 operand to compare against. For prec <= BITS_PER_WORD, I think
781 preferring REG operand is better over CONST_INT, because
782 the CONST_INT might enlarge the instruction or CSE would need
783 to figure out we'd already loaded it into a register before.
784 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
785 as then the multi-word comparison can be perhaps simplified. */
786 if (code
== PLUS_EXPR
787 && (prec
<= BITS_PER_WORD
788 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
789 : CONST_SCALAR_INT_P (op1
)))
791 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
792 true, mode
, NULL_RTX
, NULL
, done_label
,
798 if (!uns0_p
&& uns1_p
&& !unsr_p
)
800 /* Compute the operation. On RTL level, the addition is always
802 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
803 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
804 rtx tem
= expand_binop (mode
, add_optab
,
805 code
== PLUS_EXPR
? res
: op0
, sgn
,
806 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
807 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
808 done_label
, PROB_VERY_LIKELY
);
813 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
815 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
817 /* As we've changed op1, we have to avoid using the value range
818 for the original argument. */
819 arg1
= error_mark_node
;
825 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
827 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
829 /* As we've changed op0, we have to avoid using the value range
830 for the original argument. */
831 arg0
= error_mark_node
;
837 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
839 /* Compute the operation. On RTL level, the addition is always
841 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
843 int pos_neg
= get_range_pos_neg (arg0
);
845 /* If ARG0 is known to be always negative, this is always overflow. */
846 emit_jump (do_error
);
847 else if (pos_neg
== 3)
848 /* If ARG0 is not known to be always positive, check at runtime. */
849 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
850 NULL
, do_error
, PROB_VERY_UNLIKELY
);
851 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
852 done_label
, PROB_VERY_LIKELY
);
857 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
859 /* Compute the operation. On RTL level, the addition is always
861 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
863 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
865 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
866 done_label
, PROB_VERY_LIKELY
);
871 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
873 /* Compute the operation. On RTL level, the addition is always
875 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
877 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
878 NULL
, do_error
, PROB_VERY_UNLIKELY
);
880 /* The operation is commutative, so we can pick operand to compare
881 against. For prec <= BITS_PER_WORD, I think preferring REG operand
882 is better over CONST_INT, because the CONST_INT might enlarge the
883 instruction or CSE would need to figure out we'd already loaded it
884 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
885 might be more beneficial, as then the multi-word comparison can be
886 perhaps simplified. */
887 if (prec
<= BITS_PER_WORD
888 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
889 : CONST_SCALAR_INT_P (op0
))
891 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
892 done_label
, PROB_VERY_LIKELY
);
897 if (!uns0_p
&& !uns1_p
&& unsr_p
)
899 /* Compute the operation. On RTL level, the addition is always
901 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
902 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
903 int pos_neg
= get_range_pos_neg (arg1
);
904 if (code
== PLUS_EXPR
)
906 int pos_neg0
= get_range_pos_neg (arg0
);
907 if (pos_neg0
!= 3 && pos_neg
== 3)
909 std::swap (op0
, op1
);
916 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
917 ? and_optab
: ior_optab
,
918 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
919 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
920 NULL
, done_label
, PROB_VERY_LIKELY
);
924 rtx_code_label
*do_ior_label
= gen_label_rtx ();
925 do_compare_rtx_and_jump (op1
, const0_rtx
,
926 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
927 NULL_RTX
, NULL
, do_ior_label
,
929 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
931 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
932 NULL
, done_label
, PROB_VERY_LIKELY
);
933 emit_jump (do_error
);
934 emit_label (do_ior_label
);
935 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
937 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
938 NULL
, done_label
, PROB_VERY_LIKELY
);
944 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
946 /* Compute the operation. On RTL level, the addition is always
948 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
950 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
951 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
952 op0_geu_op1
, PROB_EVEN
);
953 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
954 NULL
, done_label
, PROB_VERY_LIKELY
);
955 emit_jump (do_error
);
956 emit_label (op0_geu_op1
);
957 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
958 NULL
, done_label
, PROB_VERY_LIKELY
);
962 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
967 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
968 : subv4_optab
, mode
);
969 if (icode
!= CODE_FOR_nothing
)
971 struct expand_operand ops
[4];
972 rtx_insn
*last
= get_last_insn ();
974 res
= gen_reg_rtx (mode
);
975 create_output_operand (&ops
[0], res
, mode
);
976 create_input_operand (&ops
[1], op0
, mode
);
977 create_input_operand (&ops
[2], op1
, mode
);
978 create_fixed_operand (&ops
[3], do_error
);
979 if (maybe_expand_insn (icode
, 4, ops
))
981 last
= get_last_insn ();
982 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
984 && any_condjump_p (last
)
985 && !find_reg_note (last
, REG_BR_PROB
, 0))
986 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
987 emit_jump (done_label
);
991 delete_insns_since (last
);
994 /* Compute the operation. On RTL level, the addition is always
996 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
997 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
999 /* If we can prove that one of the arguments (for MINUS_EXPR only
1000 the second operand, as subtraction is not commutative) is always
1001 non-negative or always negative, we can do just one comparison
1002 and conditional jump. */
1003 int pos_neg
= get_range_pos_neg (arg1
);
1004 if (code
== PLUS_EXPR
)
1006 int pos_neg0
= get_range_pos_neg (arg0
);
1007 if (pos_neg0
!= 3 && pos_neg
== 3)
1009 std::swap (op0
, op1
);
1014 /* Addition overflows if and only if the two operands have the same sign,
1015 and the result has the opposite sign. Subtraction overflows if and
1016 only if the two operands have opposite sign, and the subtrahend has
1017 the same sign as the result. Here 0 is counted as positive. */
1020 /* Compute op0 ^ op1 (operands have opposite sign). */
1021 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1024 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1025 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1029 if (code
== PLUS_EXPR
)
1031 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1032 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1033 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1038 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1039 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1040 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1044 /* No overflow if the result has bit sign cleared. */
1045 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1046 NULL
, done_label
, PROB_VERY_LIKELY
);
1049 /* Compare the result of the operation with the first operand.
1050 No overflow for addition if second operand is positive and result
1051 is larger or second operand is negative and result is smaller.
1052 Likewise for subtraction with sign of second operand flipped. */
1054 do_compare_rtx_and_jump (res
, op0
,
1055 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1056 false, mode
, NULL_RTX
, NULL
, done_label
,
1061 emit_label (do_error
);
1064 /* Expand the ubsan builtin call. */
1066 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1070 do_pending_stack_adjust ();
1073 expand_arith_set_overflow (lhs
, target
);
1076 emit_label (done_label
);
1081 expand_ubsan_result_store (target
, res
);
1085 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1088 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1093 /* Add negate overflow checking to the statement STMT. */
1096 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1101 rtx_code_label
*done_label
, *do_error
;
1102 rtx target
= NULL_RTX
;
1104 done_label
= gen_label_rtx ();
1105 do_error
= gen_label_rtx ();
1107 do_pending_stack_adjust ();
1108 op1
= expand_normal (arg1
);
1110 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
1113 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1115 write_complex_part (target
, const0_rtx
, true);
1118 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1119 if (icode
!= CODE_FOR_nothing
)
1121 struct expand_operand ops
[3];
1122 rtx_insn
*last
= get_last_insn ();
1124 res
= gen_reg_rtx (mode
);
1125 create_output_operand (&ops
[0], res
, mode
);
1126 create_input_operand (&ops
[1], op1
, mode
);
1127 create_fixed_operand (&ops
[2], do_error
);
1128 if (maybe_expand_insn (icode
, 3, ops
))
1130 last
= get_last_insn ();
1131 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1133 && any_condjump_p (last
)
1134 && !find_reg_note (last
, REG_BR_PROB
, 0))
1135 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1136 emit_jump (done_label
);
1140 delete_insns_since (last
);
1141 icode
= CODE_FOR_nothing
;
1145 if (icode
== CODE_FOR_nothing
)
1147 /* Compute the operation. On RTL level, the addition is always
1149 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1151 /* Compare the operand with the most negative value. */
1152 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1153 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1154 done_label
, PROB_VERY_LIKELY
);
1157 emit_label (do_error
);
1160 /* Expand the ubsan builtin call. */
1162 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1163 arg1
, NULL_TREE
, datap
);
1166 do_pending_stack_adjust ();
1169 expand_arith_set_overflow (lhs
, target
);
1172 emit_label (done_label
);
1177 expand_ubsan_result_store (target
, res
);
1179 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1183 /* Add mul overflow checking to the statement STMT. */
1186 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1187 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1192 rtx_code_label
*done_label
, *do_error
;
1193 rtx target
= NULL_RTX
;
1195 enum insn_code icode
;
1197 done_label
= gen_label_rtx ();
1198 do_error
= gen_label_rtx ();
1200 do_pending_stack_adjust ();
1201 op0
= expand_normal (arg0
);
1202 op1
= expand_normal (arg1
);
1204 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
1208 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1210 write_complex_part (target
, const0_rtx
, true);
1214 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1216 /* We assume both operands and result have the same precision
1217 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1218 with that precision, U for unsigned type with that precision,
1219 sgn for unsigned most significant bit in that precision.
1220 s1 is signed first operand, u1 is unsigned first operand,
1221 s2 is signed second operand, u2 is unsigned second operand,
1222 sr is signed result, ur is unsigned result and the following
1223 rules say how to compute result (which is always result of
1224 the operands as if both were unsigned, cast to the right
1225 signedness) and how to compute whether operation overflowed.
1226 main_ovf (false) stands for jump on signed multiplication
1227 overflow or the main algorithm with uns == false.
1228 main_ovf (true) stands for jump on unsigned multiplication
1229 overflow or the main algorithm with uns == true.
1232 res = (S) ((U) s1 * (U) s2)
1233 ovf = main_ovf (false)
1236 ovf = main_ovf (true)
1239 ovf = (s1 < 0 && u2) || main_ovf (true)
1242 ovf = res < 0 || main_ovf (true)
1244 res = (S) ((U) s1 * u2)
1245 ovf = (S) u2 >= 0 ? main_ovf (false)
1246 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1248 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1249 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1251 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1253 if (uns0_p
&& !uns1_p
)
1255 /* Multiplication is commutative, if operand signedness differs,
1256 canonicalize to the first operand being signed and second
1257 unsigned to simplify following code. */
1258 std::swap (op0
, op1
);
1259 std::swap (arg0
, arg1
);
1264 int pos_neg0
= get_range_pos_neg (arg0
);
1265 int pos_neg1
= get_range_pos_neg (arg1
);
1268 if (!uns0_p
&& uns1_p
&& unsr_p
)
1273 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1276 /* If s1 is negative, avoid the main code, just multiply and
1277 signal overflow if op1 is not 0. */
1278 struct separate_ops ops
;
1279 ops
.code
= MULT_EXPR
;
1280 ops
.type
= TREE_TYPE (arg1
);
1281 ops
.op0
= make_tree (ops
.type
, op0
);
1282 ops
.op1
= make_tree (ops
.type
, op1
);
1283 ops
.op2
= NULL_TREE
;
1285 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1286 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1287 NULL
, done_label
, PROB_VERY_LIKELY
);
1288 goto do_error_label
;
1290 rtx_code_label
*do_main_label
;
1291 do_main_label
= gen_label_rtx ();
1292 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1293 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1294 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1295 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1296 expand_arith_set_overflow (lhs
, target
);
1297 emit_label (do_main_label
);
1305 if (uns0_p
&& uns1_p
&& !unsr_p
)
1308 /* Rest of handling of this case after res is computed. */
1313 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1320 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1321 avoid the main code, just multiply and signal overflow
1322 unless 0 * u2 or -1 * ((U) Smin). */
1323 struct separate_ops ops
;
1324 ops
.code
= MULT_EXPR
;
1325 ops
.type
= TREE_TYPE (arg1
);
1326 ops
.op0
= make_tree (ops
.type
, op0
);
1327 ops
.op1
= make_tree (ops
.type
, op1
);
1328 ops
.op2
= NULL_TREE
;
1330 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1331 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1332 NULL
, done_label
, PROB_VERY_LIKELY
);
1333 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1334 NULL
, do_error
, PROB_VERY_UNLIKELY
);
1336 prec
= GET_MODE_PRECISION (mode
);
1338 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1339 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1340 NULL
, done_label
, PROB_VERY_LIKELY
);
1341 goto do_error_label
;
1343 /* Rest of handling of this case after res is computed. */
1351 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1354 switch (pos_neg0
| pos_neg1
)
1356 case 1: /* Both operands known to be non-negative. */
1358 case 2: /* Both operands known to be negative. */
1359 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1360 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1361 /* Avoid looking at arg0/arg1 ranges, as we've changed
1363 arg0
= error_mark_node
;
1364 arg1
= error_mark_node
;
1367 if ((pos_neg0
^ pos_neg1
) == 3)
1369 /* If one operand is known to be negative and the other
1370 non-negative, this overflows always, unless the non-negative
1371 one is 0. Just do normal multiply and set overflow
1372 unless one of the operands is 0. */
1373 struct separate_ops ops
;
1374 ops
.code
= MULT_EXPR
;
1376 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1378 ops
.op0
= make_tree (ops
.type
, op0
);
1379 ops
.op1
= make_tree (ops
.type
, op1
);
1380 ops
.op2
= NULL_TREE
;
1382 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1383 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1385 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1386 NULL_RTX
, NULL
, done_label
,
1388 goto do_error_label
;
1390 /* The general case, do all the needed comparisons at runtime. */
1391 rtx_code_label
*do_main_label
, *after_negate_label
;
1393 rop0
= gen_reg_rtx (mode
);
1394 rop1
= gen_reg_rtx (mode
);
1395 emit_move_insn (rop0
, op0
);
1396 emit_move_insn (rop1
, op1
);
1399 do_main_label
= gen_label_rtx ();
1400 after_negate_label
= gen_label_rtx ();
1401 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1403 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1404 NULL
, after_negate_label
, PROB_VERY_LIKELY
);
1405 /* Both arguments negative here, negate them and continue with
1406 normal unsigned overflow checking multiplication. */
1407 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1409 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1411 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1413 arg0
= error_mark_node
;
1414 arg1
= error_mark_node
;
1415 emit_jump (do_main_label
);
1416 emit_label (after_negate_label
);
1417 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1419 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1420 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1421 /* One argument is negative here, the other positive. This
1422 overflows always, unless one of the arguments is 0. But
1423 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1424 is, thus we can keep do_main code oring in overflow as is. */
1425 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1426 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1427 expand_arith_set_overflow (lhs
, target
);
1428 emit_label (do_main_label
);
1436 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1437 sign
= uns
? UNSIGNED
: SIGNED
;
1438 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1439 if (icode
!= CODE_FOR_nothing
)
1441 struct expand_operand ops
[4];
1442 rtx_insn
*last
= get_last_insn ();
1444 res
= gen_reg_rtx (mode
);
1445 create_output_operand (&ops
[0], res
, mode
);
1446 create_input_operand (&ops
[1], op0
, mode
);
1447 create_input_operand (&ops
[2], op1
, mode
);
1448 create_fixed_operand (&ops
[3], do_error
);
1449 if (maybe_expand_insn (icode
, 4, ops
))
1451 last
= get_last_insn ();
1452 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1454 && any_condjump_p (last
)
1455 && !find_reg_note (last
, REG_BR_PROB
, 0))
1456 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1457 emit_jump (done_label
);
1461 delete_insns_since (last
);
1462 icode
= CODE_FOR_nothing
;
1466 if (icode
== CODE_FOR_nothing
)
1468 struct separate_ops ops
;
1469 int prec
= GET_MODE_PRECISION (mode
);
1470 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1471 ops
.op0
= make_tree (type
, op0
);
1472 ops
.op1
= make_tree (type
, op1
);
1473 ops
.op2
= NULL_TREE
;
1475 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1476 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1478 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1479 ops
.code
= WIDEN_MULT_EXPR
;
1481 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1483 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1484 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1486 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1487 res
= convert_modes (mode
, wmode
, res
, uns
);
1489 /* For the unsigned multiplication, there was overflow if
1490 HIPART is non-zero. */
1491 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1492 NULL_RTX
, NULL
, done_label
,
1496 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1498 /* RES is low half of the double width result, HIPART
1499 the high half. There was overflow if
1500 HIPART is different from RES < 0 ? -1 : 0. */
1501 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1502 NULL_RTX
, NULL
, done_label
,
1506 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1508 rtx_code_label
*large_op0
= gen_label_rtx ();
1509 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1510 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1511 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1512 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1513 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1514 rtx_code_label
*do_overflow
= gen_label_rtx ();
1515 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1517 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1518 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1520 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1521 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1522 rtx signbit0
= const0_rtx
;
1524 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1526 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1528 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1529 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1530 rtx signbit1
= const0_rtx
;
1532 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1535 res
= gen_reg_rtx (mode
);
1537 /* True if op0 resp. op1 are known to be in the range of
1539 bool op0_small_p
= false;
1540 bool op1_small_p
= false;
1541 /* True if op0 resp. op1 are known to have all zeros or all ones
1542 in the upper half of bits, but are not known to be
1544 bool op0_medium_p
= false;
1545 bool op1_medium_p
= false;
1546 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1547 nonnegative, 1 if unknown. */
1553 else if (pos_neg0
== 2)
1557 else if (pos_neg1
== 2)
1560 unsigned int mprec0
= prec
;
1561 if (arg0
!= error_mark_node
)
1562 mprec0
= get_min_precision (arg0
, sign
);
1563 if (mprec0
<= hprec
)
1565 else if (!uns
&& mprec0
<= hprec
+ 1)
1566 op0_medium_p
= true;
1567 unsigned int mprec1
= prec
;
1568 if (arg1
!= error_mark_node
)
1569 mprec1
= get_min_precision (arg1
, sign
);
1570 if (mprec1
<= hprec
)
1572 else if (!uns
&& mprec1
<= hprec
+ 1)
1573 op1_medium_p
= true;
1575 int smaller_sign
= 1;
1576 int larger_sign
= 1;
1579 smaller_sign
= op0_sign
;
1580 larger_sign
= op1_sign
;
1582 else if (op1_small_p
)
1584 smaller_sign
= op1_sign
;
1585 larger_sign
= op0_sign
;
1587 else if (op0_sign
== op1_sign
)
1589 smaller_sign
= op0_sign
;
1590 larger_sign
= op0_sign
;
1594 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1595 NULL_RTX
, NULL
, large_op0
,
1599 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1600 NULL_RTX
, NULL
, small_op0_large_op1
,
1603 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1604 hmode to mode, the multiplication will never overflow. We can
1605 do just one hmode x hmode => mode widening multiplication. */
1606 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1607 if (GET_CODE (lopart0
) == SUBREG
)
1609 lopart0s
= shallow_copy_rtx (lopart0
);
1610 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1611 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1613 if (GET_CODE (lopart1
) == SUBREG
)
1615 lopart1s
= shallow_copy_rtx (lopart1
);
1616 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1617 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1619 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1620 ops
.op0
= make_tree (halfstype
, lopart0s
);
1621 ops
.op1
= make_tree (halfstype
, lopart1s
);
1622 ops
.code
= WIDEN_MULT_EXPR
;
1625 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1626 emit_move_insn (res
, thisres
);
1627 emit_jump (done_label
);
1629 emit_label (small_op0_large_op1
);
1631 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1632 but op1 is not, just swap the arguments and handle it as op1
1633 sign/zero extended, op0 not. */
1634 rtx larger
= gen_reg_rtx (mode
);
1635 rtx hipart
= gen_reg_rtx (hmode
);
1636 rtx lopart
= gen_reg_rtx (hmode
);
1637 emit_move_insn (larger
, op1
);
1638 emit_move_insn (hipart
, hipart1
);
1639 emit_move_insn (lopart
, lopart0
);
1640 emit_jump (one_small_one_large
);
1642 emit_label (large_op0
);
1645 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1646 NULL_RTX
, NULL
, both_ops_large
,
1649 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1650 but op0 is not, prepare larger, hipart and lopart pseudos and
1651 handle it together with small_op0_large_op1. */
1652 emit_move_insn (larger
, op0
);
1653 emit_move_insn (hipart
, hipart0
);
1654 emit_move_insn (lopart
, lopart1
);
1656 emit_label (one_small_one_large
);
1658 /* lopart is the low part of the operand that is sign extended
1659 to mode, larger is the other operand, hipart is the
1660 high part of larger and lopart0 and lopart1 are the low parts
1662 We perform lopart0 * lopart1 and lopart * hipart widening
1664 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1665 ops
.op0
= make_tree (halfutype
, lopart0
);
1666 ops
.op1
= make_tree (halfutype
, lopart1
);
1668 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1670 ops
.op0
= make_tree (halfutype
, lopart
);
1671 ops
.op1
= make_tree (halfutype
, hipart
);
1672 rtx loxhi
= gen_reg_rtx (mode
);
1673 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1674 emit_move_insn (loxhi
, tem
);
1678 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1679 if (larger_sign
== 0)
1680 emit_jump (after_hipart_neg
);
1681 else if (larger_sign
!= -1)
1682 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1683 NULL_RTX
, NULL
, after_hipart_neg
,
1686 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1687 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1688 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1690 emit_move_insn (loxhi
, tem
);
1692 emit_label (after_hipart_neg
);
1694 /* if (lopart < 0) loxhi -= larger; */
1695 if (smaller_sign
== 0)
1696 emit_jump (after_lopart_neg
);
1697 else if (smaller_sign
!= -1)
1698 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1699 NULL_RTX
, NULL
, after_lopart_neg
,
1702 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1704 emit_move_insn (loxhi
, tem
);
1706 emit_label (after_lopart_neg
);
1709 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1710 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1711 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1713 emit_move_insn (loxhi
, tem
);
1715 /* if (loxhi >> (bitsize / 2)
1716 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1717 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1718 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1720 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1721 rtx signbitloxhi
= const0_rtx
;
1723 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1724 convert_modes (hmode
, mode
,
1726 hprec
- 1, NULL_RTX
, 0);
1728 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1729 NULL_RTX
, NULL
, do_overflow
,
1730 PROB_VERY_UNLIKELY
);
1732 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1733 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1735 tem
= convert_modes (mode
, hmode
,
1736 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1738 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1741 emit_move_insn (res
, tem
);
1742 emit_jump (done_label
);
1744 emit_label (both_ops_large
);
1746 /* If both operands are large (not sign (!uns) or zero (uns)
1747 extended from hmode), then perform the full multiplication
1748 which will be the result of the operation.
1749 The only cases which don't overflow are for signed multiplication
1750 some cases where both hipart0 and highpart1 are 0 or -1.
1751 For unsigned multiplication when high parts are both non-zero
1752 this overflows always. */
1753 ops
.code
= MULT_EXPR
;
1754 ops
.op0
= make_tree (type
, op0
);
1755 ops
.op1
= make_tree (type
, op1
);
1756 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1757 emit_move_insn (res
, tem
);
1763 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1764 NULL_RTX
, 1, OPTAB_DIRECT
);
1765 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1766 NULL_RTX
, NULL
, do_error
,
1767 PROB_VERY_UNLIKELY
);
1772 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1773 NULL_RTX
, 1, OPTAB_DIRECT
);
1774 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1775 NULL_RTX
, NULL
, do_error
,
1776 PROB_VERY_UNLIKELY
);
1779 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1780 the same, overflow happened if res is negative, if they are
1781 different, overflow happened if res is positive. */
1782 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1783 emit_jump (hipart_different
);
1784 else if (op0_sign
== 1 || op1_sign
== 1)
1785 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1786 NULL_RTX
, NULL
, hipart_different
,
1789 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1790 NULL_RTX
, NULL
, do_error
,
1791 PROB_VERY_UNLIKELY
);
1792 emit_jump (done_label
);
1794 emit_label (hipart_different
);
1796 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1797 NULL_RTX
, NULL
, do_error
,
1798 PROB_VERY_UNLIKELY
);
1799 emit_jump (done_label
);
1802 emit_label (do_overflow
);
1804 /* Overflow, do full multiplication and fallthru into do_error. */
1805 ops
.op0
= make_tree (type
, op0
);
1806 ops
.op1
= make_tree (type
, op1
);
1807 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1808 emit_move_insn (res
, tem
);
1812 gcc_assert (!is_ubsan
);
1813 ops
.code
= MULT_EXPR
;
1815 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1816 emit_jump (done_label
);
1821 emit_label (do_error
);
1824 /* Expand the ubsan builtin call. */
1826 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1830 do_pending_stack_adjust ();
1833 expand_arith_set_overflow (lhs
, target
);
1836 emit_label (done_label
);
1839 if (uns0_p
&& uns1_p
&& !unsr_p
)
1841 rtx_code_label
*all_done_label
= gen_label_rtx ();
1842 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1843 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1844 expand_arith_set_overflow (lhs
, target
);
1845 emit_label (all_done_label
);
1849 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1851 rtx_code_label
*all_done_label
= gen_label_rtx ();
1852 rtx_code_label
*set_noovf
= gen_label_rtx ();
1853 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1854 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1855 expand_arith_set_overflow (lhs
, target
);
1856 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1857 NULL
, set_noovf
, PROB_VERY_LIKELY
);
1858 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1859 NULL
, all_done_label
, PROB_VERY_UNLIKELY
);
1860 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1861 all_done_label
, PROB_VERY_UNLIKELY
);
1862 emit_label (set_noovf
);
1863 write_complex_part (target
, const0_rtx
, true);
1864 emit_label (all_done_label
);
1870 expand_ubsan_result_store (target
, res
);
1872 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1876 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
1879 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
1880 tree arg0
, tree arg1
)
1882 int cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
1883 rtx_code_label
*loop_lab
= NULL
;
1884 rtx cntvar
= NULL_RTX
;
1885 tree cntv
= NULL_TREE
;
1886 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
1887 tree sz
= TYPE_SIZE (eltype
);
1888 tree data
= NULL_TREE
;
1889 tree resv
= NULL_TREE
;
1890 rtx lhsr
= NULL_RTX
;
1891 rtx resvr
= NULL_RTX
;
1896 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1897 if (GET_MODE (lhsr
) == BLKmode
1898 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
1899 optab_default
)) == unknown_optab
1900 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
1901 == CODE_FOR_nothing
))
1904 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
1907 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
1908 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
1914 do_pending_stack_adjust ();
1915 loop_lab
= gen_label_rtx ();
1916 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
1917 cntv
= make_tree (sizetype
, cntvar
);
1918 emit_move_insn (cntvar
, const0_rtx
);
1919 emit_label (loop_lab
);
1921 if (TREE_CODE (arg0
) != VECTOR_CST
)
1923 rtx arg0r
= expand_normal (arg0
);
1924 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
1926 if (TREE_CODE (arg1
) != VECTOR_CST
)
1928 rtx arg1r
= expand_normal (arg1
);
1929 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
1931 for (int i
= 0; i
< (cnt
> 4 ? 1 : cnt
); i
++)
1933 tree op0
, op1
, res
= NULL_TREE
;
1936 tree atype
= build_array_type_nelts (eltype
, cnt
);
1937 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
1938 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
1939 NULL_TREE
, NULL_TREE
);
1940 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
1941 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
1942 NULL_TREE
, NULL_TREE
);
1945 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
1946 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
1947 NULL_TREE
, NULL_TREE
);
1952 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
1953 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
1954 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
1956 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
1962 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
1963 false, false, false, true, &data
);
1966 if (cnt
> 4 ? integer_zerop (arg0
) : integer_zerop (op0
))
1967 expand_neg_overflow (loc
, res
, op1
, true, &data
);
1969 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
1970 false, false, false, true, &data
);
1973 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
1982 struct separate_ops ops
;
1983 ops
.code
= PLUS_EXPR
;
1984 ops
.type
= TREE_TYPE (cntv
);
1986 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
1987 ops
.op2
= NULL_TREE
;
1989 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
1992 emit_move_insn (cntvar
, ret
);
1993 do_compare_rtx_and_jump (cntvar
, GEN_INT (cnt
), NE
, false,
1994 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
1997 if (lhs
&& resv
== NULL_TREE
)
1999 struct separate_ops ops
;
2001 ops
.type
= TREE_TYPE (arg0
);
2004 ops
.op2
= NULL_TREE
;
2006 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2009 emit_move_insn (lhsr
, ret
);
2012 emit_move_insn (lhsr
, resvr
);
2015 /* Expand UBSAN_CHECK_ADD call STMT. */
2018 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2020 location_t loc
= gimple_location (stmt
);
2021 tree lhs
= gimple_call_lhs (stmt
);
2022 tree arg0
= gimple_call_arg (stmt
, 0);
2023 tree arg1
= gimple_call_arg (stmt
, 1);
2024 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2025 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2027 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2028 false, false, false, true, NULL
);
2031 /* Expand UBSAN_CHECK_SUB call STMT. */
2034 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2036 location_t loc
= gimple_location (stmt
);
2037 tree lhs
= gimple_call_lhs (stmt
);
2038 tree arg0
= gimple_call_arg (stmt
, 0);
2039 tree arg1
= gimple_call_arg (stmt
, 1);
2040 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2041 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2042 else if (integer_zerop (arg0
))
2043 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2045 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2046 false, false, false, true, NULL
);
2049 /* Expand UBSAN_CHECK_MUL call STMT. */
2052 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2054 location_t loc
= gimple_location (stmt
);
2055 tree lhs
= gimple_call_lhs (stmt
);
2056 tree arg0
= gimple_call_arg (stmt
, 0);
2057 tree arg1
= gimple_call_arg (stmt
, 1);
2058 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2059 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2061 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2065 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2068 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2070 tree lhs
= gimple_call_lhs (stmt
);
2071 if (lhs
== NULL_TREE
)
2073 tree arg0
= gimple_call_arg (stmt
, 0);
2074 tree arg1
= gimple_call_arg (stmt
, 1);
2075 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2076 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2077 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2078 int unsr_p
= TYPE_UNSIGNED (type
);
2079 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2080 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2081 int precres
= TYPE_PRECISION (type
);
2082 location_t loc
= gimple_location (stmt
);
2083 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2085 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2087 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2088 prec0
= MIN (prec0
, pr
);
2089 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2090 prec1
= MIN (prec1
, pr
);
2092 /* If uns0_p && uns1_p, precop is minimum needed precision
2093 of unsigned type to hold the exact result, otherwise
2094 precop is minimum needed precision of signed type to
2095 hold the exact result. */
2097 if (code
== MULT_EXPR
)
2098 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2101 if (uns0_p
== uns1_p
)
2102 precop
= MAX (prec0
, prec1
) + 1;
2104 precop
= MAX (prec0
+ 1, prec1
) + 1;
2106 precop
= MAX (prec0
, prec1
+ 1) + 1;
2108 int orig_precres
= precres
;
2112 if ((uns0_p
&& uns1_p
)
2113 ? ((precop
+ !unsr_p
) <= precres
2114 /* u1 - u2 -> ur can overflow, no matter what precision
2116 && (code
!= MINUS_EXPR
|| !unsr_p
))
2117 : (!unsr_p
&& precop
<= precres
))
2119 /* The infinity precision result will always fit into result. */
2120 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2121 write_complex_part (target
, const0_rtx
, true);
2122 enum machine_mode mode
= TYPE_MODE (type
);
2123 struct separate_ops ops
;
2126 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2127 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2128 ops
.op2
= NULL_TREE
;
2130 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2131 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2135 /* For operations with low precision, if target doesn't have them, start
2136 with precres widening right away, otherwise do it only if the most
2137 simple cases can't be used. */
2138 const int min_precision
= targetm
.min_arithmetic_precision ();
2139 if (orig_precres
== precres
&& precres
< min_precision
)
2141 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2142 && prec1
<= precres
)
2143 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2144 && prec0
+ uns0_p
<= precres
2145 && prec1
+ uns1_p
<= precres
))
2147 arg0
= fold_convert_loc (loc
, type
, arg0
);
2148 arg1
= fold_convert_loc (loc
, type
, arg1
);
2152 if (integer_zerop (arg0
) && !unsr_p
)
2154 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2159 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2160 unsr_p
, unsr_p
, false, NULL
);
2163 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2164 unsr_p
, unsr_p
, false, NULL
);
2171 /* For sub-word operations, retry with a wider type first. */
2172 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2174 int p
= MAX (min_precision
, precop
);
2175 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2176 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2179 p
= TYPE_PRECISION (optype
);
2183 unsr_p
= TYPE_UNSIGNED (optype
);
2189 if (prec0
<= precres
&& prec1
<= precres
)
2194 types
[0] = build_nonstandard_integer_type (precres
, 0);
2200 types
[1] = build_nonstandard_integer_type (precres
, 1);
2202 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2203 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2204 if (code
!= MULT_EXPR
)
2205 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2206 uns0_p
, uns1_p
, false, NULL
);
2208 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2209 uns0_p
, uns1_p
, false, NULL
);
2213 /* Retry with a wider type. */
2214 if (orig_precres
== precres
)
2216 int p
= MAX (prec0
, prec1
);
2217 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2218 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2221 p
= TYPE_PRECISION (optype
);
2225 unsr_p
= TYPE_UNSIGNED (optype
);
2236 /* Expand ADD_OVERFLOW STMT. */
2239 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2241 expand_arith_overflow (PLUS_EXPR
, stmt
);
2244 /* Expand SUB_OVERFLOW STMT. */
2247 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2249 expand_arith_overflow (MINUS_EXPR
, stmt
);
2252 /* Expand MUL_OVERFLOW STMT. */
2255 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2257 expand_arith_overflow (MULT_EXPR
, stmt
);
2260 /* This should get folded in tree-vectorizer.c. */
2263 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2268 /* Expand MASK_LOAD call STMT using optab OPTAB. */
2271 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2273 struct expand_operand ops
[3];
2274 tree type
, lhs
, rhs
, maskt
, ptr
;
2275 rtx mem
, target
, mask
;
2278 maskt
= gimple_call_arg (stmt
, 2);
2279 lhs
= gimple_call_lhs (stmt
);
2280 if (lhs
== NULL_TREE
)
2282 type
= TREE_TYPE (lhs
);
2283 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2284 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2285 if (TYPE_ALIGN (type
) != align
)
2286 type
= build_aligned_type (type
, align
);
2287 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2289 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2290 gcc_assert (MEM_P (mem
));
2291 mask
= expand_normal (maskt
);
2292 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2293 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2294 create_fixed_operand (&ops
[1], mem
);
2295 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2296 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2297 TYPE_MODE (TREE_TYPE (maskt
))),
2301 /* Expand MASK_STORE call STMT using optab OPTAB. */
2304 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2306 struct expand_operand ops
[3];
2307 tree type
, lhs
, rhs
, maskt
, ptr
;
2311 maskt
= gimple_call_arg (stmt
, 2);
2312 rhs
= gimple_call_arg (stmt
, 3);
2313 type
= TREE_TYPE (rhs
);
2314 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2315 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2316 if (TYPE_ALIGN (type
) != align
)
2317 type
= build_aligned_type (type
, align
);
2318 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2320 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2321 gcc_assert (MEM_P (mem
));
2322 mask
= expand_normal (maskt
);
2323 reg
= expand_normal (rhs
);
2324 create_fixed_operand (&ops
[0], mem
);
2325 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2326 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2327 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2328 TYPE_MODE (TREE_TYPE (maskt
))),
2333 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2338 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2340 /* When guessing was done, the hints should be already stripped away. */
2341 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2344 tree lhs
= gimple_call_lhs (stmt
);
2346 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2348 target
= const0_rtx
;
2349 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2350 if (lhs
&& val
!= target
)
2351 emit_move_insn (target
, val
);
2354 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2355 should never be called. */
2358 expand_VA_ARG (internal_fn
, gcall
*)
2363 /* Expand the IFN_UNIQUE function according to its first argument. */
2366 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2368 rtx pattern
= NULL_RTX
;
2369 enum ifn_unique_kind kind
2370 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2377 case IFN_UNIQUE_UNSPEC
:
2378 if (targetm
.have_unique ())
2379 pattern
= targetm
.gen_unique ();
2382 case IFN_UNIQUE_OACC_FORK
:
2383 case IFN_UNIQUE_OACC_JOIN
:
2384 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2386 tree lhs
= gimple_call_lhs (stmt
);
2387 rtx target
= const0_rtx
;
2390 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2392 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2393 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2395 if (kind
== IFN_UNIQUE_OACC_FORK
)
2396 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2398 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2406 emit_insn (pattern
);
2409 /* The size of an OpenACC compute dimension. */
2412 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2414 tree lhs
= gimple_call_lhs (stmt
);
2419 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2420 if (targetm
.have_oacc_dim_size ())
2422 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2423 VOIDmode
, EXPAND_NORMAL
);
2424 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2427 emit_move_insn (target
, GEN_INT (1));
2430 /* The position of an OpenACC execution engine along one compute axis. */
2433 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2435 tree lhs
= gimple_call_lhs (stmt
);
2440 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2441 if (targetm
.have_oacc_dim_pos ())
2443 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2444 VOIDmode
, EXPAND_NORMAL
);
2445 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2448 emit_move_insn (target
, const0_rtx
);
2451 /* This is expanded by oacc_device_lower pass. */
2454 expand_GOACC_LOOP (internal_fn
, gcall
*)
2459 /* This is expanded by oacc_device_lower pass. */
2462 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2467 /* Set errno to EDOM. */
2470 expand_SET_EDOM (internal_fn
, gcall
*)
2473 #ifdef GEN_ERRNO_RTX
2474 rtx errno_rtx
= GEN_ERRNO_RTX
;
2476 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2478 emit_move_insn (errno_rtx
,
2479 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2485 /* Expand atomic bit test and set. */
2488 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2490 expand_ifn_atomic_bit_test_and (call
);
2493 /* Expand atomic bit test and complement. */
2496 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2498 expand_ifn_atomic_bit_test_and (call
);
2501 /* Expand atomic bit test and reset. */
2504 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2506 expand_ifn_atomic_bit_test_and (call
);
2509 /* Expand atomic bit test and set. */
2512 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2514 expand_ifn_atomic_compare_exchange (call
);
2517 /* Expand LAUNDER to assignment, lhs = arg0. */
2520 expand_LAUNDER (internal_fn
, gcall
*call
)
2522 tree lhs
= gimple_call_lhs (call
);
2527 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2530 /* Expand DIVMOD() using:
2531 a) optab handler for udivmod/sdivmod if it is available.
2532 b) If optab_handler doesn't exist, generate call to
2533 target-specific divmod libfunc. */
2536 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2538 tree lhs
= gimple_call_lhs (call_stmt
);
2539 tree arg0
= gimple_call_arg (call_stmt
, 0);
2540 tree arg1
= gimple_call_arg (call_stmt
, 1);
2542 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2543 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2544 machine_mode mode
= TYPE_MODE (type
);
2545 bool unsignedp
= TYPE_UNSIGNED (type
);
2546 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2548 rtx op0
= expand_normal (arg0
);
2549 rtx op1
= expand_normal (arg1
);
2550 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2552 rtx quotient
, remainder
, libfunc
;
2554 /* Check if optab_handler exists for divmod_optab for given mode. */
2555 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2557 quotient
= gen_reg_rtx (mode
);
2558 remainder
= gen_reg_rtx (mode
);
2559 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2562 /* Generate call to divmod libfunc if it exists. */
2563 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2564 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2565 "ient
, &remainder
);
2570 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2571 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2572 make_tree (TREE_TYPE (arg0
), quotient
),
2573 make_tree (TREE_TYPE (arg1
), remainder
)),
2574 target
, VOIDmode
, EXPAND_NORMAL
);
2577 /* Expand a call to FN using the operands in STMT. FN has a single
2578 output operand and NARGS input operands. */
2581 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2584 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2586 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2587 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2589 tree lhs
= gimple_call_lhs (stmt
);
2590 tree lhs_type
= TREE_TYPE (lhs
);
2591 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2592 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2594 for (unsigned int i
= 0; i
< nargs
; ++i
)
2596 tree rhs
= gimple_call_arg (stmt
, i
);
2597 tree rhs_type
= TREE_TYPE (rhs
);
2598 rtx rhs_rtx
= expand_normal (rhs
);
2599 if (INTEGRAL_TYPE_P (rhs_type
))
2600 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2601 TYPE_MODE (rhs_type
),
2602 TYPE_UNSIGNED (rhs_type
));
2604 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2607 expand_insn (icode
, nargs
+ 1, ops
);
2608 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2610 /* If the return value has an integral type, convert the instruction
2611 result to that type. This is useful for things that return an
2612 int regardless of the size of the input. If the instruction result
2613 is smaller than required, assume that it is signed.
2615 If the return value has a nonintegral type, its mode must match
2616 the instruction result. */
2617 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2619 /* If this is a scalar in a register that is stored in a wider
2620 mode than the declared mode, compute the result into its
2621 declared mode and then convert to the wider mode. */
2622 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2623 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2624 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2625 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2627 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2628 emit_move_insn (lhs_rtx
, ops
[0].value
);
2631 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2632 convert_move (lhs_rtx
, ops
[0].value
, 0);
2637 /* Expanders for optabs that can use expand_direct_optab_fn. */
2639 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2640 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2642 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2643 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2645 /* RETURN_TYPE and ARGS are a return type and argument list that are
2646 in principle compatible with FN (which satisfies direct_internal_fn_p).
2647 Return the types that should be used to determine whether the
2648 target supports FN. */
2651 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2653 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2654 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2655 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2656 return tree_pair (type0
, type1
);
2659 /* CALL is a call whose return type and arguments are in principle
2660 compatible with FN (which satisfies direct_internal_fn_p). Return the
2661 types that should be used to determine whether the target supports FN. */
2664 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2666 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2667 tree op0
= (info
.type0
< 0
2668 ? gimple_call_lhs (call
)
2669 : gimple_call_arg (call
, info
.type0
));
2670 tree op1
= (info
.type1
< 0
2671 ? gimple_call_lhs (call
)
2672 : gimple_call_arg (call
, info
.type1
));
2673 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2676 /* Return true if OPTAB is supported for TYPES (whose modes should be
2677 the same) when the optimization type is OPT_TYPE. Used for simple
2681 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2682 optimization_type opt_type
)
2684 machine_mode mode
= TYPE_MODE (types
.first
);
2685 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2686 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2689 /* Return true if load/store lanes optab OPTAB is supported for
2690 array type TYPES.first when the optimization type is OPT_TYPE. */
2693 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2694 optimization_type opt_type
)
2696 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2697 machine_mode imode
= TYPE_MODE (types
.first
);
2698 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2699 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2700 != CODE_FOR_nothing
);
2703 #define direct_unary_optab_supported_p direct_optab_supported_p
2704 #define direct_binary_optab_supported_p direct_optab_supported_p
2705 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2706 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2707 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2708 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2710 /* Return true if FN is supported for the types in TYPES when the
2711 optimization type is OPT_TYPE. The types are those associated with
2712 the "type0" and "type1" fields of FN's direct_internal_fn_info
2716 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2717 optimization_type opt_type
)
2721 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2722 case IFN_##CODE: break;
2723 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2725 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2727 #include "internal-fn.def"
2735 /* Return true if FN is supported for type TYPE when the optimization
2736 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2737 fields of FN's direct_internal_fn_info structure are the same. */
2740 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2741 optimization_type opt_type
)
2743 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2744 gcc_checking_assert (info
.type0
== info
.type1
);
2745 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2748 /* Return true if IFN_SET_EDOM is supported. */
2751 set_edom_supported_p (void)
2760 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2762 expand_##CODE (internal_fn fn, gcall *stmt) \
2764 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2766 #include "internal-fn.def"
2768 /* Routines to expand each internal function, indexed by function number.
2769 Each routine has the prototype:
2771 expand_<NAME> (gcall *stmt)
2773 where STMT is the statement that performs the call. */
2774 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2775 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2776 #include "internal-fn.def"
2780 /* Expand STMT as though it were a call to internal function FN. */
2783 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2785 internal_fn_expanders
[fn
] (fn
, stmt
);
2788 /* Expand STMT, which is a call to internal function FN. */
2791 expand_internal_call (gcall
*stmt
)
2793 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
2797 expand_PHI (internal_fn
, gcall
*)