2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
46 #include "optabs-tree.h"
48 /* The names of each internal function, indexed by function number. */
49 const char *const internal_fn_name_array
[] = {
50 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
51 #include "internal-fn.def"
55 /* The ECF_* flags of each internal function, indexed by function number. */
56 const int internal_fn_flags_array
[] = {
57 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
58 #include "internal-fn.def"
62 /* Fnspec of each internal function, indexed by function number. */
63 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
69 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
70 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
71 #include "internal-fn.def"
72 internal_fn_fnspec_array
[IFN_LAST
] = 0;
75 /* Create static initializers for the information returned by
76 direct_internal_fn. */
77 #define not_direct { -2, -2, false }
78 #define mask_load_direct { -1, 2, false }
79 #define load_lanes_direct { -1, -1, false }
80 #define mask_store_direct { 3, 2, false }
81 #define store_lanes_direct { 0, 0, false }
82 #define unary_direct { 0, 0, true }
83 #define binary_direct { 0, 0, true }
85 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
86 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
87 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
88 #include "internal-fn.def"
92 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
93 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
96 get_multi_vector_move (tree array_type
, convert_optab optab
)
101 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
102 imode
= TYPE_MODE (array_type
);
103 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
105 return convert_optab_handler (optab
, imode
, vmode
);
108 /* Expand LOAD_LANES call STMT using optab OPTAB. */
111 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
113 struct expand_operand ops
[2];
117 lhs
= gimple_call_lhs (stmt
);
118 rhs
= gimple_call_arg (stmt
, 0);
119 type
= TREE_TYPE (lhs
);
121 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
122 mem
= expand_normal (rhs
);
124 gcc_assert (MEM_P (mem
));
125 PUT_MODE (mem
, TYPE_MODE (type
));
127 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
128 create_fixed_operand (&ops
[1], mem
);
129 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
132 /* Expand STORE_LANES call STMT using optab OPTAB. */
135 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
137 struct expand_operand ops
[2];
141 lhs
= gimple_call_lhs (stmt
);
142 rhs
= gimple_call_arg (stmt
, 0);
143 type
= TREE_TYPE (rhs
);
145 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
146 reg
= expand_normal (rhs
);
148 gcc_assert (MEM_P (target
));
149 PUT_MODE (target
, TYPE_MODE (type
));
151 create_fixed_operand (&ops
[0], target
);
152 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
153 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
157 expand_ANNOTATE (internal_fn
, gcall
*)
162 /* This should get expanded in omp_device_lower pass. */
165 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
170 /* This should get expanded in omp_device_lower pass. */
173 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
178 /* Allocate per-lane storage and begin non-uniform execution region. */
181 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
184 tree lhs
= gimple_call_lhs (stmt
);
186 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
188 target
= gen_reg_rtx (Pmode
);
189 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
190 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
191 struct expand_operand ops
[3];
192 create_output_operand (&ops
[0], target
, Pmode
);
193 create_input_operand (&ops
[1], size
, Pmode
);
194 create_input_operand (&ops
[2], align
, Pmode
);
195 gcc_assert (targetm
.have_omp_simt_enter ());
196 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
199 /* Deallocate per-lane storage and leave non-uniform execution region. */
202 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
204 gcc_checking_assert (!gimple_call_lhs (stmt
));
205 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
206 struct expand_operand ops
[1];
207 create_input_operand (&ops
[0], arg
, Pmode
);
208 gcc_assert (targetm
.have_omp_simt_exit ());
209 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
212 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
213 without SIMT execution this should be expanded in omp_device_lower pass. */
216 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
218 tree lhs
= gimple_call_lhs (stmt
);
222 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
223 gcc_assert (targetm
.have_omp_simt_lane ());
224 emit_insn (targetm
.gen_omp_simt_lane (target
));
227 /* This should get expanded in omp_device_lower pass. */
230 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
235 /* Lane index of the first SIMT lane that supplies a non-zero argument.
236 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
237 lane that executed the last iteration for handling OpenMP lastprivate. */
240 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
242 tree lhs
= gimple_call_lhs (stmt
);
246 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
247 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
248 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
249 struct expand_operand ops
[2];
250 create_output_operand (&ops
[0], target
, mode
);
251 create_input_operand (&ops
[1], cond
, mode
);
252 gcc_assert (targetm
.have_omp_simt_last_lane ());
253 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
256 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
259 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
261 tree lhs
= gimple_call_lhs (stmt
);
265 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
266 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
267 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
268 struct expand_operand ops
[2];
269 create_output_operand (&ops
[0], target
, mode
);
270 create_input_operand (&ops
[1], ctr
, mode
);
271 gcc_assert (targetm
.have_omp_simt_ordered ());
272 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
275 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
276 any lane supplies a non-zero argument. */
279 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
281 tree lhs
= gimple_call_lhs (stmt
);
285 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
286 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
287 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
288 struct expand_operand ops
[2];
289 create_output_operand (&ops
[0], target
, mode
);
290 create_input_operand (&ops
[1], cond
, mode
);
291 gcc_assert (targetm
.have_omp_simt_vote_any ());
292 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
295 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
296 is destination lane index XOR given offset. */
299 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
301 tree lhs
= gimple_call_lhs (stmt
);
305 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
306 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
307 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
308 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
309 struct expand_operand ops
[3];
310 create_output_operand (&ops
[0], target
, mode
);
311 create_input_operand (&ops
[1], src
, mode
);
312 create_input_operand (&ops
[2], idx
, SImode
);
313 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
314 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
317 /* Exchange between SIMT lanes according to given source lane index. */
320 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
322 tree lhs
= gimple_call_lhs (stmt
);
326 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
327 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
328 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
329 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
330 struct expand_operand ops
[3];
331 create_output_operand (&ops
[0], target
, mode
);
332 create_input_operand (&ops
[1], src
, mode
);
333 create_input_operand (&ops
[2], idx
, SImode
);
334 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
335 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
338 /* This should get expanded in adjust_simduid_builtins. */
341 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
346 /* This should get expanded in adjust_simduid_builtins. */
349 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
354 /* This should get expanded in adjust_simduid_builtins. */
357 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
362 /* This should get expanded in adjust_simduid_builtins. */
365 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
370 /* This should get expanded in adjust_simduid_builtins. */
373 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
378 /* This should get expanded in the sanopt pass. */
381 expand_UBSAN_NULL (internal_fn
, gcall
*)
386 /* This should get expanded in the sanopt pass. */
389 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
394 /* This should get expanded in the sanopt pass. */
397 expand_UBSAN_VPTR (internal_fn
, gcall
*)
402 /* This should get expanded in the sanopt pass. */
405 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
410 /* This should get expanded in the sanopt pass. */
413 expand_ASAN_CHECK (internal_fn
, gcall
*)
418 /* This should get expanded in the sanopt pass. */
421 expand_ASAN_MARK (internal_fn
, gcall
*)
426 /* This should get expanded in the sanopt pass. */
429 expand_ASAN_POISON (internal_fn
, gcall
*)
434 /* This should get expanded in the sanopt pass. */
437 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
442 /* This should get expanded in the tsan pass. */
445 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
450 /* This should get expanded in the lower pass. */
453 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
455 error_at (gimple_location (call
),
456 "invalid use of attribute %<fallthrough%>");
459 /* Return minimum precision needed to represent all values
460 of ARG in SIGNed integral type. */
463 get_min_precision (tree arg
, signop sign
)
465 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
467 signop orig_sign
= sign
;
468 if (TREE_CODE (arg
) == INTEGER_CST
)
471 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
473 widest_int w
= wi::to_widest (arg
);
474 w
= wi::ext (w
, prec
, sign
);
475 p
= wi::min_precision (w
, sign
);
478 p
= wi::min_precision (arg
, sign
);
479 return MIN (p
, prec
);
481 while (CONVERT_EXPR_P (arg
)
482 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
483 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
485 arg
= TREE_OPERAND (arg
, 0);
486 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
488 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
490 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
491 return prec
+ (orig_sign
!= sign
);
492 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
495 return prec
+ (orig_sign
!= sign
);
497 if (TREE_CODE (arg
) != SSA_NAME
)
498 return prec
+ (orig_sign
!= sign
);
499 wide_int arg_min
, arg_max
;
500 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
502 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
503 if (is_gimple_assign (g
)
504 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
506 tree t
= gimple_assign_rhs1 (g
);
507 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
508 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
511 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
513 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
515 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
516 return prec
+ (orig_sign
!= sign
);
517 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
520 return prec
+ (orig_sign
!= sign
);
524 return prec
+ (orig_sign
!= sign
);
526 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
528 int p1
= wi::min_precision (arg_min
, sign
);
529 int p2
= wi::min_precision (arg_max
, sign
);
531 prec
= MIN (prec
, p1
);
533 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
535 int p
= wi::min_precision (arg_max
, UNSIGNED
);
536 prec
= MIN (prec
, p
);
538 return prec
+ (orig_sign
!= sign
);
541 /* Helper for expand_*_overflow. Set the __imag__ part to true
542 (1 except for signed:1 type, in which case store -1). */
545 expand_arith_set_overflow (tree lhs
, rtx target
)
547 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
548 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
549 write_complex_part (target
, constm1_rtx
, true);
551 write_complex_part (target
, const1_rtx
, true);
554 /* Helper for expand_*_overflow. Store RES into the __real__ part
555 of TARGET. If RES has larger MODE than __real__ part of TARGET,
556 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
557 if LHS has smaller precision than its mode. */
560 expand_arith_overflow_result_store (tree lhs
, rtx target
,
561 machine_mode mode
, rtx res
)
563 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
567 rtx_code_label
*done_label
= gen_label_rtx ();
568 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
569 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
570 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
571 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
572 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
573 profile_probability::very_likely ());
574 expand_arith_set_overflow (lhs
, target
);
575 emit_label (done_label
);
577 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
578 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
581 rtx_code_label
*done_label
= gen_label_rtx ();
582 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
587 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
589 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
590 true, OPTAB_LIB_WIDEN
);
594 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
596 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
599 do_compare_rtx_and_jump (res
, lres
,
600 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
601 profile_probability::very_likely ());
602 expand_arith_set_overflow (lhs
, target
);
603 emit_label (done_label
);
605 write_complex_part (target
, lres
, false);
608 /* Helper for expand_*_overflow. Store RES into TARGET. */
611 expand_ubsan_result_store (rtx target
, rtx res
)
613 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
614 /* If this is a scalar in a register that is stored in a wider mode
615 than the declared mode, compute the result into its declared mode
616 and then convert to the wider mode. Our value is the computed
618 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
620 emit_move_insn (target
, res
);
623 /* Add sub/add overflow checking to the statement STMT.
624 CODE says whether the operation is +, or -. */
627 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
628 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
629 bool uns1_p
, bool is_ubsan
, tree
*datap
)
631 rtx res
, target
= NULL_RTX
;
633 rtx_code_label
*done_label
= gen_label_rtx ();
634 rtx_code_label
*do_error
= gen_label_rtx ();
635 do_pending_stack_adjust ();
636 rtx op0
= expand_normal (arg0
);
637 rtx op1
= expand_normal (arg1
);
638 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
639 int prec
= GET_MODE_PRECISION (mode
);
640 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
644 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
648 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
650 write_complex_part (target
, const0_rtx
, true);
653 /* We assume both operands and result have the same precision
654 here (GET_MODE_BITSIZE (mode)), S stands for signed type
655 with that precision, U for unsigned type with that precision,
656 sgn for unsigned most significant bit in that precision.
657 s1 is signed first operand, u1 is unsigned first operand,
658 s2 is signed second operand, u2 is unsigned second operand,
659 sr is signed result, ur is unsigned result and the following
660 rules say how to compute result (which is always result of
661 the operands as if both were unsigned, cast to the right
662 signedness) and how to compute whether operation overflowed.
665 res = (S) ((U) s1 + (U) s2)
666 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
668 res = (S) ((U) s1 - (U) s2)
669 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
672 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
675 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
677 res = (S) ((U) s1 + u2)
678 ovf = ((U) res ^ sgn) < u2
683 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
685 res = (S) ((U) s1 - u2)
686 ovf = u2 > ((U) s1 ^ sgn)
689 ovf = s1 < 0 || u2 > (U) s1
692 ovf = u1 >= ((U) s2 ^ sgn)
697 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
699 res = (U) s1 + (U) s2
700 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
703 ovf = (U) res < u2 || res < 0
706 ovf = u1 >= u2 ? res < 0 : res >= 0
708 res = (U) s1 - (U) s2
709 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
711 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
713 /* PLUS_EXPR is commutative, if operand signedness differs,
714 canonicalize to the first operand being signed and second
715 unsigned to simplify following code. */
716 std::swap (op0
, op1
);
717 std::swap (arg0
, arg1
);
723 if (uns0_p
&& uns1_p
&& unsr_p
)
725 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
726 : usubv4_optab
, mode
);
727 if (icode
!= CODE_FOR_nothing
)
729 struct expand_operand ops
[4];
730 rtx_insn
*last
= get_last_insn ();
732 res
= gen_reg_rtx (mode
);
733 create_output_operand (&ops
[0], res
, mode
);
734 create_input_operand (&ops
[1], op0
, mode
);
735 create_input_operand (&ops
[2], op1
, mode
);
736 create_fixed_operand (&ops
[3], do_error
);
737 if (maybe_expand_insn (icode
, 4, ops
))
739 last
= get_last_insn ();
740 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
742 && any_condjump_p (last
)
743 && !find_reg_note (last
, REG_BR_PROB
, 0))
744 add_reg_br_prob_note (last
,
745 profile_probability::very_unlikely ());
746 emit_jump (done_label
);
750 delete_insns_since (last
);
753 /* Compute the operation. On RTL level, the addition is always
755 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
756 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
758 /* For PLUS_EXPR, the operation is commutative, so we can pick
759 operand to compare against. For prec <= BITS_PER_WORD, I think
760 preferring REG operand is better over CONST_INT, because
761 the CONST_INT might enlarge the instruction or CSE would need
762 to figure out we'd already loaded it into a register before.
763 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
764 as then the multi-word comparison can be perhaps simplified. */
765 if (code
== PLUS_EXPR
766 && (prec
<= BITS_PER_WORD
767 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
768 : CONST_SCALAR_INT_P (op1
)))
770 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
771 true, mode
, NULL_RTX
, NULL
, done_label
,
772 profile_probability::very_likely ());
777 if (!uns0_p
&& uns1_p
&& !unsr_p
)
779 /* Compute the operation. On RTL level, the addition is always
781 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
782 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
783 rtx tem
= expand_binop (mode
, add_optab
,
784 code
== PLUS_EXPR
? res
: op0
, sgn
,
785 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
786 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
787 done_label
, profile_probability::very_likely ());
792 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
794 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
796 /* As we've changed op1, we have to avoid using the value range
797 for the original argument. */
798 arg1
= error_mark_node
;
804 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
806 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
808 /* As we've changed op0, we have to avoid using the value range
809 for the original argument. */
810 arg0
= error_mark_node
;
816 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
818 /* Compute the operation. On RTL level, the addition is always
820 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
822 int pos_neg
= get_range_pos_neg (arg0
);
824 /* If ARG0 is known to be always negative, this is always overflow. */
825 emit_jump (do_error
);
826 else if (pos_neg
== 3)
827 /* If ARG0 is not known to be always positive, check at runtime. */
828 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
829 NULL
, do_error
, profile_probability::very_unlikely ());
830 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
831 done_label
, profile_probability::very_likely ());
836 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
838 /* Compute the operation. On RTL level, the addition is always
840 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
842 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
844 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
845 done_label
, profile_probability::very_likely ());
850 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
852 /* Compute the operation. On RTL level, the addition is always
854 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
856 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
857 NULL
, do_error
, profile_probability::very_unlikely ());
859 /* The operation is commutative, so we can pick operand to compare
860 against. For prec <= BITS_PER_WORD, I think preferring REG operand
861 is better over CONST_INT, because the CONST_INT might enlarge the
862 instruction or CSE would need to figure out we'd already loaded it
863 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
864 might be more beneficial, as then the multi-word comparison can be
865 perhaps simplified. */
866 if (prec
<= BITS_PER_WORD
867 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
868 : CONST_SCALAR_INT_P (op0
))
870 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
871 done_label
, profile_probability::very_likely ());
876 if (!uns0_p
&& !uns1_p
&& unsr_p
)
878 /* Compute the operation. On RTL level, the addition is always
880 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
881 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
882 int pos_neg
= get_range_pos_neg (arg1
);
883 if (code
== PLUS_EXPR
)
885 int pos_neg0
= get_range_pos_neg (arg0
);
886 if (pos_neg0
!= 3 && pos_neg
== 3)
888 std::swap (op0
, op1
);
895 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
896 ? and_optab
: ior_optab
,
897 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
898 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
899 NULL
, done_label
, profile_probability::very_likely ());
903 rtx_code_label
*do_ior_label
= gen_label_rtx ();
904 do_compare_rtx_and_jump (op1
, const0_rtx
,
905 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
906 NULL_RTX
, NULL
, do_ior_label
,
907 profile_probability::even ());
908 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
910 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
911 NULL
, done_label
, profile_probability::very_likely ());
912 emit_jump (do_error
);
913 emit_label (do_ior_label
);
914 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
916 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
917 NULL
, done_label
, profile_probability::very_likely ());
923 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
925 /* Compute the operation. On RTL level, the addition is always
927 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
929 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
930 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
931 op0_geu_op1
, profile_probability::even ());
932 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
933 NULL
, done_label
, profile_probability::very_likely ());
934 emit_jump (do_error
);
935 emit_label (op0_geu_op1
);
936 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
937 NULL
, done_label
, profile_probability::very_likely ());
941 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
946 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
947 : subv4_optab
, mode
);
948 if (icode
!= CODE_FOR_nothing
)
950 struct expand_operand ops
[4];
951 rtx_insn
*last
= get_last_insn ();
953 res
= gen_reg_rtx (mode
);
954 create_output_operand (&ops
[0], res
, mode
);
955 create_input_operand (&ops
[1], op0
, mode
);
956 create_input_operand (&ops
[2], op1
, mode
);
957 create_fixed_operand (&ops
[3], do_error
);
958 if (maybe_expand_insn (icode
, 4, ops
))
960 last
= get_last_insn ();
961 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
963 && any_condjump_p (last
)
964 && !find_reg_note (last
, REG_BR_PROB
, 0))
965 add_reg_br_prob_note (last
,
966 profile_probability::very_unlikely ());
967 emit_jump (done_label
);
971 delete_insns_since (last
);
974 /* Compute the operation. On RTL level, the addition is always
976 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
977 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
979 /* If we can prove that one of the arguments (for MINUS_EXPR only
980 the second operand, as subtraction is not commutative) is always
981 non-negative or always negative, we can do just one comparison
982 and conditional jump. */
983 int pos_neg
= get_range_pos_neg (arg1
);
984 if (code
== PLUS_EXPR
)
986 int pos_neg0
= get_range_pos_neg (arg0
);
987 if (pos_neg0
!= 3 && pos_neg
== 3)
989 std::swap (op0
, op1
);
994 /* Addition overflows if and only if the two operands have the same sign,
995 and the result has the opposite sign. Subtraction overflows if and
996 only if the two operands have opposite sign, and the subtrahend has
997 the same sign as the result. Here 0 is counted as positive. */
1000 /* Compute op0 ^ op1 (operands have opposite sign). */
1001 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1004 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1005 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1009 if (code
== PLUS_EXPR
)
1011 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1012 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1013 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1018 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1019 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1020 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1024 /* No overflow if the result has bit sign cleared. */
1025 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1026 NULL
, done_label
, profile_probability::very_likely ());
1029 /* Compare the result of the operation with the first operand.
1030 No overflow for addition if second operand is positive and result
1031 is larger or second operand is negative and result is smaller.
1032 Likewise for subtraction with sign of second operand flipped. */
1034 do_compare_rtx_and_jump (res
, op0
,
1035 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1036 false, mode
, NULL_RTX
, NULL
, done_label
,
1037 profile_probability::very_likely ());
1041 emit_label (do_error
);
1044 /* Expand the ubsan builtin call. */
1046 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1050 do_pending_stack_adjust ();
1053 expand_arith_set_overflow (lhs
, target
);
1056 emit_label (done_label
);
1061 expand_ubsan_result_store (target
, res
);
1065 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1068 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1073 /* Add negate overflow checking to the statement STMT. */
1076 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1081 rtx_code_label
*done_label
, *do_error
;
1082 rtx target
= NULL_RTX
;
1084 done_label
= gen_label_rtx ();
1085 do_error
= gen_label_rtx ();
1087 do_pending_stack_adjust ();
1088 op1
= expand_normal (arg1
);
1090 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
1093 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1095 write_complex_part (target
, const0_rtx
, true);
1098 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1099 if (icode
!= CODE_FOR_nothing
)
1101 struct expand_operand ops
[3];
1102 rtx_insn
*last
= get_last_insn ();
1104 res
= gen_reg_rtx (mode
);
1105 create_output_operand (&ops
[0], res
, mode
);
1106 create_input_operand (&ops
[1], op1
, mode
);
1107 create_fixed_operand (&ops
[2], do_error
);
1108 if (maybe_expand_insn (icode
, 3, ops
))
1110 last
= get_last_insn ();
1111 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1113 && any_condjump_p (last
)
1114 && !find_reg_note (last
, REG_BR_PROB
, 0))
1115 add_reg_br_prob_note (last
,
1116 profile_probability::very_unlikely ());
1117 emit_jump (done_label
);
1121 delete_insns_since (last
);
1122 icode
= CODE_FOR_nothing
;
1126 if (icode
== CODE_FOR_nothing
)
1128 /* Compute the operation. On RTL level, the addition is always
1130 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1132 /* Compare the operand with the most negative value. */
1133 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1134 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1135 done_label
, profile_probability::very_likely ());
1138 emit_label (do_error
);
1141 /* Expand the ubsan builtin call. */
1143 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1144 arg1
, NULL_TREE
, datap
);
1147 do_pending_stack_adjust ();
1150 expand_arith_set_overflow (lhs
, target
);
1153 emit_label (done_label
);
1158 expand_ubsan_result_store (target
, res
);
1160 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1164 /* Add mul overflow checking to the statement STMT. */
1167 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1168 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1173 rtx_code_label
*done_label
, *do_error
;
1174 rtx target
= NULL_RTX
;
1176 enum insn_code icode
;
1178 done_label
= gen_label_rtx ();
1179 do_error
= gen_label_rtx ();
1181 do_pending_stack_adjust ();
1182 op0
= expand_normal (arg0
);
1183 op1
= expand_normal (arg1
);
1185 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
1189 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1191 write_complex_part (target
, const0_rtx
, true);
1195 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1197 /* We assume both operands and result have the same precision
1198 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1199 with that precision, U for unsigned type with that precision,
1200 sgn for unsigned most significant bit in that precision.
1201 s1 is signed first operand, u1 is unsigned first operand,
1202 s2 is signed second operand, u2 is unsigned second operand,
1203 sr is signed result, ur is unsigned result and the following
1204 rules say how to compute result (which is always result of
1205 the operands as if both were unsigned, cast to the right
1206 signedness) and how to compute whether operation overflowed.
1207 main_ovf (false) stands for jump on signed multiplication
1208 overflow or the main algorithm with uns == false.
1209 main_ovf (true) stands for jump on unsigned multiplication
1210 overflow or the main algorithm with uns == true.
1213 res = (S) ((U) s1 * (U) s2)
1214 ovf = main_ovf (false)
1217 ovf = main_ovf (true)
1220 ovf = (s1 < 0 && u2) || main_ovf (true)
1223 ovf = res < 0 || main_ovf (true)
1225 res = (S) ((U) s1 * u2)
1226 ovf = (S) u2 >= 0 ? main_ovf (false)
1227 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1229 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1230 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1232 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1234 if (uns0_p
&& !uns1_p
)
1236 /* Multiplication is commutative, if operand signedness differs,
1237 canonicalize to the first operand being signed and second
1238 unsigned to simplify following code. */
1239 std::swap (op0
, op1
);
1240 std::swap (arg0
, arg1
);
1245 int pos_neg0
= get_range_pos_neg (arg0
);
1246 int pos_neg1
= get_range_pos_neg (arg1
);
1249 if (!uns0_p
&& uns1_p
&& unsr_p
)
1254 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1257 /* If s1 is negative, avoid the main code, just multiply and
1258 signal overflow if op1 is not 0. */
1259 struct separate_ops ops
;
1260 ops
.code
= MULT_EXPR
;
1261 ops
.type
= TREE_TYPE (arg1
);
1262 ops
.op0
= make_tree (ops
.type
, op0
);
1263 ops
.op1
= make_tree (ops
.type
, op1
);
1264 ops
.op2
= NULL_TREE
;
1266 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1267 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1268 NULL
, done_label
, profile_probability::very_likely ());
1269 goto do_error_label
;
1271 rtx_code_label
*do_main_label
;
1272 do_main_label
= gen_label_rtx ();
1273 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1274 NULL
, do_main_label
, profile_probability::very_likely ());
1275 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1276 NULL
, do_main_label
, profile_probability::very_likely ());
1277 expand_arith_set_overflow (lhs
, target
);
1278 emit_label (do_main_label
);
1286 if (uns0_p
&& uns1_p
&& !unsr_p
)
1289 /* Rest of handling of this case after res is computed. */
1294 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1301 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1302 avoid the main code, just multiply and signal overflow
1303 unless 0 * u2 or -1 * ((U) Smin). */
1304 struct separate_ops ops
;
1305 ops
.code
= MULT_EXPR
;
1306 ops
.type
= TREE_TYPE (arg1
);
1307 ops
.op0
= make_tree (ops
.type
, op0
);
1308 ops
.op1
= make_tree (ops
.type
, op1
);
1309 ops
.op2
= NULL_TREE
;
1311 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1312 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1313 NULL
, done_label
, profile_probability::very_likely ());
1314 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1315 NULL
, do_error
, profile_probability::very_unlikely ());
1317 prec
= GET_MODE_PRECISION (mode
);
1319 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1320 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1321 NULL
, done_label
, profile_probability::very_likely ());
1322 goto do_error_label
;
1324 /* Rest of handling of this case after res is computed. */
1332 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1335 switch (pos_neg0
| pos_neg1
)
1337 case 1: /* Both operands known to be non-negative. */
1339 case 2: /* Both operands known to be negative. */
1340 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1341 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1342 /* Avoid looking at arg0/arg1 ranges, as we've changed
1344 arg0
= error_mark_node
;
1345 arg1
= error_mark_node
;
1348 if ((pos_neg0
^ pos_neg1
) == 3)
1350 /* If one operand is known to be negative and the other
1351 non-negative, this overflows always, unless the non-negative
1352 one is 0. Just do normal multiply and set overflow
1353 unless one of the operands is 0. */
1354 struct separate_ops ops
;
1355 ops
.code
= MULT_EXPR
;
1357 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1359 ops
.op0
= make_tree (ops
.type
, op0
);
1360 ops
.op1
= make_tree (ops
.type
, op1
);
1361 ops
.op2
= NULL_TREE
;
1363 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1364 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1366 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1367 NULL_RTX
, NULL
, done_label
,
1368 profile_probability::very_likely ());
1369 goto do_error_label
;
1371 /* The general case, do all the needed comparisons at runtime. */
1372 rtx_code_label
*do_main_label
, *after_negate_label
;
1374 rop0
= gen_reg_rtx (mode
);
1375 rop1
= gen_reg_rtx (mode
);
1376 emit_move_insn (rop0
, op0
);
1377 emit_move_insn (rop1
, op1
);
1380 do_main_label
= gen_label_rtx ();
1381 after_negate_label
= gen_label_rtx ();
1382 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1384 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1385 NULL
, after_negate_label
, profile_probability::very_likely ());
1386 /* Both arguments negative here, negate them and continue with
1387 normal unsigned overflow checking multiplication. */
1388 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1390 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1392 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1394 arg0
= error_mark_node
;
1395 arg1
= error_mark_node
;
1396 emit_jump (do_main_label
);
1397 emit_label (after_negate_label
);
1398 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1400 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1401 NULL
, do_main_label
, profile_probability::very_likely ());
1402 /* One argument is negative here, the other positive. This
1403 overflows always, unless one of the arguments is 0. But
1404 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1405 is, thus we can keep do_main code oring in overflow as is. */
1406 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1407 NULL
, do_main_label
, profile_probability::very_likely ());
1408 expand_arith_set_overflow (lhs
, target
);
1409 emit_label (do_main_label
);
1417 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1418 sign
= uns
? UNSIGNED
: SIGNED
;
1419 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1420 if (icode
!= CODE_FOR_nothing
)
1422 struct expand_operand ops
[4];
1423 rtx_insn
*last
= get_last_insn ();
1425 res
= gen_reg_rtx (mode
);
1426 create_output_operand (&ops
[0], res
, mode
);
1427 create_input_operand (&ops
[1], op0
, mode
);
1428 create_input_operand (&ops
[2], op1
, mode
);
1429 create_fixed_operand (&ops
[3], do_error
);
1430 if (maybe_expand_insn (icode
, 4, ops
))
1432 last
= get_last_insn ();
1433 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1435 && any_condjump_p (last
)
1436 && !find_reg_note (last
, REG_BR_PROB
, 0))
1437 add_reg_br_prob_note (last
,
1438 profile_probability::very_unlikely ());
1439 emit_jump (done_label
);
1443 delete_insns_since (last
);
1444 icode
= CODE_FOR_nothing
;
1448 if (icode
== CODE_FOR_nothing
)
1450 struct separate_ops ops
;
1451 int prec
= GET_MODE_PRECISION (mode
);
1452 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1453 ops
.op0
= make_tree (type
, op0
);
1454 ops
.op1
= make_tree (type
, op1
);
1455 ops
.op2
= NULL_TREE
;
1457 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1458 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1460 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1461 ops
.code
= WIDEN_MULT_EXPR
;
1463 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1465 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1466 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1468 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1469 res
= convert_modes (mode
, wmode
, res
, uns
);
1471 /* For the unsigned multiplication, there was overflow if
1472 HIPART is non-zero. */
1473 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1474 NULL_RTX
, NULL
, done_label
,
1475 profile_probability::very_likely ());
1478 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1480 /* RES is low half of the double width result, HIPART
1481 the high half. There was overflow if
1482 HIPART is different from RES < 0 ? -1 : 0. */
1483 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1484 NULL_RTX
, NULL
, done_label
,
1485 profile_probability::very_likely ());
1488 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1490 rtx_code_label
*large_op0
= gen_label_rtx ();
1491 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1492 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1493 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1494 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1495 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1496 rtx_code_label
*do_overflow
= gen_label_rtx ();
1497 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1499 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1500 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1502 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1503 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1504 rtx signbit0
= const0_rtx
;
1506 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1508 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1510 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1511 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1512 rtx signbit1
= const0_rtx
;
1514 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1517 res
= gen_reg_rtx (mode
);
1519 /* True if op0 resp. op1 are known to be in the range of
1521 bool op0_small_p
= false;
1522 bool op1_small_p
= false;
1523 /* True if op0 resp. op1 are known to have all zeros or all ones
1524 in the upper half of bits, but are not known to be
1526 bool op0_medium_p
= false;
1527 bool op1_medium_p
= false;
1528 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1529 nonnegative, 1 if unknown. */
1535 else if (pos_neg0
== 2)
1539 else if (pos_neg1
== 2)
1542 unsigned int mprec0
= prec
;
1543 if (arg0
!= error_mark_node
)
1544 mprec0
= get_min_precision (arg0
, sign
);
1545 if (mprec0
<= hprec
)
1547 else if (!uns
&& mprec0
<= hprec
+ 1)
1548 op0_medium_p
= true;
1549 unsigned int mprec1
= prec
;
1550 if (arg1
!= error_mark_node
)
1551 mprec1
= get_min_precision (arg1
, sign
);
1552 if (mprec1
<= hprec
)
1554 else if (!uns
&& mprec1
<= hprec
+ 1)
1555 op1_medium_p
= true;
1557 int smaller_sign
= 1;
1558 int larger_sign
= 1;
1561 smaller_sign
= op0_sign
;
1562 larger_sign
= op1_sign
;
1564 else if (op1_small_p
)
1566 smaller_sign
= op1_sign
;
1567 larger_sign
= op0_sign
;
1569 else if (op0_sign
== op1_sign
)
1571 smaller_sign
= op0_sign
;
1572 larger_sign
= op0_sign
;
1576 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1577 NULL_RTX
, NULL
, large_op0
,
1578 profile_probability::unlikely ());
1581 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1582 NULL_RTX
, NULL
, small_op0_large_op1
,
1583 profile_probability::unlikely ());
1585 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1586 hmode to mode, the multiplication will never overflow. We can
1587 do just one hmode x hmode => mode widening multiplication. */
1588 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1589 if (GET_CODE (lopart0
) == SUBREG
)
1591 lopart0s
= shallow_copy_rtx (lopart0
);
1592 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1593 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1595 if (GET_CODE (lopart1
) == SUBREG
)
1597 lopart1s
= shallow_copy_rtx (lopart1
);
1598 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1599 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1601 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1602 ops
.op0
= make_tree (halfstype
, lopart0s
);
1603 ops
.op1
= make_tree (halfstype
, lopart1s
);
1604 ops
.code
= WIDEN_MULT_EXPR
;
1607 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1608 emit_move_insn (res
, thisres
);
1609 emit_jump (done_label
);
1611 emit_label (small_op0_large_op1
);
1613 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1614 but op1 is not, just swap the arguments and handle it as op1
1615 sign/zero extended, op0 not. */
1616 rtx larger
= gen_reg_rtx (mode
);
1617 rtx hipart
= gen_reg_rtx (hmode
);
1618 rtx lopart
= gen_reg_rtx (hmode
);
1619 emit_move_insn (larger
, op1
);
1620 emit_move_insn (hipart
, hipart1
);
1621 emit_move_insn (lopart
, lopart0
);
1622 emit_jump (one_small_one_large
);
1624 emit_label (large_op0
);
1627 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1628 NULL_RTX
, NULL
, both_ops_large
,
1629 profile_probability::unlikely ());
1631 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1632 but op0 is not, prepare larger, hipart and lopart pseudos and
1633 handle it together with small_op0_large_op1. */
1634 emit_move_insn (larger
, op0
);
1635 emit_move_insn (hipart
, hipart0
);
1636 emit_move_insn (lopart
, lopart1
);
1638 emit_label (one_small_one_large
);
1640 /* lopart is the low part of the operand that is sign extended
1641 to mode, larger is the other operand, hipart is the
1642 high part of larger and lopart0 and lopart1 are the low parts
1644 We perform lopart0 * lopart1 and lopart * hipart widening
1646 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1647 ops
.op0
= make_tree (halfutype
, lopart0
);
1648 ops
.op1
= make_tree (halfutype
, lopart1
);
1650 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1652 ops
.op0
= make_tree (halfutype
, lopart
);
1653 ops
.op1
= make_tree (halfutype
, hipart
);
1654 rtx loxhi
= gen_reg_rtx (mode
);
1655 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1656 emit_move_insn (loxhi
, tem
);
1660 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1661 if (larger_sign
== 0)
1662 emit_jump (after_hipart_neg
);
1663 else if (larger_sign
!= -1)
1664 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1665 NULL_RTX
, NULL
, after_hipart_neg
,
1666 profile_probability::even ());
1668 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1669 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1670 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1672 emit_move_insn (loxhi
, tem
);
1674 emit_label (after_hipart_neg
);
1676 /* if (lopart < 0) loxhi -= larger; */
1677 if (smaller_sign
== 0)
1678 emit_jump (after_lopart_neg
);
1679 else if (smaller_sign
!= -1)
1680 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1681 NULL_RTX
, NULL
, after_lopart_neg
,
1682 profile_probability::even ());
1684 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1686 emit_move_insn (loxhi
, tem
);
1688 emit_label (after_lopart_neg
);
1691 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1692 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1693 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1695 emit_move_insn (loxhi
, tem
);
1697 /* if (loxhi >> (bitsize / 2)
1698 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1699 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1700 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1702 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1703 rtx signbitloxhi
= const0_rtx
;
1705 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1706 convert_modes (hmode
, mode
,
1708 hprec
- 1, NULL_RTX
, 0);
1710 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1711 NULL_RTX
, NULL
, do_overflow
,
1712 profile_probability::very_unlikely ());
1714 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1715 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1717 tem
= convert_modes (mode
, hmode
,
1718 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1720 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1723 emit_move_insn (res
, tem
);
1724 emit_jump (done_label
);
1726 emit_label (both_ops_large
);
1728 /* If both operands are large (not sign (!uns) or zero (uns)
1729 extended from hmode), then perform the full multiplication
1730 which will be the result of the operation.
1731 The only cases which don't overflow are for signed multiplication
1732 some cases where both hipart0 and highpart1 are 0 or -1.
1733 For unsigned multiplication when high parts are both non-zero
1734 this overflows always. */
1735 ops
.code
= MULT_EXPR
;
1736 ops
.op0
= make_tree (type
, op0
);
1737 ops
.op1
= make_tree (type
, op1
);
1738 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1739 emit_move_insn (res
, tem
);
1745 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1746 NULL_RTX
, 1, OPTAB_DIRECT
);
1747 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1748 NULL_RTX
, NULL
, do_error
,
1749 profile_probability::very_unlikely ());
1754 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1755 NULL_RTX
, 1, OPTAB_DIRECT
);
1756 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1757 NULL_RTX
, NULL
, do_error
,
1758 profile_probability::very_unlikely ());
1761 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1762 the same, overflow happened if res is negative, if they are
1763 different, overflow happened if res is positive. */
1764 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1765 emit_jump (hipart_different
);
1766 else if (op0_sign
== 1 || op1_sign
== 1)
1767 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1768 NULL_RTX
, NULL
, hipart_different
,
1769 profile_probability::even ());
1771 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1772 NULL_RTX
, NULL
, do_error
,
1773 profile_probability::very_unlikely ());
1774 emit_jump (done_label
);
1776 emit_label (hipart_different
);
1778 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1779 NULL_RTX
, NULL
, do_error
,
1780 profile_probability::very_unlikely ());
1781 emit_jump (done_label
);
1784 emit_label (do_overflow
);
1786 /* Overflow, do full multiplication and fallthru into do_error. */
1787 ops
.op0
= make_tree (type
, op0
);
1788 ops
.op1
= make_tree (type
, op1
);
1789 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1790 emit_move_insn (res
, tem
);
1794 gcc_assert (!is_ubsan
);
1795 ops
.code
= MULT_EXPR
;
1797 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1798 emit_jump (done_label
);
1803 emit_label (do_error
);
1806 /* Expand the ubsan builtin call. */
1808 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1812 do_pending_stack_adjust ();
1815 expand_arith_set_overflow (lhs
, target
);
1818 emit_label (done_label
);
1821 if (uns0_p
&& uns1_p
&& !unsr_p
)
1823 rtx_code_label
*all_done_label
= gen_label_rtx ();
1824 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1825 NULL
, all_done_label
, profile_probability::very_likely ());
1826 expand_arith_set_overflow (lhs
, target
);
1827 emit_label (all_done_label
);
1831 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1833 rtx_code_label
*all_done_label
= gen_label_rtx ();
1834 rtx_code_label
*set_noovf
= gen_label_rtx ();
1835 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1836 NULL
, all_done_label
, profile_probability::very_likely ());
1837 expand_arith_set_overflow (lhs
, target
);
1838 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1839 NULL
, set_noovf
, profile_probability::very_likely ());
1840 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1841 NULL
, all_done_label
, profile_probability::very_unlikely ());
1842 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1843 all_done_label
, profile_probability::very_unlikely ());
1844 emit_label (set_noovf
);
1845 write_complex_part (target
, const0_rtx
, true);
1846 emit_label (all_done_label
);
1852 expand_ubsan_result_store (target
, res
);
1854 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1858 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
1861 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
1862 tree arg0
, tree arg1
)
1864 int cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
1865 rtx_code_label
*loop_lab
= NULL
;
1866 rtx cntvar
= NULL_RTX
;
1867 tree cntv
= NULL_TREE
;
1868 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
1869 tree sz
= TYPE_SIZE (eltype
);
1870 tree data
= NULL_TREE
;
1871 tree resv
= NULL_TREE
;
1872 rtx lhsr
= NULL_RTX
;
1873 rtx resvr
= NULL_RTX
;
1878 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1879 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
1880 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
1881 optab_default
)) == unknown_optab
1882 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
1883 == CODE_FOR_nothing
))
1886 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
1889 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
1890 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
1896 do_pending_stack_adjust ();
1897 loop_lab
= gen_label_rtx ();
1898 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
1899 cntv
= make_tree (sizetype
, cntvar
);
1900 emit_move_insn (cntvar
, const0_rtx
);
1901 emit_label (loop_lab
);
1903 if (TREE_CODE (arg0
) != VECTOR_CST
)
1905 rtx arg0r
= expand_normal (arg0
);
1906 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
1908 if (TREE_CODE (arg1
) != VECTOR_CST
)
1910 rtx arg1r
= expand_normal (arg1
);
1911 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
1913 for (int i
= 0; i
< (cnt
> 4 ? 1 : cnt
); i
++)
1915 tree op0
, op1
, res
= NULL_TREE
;
1918 tree atype
= build_array_type_nelts (eltype
, cnt
);
1919 op0
= uniform_vector_p (arg0
);
1920 if (op0
== NULL_TREE
)
1922 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
1923 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
1924 NULL_TREE
, NULL_TREE
);
1926 op1
= uniform_vector_p (arg1
);
1927 if (op1
== NULL_TREE
)
1929 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
1930 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
1931 NULL_TREE
, NULL_TREE
);
1935 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
1936 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
1937 NULL_TREE
, NULL_TREE
);
1942 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
1943 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
1944 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
1946 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
1952 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
1953 false, false, false, true, &data
);
1956 if (cnt
> 4 ? integer_zerop (arg0
) : integer_zerop (op0
))
1957 expand_neg_overflow (loc
, res
, op1
, true, &data
);
1959 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
1960 false, false, false, true, &data
);
1963 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
1972 struct separate_ops ops
;
1973 ops
.code
= PLUS_EXPR
;
1974 ops
.type
= TREE_TYPE (cntv
);
1976 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
1977 ops
.op2
= NULL_TREE
;
1979 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
1982 emit_move_insn (cntvar
, ret
);
1983 do_compare_rtx_and_jump (cntvar
, GEN_INT (cnt
), NE
, false,
1984 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
1985 profile_probability::very_likely ());
1987 if (lhs
&& resv
== NULL_TREE
)
1989 struct separate_ops ops
;
1991 ops
.type
= TREE_TYPE (arg0
);
1994 ops
.op2
= NULL_TREE
;
1996 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
1999 emit_move_insn (lhsr
, ret
);
2002 emit_move_insn (lhsr
, resvr
);
2005 /* Expand UBSAN_CHECK_ADD call STMT. */
2008 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2010 location_t loc
= gimple_location (stmt
);
2011 tree lhs
= gimple_call_lhs (stmt
);
2012 tree arg0
= gimple_call_arg (stmt
, 0);
2013 tree arg1
= gimple_call_arg (stmt
, 1);
2014 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2015 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2017 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2018 false, false, false, true, NULL
);
2021 /* Expand UBSAN_CHECK_SUB call STMT. */
2024 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2026 location_t loc
= gimple_location (stmt
);
2027 tree lhs
= gimple_call_lhs (stmt
);
2028 tree arg0
= gimple_call_arg (stmt
, 0);
2029 tree arg1
= gimple_call_arg (stmt
, 1);
2030 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2031 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2032 else if (integer_zerop (arg0
))
2033 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2035 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2036 false, false, false, true, NULL
);
2039 /* Expand UBSAN_CHECK_MUL call STMT. */
2042 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2044 location_t loc
= gimple_location (stmt
);
2045 tree lhs
= gimple_call_lhs (stmt
);
2046 tree arg0
= gimple_call_arg (stmt
, 0);
2047 tree arg1
= gimple_call_arg (stmt
, 1);
2048 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2049 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2051 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2055 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2058 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2060 tree lhs
= gimple_call_lhs (stmt
);
2061 if (lhs
== NULL_TREE
)
2063 tree arg0
= gimple_call_arg (stmt
, 0);
2064 tree arg1
= gimple_call_arg (stmt
, 1);
2065 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2066 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2067 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2068 int unsr_p
= TYPE_UNSIGNED (type
);
2069 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2070 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2071 int precres
= TYPE_PRECISION (type
);
2072 location_t loc
= gimple_location (stmt
);
2073 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2075 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2077 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2078 prec0
= MIN (prec0
, pr
);
2079 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2080 prec1
= MIN (prec1
, pr
);
2082 /* If uns0_p && uns1_p, precop is minimum needed precision
2083 of unsigned type to hold the exact result, otherwise
2084 precop is minimum needed precision of signed type to
2085 hold the exact result. */
2087 if (code
== MULT_EXPR
)
2088 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2091 if (uns0_p
== uns1_p
)
2092 precop
= MAX (prec0
, prec1
) + 1;
2094 precop
= MAX (prec0
+ 1, prec1
) + 1;
2096 precop
= MAX (prec0
, prec1
+ 1) + 1;
2098 int orig_precres
= precres
;
2102 if ((uns0_p
&& uns1_p
)
2103 ? ((precop
+ !unsr_p
) <= precres
2104 /* u1 - u2 -> ur can overflow, no matter what precision
2106 && (code
!= MINUS_EXPR
|| !unsr_p
))
2107 : (!unsr_p
&& precop
<= precres
))
2109 /* The infinity precision result will always fit into result. */
2110 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2111 write_complex_part (target
, const0_rtx
, true);
2112 machine_mode mode
= TYPE_MODE (type
);
2113 struct separate_ops ops
;
2116 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2117 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2118 ops
.op2
= NULL_TREE
;
2120 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2121 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2125 /* For operations with low precision, if target doesn't have them, start
2126 with precres widening right away, otherwise do it only if the most
2127 simple cases can't be used. */
2128 const int min_precision
= targetm
.min_arithmetic_precision ();
2129 if (orig_precres
== precres
&& precres
< min_precision
)
2131 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2132 && prec1
<= precres
)
2133 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2134 && prec0
+ uns0_p
<= precres
2135 && prec1
+ uns1_p
<= precres
))
2137 arg0
= fold_convert_loc (loc
, type
, arg0
);
2138 arg1
= fold_convert_loc (loc
, type
, arg1
);
2142 if (integer_zerop (arg0
) && !unsr_p
)
2144 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2149 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2150 unsr_p
, unsr_p
, false, NULL
);
2153 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2154 unsr_p
, unsr_p
, false, NULL
);
2161 /* For sub-word operations, retry with a wider type first. */
2162 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2164 int p
= MAX (min_precision
, precop
);
2165 machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2166 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2169 p
= TYPE_PRECISION (optype
);
2173 unsr_p
= TYPE_UNSIGNED (optype
);
2179 if (prec0
<= precres
&& prec1
<= precres
)
2184 types
[0] = build_nonstandard_integer_type (precres
, 0);
2190 types
[1] = build_nonstandard_integer_type (precres
, 1);
2192 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2193 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2194 if (code
!= MULT_EXPR
)
2195 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2196 uns0_p
, uns1_p
, false, NULL
);
2198 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2199 uns0_p
, uns1_p
, false, NULL
);
2203 /* Retry with a wider type. */
2204 if (orig_precres
== precres
)
2206 int p
= MAX (prec0
, prec1
);
2207 machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2208 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2211 p
= TYPE_PRECISION (optype
);
2215 unsr_p
= TYPE_UNSIGNED (optype
);
2226 /* Expand ADD_OVERFLOW STMT. */
2229 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2231 expand_arith_overflow (PLUS_EXPR
, stmt
);
2234 /* Expand SUB_OVERFLOW STMT. */
2237 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2239 expand_arith_overflow (MINUS_EXPR
, stmt
);
2242 /* Expand MUL_OVERFLOW STMT. */
2245 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2247 expand_arith_overflow (MULT_EXPR
, stmt
);
2250 /* This should get folded in tree-vectorizer.c. */
2253 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2258 /* This should get folded in tree-vectorizer.c. */
2261 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2266 /* Expand MASK_LOAD call STMT using optab OPTAB. */
2269 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2271 struct expand_operand ops
[3];
2272 tree type
, lhs
, rhs
, maskt
, ptr
;
2273 rtx mem
, target
, mask
;
2276 maskt
= gimple_call_arg (stmt
, 2);
2277 lhs
= gimple_call_lhs (stmt
);
2278 if (lhs
== NULL_TREE
)
2280 type
= TREE_TYPE (lhs
);
2281 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2282 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2283 if (TYPE_ALIGN (type
) != align
)
2284 type
= build_aligned_type (type
, align
);
2285 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2287 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2288 gcc_assert (MEM_P (mem
));
2289 mask
= expand_normal (maskt
);
2290 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2291 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2292 create_fixed_operand (&ops
[1], mem
);
2293 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2294 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2295 TYPE_MODE (TREE_TYPE (maskt
))),
2299 /* Expand MASK_STORE call STMT using optab OPTAB. */
2302 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2304 struct expand_operand ops
[3];
2305 tree type
, lhs
, rhs
, maskt
, ptr
;
2309 maskt
= gimple_call_arg (stmt
, 2);
2310 rhs
= gimple_call_arg (stmt
, 3);
2311 type
= TREE_TYPE (rhs
);
2312 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2313 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2314 if (TYPE_ALIGN (type
) != align
)
2315 type
= build_aligned_type (type
, align
);
2316 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2318 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2319 gcc_assert (MEM_P (mem
));
2320 mask
= expand_normal (maskt
);
2321 reg
= expand_normal (rhs
);
2322 create_fixed_operand (&ops
[0], mem
);
2323 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2324 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2325 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2326 TYPE_MODE (TREE_TYPE (maskt
))),
2331 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2336 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2338 /* When guessing was done, the hints should be already stripped away. */
2339 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2342 tree lhs
= gimple_call_lhs (stmt
);
2344 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2346 target
= const0_rtx
;
2347 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2348 if (lhs
&& val
!= target
)
2349 emit_move_insn (target
, val
);
2352 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2353 should never be called. */
2356 expand_VA_ARG (internal_fn
, gcall
*)
2361 /* Expand the IFN_UNIQUE function according to its first argument. */
2364 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2366 rtx pattern
= NULL_RTX
;
2367 enum ifn_unique_kind kind
2368 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2375 case IFN_UNIQUE_UNSPEC
:
2376 if (targetm
.have_unique ())
2377 pattern
= targetm
.gen_unique ();
2380 case IFN_UNIQUE_OACC_FORK
:
2381 case IFN_UNIQUE_OACC_JOIN
:
2382 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2384 tree lhs
= gimple_call_lhs (stmt
);
2385 rtx target
= const0_rtx
;
2388 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2390 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2391 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2393 if (kind
== IFN_UNIQUE_OACC_FORK
)
2394 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2396 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2404 emit_insn (pattern
);
2407 /* The size of an OpenACC compute dimension. */
2410 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2412 tree lhs
= gimple_call_lhs (stmt
);
2417 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2418 if (targetm
.have_oacc_dim_size ())
2420 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2421 VOIDmode
, EXPAND_NORMAL
);
2422 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2425 emit_move_insn (target
, GEN_INT (1));
2428 /* The position of an OpenACC execution engine along one compute axis. */
2431 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2433 tree lhs
= gimple_call_lhs (stmt
);
2438 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2439 if (targetm
.have_oacc_dim_pos ())
2441 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2442 VOIDmode
, EXPAND_NORMAL
);
2443 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2446 emit_move_insn (target
, const0_rtx
);
2449 /* This is expanded by oacc_device_lower pass. */
2452 expand_GOACC_LOOP (internal_fn
, gcall
*)
2457 /* This is expanded by oacc_device_lower pass. */
2460 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2465 /* This is expanded by oacc_device_lower pass. */
2468 expand_GOACC_TILE (internal_fn
, gcall
*)
2473 /* Set errno to EDOM. */
2476 expand_SET_EDOM (internal_fn
, gcall
*)
2479 #ifdef GEN_ERRNO_RTX
2480 rtx errno_rtx
= GEN_ERRNO_RTX
;
2482 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2484 emit_move_insn (errno_rtx
,
2485 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2491 /* Expand atomic bit test and set. */
2494 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2496 expand_ifn_atomic_bit_test_and (call
);
2499 /* Expand atomic bit test and complement. */
2502 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2504 expand_ifn_atomic_bit_test_and (call
);
2507 /* Expand atomic bit test and reset. */
2510 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2512 expand_ifn_atomic_bit_test_and (call
);
2515 /* Expand atomic bit test and set. */
2518 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2520 expand_ifn_atomic_compare_exchange (call
);
2523 /* Expand LAUNDER to assignment, lhs = arg0. */
2526 expand_LAUNDER (internal_fn
, gcall
*call
)
2528 tree lhs
= gimple_call_lhs (call
);
2533 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2536 /* Expand DIVMOD() using:
2537 a) optab handler for udivmod/sdivmod if it is available.
2538 b) If optab_handler doesn't exist, generate call to
2539 target-specific divmod libfunc. */
2542 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2544 tree lhs
= gimple_call_lhs (call_stmt
);
2545 tree arg0
= gimple_call_arg (call_stmt
, 0);
2546 tree arg1
= gimple_call_arg (call_stmt
, 1);
2548 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2549 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2550 machine_mode mode
= TYPE_MODE (type
);
2551 bool unsignedp
= TYPE_UNSIGNED (type
);
2552 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2554 rtx op0
= expand_normal (arg0
);
2555 rtx op1
= expand_normal (arg1
);
2556 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2558 rtx quotient
, remainder
, libfunc
;
2560 /* Check if optab_handler exists for divmod_optab for given mode. */
2561 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2563 quotient
= gen_reg_rtx (mode
);
2564 remainder
= gen_reg_rtx (mode
);
2565 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2568 /* Generate call to divmod libfunc if it exists. */
2569 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2570 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2571 "ient
, &remainder
);
2576 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2577 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2578 make_tree (TREE_TYPE (arg0
), quotient
),
2579 make_tree (TREE_TYPE (arg1
), remainder
)),
2580 target
, VOIDmode
, EXPAND_NORMAL
);
2583 /* Expand a call to FN using the operands in STMT. FN has a single
2584 output operand and NARGS input operands. */
2587 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2590 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2592 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2593 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2595 tree lhs
= gimple_call_lhs (stmt
);
2596 tree lhs_type
= TREE_TYPE (lhs
);
2597 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2598 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2600 for (unsigned int i
= 0; i
< nargs
; ++i
)
2602 tree rhs
= gimple_call_arg (stmt
, i
);
2603 tree rhs_type
= TREE_TYPE (rhs
);
2604 rtx rhs_rtx
= expand_normal (rhs
);
2605 if (INTEGRAL_TYPE_P (rhs_type
))
2606 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2607 TYPE_MODE (rhs_type
),
2608 TYPE_UNSIGNED (rhs_type
));
2610 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2613 expand_insn (icode
, nargs
+ 1, ops
);
2614 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2616 /* If the return value has an integral type, convert the instruction
2617 result to that type. This is useful for things that return an
2618 int regardless of the size of the input. If the instruction result
2619 is smaller than required, assume that it is signed.
2621 If the return value has a nonintegral type, its mode must match
2622 the instruction result. */
2623 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2625 /* If this is a scalar in a register that is stored in a wider
2626 mode than the declared mode, compute the result into its
2627 declared mode and then convert to the wider mode. */
2628 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2629 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2630 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2631 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2633 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2634 emit_move_insn (lhs_rtx
, ops
[0].value
);
2637 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2638 convert_move (lhs_rtx
, ops
[0].value
, 0);
2643 /* Expanders for optabs that can use expand_direct_optab_fn. */
2645 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2646 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2648 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2649 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2651 /* RETURN_TYPE and ARGS are a return type and argument list that are
2652 in principle compatible with FN (which satisfies direct_internal_fn_p).
2653 Return the types that should be used to determine whether the
2654 target supports FN. */
2657 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2659 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2660 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2661 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2662 return tree_pair (type0
, type1
);
2665 /* CALL is a call whose return type and arguments are in principle
2666 compatible with FN (which satisfies direct_internal_fn_p). Return the
2667 types that should be used to determine whether the target supports FN. */
2670 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2672 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2673 tree op0
= (info
.type0
< 0
2674 ? gimple_call_lhs (call
)
2675 : gimple_call_arg (call
, info
.type0
));
2676 tree op1
= (info
.type1
< 0
2677 ? gimple_call_lhs (call
)
2678 : gimple_call_arg (call
, info
.type1
));
2679 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2682 /* Return true if OPTAB is supported for TYPES (whose modes should be
2683 the same) when the optimization type is OPT_TYPE. Used for simple
2687 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2688 optimization_type opt_type
)
2690 machine_mode mode
= TYPE_MODE (types
.first
);
2691 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2692 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2695 /* Return true if load/store lanes optab OPTAB is supported for
2696 array type TYPES.first when the optimization type is OPT_TYPE. */
2699 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2700 optimization_type opt_type
)
2702 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2703 machine_mode imode
= TYPE_MODE (types
.first
);
2704 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2705 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2706 != CODE_FOR_nothing
);
2709 #define direct_unary_optab_supported_p direct_optab_supported_p
2710 #define direct_binary_optab_supported_p direct_optab_supported_p
2711 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2712 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2713 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2714 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2716 /* Return true if FN is supported for the types in TYPES when the
2717 optimization type is OPT_TYPE. The types are those associated with
2718 the "type0" and "type1" fields of FN's direct_internal_fn_info
2722 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2723 optimization_type opt_type
)
2727 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2728 case IFN_##CODE: break;
2729 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2731 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2733 #include "internal-fn.def"
2741 /* Return true if FN is supported for type TYPE when the optimization
2742 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2743 fields of FN's direct_internal_fn_info structure are the same. */
2746 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2747 optimization_type opt_type
)
2749 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2750 gcc_checking_assert (info
.type0
== info
.type1
);
2751 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2754 /* Return true if IFN_SET_EDOM is supported. */
2757 set_edom_supported_p (void)
2766 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2768 expand_##CODE (internal_fn fn, gcall *stmt) \
2770 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2772 #include "internal-fn.def"
2774 /* Routines to expand each internal function, indexed by function number.
2775 Each routine has the prototype:
2777 expand_<NAME> (gcall *stmt)
2779 where STMT is the statement that performs the call. */
2780 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2781 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2782 #include "internal-fn.def"
2786 /* Expand STMT as though it were a call to internal function FN. */
2789 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2791 internal_fn_expanders
[fn
] (fn
, stmt
);
2794 /* Expand STMT, which is a call to internal function FN. */
2797 expand_internal_call (gcall
*stmt
)
2799 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
2803 expand_PHI (internal_fn
, gcall
*)