2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 /* The names of each internal function, indexed by function number. */
48 const char *const internal_fn_name_array
[] = {
49 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
50 #include "internal-fn.def"
54 /* The ECF_* flags of each internal function, indexed by function number. */
55 const int internal_fn_flags_array
[] = {
56 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
57 #include "internal-fn.def"
61 /* Fnspec of each internal function, indexed by function number. */
62 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
67 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
68 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
69 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
70 #include "internal-fn.def"
71 internal_fn_fnspec_array
[IFN_LAST
] = 0;
74 /* Create static initializers for the information returned by
75 direct_internal_fn. */
76 #define not_direct { -2, -2, false }
77 #define mask_load_direct { -1, 2, false }
78 #define load_lanes_direct { -1, -1, false }
79 #define mask_store_direct { 3, 2, false }
80 #define store_lanes_direct { 0, 0, false }
81 #define unary_direct { 0, 0, true }
82 #define binary_direct { 0, 0, true }
84 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
85 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
86 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
87 #include "internal-fn.def"
91 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
92 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
95 get_multi_vector_move (tree array_type
, convert_optab optab
)
100 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
101 imode
= TYPE_MODE (array_type
);
102 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
104 return convert_optab_handler (optab
, imode
, vmode
);
107 /* Expand LOAD_LANES call STMT using optab OPTAB. */
110 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
112 struct expand_operand ops
[2];
116 lhs
= gimple_call_lhs (stmt
);
117 rhs
= gimple_call_arg (stmt
, 0);
118 type
= TREE_TYPE (lhs
);
120 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
121 mem
= expand_normal (rhs
);
123 gcc_assert (MEM_P (mem
));
124 PUT_MODE (mem
, TYPE_MODE (type
));
126 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
127 create_fixed_operand (&ops
[1], mem
);
128 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
131 /* Expand STORE_LANES call STMT using optab OPTAB. */
134 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
136 struct expand_operand ops
[2];
140 lhs
= gimple_call_lhs (stmt
);
141 rhs
= gimple_call_arg (stmt
, 0);
142 type
= TREE_TYPE (rhs
);
144 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
145 reg
= expand_normal (rhs
);
147 gcc_assert (MEM_P (target
));
148 PUT_MODE (target
, TYPE_MODE (type
));
150 create_fixed_operand (&ops
[0], target
);
151 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
152 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
156 expand_ANNOTATE (internal_fn
, gcall
*)
161 /* This should get expanded in omp_device_lower pass. */
164 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
169 /* This should get expanded in omp_device_lower pass. */
172 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
177 /* Allocate per-lane storage and begin non-uniform execution region. */
180 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
183 tree lhs
= gimple_call_lhs (stmt
);
185 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
187 target
= gen_reg_rtx (Pmode
);
188 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
189 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
190 struct expand_operand ops
[3];
191 create_output_operand (&ops
[0], target
, Pmode
);
192 create_input_operand (&ops
[1], size
, Pmode
);
193 create_input_operand (&ops
[2], align
, Pmode
);
194 gcc_assert (targetm
.have_omp_simt_enter ());
195 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
198 /* Deallocate per-lane storage and leave non-uniform execution region. */
201 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
203 gcc_checking_assert (!gimple_call_lhs (stmt
));
204 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
205 struct expand_operand ops
[1];
206 create_input_operand (&ops
[0], arg
, Pmode
);
207 gcc_assert (targetm
.have_omp_simt_exit ());
208 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
211 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
212 without SIMT execution this should be expanded in omp_device_lower pass. */
215 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
217 tree lhs
= gimple_call_lhs (stmt
);
221 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
222 gcc_assert (targetm
.have_omp_simt_lane ());
223 emit_insn (targetm
.gen_omp_simt_lane (target
));
226 /* This should get expanded in omp_device_lower pass. */
229 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
234 /* Lane index of the first SIMT lane that supplies a non-zero argument.
235 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
236 lane that executed the last iteration for handling OpenMP lastprivate. */
239 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
241 tree lhs
= gimple_call_lhs (stmt
);
245 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
246 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
247 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
248 struct expand_operand ops
[2];
249 create_output_operand (&ops
[0], target
, mode
);
250 create_input_operand (&ops
[1], cond
, mode
);
251 gcc_assert (targetm
.have_omp_simt_last_lane ());
252 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
255 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
258 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
260 tree lhs
= gimple_call_lhs (stmt
);
264 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
265 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
266 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
267 struct expand_operand ops
[2];
268 create_output_operand (&ops
[0], target
, mode
);
269 create_input_operand (&ops
[1], ctr
, mode
);
270 gcc_assert (targetm
.have_omp_simt_ordered ());
271 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
274 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
275 any lane supplies a non-zero argument. */
278 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
280 tree lhs
= gimple_call_lhs (stmt
);
284 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
285 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
286 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
287 struct expand_operand ops
[2];
288 create_output_operand (&ops
[0], target
, mode
);
289 create_input_operand (&ops
[1], cond
, mode
);
290 gcc_assert (targetm
.have_omp_simt_vote_any ());
291 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
294 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
295 is destination lane index XOR given offset. */
298 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
300 tree lhs
= gimple_call_lhs (stmt
);
304 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
305 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
306 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
307 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
308 struct expand_operand ops
[3];
309 create_output_operand (&ops
[0], target
, mode
);
310 create_input_operand (&ops
[1], src
, mode
);
311 create_input_operand (&ops
[2], idx
, SImode
);
312 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
313 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
316 /* Exchange between SIMT lanes according to given source lane index. */
319 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
321 tree lhs
= gimple_call_lhs (stmt
);
325 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
326 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
327 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
328 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
329 struct expand_operand ops
[3];
330 create_output_operand (&ops
[0], target
, mode
);
331 create_input_operand (&ops
[1], src
, mode
);
332 create_input_operand (&ops
[2], idx
, SImode
);
333 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
334 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
337 /* This should get expanded in adjust_simduid_builtins. */
340 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
345 /* This should get expanded in adjust_simduid_builtins. */
348 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
353 /* This should get expanded in adjust_simduid_builtins. */
356 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
361 /* This should get expanded in adjust_simduid_builtins. */
364 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
369 /* This should get expanded in adjust_simduid_builtins. */
372 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
377 /* This should get expanded in the sanopt pass. */
380 expand_UBSAN_NULL (internal_fn
, gcall
*)
385 /* This should get expanded in the sanopt pass. */
388 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
393 /* This should get expanded in the sanopt pass. */
396 expand_UBSAN_VPTR (internal_fn
, gcall
*)
401 /* This should get expanded in the sanopt pass. */
404 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
409 /* This should get expanded in the sanopt pass. */
412 expand_ASAN_CHECK (internal_fn
, gcall
*)
417 /* This should get expanded in the sanopt pass. */
420 expand_ASAN_MARK (internal_fn
, gcall
*)
425 /* This should get expanded in the sanopt pass. */
428 expand_ASAN_POISON (internal_fn
, gcall
*)
433 /* This should get expanded in the sanopt pass. */
436 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
441 /* This should get expanded in the tsan pass. */
444 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
449 /* This should get expanded in the lower pass. */
452 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
454 error_at (gimple_location (call
),
455 "invalid use of attribute %<fallthrough%>");
458 /* Return minimum precision needed to represent all values
459 of ARG in SIGNed integral type. */
462 get_min_precision (tree arg
, signop sign
)
464 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
466 signop orig_sign
= sign
;
467 if (TREE_CODE (arg
) == INTEGER_CST
)
470 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
472 widest_int w
= wi::to_widest (arg
);
473 w
= wi::ext (w
, prec
, sign
);
474 p
= wi::min_precision (w
, sign
);
477 p
= wi::min_precision (arg
, sign
);
478 return MIN (p
, prec
);
480 while (CONVERT_EXPR_P (arg
)
481 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
482 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
484 arg
= TREE_OPERAND (arg
, 0);
485 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
487 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
489 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
490 return prec
+ (orig_sign
!= sign
);
491 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
494 return prec
+ (orig_sign
!= sign
);
496 if (TREE_CODE (arg
) != SSA_NAME
)
497 return prec
+ (orig_sign
!= sign
);
498 wide_int arg_min
, arg_max
;
499 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
501 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
502 if (is_gimple_assign (g
)
503 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
505 tree t
= gimple_assign_rhs1 (g
);
506 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
507 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
510 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
512 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
514 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
515 return prec
+ (orig_sign
!= sign
);
516 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
519 return prec
+ (orig_sign
!= sign
);
523 return prec
+ (orig_sign
!= sign
);
525 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
527 int p1
= wi::min_precision (arg_min
, sign
);
528 int p2
= wi::min_precision (arg_max
, sign
);
530 prec
= MIN (prec
, p1
);
532 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
534 int p
= wi::min_precision (arg_max
, UNSIGNED
);
535 prec
= MIN (prec
, p
);
537 return prec
+ (orig_sign
!= sign
);
540 /* Helper for expand_*_overflow. Set the __imag__ part to true
541 (1 except for signed:1 type, in which case store -1). */
544 expand_arith_set_overflow (tree lhs
, rtx target
)
546 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
547 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
548 write_complex_part (target
, constm1_rtx
, true);
550 write_complex_part (target
, const1_rtx
, true);
553 /* Helper for expand_*_overflow. Store RES into the __real__ part
554 of TARGET. If RES has larger MODE than __real__ part of TARGET,
555 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
556 if LHS has smaller precision than its mode. */
559 expand_arith_overflow_result_store (tree lhs
, rtx target
,
560 machine_mode mode
, rtx res
)
562 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
566 rtx_code_label
*done_label
= gen_label_rtx ();
567 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
568 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
569 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
570 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
571 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
572 profile_probability::very_likely ());
573 expand_arith_set_overflow (lhs
, target
);
574 emit_label (done_label
);
576 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
577 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
580 rtx_code_label
*done_label
= gen_label_rtx ();
581 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
586 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
588 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
589 true, OPTAB_LIB_WIDEN
);
593 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
595 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
598 do_compare_rtx_and_jump (res
, lres
,
599 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
600 profile_probability::very_likely ());
601 expand_arith_set_overflow (lhs
, target
);
602 emit_label (done_label
);
604 write_complex_part (target
, lres
, false);
607 /* Helper for expand_*_overflow. Store RES into TARGET. */
610 expand_ubsan_result_store (rtx target
, rtx res
)
612 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
613 /* If this is a scalar in a register that is stored in a wider mode
614 than the declared mode, compute the result into its declared mode
615 and then convert to the wider mode. Our value is the computed
617 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
619 emit_move_insn (target
, res
);
622 /* Add sub/add overflow checking to the statement STMT.
623 CODE says whether the operation is +, or -. */
626 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
627 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
628 bool uns1_p
, bool is_ubsan
, tree
*datap
)
630 rtx res
, target
= NULL_RTX
;
632 rtx_code_label
*done_label
= gen_label_rtx ();
633 rtx_code_label
*do_error
= gen_label_rtx ();
634 do_pending_stack_adjust ();
635 rtx op0
= expand_normal (arg0
);
636 rtx op1
= expand_normal (arg1
);
637 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
638 int prec
= GET_MODE_PRECISION (mode
);
639 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
643 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
647 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
649 write_complex_part (target
, const0_rtx
, true);
652 /* We assume both operands and result have the same precision
653 here (GET_MODE_BITSIZE (mode)), S stands for signed type
654 with that precision, U for unsigned type with that precision,
655 sgn for unsigned most significant bit in that precision.
656 s1 is signed first operand, u1 is unsigned first operand,
657 s2 is signed second operand, u2 is unsigned second operand,
658 sr is signed result, ur is unsigned result and the following
659 rules say how to compute result (which is always result of
660 the operands as if both were unsigned, cast to the right
661 signedness) and how to compute whether operation overflowed.
664 res = (S) ((U) s1 + (U) s2)
665 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
667 res = (S) ((U) s1 - (U) s2)
668 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
671 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
674 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
676 res = (S) ((U) s1 + u2)
677 ovf = ((U) res ^ sgn) < u2
682 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
684 res = (S) ((U) s1 - u2)
685 ovf = u2 > ((U) s1 ^ sgn)
688 ovf = s1 < 0 || u2 > (U) s1
691 ovf = u1 >= ((U) s2 ^ sgn)
696 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
698 res = (U) s1 + (U) s2
699 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
702 ovf = (U) res < u2 || res < 0
705 ovf = u1 >= u2 ? res < 0 : res >= 0
707 res = (U) s1 - (U) s2
708 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
710 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
712 /* PLUS_EXPR is commutative, if operand signedness differs,
713 canonicalize to the first operand being signed and second
714 unsigned to simplify following code. */
715 std::swap (op0
, op1
);
716 std::swap (arg0
, arg1
);
722 if (uns0_p
&& uns1_p
&& unsr_p
)
724 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
725 : usubv4_optab
, mode
);
726 if (icode
!= CODE_FOR_nothing
)
728 struct expand_operand ops
[4];
729 rtx_insn
*last
= get_last_insn ();
731 res
= gen_reg_rtx (mode
);
732 create_output_operand (&ops
[0], res
, mode
);
733 create_input_operand (&ops
[1], op0
, mode
);
734 create_input_operand (&ops
[2], op1
, mode
);
735 create_fixed_operand (&ops
[3], do_error
);
736 if (maybe_expand_insn (icode
, 4, ops
))
738 last
= get_last_insn ();
739 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
741 && any_condjump_p (last
)
742 && !find_reg_note (last
, REG_BR_PROB
, 0))
743 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
744 emit_jump (done_label
);
748 delete_insns_since (last
);
751 /* Compute the operation. On RTL level, the addition is always
753 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
754 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
756 /* For PLUS_EXPR, the operation is commutative, so we can pick
757 operand to compare against. For prec <= BITS_PER_WORD, I think
758 preferring REG operand is better over CONST_INT, because
759 the CONST_INT might enlarge the instruction or CSE would need
760 to figure out we'd already loaded it into a register before.
761 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
762 as then the multi-word comparison can be perhaps simplified. */
763 if (code
== PLUS_EXPR
764 && (prec
<= BITS_PER_WORD
765 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
766 : CONST_SCALAR_INT_P (op1
)))
768 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
769 true, mode
, NULL_RTX
, NULL
, done_label
,
770 profile_probability::very_likely ());
775 if (!uns0_p
&& uns1_p
&& !unsr_p
)
777 /* Compute the operation. On RTL level, the addition is always
779 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
780 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
781 rtx tem
= expand_binop (mode
, add_optab
,
782 code
== PLUS_EXPR
? res
: op0
, sgn
,
783 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
784 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
785 done_label
, profile_probability::very_likely ());
790 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
792 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
794 /* As we've changed op1, we have to avoid using the value range
795 for the original argument. */
796 arg1
= error_mark_node
;
802 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
804 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
806 /* As we've changed op0, we have to avoid using the value range
807 for the original argument. */
808 arg0
= error_mark_node
;
814 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
816 /* Compute the operation. On RTL level, the addition is always
818 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
820 int pos_neg
= get_range_pos_neg (arg0
);
822 /* If ARG0 is known to be always negative, this is always overflow. */
823 emit_jump (do_error
);
824 else if (pos_neg
== 3)
825 /* If ARG0 is not known to be always positive, check at runtime. */
826 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
827 NULL
, do_error
, profile_probability::very_unlikely ());
828 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
829 done_label
, profile_probability::very_likely ());
834 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
836 /* Compute the operation. On RTL level, the addition is always
838 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
840 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
842 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
843 done_label
, profile_probability::very_likely ());
848 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
850 /* Compute the operation. On RTL level, the addition is always
852 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
854 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
855 NULL
, do_error
, profile_probability::very_unlikely ());
857 /* The operation is commutative, so we can pick operand to compare
858 against. For prec <= BITS_PER_WORD, I think preferring REG operand
859 is better over CONST_INT, because the CONST_INT might enlarge the
860 instruction or CSE would need to figure out we'd already loaded it
861 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
862 might be more beneficial, as then the multi-word comparison can be
863 perhaps simplified. */
864 if (prec
<= BITS_PER_WORD
865 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
866 : CONST_SCALAR_INT_P (op0
))
868 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
869 done_label
, profile_probability::very_likely ());
874 if (!uns0_p
&& !uns1_p
&& unsr_p
)
876 /* Compute the operation. On RTL level, the addition is always
878 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
879 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
880 int pos_neg
= get_range_pos_neg (arg1
);
881 if (code
== PLUS_EXPR
)
883 int pos_neg0
= get_range_pos_neg (arg0
);
884 if (pos_neg0
!= 3 && pos_neg
== 3)
886 std::swap (op0
, op1
);
893 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
894 ? and_optab
: ior_optab
,
895 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
896 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
897 NULL
, done_label
, profile_probability::very_likely ());
901 rtx_code_label
*do_ior_label
= gen_label_rtx ();
902 do_compare_rtx_and_jump (op1
, const0_rtx
,
903 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
904 NULL_RTX
, NULL
, do_ior_label
,
905 profile_probability::even ());
906 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
908 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
909 NULL
, done_label
, profile_probability::very_likely ());
910 emit_jump (do_error
);
911 emit_label (do_ior_label
);
912 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
914 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
915 NULL
, done_label
, profile_probability::very_likely ());
921 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
923 /* Compute the operation. On RTL level, the addition is always
925 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
927 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
928 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
929 op0_geu_op1
, profile_probability::even ());
930 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
931 NULL
, done_label
, profile_probability::very_likely ());
932 emit_jump (do_error
);
933 emit_label (op0_geu_op1
);
934 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
935 NULL
, done_label
, profile_probability::very_likely ());
939 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
944 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
945 : subv4_optab
, mode
);
946 if (icode
!= CODE_FOR_nothing
)
948 struct expand_operand ops
[4];
949 rtx_insn
*last
= get_last_insn ();
951 res
= gen_reg_rtx (mode
);
952 create_output_operand (&ops
[0], res
, mode
);
953 create_input_operand (&ops
[1], op0
, mode
);
954 create_input_operand (&ops
[2], op1
, mode
);
955 create_fixed_operand (&ops
[3], do_error
);
956 if (maybe_expand_insn (icode
, 4, ops
))
958 last
= get_last_insn ();
959 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
961 && any_condjump_p (last
)
962 && !find_reg_note (last
, REG_BR_PROB
, 0))
963 add_int_reg_note (last
, REG_BR_PROB
, PROB_UNLIKELY
);
964 emit_jump (done_label
);
968 delete_insns_since (last
);
971 /* Compute the operation. On RTL level, the addition is always
973 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
974 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
976 /* If we can prove that one of the arguments (for MINUS_EXPR only
977 the second operand, as subtraction is not commutative) is always
978 non-negative or always negative, we can do just one comparison
979 and conditional jump. */
980 int pos_neg
= get_range_pos_neg (arg1
);
981 if (code
== PLUS_EXPR
)
983 int pos_neg0
= get_range_pos_neg (arg0
);
984 if (pos_neg0
!= 3 && pos_neg
== 3)
986 std::swap (op0
, op1
);
991 /* Addition overflows if and only if the two operands have the same sign,
992 and the result has the opposite sign. Subtraction overflows if and
993 only if the two operands have opposite sign, and the subtrahend has
994 the same sign as the result. Here 0 is counted as positive. */
997 /* Compute op0 ^ op1 (operands have opposite sign). */
998 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1001 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1002 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1006 if (code
== PLUS_EXPR
)
1008 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1009 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1010 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1015 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1016 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1017 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1021 /* No overflow if the result has bit sign cleared. */
1022 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1023 NULL
, done_label
, profile_probability::very_likely ());
1026 /* Compare the result of the operation with the first operand.
1027 No overflow for addition if second operand is positive and result
1028 is larger or second operand is negative and result is smaller.
1029 Likewise for subtraction with sign of second operand flipped. */
1031 do_compare_rtx_and_jump (res
, op0
,
1032 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1033 false, mode
, NULL_RTX
, NULL
, done_label
,
1034 profile_probability::very_likely ());
1038 emit_label (do_error
);
1041 /* Expand the ubsan builtin call. */
1043 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1047 do_pending_stack_adjust ();
1050 expand_arith_set_overflow (lhs
, target
);
1053 emit_label (done_label
);
1058 expand_ubsan_result_store (target
, res
);
1062 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1065 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1070 /* Add negate overflow checking to the statement STMT. */
1073 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1078 rtx_code_label
*done_label
, *do_error
;
1079 rtx target
= NULL_RTX
;
1081 done_label
= gen_label_rtx ();
1082 do_error
= gen_label_rtx ();
1084 do_pending_stack_adjust ();
1085 op1
= expand_normal (arg1
);
1087 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
1090 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1092 write_complex_part (target
, const0_rtx
, true);
1095 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1096 if (icode
!= CODE_FOR_nothing
)
1098 struct expand_operand ops
[3];
1099 rtx_insn
*last
= get_last_insn ();
1101 res
= gen_reg_rtx (mode
);
1102 create_output_operand (&ops
[0], res
, mode
);
1103 create_input_operand (&ops
[1], op1
, mode
);
1104 create_fixed_operand (&ops
[2], do_error
);
1105 if (maybe_expand_insn (icode
, 3, ops
))
1107 last
= get_last_insn ();
1108 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1110 && any_condjump_p (last
)
1111 && !find_reg_note (last
, REG_BR_PROB
, 0))
1112 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1113 emit_jump (done_label
);
1117 delete_insns_since (last
);
1118 icode
= CODE_FOR_nothing
;
1122 if (icode
== CODE_FOR_nothing
)
1124 /* Compute the operation. On RTL level, the addition is always
1126 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1128 /* Compare the operand with the most negative value. */
1129 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1130 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1131 done_label
, profile_probability::very_likely ());
1134 emit_label (do_error
);
1137 /* Expand the ubsan builtin call. */
1139 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1140 arg1
, NULL_TREE
, datap
);
1143 do_pending_stack_adjust ();
1146 expand_arith_set_overflow (lhs
, target
);
1149 emit_label (done_label
);
1154 expand_ubsan_result_store (target
, res
);
1156 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1160 /* Add mul overflow checking to the statement STMT. */
1163 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1164 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1169 rtx_code_label
*done_label
, *do_error
;
1170 rtx target
= NULL_RTX
;
1172 enum insn_code icode
;
1174 done_label
= gen_label_rtx ();
1175 do_error
= gen_label_rtx ();
1177 do_pending_stack_adjust ();
1178 op0
= expand_normal (arg0
);
1179 op1
= expand_normal (arg1
);
1181 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
1185 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1187 write_complex_part (target
, const0_rtx
, true);
1191 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1193 /* We assume both operands and result have the same precision
1194 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1195 with that precision, U for unsigned type with that precision,
1196 sgn for unsigned most significant bit in that precision.
1197 s1 is signed first operand, u1 is unsigned first operand,
1198 s2 is signed second operand, u2 is unsigned second operand,
1199 sr is signed result, ur is unsigned result and the following
1200 rules say how to compute result (which is always result of
1201 the operands as if both were unsigned, cast to the right
1202 signedness) and how to compute whether operation overflowed.
1203 main_ovf (false) stands for jump on signed multiplication
1204 overflow or the main algorithm with uns == false.
1205 main_ovf (true) stands for jump on unsigned multiplication
1206 overflow or the main algorithm with uns == true.
1209 res = (S) ((U) s1 * (U) s2)
1210 ovf = main_ovf (false)
1213 ovf = main_ovf (true)
1216 ovf = (s1 < 0 && u2) || main_ovf (true)
1219 ovf = res < 0 || main_ovf (true)
1221 res = (S) ((U) s1 * u2)
1222 ovf = (S) u2 >= 0 ? main_ovf (false)
1223 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1225 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1226 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1228 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1230 if (uns0_p
&& !uns1_p
)
1232 /* Multiplication is commutative, if operand signedness differs,
1233 canonicalize to the first operand being signed and second
1234 unsigned to simplify following code. */
1235 std::swap (op0
, op1
);
1236 std::swap (arg0
, arg1
);
1241 int pos_neg0
= get_range_pos_neg (arg0
);
1242 int pos_neg1
= get_range_pos_neg (arg1
);
1245 if (!uns0_p
&& uns1_p
&& unsr_p
)
1250 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1253 /* If s1 is negative, avoid the main code, just multiply and
1254 signal overflow if op1 is not 0. */
1255 struct separate_ops ops
;
1256 ops
.code
= MULT_EXPR
;
1257 ops
.type
= TREE_TYPE (arg1
);
1258 ops
.op0
= make_tree (ops
.type
, op0
);
1259 ops
.op1
= make_tree (ops
.type
, op1
);
1260 ops
.op2
= NULL_TREE
;
1262 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1263 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1264 NULL
, done_label
, profile_probability::very_likely ());
1265 goto do_error_label
;
1267 rtx_code_label
*do_main_label
;
1268 do_main_label
= gen_label_rtx ();
1269 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1270 NULL
, do_main_label
, profile_probability::very_likely ());
1271 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1272 NULL
, do_main_label
, profile_probability::very_likely ());
1273 expand_arith_set_overflow (lhs
, target
);
1274 emit_label (do_main_label
);
1282 if (uns0_p
&& uns1_p
&& !unsr_p
)
1285 /* Rest of handling of this case after res is computed. */
1290 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1297 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1298 avoid the main code, just multiply and signal overflow
1299 unless 0 * u2 or -1 * ((U) Smin). */
1300 struct separate_ops ops
;
1301 ops
.code
= MULT_EXPR
;
1302 ops
.type
= TREE_TYPE (arg1
);
1303 ops
.op0
= make_tree (ops
.type
, op0
);
1304 ops
.op1
= make_tree (ops
.type
, op1
);
1305 ops
.op2
= NULL_TREE
;
1307 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1308 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1309 NULL
, done_label
, profile_probability::very_likely ());
1310 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1311 NULL
, do_error
, profile_probability::very_unlikely ());
1313 prec
= GET_MODE_PRECISION (mode
);
1315 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1316 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1317 NULL
, done_label
, profile_probability::very_likely ());
1318 goto do_error_label
;
1320 /* Rest of handling of this case after res is computed. */
1328 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1331 switch (pos_neg0
| pos_neg1
)
1333 case 1: /* Both operands known to be non-negative. */
1335 case 2: /* Both operands known to be negative. */
1336 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1337 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1338 /* Avoid looking at arg0/arg1 ranges, as we've changed
1340 arg0
= error_mark_node
;
1341 arg1
= error_mark_node
;
1344 if ((pos_neg0
^ pos_neg1
) == 3)
1346 /* If one operand is known to be negative and the other
1347 non-negative, this overflows always, unless the non-negative
1348 one is 0. Just do normal multiply and set overflow
1349 unless one of the operands is 0. */
1350 struct separate_ops ops
;
1351 ops
.code
= MULT_EXPR
;
1353 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1355 ops
.op0
= make_tree (ops
.type
, op0
);
1356 ops
.op1
= make_tree (ops
.type
, op1
);
1357 ops
.op2
= NULL_TREE
;
1359 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1360 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1362 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1363 NULL_RTX
, NULL
, done_label
,
1364 profile_probability::very_likely ());
1365 goto do_error_label
;
1367 /* The general case, do all the needed comparisons at runtime. */
1368 rtx_code_label
*do_main_label
, *after_negate_label
;
1370 rop0
= gen_reg_rtx (mode
);
1371 rop1
= gen_reg_rtx (mode
);
1372 emit_move_insn (rop0
, op0
);
1373 emit_move_insn (rop1
, op1
);
1376 do_main_label
= gen_label_rtx ();
1377 after_negate_label
= gen_label_rtx ();
1378 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1380 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1381 NULL
, after_negate_label
, profile_probability::very_likely ());
1382 /* Both arguments negative here, negate them and continue with
1383 normal unsigned overflow checking multiplication. */
1384 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1386 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1388 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1390 arg0
= error_mark_node
;
1391 arg1
= error_mark_node
;
1392 emit_jump (do_main_label
);
1393 emit_label (after_negate_label
);
1394 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1396 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1397 NULL
, do_main_label
, profile_probability::very_likely ());
1398 /* One argument is negative here, the other positive. This
1399 overflows always, unless one of the arguments is 0. But
1400 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1401 is, thus we can keep do_main code oring in overflow as is. */
1402 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1403 NULL
, do_main_label
, profile_probability::very_likely ());
1404 expand_arith_set_overflow (lhs
, target
);
1405 emit_label (do_main_label
);
1413 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1414 sign
= uns
? UNSIGNED
: SIGNED
;
1415 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1416 if (icode
!= CODE_FOR_nothing
)
1418 struct expand_operand ops
[4];
1419 rtx_insn
*last
= get_last_insn ();
1421 res
= gen_reg_rtx (mode
);
1422 create_output_operand (&ops
[0], res
, mode
);
1423 create_input_operand (&ops
[1], op0
, mode
);
1424 create_input_operand (&ops
[2], op1
, mode
);
1425 create_fixed_operand (&ops
[3], do_error
);
1426 if (maybe_expand_insn (icode
, 4, ops
))
1428 last
= get_last_insn ();
1429 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1431 && any_condjump_p (last
)
1432 && !find_reg_note (last
, REG_BR_PROB
, 0))
1433 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1434 emit_jump (done_label
);
1438 delete_insns_since (last
);
1439 icode
= CODE_FOR_nothing
;
1443 if (icode
== CODE_FOR_nothing
)
1445 struct separate_ops ops
;
1446 int prec
= GET_MODE_PRECISION (mode
);
1447 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1448 ops
.op0
= make_tree (type
, op0
);
1449 ops
.op1
= make_tree (type
, op1
);
1450 ops
.op2
= NULL_TREE
;
1452 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1453 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1455 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1456 ops
.code
= WIDEN_MULT_EXPR
;
1458 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1460 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1461 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1463 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1464 res
= convert_modes (mode
, wmode
, res
, uns
);
1466 /* For the unsigned multiplication, there was overflow if
1467 HIPART is non-zero. */
1468 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1469 NULL_RTX
, NULL
, done_label
,
1470 profile_probability::very_likely ());
1473 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1475 /* RES is low half of the double width result, HIPART
1476 the high half. There was overflow if
1477 HIPART is different from RES < 0 ? -1 : 0. */
1478 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1479 NULL_RTX
, NULL
, done_label
,
1480 profile_probability::very_likely ());
1483 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1485 rtx_code_label
*large_op0
= gen_label_rtx ();
1486 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1487 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1488 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1489 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1490 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1491 rtx_code_label
*do_overflow
= gen_label_rtx ();
1492 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1494 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1495 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1497 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1498 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1499 rtx signbit0
= const0_rtx
;
1501 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1503 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1505 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1506 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1507 rtx signbit1
= const0_rtx
;
1509 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1512 res
= gen_reg_rtx (mode
);
1514 /* True if op0 resp. op1 are known to be in the range of
1516 bool op0_small_p
= false;
1517 bool op1_small_p
= false;
1518 /* True if op0 resp. op1 are known to have all zeros or all ones
1519 in the upper half of bits, but are not known to be
1521 bool op0_medium_p
= false;
1522 bool op1_medium_p
= false;
1523 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1524 nonnegative, 1 if unknown. */
1530 else if (pos_neg0
== 2)
1534 else if (pos_neg1
== 2)
1537 unsigned int mprec0
= prec
;
1538 if (arg0
!= error_mark_node
)
1539 mprec0
= get_min_precision (arg0
, sign
);
1540 if (mprec0
<= hprec
)
1542 else if (!uns
&& mprec0
<= hprec
+ 1)
1543 op0_medium_p
= true;
1544 unsigned int mprec1
= prec
;
1545 if (arg1
!= error_mark_node
)
1546 mprec1
= get_min_precision (arg1
, sign
);
1547 if (mprec1
<= hprec
)
1549 else if (!uns
&& mprec1
<= hprec
+ 1)
1550 op1_medium_p
= true;
1552 int smaller_sign
= 1;
1553 int larger_sign
= 1;
1556 smaller_sign
= op0_sign
;
1557 larger_sign
= op1_sign
;
1559 else if (op1_small_p
)
1561 smaller_sign
= op1_sign
;
1562 larger_sign
= op0_sign
;
1564 else if (op0_sign
== op1_sign
)
1566 smaller_sign
= op0_sign
;
1567 larger_sign
= op0_sign
;
1571 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1572 NULL_RTX
, NULL
, large_op0
,
1573 profile_probability::unlikely ());
1576 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1577 NULL_RTX
, NULL
, small_op0_large_op1
,
1578 profile_probability::unlikely ());
1580 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1581 hmode to mode, the multiplication will never overflow. We can
1582 do just one hmode x hmode => mode widening multiplication. */
1583 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1584 if (GET_CODE (lopart0
) == SUBREG
)
1586 lopart0s
= shallow_copy_rtx (lopart0
);
1587 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1588 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1590 if (GET_CODE (lopart1
) == SUBREG
)
1592 lopart1s
= shallow_copy_rtx (lopart1
);
1593 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1594 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1596 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1597 ops
.op0
= make_tree (halfstype
, lopart0s
);
1598 ops
.op1
= make_tree (halfstype
, lopart1s
);
1599 ops
.code
= WIDEN_MULT_EXPR
;
1602 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1603 emit_move_insn (res
, thisres
);
1604 emit_jump (done_label
);
1606 emit_label (small_op0_large_op1
);
1608 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1609 but op1 is not, just swap the arguments and handle it as op1
1610 sign/zero extended, op0 not. */
1611 rtx larger
= gen_reg_rtx (mode
);
1612 rtx hipart
= gen_reg_rtx (hmode
);
1613 rtx lopart
= gen_reg_rtx (hmode
);
1614 emit_move_insn (larger
, op1
);
1615 emit_move_insn (hipart
, hipart1
);
1616 emit_move_insn (lopart
, lopart0
);
1617 emit_jump (one_small_one_large
);
1619 emit_label (large_op0
);
1622 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1623 NULL_RTX
, NULL
, both_ops_large
,
1624 profile_probability::unlikely ());
1626 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1627 but op0 is not, prepare larger, hipart and lopart pseudos and
1628 handle it together with small_op0_large_op1. */
1629 emit_move_insn (larger
, op0
);
1630 emit_move_insn (hipart
, hipart0
);
1631 emit_move_insn (lopart
, lopart1
);
1633 emit_label (one_small_one_large
);
1635 /* lopart is the low part of the operand that is sign extended
1636 to mode, larger is the other operand, hipart is the
1637 high part of larger and lopart0 and lopart1 are the low parts
1639 We perform lopart0 * lopart1 and lopart * hipart widening
1641 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1642 ops
.op0
= make_tree (halfutype
, lopart0
);
1643 ops
.op1
= make_tree (halfutype
, lopart1
);
1645 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1647 ops
.op0
= make_tree (halfutype
, lopart
);
1648 ops
.op1
= make_tree (halfutype
, hipart
);
1649 rtx loxhi
= gen_reg_rtx (mode
);
1650 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1651 emit_move_insn (loxhi
, tem
);
1655 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1656 if (larger_sign
== 0)
1657 emit_jump (after_hipart_neg
);
1658 else if (larger_sign
!= -1)
1659 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1660 NULL_RTX
, NULL
, after_hipart_neg
,
1661 profile_probability::even ());
1663 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1664 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1665 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1667 emit_move_insn (loxhi
, tem
);
1669 emit_label (after_hipart_neg
);
1671 /* if (lopart < 0) loxhi -= larger; */
1672 if (smaller_sign
== 0)
1673 emit_jump (after_lopart_neg
);
1674 else if (smaller_sign
!= -1)
1675 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1676 NULL_RTX
, NULL
, after_lopart_neg
,
1677 profile_probability::even ());
1679 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1681 emit_move_insn (loxhi
, tem
);
1683 emit_label (after_lopart_neg
);
1686 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1687 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1688 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1690 emit_move_insn (loxhi
, tem
);
1692 /* if (loxhi >> (bitsize / 2)
1693 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1694 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1695 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1697 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1698 rtx signbitloxhi
= const0_rtx
;
1700 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1701 convert_modes (hmode
, mode
,
1703 hprec
- 1, NULL_RTX
, 0);
1705 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1706 NULL_RTX
, NULL
, do_overflow
,
1707 profile_probability::very_unlikely ());
1709 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1710 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1712 tem
= convert_modes (mode
, hmode
,
1713 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1715 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1718 emit_move_insn (res
, tem
);
1719 emit_jump (done_label
);
1721 emit_label (both_ops_large
);
1723 /* If both operands are large (not sign (!uns) or zero (uns)
1724 extended from hmode), then perform the full multiplication
1725 which will be the result of the operation.
1726 The only cases which don't overflow are for signed multiplication
1727 some cases where both hipart0 and highpart1 are 0 or -1.
1728 For unsigned multiplication when high parts are both non-zero
1729 this overflows always. */
1730 ops
.code
= MULT_EXPR
;
1731 ops
.op0
= make_tree (type
, op0
);
1732 ops
.op1
= make_tree (type
, op1
);
1733 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1734 emit_move_insn (res
, tem
);
1740 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1741 NULL_RTX
, 1, OPTAB_DIRECT
);
1742 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1743 NULL_RTX
, NULL
, do_error
,
1744 profile_probability::very_unlikely ());
1749 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1750 NULL_RTX
, 1, OPTAB_DIRECT
);
1751 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1752 NULL_RTX
, NULL
, do_error
,
1753 profile_probability::very_unlikely ());
1756 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1757 the same, overflow happened if res is negative, if they are
1758 different, overflow happened if res is positive. */
1759 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1760 emit_jump (hipart_different
);
1761 else if (op0_sign
== 1 || op1_sign
== 1)
1762 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1763 NULL_RTX
, NULL
, hipart_different
,
1764 profile_probability::even ());
1766 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1767 NULL_RTX
, NULL
, do_error
,
1768 profile_probability::very_unlikely ());
1769 emit_jump (done_label
);
1771 emit_label (hipart_different
);
1773 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1774 NULL_RTX
, NULL
, do_error
,
1775 profile_probability::very_unlikely ());
1776 emit_jump (done_label
);
1779 emit_label (do_overflow
);
1781 /* Overflow, do full multiplication and fallthru into do_error. */
1782 ops
.op0
= make_tree (type
, op0
);
1783 ops
.op1
= make_tree (type
, op1
);
1784 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1785 emit_move_insn (res
, tem
);
1789 gcc_assert (!is_ubsan
);
1790 ops
.code
= MULT_EXPR
;
1792 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1793 emit_jump (done_label
);
1798 emit_label (do_error
);
1801 /* Expand the ubsan builtin call. */
1803 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1807 do_pending_stack_adjust ();
1810 expand_arith_set_overflow (lhs
, target
);
1813 emit_label (done_label
);
1816 if (uns0_p
&& uns1_p
&& !unsr_p
)
1818 rtx_code_label
*all_done_label
= gen_label_rtx ();
1819 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1820 NULL
, all_done_label
, profile_probability::very_likely ());
1821 expand_arith_set_overflow (lhs
, target
);
1822 emit_label (all_done_label
);
1826 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1828 rtx_code_label
*all_done_label
= gen_label_rtx ();
1829 rtx_code_label
*set_noovf
= gen_label_rtx ();
1830 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1831 NULL
, all_done_label
, profile_probability::very_likely ());
1832 expand_arith_set_overflow (lhs
, target
);
1833 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1834 NULL
, set_noovf
, profile_probability::very_likely ());
1835 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1836 NULL
, all_done_label
, profile_probability::very_unlikely ());
1837 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1838 all_done_label
, profile_probability::very_unlikely ());
1839 emit_label (set_noovf
);
1840 write_complex_part (target
, const0_rtx
, true);
1841 emit_label (all_done_label
);
1847 expand_ubsan_result_store (target
, res
);
1849 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1853 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
1856 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
1857 tree arg0
, tree arg1
)
1859 int cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
1860 rtx_code_label
*loop_lab
= NULL
;
1861 rtx cntvar
= NULL_RTX
;
1862 tree cntv
= NULL_TREE
;
1863 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
1864 tree sz
= TYPE_SIZE (eltype
);
1865 tree data
= NULL_TREE
;
1866 tree resv
= NULL_TREE
;
1867 rtx lhsr
= NULL_RTX
;
1868 rtx resvr
= NULL_RTX
;
1873 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1874 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
1875 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
1876 optab_default
)) == unknown_optab
1877 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
1878 == CODE_FOR_nothing
))
1881 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
1884 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
1885 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
1891 do_pending_stack_adjust ();
1892 loop_lab
= gen_label_rtx ();
1893 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
1894 cntv
= make_tree (sizetype
, cntvar
);
1895 emit_move_insn (cntvar
, const0_rtx
);
1896 emit_label (loop_lab
);
1898 if (TREE_CODE (arg0
) != VECTOR_CST
)
1900 rtx arg0r
= expand_normal (arg0
);
1901 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
1903 if (TREE_CODE (arg1
) != VECTOR_CST
)
1905 rtx arg1r
= expand_normal (arg1
);
1906 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
1908 for (int i
= 0; i
< (cnt
> 4 ? 1 : cnt
); i
++)
1910 tree op0
, op1
, res
= NULL_TREE
;
1913 tree atype
= build_array_type_nelts (eltype
, cnt
);
1914 op0
= uniform_vector_p (arg0
);
1915 if (op0
== NULL_TREE
)
1917 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
1918 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
1919 NULL_TREE
, NULL_TREE
);
1921 op1
= uniform_vector_p (arg1
);
1922 if (op1
== NULL_TREE
)
1924 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
1925 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
1926 NULL_TREE
, NULL_TREE
);
1930 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
1931 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
1932 NULL_TREE
, NULL_TREE
);
1937 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
1938 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
1939 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
1941 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
1947 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
1948 false, false, false, true, &data
);
1951 if (cnt
> 4 ? integer_zerop (arg0
) : integer_zerop (op0
))
1952 expand_neg_overflow (loc
, res
, op1
, true, &data
);
1954 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
1955 false, false, false, true, &data
);
1958 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
1967 struct separate_ops ops
;
1968 ops
.code
= PLUS_EXPR
;
1969 ops
.type
= TREE_TYPE (cntv
);
1971 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
1972 ops
.op2
= NULL_TREE
;
1974 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
1977 emit_move_insn (cntvar
, ret
);
1978 do_compare_rtx_and_jump (cntvar
, GEN_INT (cnt
), NE
, false,
1979 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
1980 profile_probability::very_likely ());
1982 if (lhs
&& resv
== NULL_TREE
)
1984 struct separate_ops ops
;
1986 ops
.type
= TREE_TYPE (arg0
);
1989 ops
.op2
= NULL_TREE
;
1991 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
1994 emit_move_insn (lhsr
, ret
);
1997 emit_move_insn (lhsr
, resvr
);
2000 /* Expand UBSAN_CHECK_ADD call STMT. */
2003 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2005 location_t loc
= gimple_location (stmt
);
2006 tree lhs
= gimple_call_lhs (stmt
);
2007 tree arg0
= gimple_call_arg (stmt
, 0);
2008 tree arg1
= gimple_call_arg (stmt
, 1);
2009 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2010 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2012 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2013 false, false, false, true, NULL
);
2016 /* Expand UBSAN_CHECK_SUB call STMT. */
2019 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2021 location_t loc
= gimple_location (stmt
);
2022 tree lhs
= gimple_call_lhs (stmt
);
2023 tree arg0
= gimple_call_arg (stmt
, 0);
2024 tree arg1
= gimple_call_arg (stmt
, 1);
2025 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2026 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2027 else if (integer_zerop (arg0
))
2028 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2030 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2031 false, false, false, true, NULL
);
2034 /* Expand UBSAN_CHECK_MUL call STMT. */
2037 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2039 location_t loc
= gimple_location (stmt
);
2040 tree lhs
= gimple_call_lhs (stmt
);
2041 tree arg0
= gimple_call_arg (stmt
, 0);
2042 tree arg1
= gimple_call_arg (stmt
, 1);
2043 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2044 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2046 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2050 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2053 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2055 tree lhs
= gimple_call_lhs (stmt
);
2056 if (lhs
== NULL_TREE
)
2058 tree arg0
= gimple_call_arg (stmt
, 0);
2059 tree arg1
= gimple_call_arg (stmt
, 1);
2060 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2061 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2062 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2063 int unsr_p
= TYPE_UNSIGNED (type
);
2064 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2065 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2066 int precres
= TYPE_PRECISION (type
);
2067 location_t loc
= gimple_location (stmt
);
2068 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2070 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2072 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2073 prec0
= MIN (prec0
, pr
);
2074 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2075 prec1
= MIN (prec1
, pr
);
2077 /* If uns0_p && uns1_p, precop is minimum needed precision
2078 of unsigned type to hold the exact result, otherwise
2079 precop is minimum needed precision of signed type to
2080 hold the exact result. */
2082 if (code
== MULT_EXPR
)
2083 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2086 if (uns0_p
== uns1_p
)
2087 precop
= MAX (prec0
, prec1
) + 1;
2089 precop
= MAX (prec0
+ 1, prec1
) + 1;
2091 precop
= MAX (prec0
, prec1
+ 1) + 1;
2093 int orig_precres
= precres
;
2097 if ((uns0_p
&& uns1_p
)
2098 ? ((precop
+ !unsr_p
) <= precres
2099 /* u1 - u2 -> ur can overflow, no matter what precision
2101 && (code
!= MINUS_EXPR
|| !unsr_p
))
2102 : (!unsr_p
&& precop
<= precres
))
2104 /* The infinity precision result will always fit into result. */
2105 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2106 write_complex_part (target
, const0_rtx
, true);
2107 machine_mode mode
= TYPE_MODE (type
);
2108 struct separate_ops ops
;
2111 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2112 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2113 ops
.op2
= NULL_TREE
;
2115 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2116 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2120 /* For operations with low precision, if target doesn't have them, start
2121 with precres widening right away, otherwise do it only if the most
2122 simple cases can't be used. */
2123 const int min_precision
= targetm
.min_arithmetic_precision ();
2124 if (orig_precres
== precres
&& precres
< min_precision
)
2126 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2127 && prec1
<= precres
)
2128 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2129 && prec0
+ uns0_p
<= precres
2130 && prec1
+ uns1_p
<= precres
))
2132 arg0
= fold_convert_loc (loc
, type
, arg0
);
2133 arg1
= fold_convert_loc (loc
, type
, arg1
);
2137 if (integer_zerop (arg0
) && !unsr_p
)
2139 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2144 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2145 unsr_p
, unsr_p
, false, NULL
);
2148 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2149 unsr_p
, unsr_p
, false, NULL
);
2156 /* For sub-word operations, retry with a wider type first. */
2157 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2159 int p
= MAX (min_precision
, precop
);
2160 machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2161 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2164 p
= TYPE_PRECISION (optype
);
2168 unsr_p
= TYPE_UNSIGNED (optype
);
2174 if (prec0
<= precres
&& prec1
<= precres
)
2179 types
[0] = build_nonstandard_integer_type (precres
, 0);
2185 types
[1] = build_nonstandard_integer_type (precres
, 1);
2187 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2188 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2189 if (code
!= MULT_EXPR
)
2190 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2191 uns0_p
, uns1_p
, false, NULL
);
2193 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2194 uns0_p
, uns1_p
, false, NULL
);
2198 /* Retry with a wider type. */
2199 if (orig_precres
== precres
)
2201 int p
= MAX (prec0
, prec1
);
2202 machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
2203 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2206 p
= TYPE_PRECISION (optype
);
2210 unsr_p
= TYPE_UNSIGNED (optype
);
2221 /* Expand ADD_OVERFLOW STMT. */
2224 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2226 expand_arith_overflow (PLUS_EXPR
, stmt
);
2229 /* Expand SUB_OVERFLOW STMT. */
2232 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2234 expand_arith_overflow (MINUS_EXPR
, stmt
);
2237 /* Expand MUL_OVERFLOW STMT. */
2240 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2242 expand_arith_overflow (MULT_EXPR
, stmt
);
2245 /* This should get folded in tree-vectorizer.c. */
2248 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2253 /* This should get folded in tree-vectorizer.c. */
2256 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2261 /* Expand MASK_LOAD call STMT using optab OPTAB. */
2264 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2266 struct expand_operand ops
[3];
2267 tree type
, lhs
, rhs
, maskt
, ptr
;
2268 rtx mem
, target
, mask
;
2271 maskt
= gimple_call_arg (stmt
, 2);
2272 lhs
= gimple_call_lhs (stmt
);
2273 if (lhs
== NULL_TREE
)
2275 type
= TREE_TYPE (lhs
);
2276 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2277 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2278 if (TYPE_ALIGN (type
) != align
)
2279 type
= build_aligned_type (type
, align
);
2280 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2282 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2283 gcc_assert (MEM_P (mem
));
2284 mask
= expand_normal (maskt
);
2285 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2286 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2287 create_fixed_operand (&ops
[1], mem
);
2288 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2289 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2290 TYPE_MODE (TREE_TYPE (maskt
))),
2294 /* Expand MASK_STORE call STMT using optab OPTAB. */
2297 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2299 struct expand_operand ops
[3];
2300 tree type
, lhs
, rhs
, maskt
, ptr
;
2304 maskt
= gimple_call_arg (stmt
, 2);
2305 rhs
= gimple_call_arg (stmt
, 3);
2306 type
= TREE_TYPE (rhs
);
2307 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2308 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2309 if (TYPE_ALIGN (type
) != align
)
2310 type
= build_aligned_type (type
, align
);
2311 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2313 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2314 gcc_assert (MEM_P (mem
));
2315 mask
= expand_normal (maskt
);
2316 reg
= expand_normal (rhs
);
2317 create_fixed_operand (&ops
[0], mem
);
2318 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2319 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2320 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2321 TYPE_MODE (TREE_TYPE (maskt
))),
2326 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2331 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2333 /* When guessing was done, the hints should be already stripped away. */
2334 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2337 tree lhs
= gimple_call_lhs (stmt
);
2339 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2341 target
= const0_rtx
;
2342 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2343 if (lhs
&& val
!= target
)
2344 emit_move_insn (target
, val
);
2347 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2348 should never be called. */
2351 expand_VA_ARG (internal_fn
, gcall
*)
2356 /* Expand the IFN_UNIQUE function according to its first argument. */
2359 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2361 rtx pattern
= NULL_RTX
;
2362 enum ifn_unique_kind kind
2363 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2370 case IFN_UNIQUE_UNSPEC
:
2371 if (targetm
.have_unique ())
2372 pattern
= targetm
.gen_unique ();
2375 case IFN_UNIQUE_OACC_FORK
:
2376 case IFN_UNIQUE_OACC_JOIN
:
2377 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2379 tree lhs
= gimple_call_lhs (stmt
);
2380 rtx target
= const0_rtx
;
2383 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2385 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2386 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2388 if (kind
== IFN_UNIQUE_OACC_FORK
)
2389 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2391 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2399 emit_insn (pattern
);
2402 /* The size of an OpenACC compute dimension. */
2405 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2407 tree lhs
= gimple_call_lhs (stmt
);
2412 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2413 if (targetm
.have_oacc_dim_size ())
2415 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2416 VOIDmode
, EXPAND_NORMAL
);
2417 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2420 emit_move_insn (target
, GEN_INT (1));
2423 /* The position of an OpenACC execution engine along one compute axis. */
2426 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2428 tree lhs
= gimple_call_lhs (stmt
);
2433 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2434 if (targetm
.have_oacc_dim_pos ())
2436 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2437 VOIDmode
, EXPAND_NORMAL
);
2438 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2441 emit_move_insn (target
, const0_rtx
);
2444 /* This is expanded by oacc_device_lower pass. */
2447 expand_GOACC_LOOP (internal_fn
, gcall
*)
2452 /* This is expanded by oacc_device_lower pass. */
2455 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2460 /* This is expanded by oacc_device_lower pass. */
2463 expand_GOACC_TILE (internal_fn
, gcall
*)
2468 /* Set errno to EDOM. */
2471 expand_SET_EDOM (internal_fn
, gcall
*)
2474 #ifdef GEN_ERRNO_RTX
2475 rtx errno_rtx
= GEN_ERRNO_RTX
;
2477 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2479 emit_move_insn (errno_rtx
,
2480 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2486 /* Expand atomic bit test and set. */
2489 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2491 expand_ifn_atomic_bit_test_and (call
);
2494 /* Expand atomic bit test and complement. */
2497 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2499 expand_ifn_atomic_bit_test_and (call
);
2502 /* Expand atomic bit test and reset. */
2505 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2507 expand_ifn_atomic_bit_test_and (call
);
2510 /* Expand atomic bit test and set. */
2513 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2515 expand_ifn_atomic_compare_exchange (call
);
2518 /* Expand LAUNDER to assignment, lhs = arg0. */
2521 expand_LAUNDER (internal_fn
, gcall
*call
)
2523 tree lhs
= gimple_call_lhs (call
);
2528 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2531 /* Expand DIVMOD() using:
2532 a) optab handler for udivmod/sdivmod if it is available.
2533 b) If optab_handler doesn't exist, generate call to
2534 target-specific divmod libfunc. */
2537 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2539 tree lhs
= gimple_call_lhs (call_stmt
);
2540 tree arg0
= gimple_call_arg (call_stmt
, 0);
2541 tree arg1
= gimple_call_arg (call_stmt
, 1);
2543 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2544 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2545 machine_mode mode
= TYPE_MODE (type
);
2546 bool unsignedp
= TYPE_UNSIGNED (type
);
2547 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2549 rtx op0
= expand_normal (arg0
);
2550 rtx op1
= expand_normal (arg1
);
2551 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2553 rtx quotient
, remainder
, libfunc
;
2555 /* Check if optab_handler exists for divmod_optab for given mode. */
2556 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2558 quotient
= gen_reg_rtx (mode
);
2559 remainder
= gen_reg_rtx (mode
);
2560 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2563 /* Generate call to divmod libfunc if it exists. */
2564 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2565 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2566 "ient
, &remainder
);
2571 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2572 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2573 make_tree (TREE_TYPE (arg0
), quotient
),
2574 make_tree (TREE_TYPE (arg1
), remainder
)),
2575 target
, VOIDmode
, EXPAND_NORMAL
);
2578 /* Expand a call to FN using the operands in STMT. FN has a single
2579 output operand and NARGS input operands. */
2582 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2585 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2587 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2588 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2590 tree lhs
= gimple_call_lhs (stmt
);
2591 tree lhs_type
= TREE_TYPE (lhs
);
2592 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2593 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2595 for (unsigned int i
= 0; i
< nargs
; ++i
)
2597 tree rhs
= gimple_call_arg (stmt
, i
);
2598 tree rhs_type
= TREE_TYPE (rhs
);
2599 rtx rhs_rtx
= expand_normal (rhs
);
2600 if (INTEGRAL_TYPE_P (rhs_type
))
2601 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2602 TYPE_MODE (rhs_type
),
2603 TYPE_UNSIGNED (rhs_type
));
2605 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2608 expand_insn (icode
, nargs
+ 1, ops
);
2609 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2611 /* If the return value has an integral type, convert the instruction
2612 result to that type. This is useful for things that return an
2613 int regardless of the size of the input. If the instruction result
2614 is smaller than required, assume that it is signed.
2616 If the return value has a nonintegral type, its mode must match
2617 the instruction result. */
2618 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2620 /* If this is a scalar in a register that is stored in a wider
2621 mode than the declared mode, compute the result into its
2622 declared mode and then convert to the wider mode. */
2623 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2624 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2625 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2626 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2628 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2629 emit_move_insn (lhs_rtx
, ops
[0].value
);
2632 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2633 convert_move (lhs_rtx
, ops
[0].value
, 0);
2638 /* Expanders for optabs that can use expand_direct_optab_fn. */
2640 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2641 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2643 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2644 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2646 /* RETURN_TYPE and ARGS are a return type and argument list that are
2647 in principle compatible with FN (which satisfies direct_internal_fn_p).
2648 Return the types that should be used to determine whether the
2649 target supports FN. */
2652 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2654 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2655 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2656 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2657 return tree_pair (type0
, type1
);
2660 /* CALL is a call whose return type and arguments are in principle
2661 compatible with FN (which satisfies direct_internal_fn_p). Return the
2662 types that should be used to determine whether the target supports FN. */
2665 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2667 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2668 tree op0
= (info
.type0
< 0
2669 ? gimple_call_lhs (call
)
2670 : gimple_call_arg (call
, info
.type0
));
2671 tree op1
= (info
.type1
< 0
2672 ? gimple_call_lhs (call
)
2673 : gimple_call_arg (call
, info
.type1
));
2674 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2677 /* Return true if OPTAB is supported for TYPES (whose modes should be
2678 the same) when the optimization type is OPT_TYPE. Used for simple
2682 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2683 optimization_type opt_type
)
2685 machine_mode mode
= TYPE_MODE (types
.first
);
2686 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2687 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2690 /* Return true if load/store lanes optab OPTAB is supported for
2691 array type TYPES.first when the optimization type is OPT_TYPE. */
2694 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2695 optimization_type opt_type
)
2697 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2698 machine_mode imode
= TYPE_MODE (types
.first
);
2699 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2700 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2701 != CODE_FOR_nothing
);
2704 #define direct_unary_optab_supported_p direct_optab_supported_p
2705 #define direct_binary_optab_supported_p direct_optab_supported_p
2706 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2707 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2708 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2709 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2711 /* Return true if FN is supported for the types in TYPES when the
2712 optimization type is OPT_TYPE. The types are those associated with
2713 the "type0" and "type1" fields of FN's direct_internal_fn_info
2717 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2718 optimization_type opt_type
)
2722 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2723 case IFN_##CODE: break;
2724 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2726 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2728 #include "internal-fn.def"
2736 /* Return true if FN is supported for type TYPE when the optimization
2737 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2738 fields of FN's direct_internal_fn_info structure are the same. */
2741 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2742 optimization_type opt_type
)
2744 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2745 gcc_checking_assert (info
.type0
== info
.type1
);
2746 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2749 /* Return true if IFN_SET_EDOM is supported. */
2752 set_edom_supported_p (void)
2761 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2763 expand_##CODE (internal_fn fn, gcall *stmt) \
2765 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2767 #include "internal-fn.def"
2769 /* Routines to expand each internal function, indexed by function number.
2770 Each routine has the prototype:
2772 expand_<NAME> (gcall *stmt)
2774 where STMT is the statement that performs the call. */
2775 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2776 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2777 #include "internal-fn.def"
2781 /* Expand STMT as though it were a call to internal function FN. */
2784 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2786 internal_fn_expanders
[fn
] (fn
, stmt
);
2789 /* Expand STMT, which is a call to internal function FN. */
2792 expand_internal_call (gcall
*stmt
)
2794 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
2798 expand_PHI (internal_fn
, gcall
*)