2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
67 /* Fnspec of each internal function, indexed by function number. */
68 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
73 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
74 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
75 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
76 #include "internal-fn.def"
77 internal_fn_fnspec_array
[IFN_LAST
] = 0;
80 /* Create static initializers for the information returned by
81 direct_internal_fn. */
82 #define not_direct { -2, -2, false }
83 #define mask_load_direct { -1, 2, false }
84 #define load_lanes_direct { -1, -1, false }
85 #define mask_store_direct { 3, 2, false }
86 #define store_lanes_direct { 0, 0, false }
87 #define unary_direct { 0, 0, true }
88 #define binary_direct { 0, 0, true }
90 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
91 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
92 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
93 #include "internal-fn.def"
97 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
98 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
100 static enum insn_code
101 get_multi_vector_move (tree array_type
, convert_optab optab
)
106 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
107 imode
= TYPE_MODE (array_type
);
108 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
110 return convert_optab_handler (optab
, imode
, vmode
);
113 /* Expand LOAD_LANES call STMT using optab OPTAB. */
116 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
118 struct expand_operand ops
[2];
122 lhs
= gimple_call_lhs (stmt
);
123 rhs
= gimple_call_arg (stmt
, 0);
124 type
= TREE_TYPE (lhs
);
126 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
127 mem
= expand_normal (rhs
);
129 gcc_assert (MEM_P (mem
));
130 PUT_MODE (mem
, TYPE_MODE (type
));
132 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
133 create_fixed_operand (&ops
[1], mem
);
134 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
137 /* Expand STORE_LANES call STMT using optab OPTAB. */
140 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
142 struct expand_operand ops
[2];
146 lhs
= gimple_call_lhs (stmt
);
147 rhs
= gimple_call_arg (stmt
, 0);
148 type
= TREE_TYPE (rhs
);
150 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
151 reg
= expand_normal (rhs
);
153 gcc_assert (MEM_P (target
));
154 PUT_MODE (target
, TYPE_MODE (type
));
156 create_fixed_operand (&ops
[0], target
);
157 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
158 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
162 expand_ANNOTATE (internal_fn
, gcall
*)
167 /* This should get expanded in omp_device_lower pass. */
170 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
175 /* This should get expanded in omp_device_lower pass. */
178 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
183 /* Allocate per-lane storage and begin non-uniform execution region. */
186 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
189 tree lhs
= gimple_call_lhs (stmt
);
191 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
193 target
= gen_reg_rtx (Pmode
);
194 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
195 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
196 struct expand_operand ops
[3];
197 create_output_operand (&ops
[0], target
, Pmode
);
198 create_input_operand (&ops
[1], size
, Pmode
);
199 create_input_operand (&ops
[2], align
, Pmode
);
200 gcc_assert (targetm
.have_omp_simt_enter ());
201 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
204 /* Deallocate per-lane storage and leave non-uniform execution region. */
207 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
209 gcc_checking_assert (!gimple_call_lhs (stmt
));
210 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
211 struct expand_operand ops
[1];
212 create_input_operand (&ops
[0], arg
, Pmode
);
213 gcc_assert (targetm
.have_omp_simt_exit ());
214 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
217 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
218 without SIMT execution this should be expanded in omp_device_lower pass. */
221 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
223 tree lhs
= gimple_call_lhs (stmt
);
227 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
228 gcc_assert (targetm
.have_omp_simt_lane ());
229 emit_insn (targetm
.gen_omp_simt_lane (target
));
232 /* This should get expanded in omp_device_lower pass. */
235 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
240 /* Lane index of the first SIMT lane that supplies a non-zero argument.
241 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
242 lane that executed the last iteration for handling OpenMP lastprivate. */
245 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
247 tree lhs
= gimple_call_lhs (stmt
);
251 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
252 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
253 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
254 struct expand_operand ops
[2];
255 create_output_operand (&ops
[0], target
, mode
);
256 create_input_operand (&ops
[1], cond
, mode
);
257 gcc_assert (targetm
.have_omp_simt_last_lane ());
258 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
261 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
264 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
266 tree lhs
= gimple_call_lhs (stmt
);
270 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
271 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
272 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
273 struct expand_operand ops
[2];
274 create_output_operand (&ops
[0], target
, mode
);
275 create_input_operand (&ops
[1], ctr
, mode
);
276 gcc_assert (targetm
.have_omp_simt_ordered ());
277 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
280 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
281 any lane supplies a non-zero argument. */
284 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
286 tree lhs
= gimple_call_lhs (stmt
);
290 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
291 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
292 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
293 struct expand_operand ops
[2];
294 create_output_operand (&ops
[0], target
, mode
);
295 create_input_operand (&ops
[1], cond
, mode
);
296 gcc_assert (targetm
.have_omp_simt_vote_any ());
297 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
300 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
301 is destination lane index XOR given offset. */
304 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
306 tree lhs
= gimple_call_lhs (stmt
);
310 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
311 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
312 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
313 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
314 struct expand_operand ops
[3];
315 create_output_operand (&ops
[0], target
, mode
);
316 create_input_operand (&ops
[1], src
, mode
);
317 create_input_operand (&ops
[2], idx
, SImode
);
318 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
319 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
322 /* Exchange between SIMT lanes according to given source lane index. */
325 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
327 tree lhs
= gimple_call_lhs (stmt
);
331 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
332 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
333 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
334 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
335 struct expand_operand ops
[3];
336 create_output_operand (&ops
[0], target
, mode
);
337 create_input_operand (&ops
[1], src
, mode
);
338 create_input_operand (&ops
[2], idx
, SImode
);
339 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
340 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
343 /* This should get expanded in adjust_simduid_builtins. */
346 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
351 /* This should get expanded in adjust_simduid_builtins. */
354 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
359 /* This should get expanded in adjust_simduid_builtins. */
362 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
367 /* This should get expanded in adjust_simduid_builtins. */
370 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
375 /* This should get expanded in adjust_simduid_builtins. */
378 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
383 /* This should get expanded in the sanopt pass. */
386 expand_UBSAN_NULL (internal_fn
, gcall
*)
391 /* This should get expanded in the sanopt pass. */
394 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
399 /* This should get expanded in the sanopt pass. */
402 expand_UBSAN_VPTR (internal_fn
, gcall
*)
407 /* This should get expanded in the sanopt pass. */
410 expand_UBSAN_PTR (internal_fn
, gcall
*)
415 /* This should get expanded in the sanopt pass. */
418 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
423 /* This should get expanded in the sanopt pass. */
426 expand_ASAN_CHECK (internal_fn
, gcall
*)
431 /* This should get expanded in the sanopt pass. */
434 expand_ASAN_MARK (internal_fn
, gcall
*)
439 /* This should get expanded in the sanopt pass. */
442 expand_ASAN_POISON (internal_fn
, gcall
*)
447 /* This should get expanded in the sanopt pass. */
450 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
455 /* This should get expanded in the tsan pass. */
458 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
463 /* This should get expanded in the lower pass. */
466 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
468 error_at (gimple_location (call
),
469 "invalid use of attribute %<fallthrough%>");
472 /* Return minimum precision needed to represent all values
473 of ARG in SIGNed integral type. */
476 get_min_precision (tree arg
, signop sign
)
478 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
480 signop orig_sign
= sign
;
481 if (TREE_CODE (arg
) == INTEGER_CST
)
484 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
486 widest_int w
= wi::to_widest (arg
);
487 w
= wi::ext (w
, prec
, sign
);
488 p
= wi::min_precision (w
, sign
);
491 p
= wi::min_precision (wi::to_wide (arg
), sign
);
492 return MIN (p
, prec
);
494 while (CONVERT_EXPR_P (arg
)
495 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
496 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
498 arg
= TREE_OPERAND (arg
, 0);
499 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
501 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
503 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
504 return prec
+ (orig_sign
!= sign
);
505 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
508 return prec
+ (orig_sign
!= sign
);
510 if (TREE_CODE (arg
) != SSA_NAME
)
511 return prec
+ (orig_sign
!= sign
);
512 wide_int arg_min
, arg_max
;
513 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
515 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
516 if (is_gimple_assign (g
)
517 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
519 tree t
= gimple_assign_rhs1 (g
);
520 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
521 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
524 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
526 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
528 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
529 return prec
+ (orig_sign
!= sign
);
530 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
533 return prec
+ (orig_sign
!= sign
);
537 return prec
+ (orig_sign
!= sign
);
539 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
541 int p1
= wi::min_precision (arg_min
, sign
);
542 int p2
= wi::min_precision (arg_max
, sign
);
544 prec
= MIN (prec
, p1
);
546 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
548 int p
= wi::min_precision (arg_max
, UNSIGNED
);
549 prec
= MIN (prec
, p
);
551 return prec
+ (orig_sign
!= sign
);
554 /* Helper for expand_*_overflow. Set the __imag__ part to true
555 (1 except for signed:1 type, in which case store -1). */
558 expand_arith_set_overflow (tree lhs
, rtx target
)
560 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
561 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
562 write_complex_part (target
, constm1_rtx
, true);
564 write_complex_part (target
, const1_rtx
, true);
567 /* Helper for expand_*_overflow. Store RES into the __real__ part
568 of TARGET. If RES has larger MODE than __real__ part of TARGET,
569 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
570 if LHS has smaller precision than its mode. */
573 expand_arith_overflow_result_store (tree lhs
, rtx target
,
574 scalar_int_mode mode
, rtx res
)
576 scalar_int_mode tgtmode
577 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
581 rtx_code_label
*done_label
= gen_label_rtx ();
582 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
583 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
584 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
585 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
586 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
587 profile_probability::very_likely ());
588 expand_arith_set_overflow (lhs
, target
);
589 emit_label (done_label
);
591 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
592 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
595 rtx_code_label
*done_label
= gen_label_rtx ();
596 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
601 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
603 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
604 true, OPTAB_LIB_WIDEN
);
608 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
610 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
613 do_compare_rtx_and_jump (res
, lres
,
614 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
615 profile_probability::very_likely ());
616 expand_arith_set_overflow (lhs
, target
);
617 emit_label (done_label
);
619 write_complex_part (target
, lres
, false);
622 /* Helper for expand_*_overflow. Store RES into TARGET. */
625 expand_ubsan_result_store (rtx target
, rtx res
)
627 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
628 /* If this is a scalar in a register that is stored in a wider mode
629 than the declared mode, compute the result into its declared mode
630 and then convert to the wider mode. Our value is the computed
632 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
634 emit_move_insn (target
, res
);
637 /* Add sub/add overflow checking to the statement STMT.
638 CODE says whether the operation is +, or -. */
641 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
642 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
643 bool uns1_p
, bool is_ubsan
, tree
*datap
)
645 rtx res
, target
= NULL_RTX
;
647 rtx_code_label
*done_label
= gen_label_rtx ();
648 rtx_code_label
*do_error
= gen_label_rtx ();
649 do_pending_stack_adjust ();
650 rtx op0
= expand_normal (arg0
);
651 rtx op1
= expand_normal (arg1
);
652 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
653 int prec
= GET_MODE_PRECISION (mode
);
654 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
658 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
662 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
664 write_complex_part (target
, const0_rtx
, true);
667 /* We assume both operands and result have the same precision
668 here (GET_MODE_BITSIZE (mode)), S stands for signed type
669 with that precision, U for unsigned type with that precision,
670 sgn for unsigned most significant bit in that precision.
671 s1 is signed first operand, u1 is unsigned first operand,
672 s2 is signed second operand, u2 is unsigned second operand,
673 sr is signed result, ur is unsigned result and the following
674 rules say how to compute result (which is always result of
675 the operands as if both were unsigned, cast to the right
676 signedness) and how to compute whether operation overflowed.
679 res = (S) ((U) s1 + (U) s2)
680 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
682 res = (S) ((U) s1 - (U) s2)
683 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
686 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
689 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
691 res = (S) ((U) s1 + u2)
692 ovf = ((U) res ^ sgn) < u2
697 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
699 res = (S) ((U) s1 - u2)
700 ovf = u2 > ((U) s1 ^ sgn)
703 ovf = s1 < 0 || u2 > (U) s1
706 ovf = u1 >= ((U) s2 ^ sgn)
711 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
713 res = (U) s1 + (U) s2
714 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
717 ovf = (U) res < u2 || res < 0
720 ovf = u1 >= u2 ? res < 0 : res >= 0
722 res = (U) s1 - (U) s2
723 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
725 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
727 /* PLUS_EXPR is commutative, if operand signedness differs,
728 canonicalize to the first operand being signed and second
729 unsigned to simplify following code. */
730 std::swap (op0
, op1
);
731 std::swap (arg0
, arg1
);
737 if (uns0_p
&& uns1_p
&& unsr_p
)
739 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
740 : usubv4_optab
, mode
);
741 if (icode
!= CODE_FOR_nothing
)
743 struct expand_operand ops
[4];
744 rtx_insn
*last
= get_last_insn ();
746 res
= gen_reg_rtx (mode
);
747 create_output_operand (&ops
[0], res
, mode
);
748 create_input_operand (&ops
[1], op0
, mode
);
749 create_input_operand (&ops
[2], op1
, mode
);
750 create_fixed_operand (&ops
[3], do_error
);
751 if (maybe_expand_insn (icode
, 4, ops
))
753 last
= get_last_insn ();
754 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
756 && any_condjump_p (last
)
757 && !find_reg_note (last
, REG_BR_PROB
, 0))
758 add_reg_br_prob_note (last
,
759 profile_probability::very_unlikely ());
760 emit_jump (done_label
);
764 delete_insns_since (last
);
767 /* Compute the operation. On RTL level, the addition is always
769 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
770 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
772 /* For PLUS_EXPR, the operation is commutative, so we can pick
773 operand to compare against. For prec <= BITS_PER_WORD, I think
774 preferring REG operand is better over CONST_INT, because
775 the CONST_INT might enlarge the instruction or CSE would need
776 to figure out we'd already loaded it into a register before.
777 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
778 as then the multi-word comparison can be perhaps simplified. */
779 if (code
== PLUS_EXPR
780 && (prec
<= BITS_PER_WORD
781 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
782 : CONST_SCALAR_INT_P (op1
)))
784 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
785 true, mode
, NULL_RTX
, NULL
, done_label
,
786 profile_probability::very_likely ());
791 if (!uns0_p
&& uns1_p
&& !unsr_p
)
793 /* Compute the operation. On RTL level, the addition is always
795 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
796 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
797 rtx tem
= expand_binop (mode
, add_optab
,
798 code
== PLUS_EXPR
? res
: op0
, sgn
,
799 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
800 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
801 done_label
, profile_probability::very_likely ());
806 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
808 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
810 /* As we've changed op1, we have to avoid using the value range
811 for the original argument. */
812 arg1
= error_mark_node
;
818 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
820 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
822 /* As we've changed op0, we have to avoid using the value range
823 for the original argument. */
824 arg0
= error_mark_node
;
830 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
832 /* Compute the operation. On RTL level, the addition is always
834 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
836 int pos_neg
= get_range_pos_neg (arg0
);
838 /* If ARG0 is known to be always negative, this is always overflow. */
839 emit_jump (do_error
);
840 else if (pos_neg
== 3)
841 /* If ARG0 is not known to be always positive, check at runtime. */
842 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
843 NULL
, do_error
, profile_probability::very_unlikely ());
844 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
845 done_label
, profile_probability::very_likely ());
850 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
852 /* Compute the operation. On RTL level, the addition is always
854 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
856 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
858 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
859 done_label
, profile_probability::very_likely ());
864 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
866 /* Compute the operation. On RTL level, the addition is always
868 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
870 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
871 NULL
, do_error
, profile_probability::very_unlikely ());
873 /* The operation is commutative, so we can pick operand to compare
874 against. For prec <= BITS_PER_WORD, I think preferring REG operand
875 is better over CONST_INT, because the CONST_INT might enlarge the
876 instruction or CSE would need to figure out we'd already loaded it
877 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
878 might be more beneficial, as then the multi-word comparison can be
879 perhaps simplified. */
880 if (prec
<= BITS_PER_WORD
881 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
882 : CONST_SCALAR_INT_P (op0
))
884 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
885 done_label
, profile_probability::very_likely ());
890 if (!uns0_p
&& !uns1_p
&& unsr_p
)
892 /* Compute the operation. On RTL level, the addition is always
894 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
895 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
896 int pos_neg
= get_range_pos_neg (arg1
);
897 if (code
== PLUS_EXPR
)
899 int pos_neg0
= get_range_pos_neg (arg0
);
900 if (pos_neg0
!= 3 && pos_neg
== 3)
902 std::swap (op0
, op1
);
909 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
910 ? and_optab
: ior_optab
,
911 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
912 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
913 NULL
, done_label
, profile_probability::very_likely ());
917 rtx_code_label
*do_ior_label
= gen_label_rtx ();
918 do_compare_rtx_and_jump (op1
, const0_rtx
,
919 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
920 NULL_RTX
, NULL
, do_ior_label
,
921 profile_probability::even ());
922 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
924 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
925 NULL
, done_label
, profile_probability::very_likely ());
926 emit_jump (do_error
);
927 emit_label (do_ior_label
);
928 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
930 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
931 NULL
, done_label
, profile_probability::very_likely ());
937 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
939 /* Compute the operation. On RTL level, the addition is always
941 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
943 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
944 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
945 op0_geu_op1
, profile_probability::even ());
946 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
947 NULL
, done_label
, profile_probability::very_likely ());
948 emit_jump (do_error
);
949 emit_label (op0_geu_op1
);
950 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
951 NULL
, done_label
, profile_probability::very_likely ());
955 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
960 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
961 : subv4_optab
, mode
);
962 if (icode
!= CODE_FOR_nothing
)
964 struct expand_operand ops
[4];
965 rtx_insn
*last
= get_last_insn ();
967 res
= gen_reg_rtx (mode
);
968 create_output_operand (&ops
[0], res
, mode
);
969 create_input_operand (&ops
[1], op0
, mode
);
970 create_input_operand (&ops
[2], op1
, mode
);
971 create_fixed_operand (&ops
[3], do_error
);
972 if (maybe_expand_insn (icode
, 4, ops
))
974 last
= get_last_insn ();
975 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
977 && any_condjump_p (last
)
978 && !find_reg_note (last
, REG_BR_PROB
, 0))
979 add_reg_br_prob_note (last
,
980 profile_probability::very_unlikely ());
981 emit_jump (done_label
);
985 delete_insns_since (last
);
988 /* Compute the operation. On RTL level, the addition is always
990 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
991 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
993 /* If we can prove that one of the arguments (for MINUS_EXPR only
994 the second operand, as subtraction is not commutative) is always
995 non-negative or always negative, we can do just one comparison
996 and conditional jump. */
997 int pos_neg
= get_range_pos_neg (arg1
);
998 if (code
== PLUS_EXPR
)
1000 int pos_neg0
= get_range_pos_neg (arg0
);
1001 if (pos_neg0
!= 3 && pos_neg
== 3)
1003 std::swap (op0
, op1
);
1008 /* Addition overflows if and only if the two operands have the same sign,
1009 and the result has the opposite sign. Subtraction overflows if and
1010 only if the two operands have opposite sign, and the subtrahend has
1011 the same sign as the result. Here 0 is counted as positive. */
1014 /* Compute op0 ^ op1 (operands have opposite sign). */
1015 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1018 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1019 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1023 if (code
== PLUS_EXPR
)
1025 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1026 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1027 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1032 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1033 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1034 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1038 /* No overflow if the result has bit sign cleared. */
1039 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1040 NULL
, done_label
, profile_probability::very_likely ());
1043 /* Compare the result of the operation with the first operand.
1044 No overflow for addition if second operand is positive and result
1045 is larger or second operand is negative and result is smaller.
1046 Likewise for subtraction with sign of second operand flipped. */
1048 do_compare_rtx_and_jump (res
, op0
,
1049 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1050 false, mode
, NULL_RTX
, NULL
, done_label
,
1051 profile_probability::very_likely ());
1055 emit_label (do_error
);
1058 /* Expand the ubsan builtin call. */
1060 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1064 do_pending_stack_adjust ();
1067 expand_arith_set_overflow (lhs
, target
);
1070 emit_label (done_label
);
1075 expand_ubsan_result_store (target
, res
);
1079 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1082 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1087 /* Add negate overflow checking to the statement STMT. */
1090 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1095 rtx_code_label
*done_label
, *do_error
;
1096 rtx target
= NULL_RTX
;
1098 done_label
= gen_label_rtx ();
1099 do_error
= gen_label_rtx ();
1101 do_pending_stack_adjust ();
1102 op1
= expand_normal (arg1
);
1104 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1107 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1109 write_complex_part (target
, const0_rtx
, true);
1112 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1113 if (icode
!= CODE_FOR_nothing
)
1115 struct expand_operand ops
[3];
1116 rtx_insn
*last
= get_last_insn ();
1118 res
= gen_reg_rtx (mode
);
1119 create_output_operand (&ops
[0], res
, mode
);
1120 create_input_operand (&ops
[1], op1
, mode
);
1121 create_fixed_operand (&ops
[2], do_error
);
1122 if (maybe_expand_insn (icode
, 3, ops
))
1124 last
= get_last_insn ();
1125 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1127 && any_condjump_p (last
)
1128 && !find_reg_note (last
, REG_BR_PROB
, 0))
1129 add_reg_br_prob_note (last
,
1130 profile_probability::very_unlikely ());
1131 emit_jump (done_label
);
1135 delete_insns_since (last
);
1136 icode
= CODE_FOR_nothing
;
1140 if (icode
== CODE_FOR_nothing
)
1142 /* Compute the operation. On RTL level, the addition is always
1144 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1146 /* Compare the operand with the most negative value. */
1147 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1148 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1149 done_label
, profile_probability::very_likely ());
1152 emit_label (do_error
);
1155 /* Expand the ubsan builtin call. */
1157 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1158 arg1
, NULL_TREE
, datap
);
1161 do_pending_stack_adjust ();
1164 expand_arith_set_overflow (lhs
, target
);
1167 emit_label (done_label
);
1172 expand_ubsan_result_store (target
, res
);
1174 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1178 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1179 mode MODE can be expanded without using a libcall. */
1182 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1183 rtx op0
, rtx op1
, bool uns
)
1185 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1186 != CODE_FOR_nothing
)
1189 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1190 != CODE_FOR_nothing
)
1193 rtx_insn
*last
= get_last_insn ();
1194 if (CONSTANT_P (op0
))
1195 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1197 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1198 if (CONSTANT_P (op1
))
1199 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1201 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1202 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1203 delete_insns_since (last
);
1204 return ret
!= NULL_RTX
;
1207 /* Add mul overflow checking to the statement STMT. */
1210 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1211 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1216 rtx_code_label
*done_label
, *do_error
;
1217 rtx target
= NULL_RTX
;
1219 enum insn_code icode
;
1221 done_label
= gen_label_rtx ();
1222 do_error
= gen_label_rtx ();
1224 do_pending_stack_adjust ();
1225 op0
= expand_normal (arg0
);
1226 op1
= expand_normal (arg1
);
1228 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1232 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1234 write_complex_part (target
, const0_rtx
, true);
1238 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1240 /* We assume both operands and result have the same precision
1241 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1242 with that precision, U for unsigned type with that precision,
1243 sgn for unsigned most significant bit in that precision.
1244 s1 is signed first operand, u1 is unsigned first operand,
1245 s2 is signed second operand, u2 is unsigned second operand,
1246 sr is signed result, ur is unsigned result and the following
1247 rules say how to compute result (which is always result of
1248 the operands as if both were unsigned, cast to the right
1249 signedness) and how to compute whether operation overflowed.
1250 main_ovf (false) stands for jump on signed multiplication
1251 overflow or the main algorithm with uns == false.
1252 main_ovf (true) stands for jump on unsigned multiplication
1253 overflow or the main algorithm with uns == true.
1256 res = (S) ((U) s1 * (U) s2)
1257 ovf = main_ovf (false)
1260 ovf = main_ovf (true)
1263 ovf = (s1 < 0 && u2) || main_ovf (true)
1266 ovf = res < 0 || main_ovf (true)
1268 res = (S) ((U) s1 * u2)
1269 ovf = (S) u2 >= 0 ? main_ovf (false)
1270 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1272 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1273 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1275 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1277 if (uns0_p
&& !uns1_p
)
1279 /* Multiplication is commutative, if operand signedness differs,
1280 canonicalize to the first operand being signed and second
1281 unsigned to simplify following code. */
1282 std::swap (op0
, op1
);
1283 std::swap (arg0
, arg1
);
1288 int pos_neg0
= get_range_pos_neg (arg0
);
1289 int pos_neg1
= get_range_pos_neg (arg1
);
1292 if (!uns0_p
&& uns1_p
&& unsr_p
)
1297 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1300 /* If s1 is negative, avoid the main code, just multiply and
1301 signal overflow if op1 is not 0. */
1302 struct separate_ops ops
;
1303 ops
.code
= MULT_EXPR
;
1304 ops
.type
= TREE_TYPE (arg1
);
1305 ops
.op0
= make_tree (ops
.type
, op0
);
1306 ops
.op1
= make_tree (ops
.type
, op1
);
1307 ops
.op2
= NULL_TREE
;
1309 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1310 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1311 NULL
, done_label
, profile_probability::very_likely ());
1312 goto do_error_label
;
1314 rtx_code_label
*do_main_label
;
1315 do_main_label
= gen_label_rtx ();
1316 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1317 NULL
, do_main_label
, profile_probability::very_likely ());
1318 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1319 NULL
, do_main_label
, profile_probability::very_likely ());
1320 expand_arith_set_overflow (lhs
, target
);
1321 emit_label (do_main_label
);
1329 if (uns0_p
&& uns1_p
&& !unsr_p
)
1332 /* Rest of handling of this case after res is computed. */
1337 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1344 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1345 avoid the main code, just multiply and signal overflow
1346 unless 0 * u2 or -1 * ((U) Smin). */
1347 struct separate_ops ops
;
1348 ops
.code
= MULT_EXPR
;
1349 ops
.type
= TREE_TYPE (arg1
);
1350 ops
.op0
= make_tree (ops
.type
, op0
);
1351 ops
.op1
= make_tree (ops
.type
, op1
);
1352 ops
.op2
= NULL_TREE
;
1354 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1355 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1356 NULL
, done_label
, profile_probability::very_likely ());
1357 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1358 NULL
, do_error
, profile_probability::very_unlikely ());
1360 prec
= GET_MODE_PRECISION (mode
);
1362 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1363 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1364 NULL
, done_label
, profile_probability::very_likely ());
1365 goto do_error_label
;
1367 /* Rest of handling of this case after res is computed. */
1375 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1378 switch (pos_neg0
| pos_neg1
)
1380 case 1: /* Both operands known to be non-negative. */
1382 case 2: /* Both operands known to be negative. */
1383 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1384 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1385 /* Avoid looking at arg0/arg1 ranges, as we've changed
1387 arg0
= error_mark_node
;
1388 arg1
= error_mark_node
;
1391 if ((pos_neg0
^ pos_neg1
) == 3)
1393 /* If one operand is known to be negative and the other
1394 non-negative, this overflows always, unless the non-negative
1395 one is 0. Just do normal multiply and set overflow
1396 unless one of the operands is 0. */
1397 struct separate_ops ops
;
1398 ops
.code
= MULT_EXPR
;
1400 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1402 ops
.op0
= make_tree (ops
.type
, op0
);
1403 ops
.op1
= make_tree (ops
.type
, op1
);
1404 ops
.op2
= NULL_TREE
;
1406 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1407 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1409 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1410 NULL_RTX
, NULL
, done_label
,
1411 profile_probability::very_likely ());
1412 goto do_error_label
;
1414 /* The general case, do all the needed comparisons at runtime. */
1415 rtx_code_label
*do_main_label
, *after_negate_label
;
1417 rop0
= gen_reg_rtx (mode
);
1418 rop1
= gen_reg_rtx (mode
);
1419 emit_move_insn (rop0
, op0
);
1420 emit_move_insn (rop1
, op1
);
1423 do_main_label
= gen_label_rtx ();
1424 after_negate_label
= gen_label_rtx ();
1425 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1427 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1428 NULL
, after_negate_label
, profile_probability::very_likely ());
1429 /* Both arguments negative here, negate them and continue with
1430 normal unsigned overflow checking multiplication. */
1431 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1433 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1435 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1437 arg0
= error_mark_node
;
1438 arg1
= error_mark_node
;
1439 emit_jump (do_main_label
);
1440 emit_label (after_negate_label
);
1441 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1443 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1444 NULL
, do_main_label
, profile_probability::very_likely ());
1445 /* One argument is negative here, the other positive. This
1446 overflows always, unless one of the arguments is 0. But
1447 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1448 is, thus we can keep do_main code oring in overflow as is. */
1449 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1450 NULL
, do_main_label
, profile_probability::very_likely ());
1451 expand_arith_set_overflow (lhs
, target
);
1452 emit_label (do_main_label
);
1460 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1461 sign
= uns
? UNSIGNED
: SIGNED
;
1462 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1463 if (icode
!= CODE_FOR_nothing
)
1465 struct expand_operand ops
[4];
1466 rtx_insn
*last
= get_last_insn ();
1468 res
= gen_reg_rtx (mode
);
1469 create_output_operand (&ops
[0], res
, mode
);
1470 create_input_operand (&ops
[1], op0
, mode
);
1471 create_input_operand (&ops
[2], op1
, mode
);
1472 create_fixed_operand (&ops
[3], do_error
);
1473 if (maybe_expand_insn (icode
, 4, ops
))
1475 last
= get_last_insn ();
1476 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1478 && any_condjump_p (last
)
1479 && !find_reg_note (last
, REG_BR_PROB
, 0))
1480 add_reg_br_prob_note (last
,
1481 profile_probability::very_unlikely ());
1482 emit_jump (done_label
);
1486 delete_insns_since (last
);
1487 icode
= CODE_FOR_nothing
;
1491 if (icode
== CODE_FOR_nothing
)
1493 struct separate_ops ops
;
1494 int prec
= GET_MODE_PRECISION (mode
);
1495 scalar_int_mode hmode
, wmode
;
1496 ops
.op0
= make_tree (type
, op0
);
1497 ops
.op1
= make_tree (type
, op1
);
1498 ops
.op2
= NULL_TREE
;
1501 /* Optimize unsigned overflow check where we don't use the
1502 multiplication result, just whether overflow happened.
1503 If we can do MULT_HIGHPART_EXPR, that followed by
1504 comparison of the result against zero is cheapest.
1505 We'll still compute res, but it should be DCEd later. */
1511 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1512 && can_mult_highpart_p (mode
, uns
) == 1
1513 && single_imm_use (lhs
, &use
, &use_stmt
)
1514 && is_gimple_assign (use_stmt
)
1515 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1518 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1519 && targetm
.scalar_mode_supported_p (wmode
)
1520 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1523 ops
.code
= WIDEN_MULT_EXPR
;
1525 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1527 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1528 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1530 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1531 res
= convert_modes (mode
, wmode
, res
, uns
);
1533 /* For the unsigned multiplication, there was overflow if
1534 HIPART is non-zero. */
1535 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1536 NULL_RTX
, NULL
, done_label
,
1537 profile_probability::very_likely ());
1540 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1542 /* RES is low half of the double width result, HIPART
1543 the high half. There was overflow if
1544 HIPART is different from RES < 0 ? -1 : 0. */
1545 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1546 NULL_RTX
, NULL
, done_label
,
1547 profile_probability::very_likely ());
1550 else if (can_mult_highpart_p (mode
, uns
) == 1)
1553 ops
.code
= MULT_HIGHPART_EXPR
;
1556 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1558 ops
.code
= MULT_EXPR
;
1559 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1561 /* For the unsigned multiplication, there was overflow if
1562 HIPART is non-zero. */
1563 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1564 NULL_RTX
, NULL
, done_label
,
1565 profile_probability::very_likely ());
1568 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1570 /* RES is low half of the double width result, HIPART
1571 the high half. There was overflow if
1572 HIPART is different from RES < 0 ? -1 : 0. */
1573 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1574 NULL_RTX
, NULL
, done_label
,
1575 profile_probability::very_likely ());
1579 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1580 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1582 rtx_code_label
*large_op0
= gen_label_rtx ();
1583 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1584 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1585 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1586 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1587 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1588 rtx_code_label
*do_overflow
= gen_label_rtx ();
1589 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1591 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1592 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1594 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1595 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1596 rtx signbit0
= const0_rtx
;
1598 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1600 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1602 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1603 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1604 rtx signbit1
= const0_rtx
;
1606 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1609 res
= gen_reg_rtx (mode
);
1611 /* True if op0 resp. op1 are known to be in the range of
1613 bool op0_small_p
= false;
1614 bool op1_small_p
= false;
1615 /* True if op0 resp. op1 are known to have all zeros or all ones
1616 in the upper half of bits, but are not known to be
1618 bool op0_medium_p
= false;
1619 bool op1_medium_p
= false;
1620 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1621 nonnegative, 1 if unknown. */
1627 else if (pos_neg0
== 2)
1631 else if (pos_neg1
== 2)
1634 unsigned int mprec0
= prec
;
1635 if (arg0
!= error_mark_node
)
1636 mprec0
= get_min_precision (arg0
, sign
);
1637 if (mprec0
<= hprec
)
1639 else if (!uns
&& mprec0
<= hprec
+ 1)
1640 op0_medium_p
= true;
1641 unsigned int mprec1
= prec
;
1642 if (arg1
!= error_mark_node
)
1643 mprec1
= get_min_precision (arg1
, sign
);
1644 if (mprec1
<= hprec
)
1646 else if (!uns
&& mprec1
<= hprec
+ 1)
1647 op1_medium_p
= true;
1649 int smaller_sign
= 1;
1650 int larger_sign
= 1;
1653 smaller_sign
= op0_sign
;
1654 larger_sign
= op1_sign
;
1656 else if (op1_small_p
)
1658 smaller_sign
= op1_sign
;
1659 larger_sign
= op0_sign
;
1661 else if (op0_sign
== op1_sign
)
1663 smaller_sign
= op0_sign
;
1664 larger_sign
= op0_sign
;
1668 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1669 NULL_RTX
, NULL
, large_op0
,
1670 profile_probability::unlikely ());
1673 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1674 NULL_RTX
, NULL
, small_op0_large_op1
,
1675 profile_probability::unlikely ());
1677 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1678 hmode to mode, the multiplication will never overflow. We can
1679 do just one hmode x hmode => mode widening multiplication. */
1680 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1681 if (GET_CODE (lopart0
) == SUBREG
)
1683 lopart0s
= shallow_copy_rtx (lopart0
);
1684 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1685 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1687 if (GET_CODE (lopart1
) == SUBREG
)
1689 lopart1s
= shallow_copy_rtx (lopart1
);
1690 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1691 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1693 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1694 ops
.op0
= make_tree (halfstype
, lopart0s
);
1695 ops
.op1
= make_tree (halfstype
, lopart1s
);
1696 ops
.code
= WIDEN_MULT_EXPR
;
1699 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1700 emit_move_insn (res
, thisres
);
1701 emit_jump (done_label
);
1703 emit_label (small_op0_large_op1
);
1705 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1706 but op1 is not, just swap the arguments and handle it as op1
1707 sign/zero extended, op0 not. */
1708 rtx larger
= gen_reg_rtx (mode
);
1709 rtx hipart
= gen_reg_rtx (hmode
);
1710 rtx lopart
= gen_reg_rtx (hmode
);
1711 emit_move_insn (larger
, op1
);
1712 emit_move_insn (hipart
, hipart1
);
1713 emit_move_insn (lopart
, lopart0
);
1714 emit_jump (one_small_one_large
);
1716 emit_label (large_op0
);
1719 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1720 NULL_RTX
, NULL
, both_ops_large
,
1721 profile_probability::unlikely ());
1723 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1724 but op0 is not, prepare larger, hipart and lopart pseudos and
1725 handle it together with small_op0_large_op1. */
1726 emit_move_insn (larger
, op0
);
1727 emit_move_insn (hipart
, hipart0
);
1728 emit_move_insn (lopart
, lopart1
);
1730 emit_label (one_small_one_large
);
1732 /* lopart is the low part of the operand that is sign extended
1733 to mode, larger is the other operand, hipart is the
1734 high part of larger and lopart0 and lopart1 are the low parts
1736 We perform lopart0 * lopart1 and lopart * hipart widening
1738 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1739 ops
.op0
= make_tree (halfutype
, lopart0
);
1740 ops
.op1
= make_tree (halfutype
, lopart1
);
1742 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1744 ops
.op0
= make_tree (halfutype
, lopart
);
1745 ops
.op1
= make_tree (halfutype
, hipart
);
1746 rtx loxhi
= gen_reg_rtx (mode
);
1747 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1748 emit_move_insn (loxhi
, tem
);
1752 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1753 if (larger_sign
== 0)
1754 emit_jump (after_hipart_neg
);
1755 else if (larger_sign
!= -1)
1756 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1757 NULL_RTX
, NULL
, after_hipart_neg
,
1758 profile_probability::even ());
1760 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1761 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1762 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1764 emit_move_insn (loxhi
, tem
);
1766 emit_label (after_hipart_neg
);
1768 /* if (lopart < 0) loxhi -= larger; */
1769 if (smaller_sign
== 0)
1770 emit_jump (after_lopart_neg
);
1771 else if (smaller_sign
!= -1)
1772 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1773 NULL_RTX
, NULL
, after_lopart_neg
,
1774 profile_probability::even ());
1776 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1778 emit_move_insn (loxhi
, tem
);
1780 emit_label (after_lopart_neg
);
1783 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1784 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1785 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1787 emit_move_insn (loxhi
, tem
);
1789 /* if (loxhi >> (bitsize / 2)
1790 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1791 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1792 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1794 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1795 rtx signbitloxhi
= const0_rtx
;
1797 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1798 convert_modes (hmode
, mode
,
1800 hprec
- 1, NULL_RTX
, 0);
1802 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1803 NULL_RTX
, NULL
, do_overflow
,
1804 profile_probability::very_unlikely ());
1806 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1807 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1809 tem
= convert_modes (mode
, hmode
,
1810 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1812 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1815 emit_move_insn (res
, tem
);
1816 emit_jump (done_label
);
1818 emit_label (both_ops_large
);
1820 /* If both operands are large (not sign (!uns) or zero (uns)
1821 extended from hmode), then perform the full multiplication
1822 which will be the result of the operation.
1823 The only cases which don't overflow are for signed multiplication
1824 some cases where both hipart0 and highpart1 are 0 or -1.
1825 For unsigned multiplication when high parts are both non-zero
1826 this overflows always. */
1827 ops
.code
= MULT_EXPR
;
1828 ops
.op0
= make_tree (type
, op0
);
1829 ops
.op1
= make_tree (type
, op1
);
1830 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1831 emit_move_insn (res
, tem
);
1837 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1838 NULL_RTX
, 1, OPTAB_DIRECT
);
1839 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1840 NULL_RTX
, NULL
, do_error
,
1841 profile_probability::very_unlikely ());
1846 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1847 NULL_RTX
, 1, OPTAB_DIRECT
);
1848 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1849 NULL_RTX
, NULL
, do_error
,
1850 profile_probability::very_unlikely ());
1853 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1854 the same, overflow happened if res is non-positive, if they
1855 are different, overflow happened if res is positive. */
1856 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1857 emit_jump (hipart_different
);
1858 else if (op0_sign
== 1 || op1_sign
== 1)
1859 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1860 NULL_RTX
, NULL
, hipart_different
,
1861 profile_probability::even ());
1863 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1864 NULL_RTX
, NULL
, do_error
,
1865 profile_probability::very_unlikely ());
1866 emit_jump (done_label
);
1868 emit_label (hipart_different
);
1870 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1871 NULL_RTX
, NULL
, do_error
,
1872 profile_probability::very_unlikely ());
1873 emit_jump (done_label
);
1876 emit_label (do_overflow
);
1878 /* Overflow, do full multiplication and fallthru into do_error. */
1879 ops
.op0
= make_tree (type
, op0
);
1880 ops
.op1
= make_tree (type
, op1
);
1881 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1882 emit_move_insn (res
, tem
);
1884 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1885 && targetm
.scalar_mode_supported_p (wmode
))
1886 /* Even emitting a libcall is better than not detecting overflow
1891 gcc_assert (!is_ubsan
);
1892 ops
.code
= MULT_EXPR
;
1894 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1895 emit_jump (done_label
);
1900 emit_label (do_error
);
1903 /* Expand the ubsan builtin call. */
1905 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1909 do_pending_stack_adjust ();
1912 expand_arith_set_overflow (lhs
, target
);
1915 emit_label (done_label
);
1918 if (uns0_p
&& uns1_p
&& !unsr_p
)
1920 rtx_code_label
*all_done_label
= gen_label_rtx ();
1921 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1922 NULL
, all_done_label
, profile_probability::very_likely ());
1923 expand_arith_set_overflow (lhs
, target
);
1924 emit_label (all_done_label
);
1928 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1930 rtx_code_label
*all_done_label
= gen_label_rtx ();
1931 rtx_code_label
*set_noovf
= gen_label_rtx ();
1932 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1933 NULL
, all_done_label
, profile_probability::very_likely ());
1934 expand_arith_set_overflow (lhs
, target
);
1935 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1936 NULL
, set_noovf
, profile_probability::very_likely ());
1937 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1938 NULL
, all_done_label
, profile_probability::very_unlikely ());
1939 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1940 all_done_label
, profile_probability::very_unlikely ());
1941 emit_label (set_noovf
);
1942 write_complex_part (target
, const0_rtx
, true);
1943 emit_label (all_done_label
);
1949 expand_ubsan_result_store (target
, res
);
1951 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1955 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
1958 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
1959 tree arg0
, tree arg1
)
1961 int cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
1962 rtx_code_label
*loop_lab
= NULL
;
1963 rtx cntvar
= NULL_RTX
;
1964 tree cntv
= NULL_TREE
;
1965 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
1966 tree sz
= TYPE_SIZE (eltype
);
1967 tree data
= NULL_TREE
;
1968 tree resv
= NULL_TREE
;
1969 rtx lhsr
= NULL_RTX
;
1970 rtx resvr
= NULL_RTX
;
1975 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1976 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
1977 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
1978 optab_default
)) == unknown_optab
1979 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
1980 == CODE_FOR_nothing
))
1983 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
1986 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
1987 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
1993 do_pending_stack_adjust ();
1994 loop_lab
= gen_label_rtx ();
1995 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
1996 cntv
= make_tree (sizetype
, cntvar
);
1997 emit_move_insn (cntvar
, const0_rtx
);
1998 emit_label (loop_lab
);
2000 if (TREE_CODE (arg0
) != VECTOR_CST
)
2002 rtx arg0r
= expand_normal (arg0
);
2003 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2005 if (TREE_CODE (arg1
) != VECTOR_CST
)
2007 rtx arg1r
= expand_normal (arg1
);
2008 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2010 for (int i
= 0; i
< (cnt
> 4 ? 1 : cnt
); i
++)
2012 tree op0
, op1
, res
= NULL_TREE
;
2015 tree atype
= build_array_type_nelts (eltype
, cnt
);
2016 op0
= uniform_vector_p (arg0
);
2017 if (op0
== NULL_TREE
)
2019 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2020 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2021 NULL_TREE
, NULL_TREE
);
2023 op1
= uniform_vector_p (arg1
);
2024 if (op1
== NULL_TREE
)
2026 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2027 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2028 NULL_TREE
, NULL_TREE
);
2032 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2033 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2034 NULL_TREE
, NULL_TREE
);
2039 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2040 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2041 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2043 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2049 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2050 false, false, false, true, &data
);
2053 if (cnt
> 4 ? integer_zerop (arg0
) : integer_zerop (op0
))
2054 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2056 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2057 false, false, false, true, &data
);
2060 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2069 struct separate_ops ops
;
2070 ops
.code
= PLUS_EXPR
;
2071 ops
.type
= TREE_TYPE (cntv
);
2073 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2074 ops
.op2
= NULL_TREE
;
2076 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2079 emit_move_insn (cntvar
, ret
);
2080 do_compare_rtx_and_jump (cntvar
, GEN_INT (cnt
), NE
, false,
2081 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2082 profile_probability::very_likely ());
2084 if (lhs
&& resv
== NULL_TREE
)
2086 struct separate_ops ops
;
2088 ops
.type
= TREE_TYPE (arg0
);
2091 ops
.op2
= NULL_TREE
;
2093 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2096 emit_move_insn (lhsr
, ret
);
2099 emit_move_insn (lhsr
, resvr
);
2102 /* Expand UBSAN_CHECK_ADD call STMT. */
2105 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2107 location_t loc
= gimple_location (stmt
);
2108 tree lhs
= gimple_call_lhs (stmt
);
2109 tree arg0
= gimple_call_arg (stmt
, 0);
2110 tree arg1
= gimple_call_arg (stmt
, 1);
2111 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2112 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2114 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2115 false, false, false, true, NULL
);
2118 /* Expand UBSAN_CHECK_SUB call STMT. */
2121 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2123 location_t loc
= gimple_location (stmt
);
2124 tree lhs
= gimple_call_lhs (stmt
);
2125 tree arg0
= gimple_call_arg (stmt
, 0);
2126 tree arg1
= gimple_call_arg (stmt
, 1);
2127 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2128 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2129 else if (integer_zerop (arg0
))
2130 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2132 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2133 false, false, false, true, NULL
);
2136 /* Expand UBSAN_CHECK_MUL call STMT. */
2139 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2141 location_t loc
= gimple_location (stmt
);
2142 tree lhs
= gimple_call_lhs (stmt
);
2143 tree arg0
= gimple_call_arg (stmt
, 0);
2144 tree arg1
= gimple_call_arg (stmt
, 1);
2145 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2146 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2148 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2152 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2155 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2157 tree lhs
= gimple_call_lhs (stmt
);
2158 if (lhs
== NULL_TREE
)
2160 tree arg0
= gimple_call_arg (stmt
, 0);
2161 tree arg1
= gimple_call_arg (stmt
, 1);
2162 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2163 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2164 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2165 int unsr_p
= TYPE_UNSIGNED (type
);
2166 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2167 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2168 int precres
= TYPE_PRECISION (type
);
2169 location_t loc
= gimple_location (stmt
);
2170 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2172 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2174 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2175 prec0
= MIN (prec0
, pr
);
2176 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2177 prec1
= MIN (prec1
, pr
);
2179 /* If uns0_p && uns1_p, precop is minimum needed precision
2180 of unsigned type to hold the exact result, otherwise
2181 precop is minimum needed precision of signed type to
2182 hold the exact result. */
2184 if (code
== MULT_EXPR
)
2185 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2188 if (uns0_p
== uns1_p
)
2189 precop
= MAX (prec0
, prec1
) + 1;
2191 precop
= MAX (prec0
+ 1, prec1
) + 1;
2193 precop
= MAX (prec0
, prec1
+ 1) + 1;
2195 int orig_precres
= precres
;
2199 if ((uns0_p
&& uns1_p
)
2200 ? ((precop
+ !unsr_p
) <= precres
2201 /* u1 - u2 -> ur can overflow, no matter what precision
2203 && (code
!= MINUS_EXPR
|| !unsr_p
))
2204 : (!unsr_p
&& precop
<= precres
))
2206 /* The infinity precision result will always fit into result. */
2207 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2208 write_complex_part (target
, const0_rtx
, true);
2209 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2210 struct separate_ops ops
;
2213 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2214 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2215 ops
.op2
= NULL_TREE
;
2217 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2218 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2222 /* For operations with low precision, if target doesn't have them, start
2223 with precres widening right away, otherwise do it only if the most
2224 simple cases can't be used. */
2225 const int min_precision
= targetm
.min_arithmetic_precision ();
2226 if (orig_precres
== precres
&& precres
< min_precision
)
2228 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2229 && prec1
<= precres
)
2230 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2231 && prec0
+ uns0_p
<= precres
2232 && prec1
+ uns1_p
<= precres
))
2234 arg0
= fold_convert_loc (loc
, type
, arg0
);
2235 arg1
= fold_convert_loc (loc
, type
, arg1
);
2239 if (integer_zerop (arg0
) && !unsr_p
)
2241 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2246 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2247 unsr_p
, unsr_p
, false, NULL
);
2250 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2251 unsr_p
, unsr_p
, false, NULL
);
2258 /* For sub-word operations, retry with a wider type first. */
2259 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2261 int p
= MAX (min_precision
, precop
);
2262 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2263 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2266 p
= TYPE_PRECISION (optype
);
2270 unsr_p
= TYPE_UNSIGNED (optype
);
2276 if (prec0
<= precres
&& prec1
<= precres
)
2281 types
[0] = build_nonstandard_integer_type (precres
, 0);
2287 types
[1] = build_nonstandard_integer_type (precres
, 1);
2289 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2290 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2291 if (code
!= MULT_EXPR
)
2292 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2293 uns0_p
, uns1_p
, false, NULL
);
2295 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2296 uns0_p
, uns1_p
, false, NULL
);
2300 /* Retry with a wider type. */
2301 if (orig_precres
== precres
)
2303 int p
= MAX (prec0
, prec1
);
2304 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2305 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2308 p
= TYPE_PRECISION (optype
);
2312 unsr_p
= TYPE_UNSIGNED (optype
);
2323 /* Expand ADD_OVERFLOW STMT. */
2326 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2328 expand_arith_overflow (PLUS_EXPR
, stmt
);
2331 /* Expand SUB_OVERFLOW STMT. */
2334 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2336 expand_arith_overflow (MINUS_EXPR
, stmt
);
2339 /* Expand MUL_OVERFLOW STMT. */
2342 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2344 expand_arith_overflow (MULT_EXPR
, stmt
);
2347 /* This should get folded in tree-vectorizer.c. */
2350 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2355 /* This should get folded in tree-vectorizer.c. */
2358 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2363 /* Expand MASK_LOAD call STMT using optab OPTAB. */
2366 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2368 struct expand_operand ops
[3];
2369 tree type
, lhs
, rhs
, maskt
, ptr
;
2370 rtx mem
, target
, mask
;
2373 maskt
= gimple_call_arg (stmt
, 2);
2374 lhs
= gimple_call_lhs (stmt
);
2375 if (lhs
== NULL_TREE
)
2377 type
= TREE_TYPE (lhs
);
2378 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2379 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2380 if (TYPE_ALIGN (type
) != align
)
2381 type
= build_aligned_type (type
, align
);
2382 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2384 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2385 gcc_assert (MEM_P (mem
));
2386 mask
= expand_normal (maskt
);
2387 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2388 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2389 create_fixed_operand (&ops
[1], mem
);
2390 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2391 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2392 TYPE_MODE (TREE_TYPE (maskt
))),
2396 /* Expand MASK_STORE call STMT using optab OPTAB. */
2399 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2401 struct expand_operand ops
[3];
2402 tree type
, lhs
, rhs
, maskt
, ptr
;
2406 maskt
= gimple_call_arg (stmt
, 2);
2407 rhs
= gimple_call_arg (stmt
, 3);
2408 type
= TREE_TYPE (rhs
);
2409 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
2410 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
2411 if (TYPE_ALIGN (type
) != align
)
2412 type
= build_aligned_type (type
, align
);
2413 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
2415 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2416 gcc_assert (MEM_P (mem
));
2417 mask
= expand_normal (maskt
);
2418 reg
= expand_normal (rhs
);
2419 create_fixed_operand (&ops
[0], mem
);
2420 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2421 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2422 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
2423 TYPE_MODE (TREE_TYPE (maskt
))),
2428 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2433 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2435 /* When guessing was done, the hints should be already stripped away. */
2436 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2439 tree lhs
= gimple_call_lhs (stmt
);
2441 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2443 target
= const0_rtx
;
2444 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2445 if (lhs
&& val
!= target
)
2446 emit_move_insn (target
, val
);
2449 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2450 should never be called. */
2453 expand_VA_ARG (internal_fn
, gcall
*)
2458 /* Expand the IFN_UNIQUE function according to its first argument. */
2461 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2463 rtx pattern
= NULL_RTX
;
2464 enum ifn_unique_kind kind
2465 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2472 case IFN_UNIQUE_UNSPEC
:
2473 if (targetm
.have_unique ())
2474 pattern
= targetm
.gen_unique ();
2477 case IFN_UNIQUE_OACC_FORK
:
2478 case IFN_UNIQUE_OACC_JOIN
:
2479 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2481 tree lhs
= gimple_call_lhs (stmt
);
2482 rtx target
= const0_rtx
;
2485 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2487 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2488 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2490 if (kind
== IFN_UNIQUE_OACC_FORK
)
2491 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2493 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2501 emit_insn (pattern
);
2504 /* The size of an OpenACC compute dimension. */
2507 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2509 tree lhs
= gimple_call_lhs (stmt
);
2514 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2515 if (targetm
.have_oacc_dim_size ())
2517 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2518 VOIDmode
, EXPAND_NORMAL
);
2519 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2522 emit_move_insn (target
, GEN_INT (1));
2525 /* The position of an OpenACC execution engine along one compute axis. */
2528 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2530 tree lhs
= gimple_call_lhs (stmt
);
2535 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2536 if (targetm
.have_oacc_dim_pos ())
2538 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2539 VOIDmode
, EXPAND_NORMAL
);
2540 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2543 emit_move_insn (target
, const0_rtx
);
2546 /* This is expanded by oacc_device_lower pass. */
2549 expand_GOACC_LOOP (internal_fn
, gcall
*)
2554 /* This is expanded by oacc_device_lower pass. */
2557 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2562 /* This is expanded by oacc_device_lower pass. */
2565 expand_GOACC_TILE (internal_fn
, gcall
*)
2570 /* Set errno to EDOM. */
2573 expand_SET_EDOM (internal_fn
, gcall
*)
2576 #ifdef GEN_ERRNO_RTX
2577 rtx errno_rtx
= GEN_ERRNO_RTX
;
2579 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2581 emit_move_insn (errno_rtx
,
2582 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2588 /* Expand atomic bit test and set. */
2591 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2593 expand_ifn_atomic_bit_test_and (call
);
2596 /* Expand atomic bit test and complement. */
2599 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2601 expand_ifn_atomic_bit_test_and (call
);
2604 /* Expand atomic bit test and reset. */
2607 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2609 expand_ifn_atomic_bit_test_and (call
);
2612 /* Expand atomic bit test and set. */
2615 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2617 expand_ifn_atomic_compare_exchange (call
);
2620 /* Expand LAUNDER to assignment, lhs = arg0. */
2623 expand_LAUNDER (internal_fn
, gcall
*call
)
2625 tree lhs
= gimple_call_lhs (call
);
2630 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2633 /* Expand DIVMOD() using:
2634 a) optab handler for udivmod/sdivmod if it is available.
2635 b) If optab_handler doesn't exist, generate call to
2636 target-specific divmod libfunc. */
2639 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2641 tree lhs
= gimple_call_lhs (call_stmt
);
2642 tree arg0
= gimple_call_arg (call_stmt
, 0);
2643 tree arg1
= gimple_call_arg (call_stmt
, 1);
2645 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2646 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2647 machine_mode mode
= TYPE_MODE (type
);
2648 bool unsignedp
= TYPE_UNSIGNED (type
);
2649 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2651 rtx op0
= expand_normal (arg0
);
2652 rtx op1
= expand_normal (arg1
);
2653 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2655 rtx quotient
, remainder
, libfunc
;
2657 /* Check if optab_handler exists for divmod_optab for given mode. */
2658 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2660 quotient
= gen_reg_rtx (mode
);
2661 remainder
= gen_reg_rtx (mode
);
2662 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2665 /* Generate call to divmod libfunc if it exists. */
2666 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2667 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2668 "ient
, &remainder
);
2673 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2674 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2675 make_tree (TREE_TYPE (arg0
), quotient
),
2676 make_tree (TREE_TYPE (arg1
), remainder
)),
2677 target
, VOIDmode
, EXPAND_NORMAL
);
2680 /* Expand a call to FN using the operands in STMT. FN has a single
2681 output operand and NARGS input operands. */
2684 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2687 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2689 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2690 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2692 tree lhs
= gimple_call_lhs (stmt
);
2693 tree lhs_type
= TREE_TYPE (lhs
);
2694 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2696 /* Do not assign directly to a promoted subreg, since there is no
2697 guarantee that the instruction will leave the upper bits of the
2698 register in the state required by SUBREG_PROMOTED_SIGN. */
2700 if (GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
2703 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
2705 for (unsigned int i
= 0; i
< nargs
; ++i
)
2707 tree rhs
= gimple_call_arg (stmt
, i
);
2708 tree rhs_type
= TREE_TYPE (rhs
);
2709 rtx rhs_rtx
= expand_normal (rhs
);
2710 if (INTEGRAL_TYPE_P (rhs_type
))
2711 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2712 TYPE_MODE (rhs_type
),
2713 TYPE_UNSIGNED (rhs_type
));
2715 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2718 expand_insn (icode
, nargs
+ 1, ops
);
2719 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2721 /* If the return value has an integral type, convert the instruction
2722 result to that type. This is useful for things that return an
2723 int regardless of the size of the input. If the instruction result
2724 is smaller than required, assume that it is signed.
2726 If the return value has a nonintegral type, its mode must match
2727 the instruction result. */
2728 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2730 /* If this is a scalar in a register that is stored in a wider
2731 mode than the declared mode, compute the result into its
2732 declared mode and then convert to the wider mode. */
2733 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2734 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2735 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2736 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2738 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2739 emit_move_insn (lhs_rtx
, ops
[0].value
);
2742 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2743 convert_move (lhs_rtx
, ops
[0].value
, 0);
2748 /* Expanders for optabs that can use expand_direct_optab_fn. */
2750 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2751 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2753 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2754 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2756 /* RETURN_TYPE and ARGS are a return type and argument list that are
2757 in principle compatible with FN (which satisfies direct_internal_fn_p).
2758 Return the types that should be used to determine whether the
2759 target supports FN. */
2762 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2764 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2765 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2766 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2767 return tree_pair (type0
, type1
);
2770 /* CALL is a call whose return type and arguments are in principle
2771 compatible with FN (which satisfies direct_internal_fn_p). Return the
2772 types that should be used to determine whether the target supports FN. */
2775 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2777 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2778 tree op0
= (info
.type0
< 0
2779 ? gimple_call_lhs (call
)
2780 : gimple_call_arg (call
, info
.type0
));
2781 tree op1
= (info
.type1
< 0
2782 ? gimple_call_lhs (call
)
2783 : gimple_call_arg (call
, info
.type1
));
2784 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2787 /* Return true if OPTAB is supported for TYPES (whose modes should be
2788 the same) when the optimization type is OPT_TYPE. Used for simple
2792 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2793 optimization_type opt_type
)
2795 machine_mode mode
= TYPE_MODE (types
.first
);
2796 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2797 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2800 /* Return true if load/store lanes optab OPTAB is supported for
2801 array type TYPES.first when the optimization type is OPT_TYPE. */
2804 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2805 optimization_type opt_type
)
2807 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2808 machine_mode imode
= TYPE_MODE (types
.first
);
2809 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2810 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2811 != CODE_FOR_nothing
);
2814 #define direct_unary_optab_supported_p direct_optab_supported_p
2815 #define direct_binary_optab_supported_p direct_optab_supported_p
2816 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2817 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2818 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2819 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2821 /* Return true if FN is supported for the types in TYPES when the
2822 optimization type is OPT_TYPE. The types are those associated with
2823 the "type0" and "type1" fields of FN's direct_internal_fn_info
2827 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2828 optimization_type opt_type
)
2832 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2833 case IFN_##CODE: break;
2834 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2836 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2838 #include "internal-fn.def"
2846 /* Return true if FN is supported for type TYPE when the optimization
2847 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2848 fields of FN's direct_internal_fn_info structure are the same. */
2851 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2852 optimization_type opt_type
)
2854 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2855 gcc_checking_assert (info
.type0
== info
.type1
);
2856 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2859 /* Return true if IFN_SET_EDOM is supported. */
2862 set_edom_supported_p (void)
2871 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2873 expand_##CODE (internal_fn fn, gcall *stmt) \
2875 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2877 #include "internal-fn.def"
2879 /* Routines to expand each internal function, indexed by function number.
2880 Each routine has the prototype:
2882 expand_<NAME> (gcall *stmt)
2884 where STMT is the statement that performs the call. */
2885 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2886 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2887 #include "internal-fn.def"
2891 /* Expand STMT as though it were a call to internal function FN. */
2894 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2896 internal_fn_expanders
[fn
] (fn
, stmt
);
2899 /* Expand STMT, which is a call to internal function FN. */
2902 expand_internal_call (gcall
*stmt
)
2904 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
2908 expand_PHI (internal_fn
, gcall
*)