2 Copyright (C) 2011-2016 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "tree-ssanames.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "internal-fn.h"
37 #include "stor-layout.h"
44 /* The names of each internal function, indexed by function number. */
45 const char *const internal_fn_name_array
[] = {
46 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
47 #include "internal-fn.def"
51 /* The ECF_* flags of each internal function, indexed by function number. */
52 const int internal_fn_flags_array
[] = {
53 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
54 #include "internal-fn.def"
58 /* Fnspec of each internal function, indexed by function number. */
59 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
64 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
65 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
66 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
67 #include "internal-fn.def"
68 internal_fn_fnspec_array
[IFN_LAST
] = 0;
71 /* Create static initializers for the information returned by
72 direct_internal_fn. */
73 #define not_direct { -2, -2, false }
74 #define mask_load_direct { -1, 2, false }
75 #define load_lanes_direct { -1, -1, false }
76 #define mask_store_direct { 3, 2, false }
77 #define store_lanes_direct { 0, 0, false }
78 #define unary_direct { 0, 0, true }
79 #define binary_direct { 0, 0, true }
81 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
82 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
83 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
84 #include "internal-fn.def"
88 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
89 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
92 get_multi_vector_move (tree array_type
, convert_optab optab
)
97 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
98 imode
= TYPE_MODE (array_type
);
99 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
101 return convert_optab_handler (optab
, imode
, vmode
);
104 /* Expand LOAD_LANES call STMT using optab OPTAB. */
107 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
109 struct expand_operand ops
[2];
113 lhs
= gimple_call_lhs (stmt
);
114 rhs
= gimple_call_arg (stmt
, 0);
115 type
= TREE_TYPE (lhs
);
117 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
118 mem
= expand_normal (rhs
);
120 gcc_assert (MEM_P (mem
));
121 PUT_MODE (mem
, TYPE_MODE (type
));
123 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
124 create_fixed_operand (&ops
[1], mem
);
125 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
128 /* Expand STORE_LANES call STMT using optab OPTAB. */
131 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
133 struct expand_operand ops
[2];
137 lhs
= gimple_call_lhs (stmt
);
138 rhs
= gimple_call_arg (stmt
, 0);
139 type
= TREE_TYPE (rhs
);
141 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
142 reg
= expand_normal (rhs
);
144 gcc_assert (MEM_P (target
));
145 PUT_MODE (target
, TYPE_MODE (type
));
147 create_fixed_operand (&ops
[0], target
);
148 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
149 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
153 expand_ANNOTATE (internal_fn
, gcall
*)
158 /* This should get expanded in adjust_simduid_builtins. */
161 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
166 /* This should get expanded in adjust_simduid_builtins. */
169 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
174 /* This should get expanded in adjust_simduid_builtins. */
177 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
182 /* This should get expanded in adjust_simduid_builtins. */
185 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
190 /* This should get expanded in adjust_simduid_builtins. */
193 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
198 /* This should get expanded in the sanopt pass. */
201 expand_UBSAN_NULL (internal_fn
, gcall
*)
206 /* This should get expanded in the sanopt pass. */
209 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
214 /* This should get expanded in the sanopt pass. */
217 expand_UBSAN_VPTR (internal_fn
, gcall
*)
222 /* This should get expanded in the sanopt pass. */
225 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
230 /* This should get expanded in the sanopt pass. */
233 expand_ASAN_CHECK (internal_fn
, gcall
*)
238 /* This should get expanded in the tsan pass. */
241 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
246 /* Helper function for expand_addsub_overflow. Return 1
247 if ARG interpreted as signed in its precision is known to be always
248 positive or 2 if ARG is known to be always negative, or 3 if ARG may
249 be positive or negative. */
252 get_range_pos_neg (tree arg
)
254 if (arg
== error_mark_node
)
257 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
259 if (TREE_CODE (arg
) == INTEGER_CST
)
261 wide_int w
= wi::sext (arg
, prec
);
267 while (CONVERT_EXPR_P (arg
)
268 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
269 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
271 arg
= TREE_OPERAND (arg
, 0);
272 /* Narrower value zero extended into wider type
273 will always result in positive values. */
274 if (TYPE_UNSIGNED (TREE_TYPE (arg
))
275 && TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
277 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
282 if (TREE_CODE (arg
) != SSA_NAME
)
284 wide_int arg_min
, arg_max
;
285 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
287 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
288 if (is_gimple_assign (g
)
289 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
291 tree t
= gimple_assign_rhs1 (g
);
292 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
293 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
295 if (TYPE_UNSIGNED (TREE_TYPE (t
))
296 && TYPE_PRECISION (TREE_TYPE (t
)) < prec
)
298 prec
= TYPE_PRECISION (TREE_TYPE (t
));
307 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
309 /* For unsigned values, the "positive" range comes
310 below the "negative" range. */
311 if (!wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
313 if (wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
318 if (!wi::neg_p (wi::sext (arg_min
, prec
), SIGNED
))
320 if (wi::neg_p (wi::sext (arg_max
, prec
), SIGNED
))
326 /* Return minimum precision needed to represent all values
327 of ARG in SIGNed integral type. */
330 get_min_precision (tree arg
, signop sign
)
332 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
334 signop orig_sign
= sign
;
335 if (TREE_CODE (arg
) == INTEGER_CST
)
338 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
340 widest_int w
= wi::to_widest (arg
);
341 w
= wi::ext (w
, prec
, sign
);
342 p
= wi::min_precision (w
, sign
);
345 p
= wi::min_precision (arg
, sign
);
346 return MIN (p
, prec
);
348 while (CONVERT_EXPR_P (arg
)
349 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
350 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
352 arg
= TREE_OPERAND (arg
, 0);
353 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
355 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
357 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
358 return prec
+ (orig_sign
!= sign
);
359 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
362 return prec
+ (orig_sign
!= sign
);
364 if (TREE_CODE (arg
) != SSA_NAME
)
365 return prec
+ (orig_sign
!= sign
);
366 wide_int arg_min
, arg_max
;
367 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
369 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
370 if (is_gimple_assign (g
)
371 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
373 tree t
= gimple_assign_rhs1 (g
);
374 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
375 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
378 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
380 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
382 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
383 return prec
+ (orig_sign
!= sign
);
384 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
387 return prec
+ (orig_sign
!= sign
);
391 return prec
+ (orig_sign
!= sign
);
393 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
395 int p1
= wi::min_precision (arg_min
, sign
);
396 int p2
= wi::min_precision (arg_max
, sign
);
398 prec
= MIN (prec
, p1
);
400 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
402 int p
= wi::min_precision (arg_max
, UNSIGNED
);
403 prec
= MIN (prec
, p
);
405 return prec
+ (orig_sign
!= sign
);
408 /* Helper for expand_*_overflow. Store RES into the __real__ part
409 of TARGET. If RES has larger MODE than __real__ part of TARGET,
410 set the __imag__ part to 1 if RES doesn't fit into it. */
413 expand_arith_overflow_result_store (tree lhs
, rtx target
,
414 machine_mode mode
, rtx res
)
416 machine_mode tgtmode
= GET_MODE_INNER (GET_MODE (target
));
420 rtx_code_label
*done_label
= gen_label_rtx ();
421 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
422 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
423 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
424 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
425 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
427 write_complex_part (target
, const1_rtx
, true);
428 emit_label (done_label
);
430 write_complex_part (target
, lres
, false);
433 /* Helper for expand_*_overflow. Store RES into TARGET. */
436 expand_ubsan_result_store (rtx target
, rtx res
)
438 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
439 /* If this is a scalar in a register that is stored in a wider mode
440 than the declared mode, compute the result into its declared mode
441 and then convert to the wider mode. Our value is the computed
443 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
445 emit_move_insn (target
, res
);
448 /* Add sub/add overflow checking to the statement STMT.
449 CODE says whether the operation is +, or -. */
452 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
453 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
454 bool uns1_p
, bool is_ubsan
)
456 rtx res
, target
= NULL_RTX
;
458 rtx_code_label
*done_label
= gen_label_rtx ();
459 rtx_code_label
*do_error
= gen_label_rtx ();
460 do_pending_stack_adjust ();
461 rtx op0
= expand_normal (arg0
);
462 rtx op1
= expand_normal (arg1
);
463 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
464 int prec
= GET_MODE_PRECISION (mode
);
465 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
469 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
473 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
475 write_complex_part (target
, const0_rtx
, true);
478 /* We assume both operands and result have the same precision
479 here (GET_MODE_BITSIZE (mode)), S stands for signed type
480 with that precision, U for unsigned type with that precision,
481 sgn for unsigned most significant bit in that precision.
482 s1 is signed first operand, u1 is unsigned first operand,
483 s2 is signed second operand, u2 is unsigned second operand,
484 sr is signed result, ur is unsigned result and the following
485 rules say how to compute result (which is always result of
486 the operands as if both were unsigned, cast to the right
487 signedness) and how to compute whether operation overflowed.
490 res = (S) ((U) s1 + (U) s2)
491 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
493 res = (S) ((U) s1 - (U) s2)
494 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
497 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
500 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
502 res = (S) ((U) s1 + u2)
503 ovf = ((U) res ^ sgn) < u2
508 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
510 res = (S) ((U) s1 - u2)
511 ovf = u2 > ((U) s1 ^ sgn)
514 ovf = s1 < 0 || u2 > (U) s1
517 ovf = u1 >= ((U) s2 ^ sgn)
522 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
524 res = (U) s1 + (U) s2
525 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
528 ovf = (U) res < u2 || res < 0
531 ovf = u1 >= u2 ? res < 0 : res >= 0
533 res = (U) s1 - (U) s2
534 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
536 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
538 /* PLUS_EXPR is commutative, if operand signedness differs,
539 canonicalize to the first operand being signed and second
540 unsigned to simplify following code. */
541 std::swap (op0
, op1
);
542 std::swap (arg0
, arg1
);
548 if (uns0_p
&& uns1_p
&& unsr_p
)
550 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
551 : usubv4_optab
, mode
);
552 if (icode
!= CODE_FOR_nothing
)
554 struct expand_operand ops
[4];
555 rtx_insn
*last
= get_last_insn ();
557 res
= gen_reg_rtx (mode
);
558 create_output_operand (&ops
[0], res
, mode
);
559 create_input_operand (&ops
[1], op0
, mode
);
560 create_input_operand (&ops
[2], op1
, mode
);
561 create_fixed_operand (&ops
[3], do_error
);
562 if (maybe_expand_insn (icode
, 4, ops
))
564 last
= get_last_insn ();
565 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
567 && any_condjump_p (last
)
568 && !find_reg_note (last
, REG_BR_PROB
, 0))
569 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
570 emit_jump (done_label
);
574 delete_insns_since (last
);
577 /* Compute the operation. On RTL level, the addition is always
579 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
580 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
582 /* For PLUS_EXPR, the operation is commutative, so we can pick
583 operand to compare against. For prec <= BITS_PER_WORD, I think
584 preferring REG operand is better over CONST_INT, because
585 the CONST_INT might enlarge the instruction or CSE would need
586 to figure out we'd already loaded it into a register before.
587 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
588 as then the multi-word comparison can be perhaps simplified. */
589 if (code
== PLUS_EXPR
590 && (prec
<= BITS_PER_WORD
591 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
592 : CONST_SCALAR_INT_P (op1
)))
594 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
595 true, mode
, NULL_RTX
, NULL
, done_label
,
601 if (!uns0_p
&& uns1_p
&& !unsr_p
)
603 /* Compute the operation. On RTL level, the addition is always
605 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
606 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
607 rtx tem
= expand_binop (mode
, add_optab
,
608 code
== PLUS_EXPR
? res
: op0
, sgn
,
609 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
610 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
611 done_label
, PROB_VERY_LIKELY
);
616 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
618 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
620 /* As we've changed op1, we have to avoid using the value range
621 for the original argument. */
622 arg1
= error_mark_node
;
628 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
630 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
632 /* As we've changed op0, we have to avoid using the value range
633 for the original argument. */
634 arg0
= error_mark_node
;
640 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
642 /* Compute the operation. On RTL level, the addition is always
644 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
646 int pos_neg
= get_range_pos_neg (arg0
);
648 /* If ARG0 is known to be always negative, this is always overflow. */
649 emit_jump (do_error
);
650 else if (pos_neg
== 3)
651 /* If ARG0 is not known to be always positive, check at runtime. */
652 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
653 NULL
, do_error
, PROB_VERY_UNLIKELY
);
654 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
655 done_label
, PROB_VERY_LIKELY
);
660 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
662 /* Compute the operation. On RTL level, the addition is always
664 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
666 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
668 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
669 done_label
, PROB_VERY_LIKELY
);
674 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
676 /* Compute the operation. On RTL level, the addition is always
678 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
680 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
681 NULL
, do_error
, PROB_VERY_UNLIKELY
);
683 /* The operation is commutative, so we can pick operand to compare
684 against. For prec <= BITS_PER_WORD, I think preferring REG operand
685 is better over CONST_INT, because the CONST_INT might enlarge the
686 instruction or CSE would need to figure out we'd already loaded it
687 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
688 might be more beneficial, as then the multi-word comparison can be
689 perhaps simplified. */
690 if (prec
<= BITS_PER_WORD
691 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
692 : CONST_SCALAR_INT_P (op0
))
694 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
695 done_label
, PROB_VERY_LIKELY
);
700 if (!uns0_p
&& !uns1_p
&& unsr_p
)
702 /* Compute the operation. On RTL level, the addition is always
704 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
705 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
706 int pos_neg
= get_range_pos_neg (arg1
);
707 if (code
== PLUS_EXPR
)
709 int pos_neg0
= get_range_pos_neg (arg0
);
710 if (pos_neg0
!= 3 && pos_neg
== 3)
712 std::swap (op0
, op1
);
719 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
720 ? and_optab
: ior_optab
,
721 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
722 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
723 NULL
, done_label
, PROB_VERY_LIKELY
);
727 rtx_code_label
*do_ior_label
= gen_label_rtx ();
728 do_compare_rtx_and_jump (op1
, const0_rtx
,
729 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
730 NULL_RTX
, NULL
, do_ior_label
,
732 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
734 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
735 NULL
, done_label
, PROB_VERY_LIKELY
);
736 emit_jump (do_error
);
737 emit_label (do_ior_label
);
738 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
740 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
741 NULL
, done_label
, PROB_VERY_LIKELY
);
747 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
749 /* Compute the operation. On RTL level, the addition is always
751 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
753 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
754 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
755 op0_geu_op1
, PROB_EVEN
);
756 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
757 NULL
, done_label
, PROB_VERY_LIKELY
);
758 emit_jump (do_error
);
759 emit_label (op0_geu_op1
);
760 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
761 NULL
, done_label
, PROB_VERY_LIKELY
);
765 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
770 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
771 : subv4_optab
, mode
);
772 if (icode
!= CODE_FOR_nothing
)
774 struct expand_operand ops
[4];
775 rtx_insn
*last
= get_last_insn ();
777 res
= gen_reg_rtx (mode
);
778 create_output_operand (&ops
[0], res
, mode
);
779 create_input_operand (&ops
[1], op0
, mode
);
780 create_input_operand (&ops
[2], op1
, mode
);
781 create_fixed_operand (&ops
[3], do_error
);
782 if (maybe_expand_insn (icode
, 4, ops
))
784 last
= get_last_insn ();
785 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
787 && any_condjump_p (last
)
788 && !find_reg_note (last
, REG_BR_PROB
, 0))
789 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
790 emit_jump (done_label
);
794 delete_insns_since (last
);
797 rtx_code_label
*sub_check
= gen_label_rtx ();
800 /* Compute the operation. On RTL level, the addition is always
802 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
803 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
805 /* If we can prove one of the arguments (for MINUS_EXPR only
806 the second operand, as subtraction is not commutative) is always
807 non-negative or always negative, we can do just one comparison
808 and conditional jump instead of 2 at runtime, 3 present in the
809 emitted code. If one of the arguments is CONST_INT, all we
810 need is to make sure it is op1, then the first
811 do_compare_rtx_and_jump will be just folded. Otherwise try
812 to use range info if available. */
813 if (code
== PLUS_EXPR
&& CONST_INT_P (op0
))
814 std::swap (op0
, op1
);
815 else if (CONST_INT_P (op1
))
817 else if (code
== PLUS_EXPR
&& TREE_CODE (arg0
) == SSA_NAME
)
819 pos_neg
= get_range_pos_neg (arg0
);
821 std::swap (op0
, op1
);
823 if (pos_neg
== 3 && !CONST_INT_P (op1
) && TREE_CODE (arg1
) == SSA_NAME
)
824 pos_neg
= get_range_pos_neg (arg1
);
826 /* If the op1 is negative, we have to use a different check. */
828 do_compare_rtx_and_jump (op1
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
829 NULL
, sub_check
, PROB_EVEN
);
831 /* Compare the result of the operation with one of the operands. */
833 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? GE
: LE
,
834 false, mode
, NULL_RTX
, NULL
, done_label
,
837 /* If we get here, we have to print the error. */
840 emit_jump (do_error
);
841 emit_label (sub_check
);
844 /* We have k = a + b for b < 0 here. k <= a must hold. */
846 do_compare_rtx_and_jump (res
, op0
, code
== PLUS_EXPR
? LE
: GE
,
847 false, mode
, NULL_RTX
, NULL
, done_label
,
852 emit_label (do_error
);
855 /* Expand the ubsan builtin call. */
857 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
861 do_pending_stack_adjust ();
864 write_complex_part (target
, const1_rtx
, true);
867 emit_label (done_label
);
872 expand_ubsan_result_store (target
, res
);
876 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
879 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
884 /* Add negate overflow checking to the statement STMT. */
887 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
)
891 rtx_code_label
*done_label
, *do_error
;
892 rtx target
= NULL_RTX
;
894 done_label
= gen_label_rtx ();
895 do_error
= gen_label_rtx ();
897 do_pending_stack_adjust ();
898 op1
= expand_normal (arg1
);
900 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
903 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
905 write_complex_part (target
, const0_rtx
, true);
908 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
909 if (icode
!= CODE_FOR_nothing
)
911 struct expand_operand ops
[3];
912 rtx_insn
*last
= get_last_insn ();
914 res
= gen_reg_rtx (mode
);
915 create_output_operand (&ops
[0], res
, mode
);
916 create_input_operand (&ops
[1], op1
, mode
);
917 create_fixed_operand (&ops
[2], do_error
);
918 if (maybe_expand_insn (icode
, 3, ops
))
920 last
= get_last_insn ();
921 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
923 && any_condjump_p (last
)
924 && !find_reg_note (last
, REG_BR_PROB
, 0))
925 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
926 emit_jump (done_label
);
930 delete_insns_since (last
);
931 icode
= CODE_FOR_nothing
;
935 if (icode
== CODE_FOR_nothing
)
937 /* Compute the operation. On RTL level, the addition is always
939 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
941 /* Compare the operand with the most negative value. */
942 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
943 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
944 done_label
, PROB_VERY_LIKELY
);
947 emit_label (do_error
);
950 /* Expand the ubsan builtin call. */
952 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
956 do_pending_stack_adjust ();
959 write_complex_part (target
, const1_rtx
, true);
962 emit_label (done_label
);
967 expand_ubsan_result_store (target
, res
);
969 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
973 /* Add mul overflow checking to the statement STMT. */
976 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
977 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
)
981 rtx_code_label
*done_label
, *do_error
;
982 rtx target
= NULL_RTX
;
984 enum insn_code icode
;
986 done_label
= gen_label_rtx ();
987 do_error
= gen_label_rtx ();
989 do_pending_stack_adjust ();
990 op0
= expand_normal (arg0
);
991 op1
= expand_normal (arg1
);
993 machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
997 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
999 write_complex_part (target
, const0_rtx
, true);
1003 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1005 /* We assume both operands and result have the same precision
1006 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1007 with that precision, U for unsigned type with that precision,
1008 sgn for unsigned most significant bit in that precision.
1009 s1 is signed first operand, u1 is unsigned first operand,
1010 s2 is signed second operand, u2 is unsigned second operand,
1011 sr is signed result, ur is unsigned result and the following
1012 rules say how to compute result (which is always result of
1013 the operands as if both were unsigned, cast to the right
1014 signedness) and how to compute whether operation overflowed.
1015 main_ovf (false) stands for jump on signed multiplication
1016 overflow or the main algorithm with uns == false.
1017 main_ovf (true) stands for jump on unsigned multiplication
1018 overflow or the main algorithm with uns == true.
1021 res = (S) ((U) s1 * (U) s2)
1022 ovf = main_ovf (false)
1025 ovf = main_ovf (true)
1028 ovf = (s1 < 0 && u2) || main_ovf (true)
1031 ovf = res < 0 || main_ovf (true)
1033 res = (S) ((U) s1 * u2)
1034 ovf = (S) u2 >= 0 ? main_ovf (false)
1035 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1037 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1038 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1040 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1042 if (uns0_p
&& !uns1_p
)
1044 /* Multiplication is commutative, if operand signedness differs,
1045 canonicalize to the first operand being signed and second
1046 unsigned to simplify following code. */
1047 std::swap (op0
, op1
);
1048 std::swap (arg0
, arg1
);
1053 int pos_neg0
= get_range_pos_neg (arg0
);
1054 int pos_neg1
= get_range_pos_neg (arg1
);
1057 if (!uns0_p
&& uns1_p
&& unsr_p
)
1062 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1065 /* If s1 is negative, avoid the main code, just multiply and
1066 signal overflow if op1 is not 0. */
1067 struct separate_ops ops
;
1068 ops
.code
= MULT_EXPR
;
1069 ops
.type
= TREE_TYPE (arg1
);
1070 ops
.op0
= make_tree (ops
.type
, op0
);
1071 ops
.op1
= make_tree (ops
.type
, op1
);
1072 ops
.op2
= NULL_TREE
;
1074 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1075 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1076 NULL
, done_label
, PROB_VERY_LIKELY
);
1077 goto do_error_label
;
1079 rtx_code_label
*do_main_label
;
1080 do_main_label
= gen_label_rtx ();
1081 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1082 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1083 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1084 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1085 write_complex_part (target
, const1_rtx
, true);
1086 emit_label (do_main_label
);
1094 if (uns0_p
&& uns1_p
&& !unsr_p
)
1097 /* Rest of handling of this case after res is computed. */
1102 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1109 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1110 avoid the main code, just multiply and signal overflow
1111 unless 0 * u2 or -1 * ((U) Smin). */
1112 struct separate_ops ops
;
1113 ops
.code
= MULT_EXPR
;
1114 ops
.type
= TREE_TYPE (arg1
);
1115 ops
.op0
= make_tree (ops
.type
, op0
);
1116 ops
.op1
= make_tree (ops
.type
, op1
);
1117 ops
.op2
= NULL_TREE
;
1119 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1120 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1121 NULL
, done_label
, PROB_VERY_LIKELY
);
1122 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1123 NULL
, do_error
, PROB_VERY_UNLIKELY
);
1125 prec
= GET_MODE_PRECISION (mode
);
1127 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1128 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1129 NULL
, done_label
, PROB_VERY_LIKELY
);
1130 goto do_error_label
;
1132 /* Rest of handling of this case after res is computed. */
1140 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1143 switch (pos_neg0
| pos_neg1
)
1145 case 1: /* Both operands known to be non-negative. */
1147 case 2: /* Both operands known to be negative. */
1148 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1149 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1150 /* Avoid looking at arg0/arg1 ranges, as we've changed
1152 arg0
= error_mark_node
;
1153 arg1
= error_mark_node
;
1156 if ((pos_neg0
^ pos_neg1
) == 3)
1158 /* If one operand is known to be negative and the other
1159 non-negative, this overflows always, unless the non-negative
1160 one is 0. Just do normal multiply and set overflow
1161 unless one of the operands is 0. */
1162 struct separate_ops ops
;
1163 ops
.code
= MULT_EXPR
;
1165 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1167 ops
.op0
= make_tree (ops
.type
, op0
);
1168 ops
.op1
= make_tree (ops
.type
, op1
);
1169 ops
.op2
= NULL_TREE
;
1171 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1172 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1174 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
,
1175 NULL_RTX
, NULL
, done_label
,
1177 goto do_error_label
;
1179 /* The general case, do all the needed comparisons at runtime. */
1180 rtx_code_label
*do_main_label
, *after_negate_label
;
1182 rop0
= gen_reg_rtx (mode
);
1183 rop1
= gen_reg_rtx (mode
);
1184 emit_move_insn (rop0
, op0
);
1185 emit_move_insn (rop1
, op1
);
1188 do_main_label
= gen_label_rtx ();
1189 after_negate_label
= gen_label_rtx ();
1190 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1192 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1193 NULL
, after_negate_label
, PROB_VERY_LIKELY
);
1194 /* Both arguments negative here, negate them and continue with
1195 normal unsigned overflow checking multiplication. */
1196 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1198 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1200 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1202 arg0
= error_mark_node
;
1203 arg1
= error_mark_node
;
1204 emit_jump (do_main_label
);
1205 emit_label (after_negate_label
);
1206 tem2
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1208 do_compare_rtx_and_jump (tem2
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1209 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1210 /* One argument is negative here, the other positive. This
1211 overflows always, unless one of the arguments is 0. But
1212 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1213 is, thus we can keep do_main code oring in overflow as is. */
1214 do_compare_rtx_and_jump (tem
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1215 NULL
, do_main_label
, PROB_VERY_LIKELY
);
1216 write_complex_part (target
, const1_rtx
, true);
1217 emit_label (do_main_label
);
1225 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1226 sign
= uns
? UNSIGNED
: SIGNED
;
1227 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1228 if (icode
!= CODE_FOR_nothing
)
1230 struct expand_operand ops
[4];
1231 rtx_insn
*last
= get_last_insn ();
1233 res
= gen_reg_rtx (mode
);
1234 create_output_operand (&ops
[0], res
, mode
);
1235 create_input_operand (&ops
[1], op0
, mode
);
1236 create_input_operand (&ops
[2], op1
, mode
);
1237 create_fixed_operand (&ops
[3], do_error
);
1238 if (maybe_expand_insn (icode
, 4, ops
))
1240 last
= get_last_insn ();
1241 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1243 && any_condjump_p (last
)
1244 && !find_reg_note (last
, REG_BR_PROB
, 0))
1245 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
1246 emit_jump (done_label
);
1250 delete_insns_since (last
);
1251 icode
= CODE_FOR_nothing
;
1255 if (icode
== CODE_FOR_nothing
)
1257 struct separate_ops ops
;
1258 int prec
= GET_MODE_PRECISION (mode
);
1259 machine_mode hmode
= mode_for_size (prec
/ 2, MODE_INT
, 1);
1260 ops
.op0
= make_tree (type
, op0
);
1261 ops
.op1
= make_tree (type
, op1
);
1262 ops
.op2
= NULL_TREE
;
1264 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1265 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
1267 machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
1268 ops
.code
= WIDEN_MULT_EXPR
;
1270 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1272 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1273 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1275 hipart
= gen_lowpart (mode
, hipart
);
1276 res
= gen_lowpart (mode
, res
);
1278 /* For the unsigned multiplication, there was overflow if
1279 HIPART is non-zero. */
1280 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1281 NULL_RTX
, NULL
, done_label
,
1285 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1287 /* RES is low half of the double width result, HIPART
1288 the high half. There was overflow if
1289 HIPART is different from RES < 0 ? -1 : 0. */
1290 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1291 NULL_RTX
, NULL
, done_label
,
1295 else if (hmode
!= BLKmode
&& 2 * GET_MODE_PRECISION (hmode
) == prec
)
1297 rtx_code_label
*large_op0
= gen_label_rtx ();
1298 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1299 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1300 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1301 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1302 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1303 rtx_code_label
*do_overflow
= gen_label_rtx ();
1304 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1306 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1307 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1309 hipart0
= gen_lowpart (hmode
, hipart0
);
1310 rtx lopart0
= gen_lowpart (hmode
, op0
);
1311 rtx signbit0
= const0_rtx
;
1313 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1315 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1317 hipart1
= gen_lowpart (hmode
, hipart1
);
1318 rtx lopart1
= gen_lowpart (hmode
, op1
);
1319 rtx signbit1
= const0_rtx
;
1321 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1324 res
= gen_reg_rtx (mode
);
1326 /* True if op0 resp. op1 are known to be in the range of
1328 bool op0_small_p
= false;
1329 bool op1_small_p
= false;
1330 /* True if op0 resp. op1 are known to have all zeros or all ones
1331 in the upper half of bits, but are not known to be
1333 bool op0_medium_p
= false;
1334 bool op1_medium_p
= false;
1335 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1336 nonnegative, 1 if unknown. */
1342 else if (pos_neg0
== 2)
1346 else if (pos_neg1
== 2)
1349 unsigned int mprec0
= prec
;
1350 if (arg0
!= error_mark_node
)
1351 mprec0
= get_min_precision (arg0
, sign
);
1352 if (mprec0
<= hprec
)
1354 else if (!uns
&& mprec0
<= hprec
+ 1)
1355 op0_medium_p
= true;
1356 unsigned int mprec1
= prec
;
1357 if (arg1
!= error_mark_node
)
1358 mprec1
= get_min_precision (arg1
, sign
);
1359 if (mprec1
<= hprec
)
1361 else if (!uns
&& mprec1
<= hprec
+ 1)
1362 op1_medium_p
= true;
1364 int smaller_sign
= 1;
1365 int larger_sign
= 1;
1368 smaller_sign
= op0_sign
;
1369 larger_sign
= op1_sign
;
1371 else if (op1_small_p
)
1373 smaller_sign
= op1_sign
;
1374 larger_sign
= op0_sign
;
1376 else if (op0_sign
== op1_sign
)
1378 smaller_sign
= op0_sign
;
1379 larger_sign
= op0_sign
;
1383 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1384 NULL_RTX
, NULL
, large_op0
,
1388 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1389 NULL_RTX
, NULL
, small_op0_large_op1
,
1392 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1393 hmode to mode, the multiplication will never overflow. We can
1394 do just one hmode x hmode => mode widening multiplication. */
1395 rtx lopart0s
= lopart0
, lopart1s
= lopart1
;
1396 if (GET_CODE (lopart0
) == SUBREG
)
1398 lopart0s
= shallow_copy_rtx (lopart0
);
1399 SUBREG_PROMOTED_VAR_P (lopart0s
) = 1;
1400 SUBREG_PROMOTED_SET (lopart0s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1402 if (GET_CODE (lopart1
) == SUBREG
)
1404 lopart1s
= shallow_copy_rtx (lopart1
);
1405 SUBREG_PROMOTED_VAR_P (lopart1s
) = 1;
1406 SUBREG_PROMOTED_SET (lopart1s
, uns
? SRP_UNSIGNED
: SRP_SIGNED
);
1408 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1409 ops
.op0
= make_tree (halfstype
, lopart0s
);
1410 ops
.op1
= make_tree (halfstype
, lopart1s
);
1411 ops
.code
= WIDEN_MULT_EXPR
;
1414 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1415 emit_move_insn (res
, thisres
);
1416 emit_jump (done_label
);
1418 emit_label (small_op0_large_op1
);
1420 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1421 but op1 is not, just swap the arguments and handle it as op1
1422 sign/zero extended, op0 not. */
1423 rtx larger
= gen_reg_rtx (mode
);
1424 rtx hipart
= gen_reg_rtx (hmode
);
1425 rtx lopart
= gen_reg_rtx (hmode
);
1426 emit_move_insn (larger
, op1
);
1427 emit_move_insn (hipart
, hipart1
);
1428 emit_move_insn (lopart
, lopart0
);
1429 emit_jump (one_small_one_large
);
1431 emit_label (large_op0
);
1434 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1435 NULL_RTX
, NULL
, both_ops_large
,
1438 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1439 but op0 is not, prepare larger, hipart and lopart pseudos and
1440 handle it together with small_op0_large_op1. */
1441 emit_move_insn (larger
, op0
);
1442 emit_move_insn (hipart
, hipart0
);
1443 emit_move_insn (lopart
, lopart1
);
1445 emit_label (one_small_one_large
);
1447 /* lopart is the low part of the operand that is sign extended
1448 to mode, larger is the other operand, hipart is the
1449 high part of larger and lopart0 and lopart1 are the low parts
1451 We perform lopart0 * lopart1 and lopart * hipart widening
1453 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1454 ops
.op0
= make_tree (halfutype
, lopart0
);
1455 ops
.op1
= make_tree (halfutype
, lopart1
);
1457 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1459 ops
.op0
= make_tree (halfutype
, lopart
);
1460 ops
.op1
= make_tree (halfutype
, hipart
);
1461 rtx loxhi
= gen_reg_rtx (mode
);
1462 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1463 emit_move_insn (loxhi
, tem
);
1467 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1468 if (larger_sign
== 0)
1469 emit_jump (after_hipart_neg
);
1470 else if (larger_sign
!= -1)
1471 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1472 NULL_RTX
, NULL
, after_hipart_neg
,
1475 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1476 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1477 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1479 emit_move_insn (loxhi
, tem
);
1481 emit_label (after_hipart_neg
);
1483 /* if (lopart < 0) loxhi -= larger; */
1484 if (smaller_sign
== 0)
1485 emit_jump (after_lopart_neg
);
1486 else if (smaller_sign
!= -1)
1487 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1488 NULL_RTX
, NULL
, after_lopart_neg
,
1491 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1493 emit_move_insn (loxhi
, tem
);
1495 emit_label (after_lopart_neg
);
1498 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1499 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1500 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1502 emit_move_insn (loxhi
, tem
);
1504 /* if (loxhi >> (bitsize / 2)
1505 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1506 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1507 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1509 hipartloxhi
= gen_lowpart (hmode
, hipartloxhi
);
1510 rtx signbitloxhi
= const0_rtx
;
1512 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1513 gen_lowpart (hmode
, loxhi
),
1514 hprec
- 1, NULL_RTX
, 0);
1516 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1517 NULL_RTX
, NULL
, do_overflow
,
1518 PROB_VERY_UNLIKELY
);
1520 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1521 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1523 tem
= convert_modes (mode
, hmode
, gen_lowpart (hmode
, lo0xlo1
), 1);
1525 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1528 emit_move_insn (res
, tem
);
1529 emit_jump (done_label
);
1531 emit_label (both_ops_large
);
1533 /* If both operands are large (not sign (!uns) or zero (uns)
1534 extended from hmode), then perform the full multiplication
1535 which will be the result of the operation.
1536 The only cases which don't overflow are for signed multiplication
1537 some cases where both hipart0 and highpart1 are 0 or -1.
1538 For unsigned multiplication when high parts are both non-zero
1539 this overflows always. */
1540 ops
.code
= MULT_EXPR
;
1541 ops
.op0
= make_tree (type
, op0
);
1542 ops
.op1
= make_tree (type
, op1
);
1543 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1544 emit_move_insn (res
, tem
);
1550 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1551 NULL_RTX
, 1, OPTAB_DIRECT
);
1552 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1553 NULL_RTX
, NULL
, do_error
,
1554 PROB_VERY_UNLIKELY
);
1559 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1560 NULL_RTX
, 1, OPTAB_DIRECT
);
1561 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1562 NULL_RTX
, NULL
, do_error
,
1563 PROB_VERY_UNLIKELY
);
1566 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1567 the same, overflow happened if res is negative, if they are
1568 different, overflow happened if res is positive. */
1569 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1570 emit_jump (hipart_different
);
1571 else if (op0_sign
== 1 || op1_sign
== 1)
1572 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1573 NULL_RTX
, NULL
, hipart_different
,
1576 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
,
1577 NULL_RTX
, NULL
, do_error
,
1578 PROB_VERY_UNLIKELY
);
1579 emit_jump (done_label
);
1581 emit_label (hipart_different
);
1583 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1584 NULL_RTX
, NULL
, do_error
,
1585 PROB_VERY_UNLIKELY
);
1586 emit_jump (done_label
);
1589 emit_label (do_overflow
);
1591 /* Overflow, do full multiplication and fallthru into do_error. */
1592 ops
.op0
= make_tree (type
, op0
);
1593 ops
.op1
= make_tree (type
, op1
);
1594 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1595 emit_move_insn (res
, tem
);
1599 gcc_assert (!is_ubsan
);
1600 ops
.code
= MULT_EXPR
;
1602 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1603 emit_jump (done_label
);
1608 emit_label (do_error
);
1611 /* Expand the ubsan builtin call. */
1613 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1617 do_pending_stack_adjust ();
1620 write_complex_part (target
, const1_rtx
, true);
1623 emit_label (done_label
);
1626 if (uns0_p
&& uns1_p
&& !unsr_p
)
1628 rtx_code_label
*all_done_label
= gen_label_rtx ();
1629 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1630 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1631 write_complex_part (target
, const1_rtx
, true);
1632 emit_label (all_done_label
);
1636 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
1638 rtx_code_label
*all_done_label
= gen_label_rtx ();
1639 rtx_code_label
*set_noovf
= gen_label_rtx ();
1640 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1641 NULL
, all_done_label
, PROB_VERY_LIKELY
);
1642 write_complex_part (target
, const1_rtx
, true);
1643 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1644 NULL
, set_noovf
, PROB_VERY_LIKELY
);
1645 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1646 NULL
, all_done_label
, PROB_VERY_UNLIKELY
);
1647 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
1648 all_done_label
, PROB_VERY_UNLIKELY
);
1649 emit_label (set_noovf
);
1650 write_complex_part (target
, const0_rtx
, true);
1651 emit_label (all_done_label
);
1657 expand_ubsan_result_store (target
, res
);
1659 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1663 /* Expand UBSAN_CHECK_ADD call STMT. */
1666 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
1668 location_t loc
= gimple_location (stmt
);
1669 tree lhs
= gimple_call_lhs (stmt
);
1670 tree arg0
= gimple_call_arg (stmt
, 0);
1671 tree arg1
= gimple_call_arg (stmt
, 1);
1672 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
1673 false, false, false, true);
1676 /* Expand UBSAN_CHECK_SUB call STMT. */
1679 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
1681 location_t loc
= gimple_location (stmt
);
1682 tree lhs
= gimple_call_lhs (stmt
);
1683 tree arg0
= gimple_call_arg (stmt
, 0);
1684 tree arg1
= gimple_call_arg (stmt
, 1);
1685 if (integer_zerop (arg0
))
1686 expand_neg_overflow (loc
, lhs
, arg1
, true);
1688 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
1689 false, false, false, true);
1692 /* Expand UBSAN_CHECK_MUL call STMT. */
1695 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
1697 location_t loc
= gimple_location (stmt
);
1698 tree lhs
= gimple_call_lhs (stmt
);
1699 tree arg0
= gimple_call_arg (stmt
, 0);
1700 tree arg1
= gimple_call_arg (stmt
, 1);
1701 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true);
1704 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1707 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
1709 tree lhs
= gimple_call_lhs (stmt
);
1710 if (lhs
== NULL_TREE
)
1712 tree arg0
= gimple_call_arg (stmt
, 0);
1713 tree arg1
= gimple_call_arg (stmt
, 1);
1714 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
1715 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
1716 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
1717 int unsr_p
= TYPE_UNSIGNED (type
);
1718 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
1719 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
1720 int precres
= TYPE_PRECISION (type
);
1721 location_t loc
= gimple_location (stmt
);
1722 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
1724 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
1726 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
1727 prec0
= MIN (prec0
, pr
);
1728 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
1729 prec1
= MIN (prec1
, pr
);
1731 /* If uns0_p && uns1_p, precop is minimum needed precision
1732 of unsigned type to hold the exact result, otherwise
1733 precop is minimum needed precision of signed type to
1734 hold the exact result. */
1736 if (code
== MULT_EXPR
)
1737 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
1740 if (uns0_p
== uns1_p
)
1741 precop
= MAX (prec0
, prec1
) + 1;
1743 precop
= MAX (prec0
+ 1, prec1
) + 1;
1745 precop
= MAX (prec0
, prec1
+ 1) + 1;
1747 int orig_precres
= precres
;
1751 if ((uns0_p
&& uns1_p
)
1752 ? ((precop
+ !unsr_p
) <= precres
1753 /* u1 - u2 -> ur can overflow, no matter what precision
1755 && (code
!= MINUS_EXPR
|| !unsr_p
))
1756 : (!unsr_p
&& precop
<= precres
))
1758 /* The infinity precision result will always fit into result. */
1759 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1760 write_complex_part (target
, const0_rtx
, true);
1761 enum machine_mode mode
= TYPE_MODE (type
);
1762 struct separate_ops ops
;
1765 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
1766 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
1767 ops
.op2
= NULL_TREE
;
1769 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1770 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
1774 /* For sub-word operations, if target doesn't have them, start
1775 with precres widening right away, otherwise do it only
1776 if the most simple cases can't be used. */
1777 if (WORD_REGISTER_OPERATIONS
1778 && orig_precres
== precres
1779 && precres
< BITS_PER_WORD
)
1781 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
1782 && prec1
<= precres
)
1783 || ((!uns0_p
|| !uns1_p
) && !unsr_p
1784 && prec0
+ uns0_p
<= precres
1785 && prec1
+ uns1_p
<= precres
))
1787 arg0
= fold_convert_loc (loc
, type
, arg0
);
1788 arg1
= fold_convert_loc (loc
, type
, arg1
);
1792 if (integer_zerop (arg0
) && !unsr_p
)
1793 expand_neg_overflow (loc
, lhs
, arg1
, false);
1796 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
,
1797 unsr_p
, unsr_p
, unsr_p
, false);
1800 expand_mul_overflow (loc
, lhs
, arg0
, arg1
,
1801 unsr_p
, unsr_p
, unsr_p
, false);
1808 /* For sub-word operations, retry with a wider type first. */
1809 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
1811 int p
= WORD_REGISTER_OPERATIONS
? BITS_PER_WORD
: precop
;
1812 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1813 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1816 p
= TYPE_PRECISION (optype
);
1820 unsr_p
= TYPE_UNSIGNED (optype
);
1826 if (prec0
<= precres
&& prec1
<= precres
)
1831 types
[0] = build_nonstandard_integer_type (precres
, 0);
1837 types
[1] = build_nonstandard_integer_type (precres
, 1);
1839 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
1840 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
1841 if (code
!= MULT_EXPR
)
1842 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
1843 uns0_p
, uns1_p
, false);
1845 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
1846 uns0_p
, uns1_p
, false);
1850 /* Retry with a wider type. */
1851 if (orig_precres
== precres
)
1853 int p
= MAX (prec0
, prec1
);
1854 enum machine_mode m
= smallest_mode_for_size (p
, MODE_INT
);
1855 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
1858 p
= TYPE_PRECISION (optype
);
1862 unsr_p
= TYPE_UNSIGNED (optype
);
1873 /* Expand ADD_OVERFLOW STMT. */
1876 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
1878 expand_arith_overflow (PLUS_EXPR
, stmt
);
1881 /* Expand SUB_OVERFLOW STMT. */
1884 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
1886 expand_arith_overflow (MINUS_EXPR
, stmt
);
1889 /* Expand MUL_OVERFLOW STMT. */
1892 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
1894 expand_arith_overflow (MULT_EXPR
, stmt
);
1897 /* This should get folded in tree-vectorizer.c. */
1900 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
1905 /* Expand MASK_LOAD call STMT using optab OPTAB. */
1908 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
1910 struct expand_operand ops
[3];
1911 tree type
, lhs
, rhs
, maskt
, ptr
;
1912 rtx mem
, target
, mask
;
1915 maskt
= gimple_call_arg (stmt
, 2);
1916 lhs
= gimple_call_lhs (stmt
);
1917 if (lhs
== NULL_TREE
)
1919 type
= TREE_TYPE (lhs
);
1920 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
1921 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
1922 if (TYPE_ALIGN (type
) != align
)
1923 type
= build_aligned_type (type
, align
);
1924 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
1926 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1927 gcc_assert (MEM_P (mem
));
1928 mask
= expand_normal (maskt
);
1929 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1930 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
1931 create_fixed_operand (&ops
[1], mem
);
1932 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1933 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
1934 TYPE_MODE (TREE_TYPE (maskt
))),
1938 /* Expand MASK_STORE call STMT using optab OPTAB. */
1941 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
1943 struct expand_operand ops
[3];
1944 tree type
, lhs
, rhs
, maskt
, ptr
;
1948 maskt
= gimple_call_arg (stmt
, 2);
1949 rhs
= gimple_call_arg (stmt
, 3);
1950 type
= TREE_TYPE (rhs
);
1951 ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)), 0);
1952 align
= tree_to_shwi (gimple_call_arg (stmt
, 1));
1953 if (TYPE_ALIGN (type
) != align
)
1954 type
= build_aligned_type (type
, align
);
1955 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0), ptr
);
1957 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1958 gcc_assert (MEM_P (mem
));
1959 mask
= expand_normal (maskt
);
1960 reg
= expand_normal (rhs
);
1961 create_fixed_operand (&ops
[0], mem
);
1962 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
1963 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
1964 expand_insn (convert_optab_handler (optab
, TYPE_MODE (type
),
1965 TYPE_MODE (TREE_TYPE (maskt
))),
1970 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
1975 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
1977 /* When guessing was done, the hints should be already stripped away. */
1978 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
1981 tree lhs
= gimple_call_lhs (stmt
);
1983 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1985 target
= const0_rtx
;
1986 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
1987 if (lhs
&& val
!= target
)
1988 emit_move_insn (target
, val
);
1991 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
1992 should never be called. */
1995 expand_VA_ARG (internal_fn
, gcall
*)
2000 /* Expand the IFN_UNIQUE function according to its first argument. */
2003 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2005 rtx pattern
= NULL_RTX
;
2006 enum ifn_unique_kind kind
2007 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2014 case IFN_UNIQUE_UNSPEC
:
2015 if (targetm
.have_unique ())
2016 pattern
= targetm
.gen_unique ();
2019 case IFN_UNIQUE_OACC_FORK
:
2020 case IFN_UNIQUE_OACC_JOIN
:
2021 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2023 tree lhs
= gimple_call_lhs (stmt
);
2024 rtx target
= const0_rtx
;
2027 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2029 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2030 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2032 if (kind
== IFN_UNIQUE_OACC_FORK
)
2033 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2035 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2043 emit_insn (pattern
);
2046 /* The size of an OpenACC compute dimension. */
2049 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2051 tree lhs
= gimple_call_lhs (stmt
);
2056 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2057 if (targetm
.have_oacc_dim_size ())
2059 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2060 VOIDmode
, EXPAND_NORMAL
);
2061 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2064 emit_move_insn (target
, GEN_INT (1));
2067 /* The position of an OpenACC execution engine along one compute axis. */
2070 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2072 tree lhs
= gimple_call_lhs (stmt
);
2077 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2078 if (targetm
.have_oacc_dim_pos ())
2080 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2081 VOIDmode
, EXPAND_NORMAL
);
2082 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2085 emit_move_insn (target
, const0_rtx
);
2088 /* This is expanded by oacc_device_lower pass. */
2091 expand_GOACC_LOOP (internal_fn
, gcall
*)
2096 /* This is expanded by oacc_device_lower pass. */
2099 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2104 /* Set errno to EDOM. */
2107 expand_SET_EDOM (internal_fn
, gcall
*)
2110 #ifdef GEN_ERRNO_RTX
2111 rtx errno_rtx
= GEN_ERRNO_RTX
;
2113 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2115 emit_move_insn (errno_rtx
,
2116 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2122 /* Expand atomic bit test and set. */
2125 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2127 expand_ifn_atomic_bit_test_and (call
);
2130 /* Expand atomic bit test and complement. */
2133 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2135 expand_ifn_atomic_bit_test_and (call
);
2138 /* Expand atomic bit test and reset. */
2141 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2143 expand_ifn_atomic_bit_test_and (call
);
2146 /* Expand a call to FN using the operands in STMT. FN has a single
2147 output operand and NARGS input operands. */
2150 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2153 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2155 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2156 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2158 tree lhs
= gimple_call_lhs (stmt
);
2159 tree lhs_type
= TREE_TYPE (lhs
);
2160 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2161 create_output_operand (&ops
[0], lhs_rtx
, insn_data
[icode
].operand
[0].mode
);
2163 for (unsigned int i
= 0; i
< nargs
; ++i
)
2165 tree rhs
= gimple_call_arg (stmt
, i
);
2166 tree rhs_type
= TREE_TYPE (rhs
);
2167 rtx rhs_rtx
= expand_normal (rhs
);
2168 if (INTEGRAL_TYPE_P (rhs_type
))
2169 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2170 TYPE_MODE (rhs_type
),
2171 TYPE_UNSIGNED (rhs_type
));
2173 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2176 expand_insn (icode
, nargs
+ 1, ops
);
2177 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
2179 /* If the return value has an integral type, convert the instruction
2180 result to that type. This is useful for things that return an
2181 int regardless of the size of the input. If the instruction result
2182 is smaller than required, assume that it is signed.
2184 If the return value has a nonintegral type, its mode must match
2185 the instruction result. */
2186 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2188 /* If this is a scalar in a register that is stored in a wider
2189 mode than the declared mode, compute the result into its
2190 declared mode and then convert to the wider mode. */
2191 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2192 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2193 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2194 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2196 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2197 emit_move_insn (lhs_rtx
, ops
[0].value
);
2200 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type
));
2201 convert_move (lhs_rtx
, ops
[0].value
, 0);
2206 /* Expanders for optabs that can use expand_direct_optab_fn. */
2208 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2209 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2211 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2212 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2214 /* RETURN_TYPE and ARGS are a return type and argument list that are
2215 in principle compatible with FN (which satisfies direct_internal_fn_p).
2216 Return the types that should be used to determine whether the
2217 target supports FN. */
2220 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
2222 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2223 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
2224 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
2225 return tree_pair (type0
, type1
);
2228 /* CALL is a call whose return type and arguments are in principle
2229 compatible with FN (which satisfies direct_internal_fn_p). Return the
2230 types that should be used to determine whether the target supports FN. */
2233 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
2235 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2236 tree op0
= (info
.type0
< 0
2237 ? gimple_call_lhs (call
)
2238 : gimple_call_arg (call
, info
.type0
));
2239 tree op1
= (info
.type1
< 0
2240 ? gimple_call_lhs (call
)
2241 : gimple_call_arg (call
, info
.type1
));
2242 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
2245 /* Return true if OPTAB is supported for TYPES (whose modes should be
2246 the same) when the optimization type is OPT_TYPE. Used for simple
2250 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
2251 optimization_type opt_type
)
2253 machine_mode mode
= TYPE_MODE (types
.first
);
2254 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
2255 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
2258 /* Return true if load/store lanes optab OPTAB is supported for
2259 array type TYPES.first when the optimization type is OPT_TYPE. */
2262 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
2263 optimization_type opt_type
)
2265 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
2266 machine_mode imode
= TYPE_MODE (types
.first
);
2267 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
2268 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
2269 != CODE_FOR_nothing
);
2272 #define direct_unary_optab_supported_p direct_optab_supported_p
2273 #define direct_binary_optab_supported_p direct_optab_supported_p
2274 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2275 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2276 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2277 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2279 /* Return true if FN is supported for the types in TYPES when the
2280 optimization type is OPT_TYPE. The types are those associated with
2281 the "type0" and "type1" fields of FN's direct_internal_fn_info
2285 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
2286 optimization_type opt_type
)
2290 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2291 case IFN_##CODE: break;
2292 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2294 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2296 #include "internal-fn.def"
2304 /* Return true if FN is supported for type TYPE when the optimization
2305 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2306 fields of FN's direct_internal_fn_info structure are the same. */
2309 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
2310 optimization_type opt_type
)
2312 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
2313 gcc_checking_assert (info
.type0
== info
.type1
);
2314 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
2317 /* Return true if IFN_SET_EDOM is supported. */
2320 set_edom_supported_p (void)
2329 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2331 expand_##CODE (internal_fn fn, gcall *stmt) \
2333 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2335 #include "internal-fn.def"
2337 /* Routines to expand each internal function, indexed by function number.
2338 Each routine has the prototype:
2340 expand_<NAME> (gcall *stmt)
2342 where STMT is the statement that performs the call. */
2343 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
2344 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2345 #include "internal-fn.def"
2349 /* Expand STMT as though it were a call to internal function FN. */
2352 expand_internal_call (internal_fn fn
, gcall
*stmt
)
2354 internal_fn_expanders
[fn
] (fn
, stmt
);
2357 /* Expand STMT, which is a call to internal function FN. */
2360 expand_internal_call (gcall
*stmt
)
2362 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);