2 Copyright (C) 2011-2021 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
56 /* The names of each internal function, indexed by function number. */
57 const char *const internal_fn_name_array
[] = {
58 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
59 #include "internal-fn.def"
63 /* The ECF_* flags of each internal function, indexed by function number. */
64 const int internal_fn_flags_array
[] = {
65 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
66 #include "internal-fn.def"
70 /* Return the internal function called NAME, or IFN_LAST if there's
74 lookup_internal_fn (const char *name
)
76 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
77 static name_to_fn_map_type
*name_to_fn_map
;
81 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
82 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
83 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
86 internal_fn
*entry
= name_to_fn_map
->get (name
);
87 return entry
? *entry
: IFN_LAST
;
90 /* Fnspec of each internal function, indexed by function number. */
91 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
96 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
97 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
98 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
99 #include "internal-fn.def"
100 internal_fn_fnspec_array
[IFN_LAST
] = 0;
103 /* Create static initializers for the information returned by
104 direct_internal_fn. */
105 #define not_direct { -2, -2, false }
106 #define mask_load_direct { -1, 2, false }
107 #define load_lanes_direct { -1, -1, false }
108 #define mask_load_lanes_direct { -1, -1, false }
109 #define gather_load_direct { 3, 1, false }
110 #define len_load_direct { -1, -1, false }
111 #define mask_store_direct { 3, 2, false }
112 #define store_lanes_direct { 0, 0, false }
113 #define mask_store_lanes_direct { 0, 0, false }
114 #define vec_cond_mask_direct { 1, 0, false }
115 #define vec_cond_direct { 2, 0, false }
116 #define scatter_store_direct { 3, 1, false }
117 #define len_store_direct { 3, 3, false }
118 #define vec_set_direct { 3, 3, false }
119 #define unary_direct { 0, 0, true }
120 #define binary_direct { 0, 0, true }
121 #define ternary_direct { 0, 0, true }
122 #define cond_unary_direct { 1, 1, true }
123 #define cond_binary_direct { 1, 1, true }
124 #define cond_ternary_direct { 1, 1, true }
125 #define while_direct { 0, 2, false }
126 #define fold_extract_direct { 2, 2, false }
127 #define fold_left_direct { 1, 1, false }
128 #define mask_fold_left_direct { 1, 1, false }
129 #define check_ptrs_direct { 0, 0, false }
131 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
132 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
133 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
134 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
135 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
136 #include "internal-fn.def"
140 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
141 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
143 static enum insn_code
144 get_multi_vector_move (tree array_type
, convert_optab optab
)
149 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
150 imode
= TYPE_MODE (array_type
);
151 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
153 return convert_optab_handler (optab
, imode
, vmode
);
156 /* Expand LOAD_LANES call STMT using optab OPTAB. */
159 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
161 class expand_operand ops
[2];
165 lhs
= gimple_call_lhs (stmt
);
166 rhs
= gimple_call_arg (stmt
, 0);
167 type
= TREE_TYPE (lhs
);
169 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
170 mem
= expand_normal (rhs
);
172 gcc_assert (MEM_P (mem
));
173 PUT_MODE (mem
, TYPE_MODE (type
));
175 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
176 create_fixed_operand (&ops
[1], mem
);
177 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
178 if (!rtx_equal_p (target
, ops
[0].value
))
179 emit_move_insn (target
, ops
[0].value
);
182 /* Expand STORE_LANES call STMT using optab OPTAB. */
185 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
187 class expand_operand ops
[2];
191 lhs
= gimple_call_lhs (stmt
);
192 rhs
= gimple_call_arg (stmt
, 0);
193 type
= TREE_TYPE (rhs
);
195 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
196 reg
= expand_normal (rhs
);
198 gcc_assert (MEM_P (target
));
199 PUT_MODE (target
, TYPE_MODE (type
));
201 create_fixed_operand (&ops
[0], target
);
202 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
203 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
207 expand_ANNOTATE (internal_fn
, gcall
*)
212 /* This should get expanded in omp_device_lower pass. */
215 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
220 /* This should get expanded in omp_device_lower pass. */
223 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
228 /* Allocate per-lane storage and begin non-uniform execution region. */
231 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
234 tree lhs
= gimple_call_lhs (stmt
);
236 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
238 target
= gen_reg_rtx (Pmode
);
239 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
240 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
241 class expand_operand ops
[3];
242 create_output_operand (&ops
[0], target
, Pmode
);
243 create_input_operand (&ops
[1], size
, Pmode
);
244 create_input_operand (&ops
[2], align
, Pmode
);
245 gcc_assert (targetm
.have_omp_simt_enter ());
246 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
247 if (!rtx_equal_p (target
, ops
[0].value
))
248 emit_move_insn (target
, ops
[0].value
);
251 /* Deallocate per-lane storage and leave non-uniform execution region. */
254 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
256 gcc_checking_assert (!gimple_call_lhs (stmt
));
257 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
258 class expand_operand ops
[1];
259 create_input_operand (&ops
[0], arg
, Pmode
);
260 gcc_assert (targetm
.have_omp_simt_exit ());
261 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
264 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
265 without SIMT execution this should be expanded in omp_device_lower pass. */
268 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
270 tree lhs
= gimple_call_lhs (stmt
);
274 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
275 gcc_assert (targetm
.have_omp_simt_lane ());
276 emit_insn (targetm
.gen_omp_simt_lane (target
));
279 /* This should get expanded in omp_device_lower pass. */
282 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
287 /* Lane index of the first SIMT lane that supplies a non-zero argument.
288 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
289 lane that executed the last iteration for handling OpenMP lastprivate. */
292 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
294 tree lhs
= gimple_call_lhs (stmt
);
298 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
299 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
300 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
301 class expand_operand ops
[2];
302 create_output_operand (&ops
[0], target
, mode
);
303 create_input_operand (&ops
[1], cond
, mode
);
304 gcc_assert (targetm
.have_omp_simt_last_lane ());
305 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
306 if (!rtx_equal_p (target
, ops
[0].value
))
307 emit_move_insn (target
, ops
[0].value
);
310 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
313 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
315 tree lhs
= gimple_call_lhs (stmt
);
319 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
320 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
321 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
322 class expand_operand ops
[2];
323 create_output_operand (&ops
[0], target
, mode
);
324 create_input_operand (&ops
[1], ctr
, mode
);
325 gcc_assert (targetm
.have_omp_simt_ordered ());
326 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
327 if (!rtx_equal_p (target
, ops
[0].value
))
328 emit_move_insn (target
, ops
[0].value
);
331 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
332 any lane supplies a non-zero argument. */
335 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
337 tree lhs
= gimple_call_lhs (stmt
);
341 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
342 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
343 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
344 class expand_operand ops
[2];
345 create_output_operand (&ops
[0], target
, mode
);
346 create_input_operand (&ops
[1], cond
, mode
);
347 gcc_assert (targetm
.have_omp_simt_vote_any ());
348 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
349 if (!rtx_equal_p (target
, ops
[0].value
))
350 emit_move_insn (target
, ops
[0].value
);
353 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
354 is destination lane index XOR given offset. */
357 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
359 tree lhs
= gimple_call_lhs (stmt
);
363 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
364 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
365 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
366 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
367 class expand_operand ops
[3];
368 create_output_operand (&ops
[0], target
, mode
);
369 create_input_operand (&ops
[1], src
, mode
);
370 create_input_operand (&ops
[2], idx
, SImode
);
371 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
372 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
373 if (!rtx_equal_p (target
, ops
[0].value
))
374 emit_move_insn (target
, ops
[0].value
);
377 /* Exchange between SIMT lanes according to given source lane index. */
380 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
382 tree lhs
= gimple_call_lhs (stmt
);
386 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
387 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
388 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
389 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
390 class expand_operand ops
[3];
391 create_output_operand (&ops
[0], target
, mode
);
392 create_input_operand (&ops
[1], src
, mode
);
393 create_input_operand (&ops
[2], idx
, SImode
);
394 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
395 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
396 if (!rtx_equal_p (target
, ops
[0].value
))
397 emit_move_insn (target
, ops
[0].value
);
400 /* This should get expanded in adjust_simduid_builtins. */
403 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
408 /* This should get expanded in adjust_simduid_builtins. */
411 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
416 /* This should get expanded in adjust_simduid_builtins. */
419 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
424 /* This should get expanded in adjust_simduid_builtins. */
427 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
432 /* This should get expanded in adjust_simduid_builtins. */
435 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
440 /* This should get expanded in the sanopt pass. */
443 expand_UBSAN_NULL (internal_fn
, gcall
*)
448 /* This should get expanded in the sanopt pass. */
451 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
456 /* This should get expanded in the sanopt pass. */
459 expand_UBSAN_VPTR (internal_fn
, gcall
*)
464 /* This should get expanded in the sanopt pass. */
467 expand_UBSAN_PTR (internal_fn
, gcall
*)
472 /* This should get expanded in the sanopt pass. */
475 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
480 /* This should get expanded in the sanopt pass. */
483 expand_HWASAN_CHECK (internal_fn
, gcall
*)
488 /* For hwasan stack tagging:
489 Clear tags on the dynamically allocated space.
490 For use after an object dynamically allocated on the stack goes out of
493 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
495 gcc_assert (Pmode
== ptr_mode
);
496 tree restored_position
= gimple_call_arg (gc
, 0);
497 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
499 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
500 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
501 stack_pointer_rtx
, NULL_RTX
, 0,
503 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
504 virtual_stack_dynamic_rtx
, Pmode
,
505 HWASAN_STACK_BACKGROUND
, QImode
,
509 /* For hwasan stack tagging:
510 Return a tag to be used for a dynamic allocation. */
512 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
514 tree tag
= gimple_call_lhs (gc
);
515 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
516 machine_mode mode
= GET_MODE (target
);
517 gcc_assert (mode
== QImode
);
519 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
520 gcc_assert (base_tag
);
521 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
522 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
523 target
, /* unsignedp = */1,
525 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
527 /* Really need to put the tag into the `target` RTX. */
528 if (chosen_tag
!= target
)
530 rtx temp
= chosen_tag
;
531 gcc_assert (GET_MODE (chosen_tag
) == mode
);
532 emit_move_insn (target
, temp
);
535 hwasan_increment_frame_tag ();
538 /* For hwasan stack tagging:
539 Tag a region of space in the shadow stack according to the base pointer of
540 an object on the stack. N.b. the length provided in the internal call is
541 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
543 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
545 gcc_assert (ptr_mode
== Pmode
);
546 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
547 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
549 tree base
= gimple_call_arg (gc
, 1);
550 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
551 rtx base_rtx
= expand_normal (base
);
553 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
554 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
555 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
557 tree len
= gimple_call_arg (gc
, 2);
558 rtx r_len
= expand_normal (len
);
560 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
561 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
562 tag
, QImode
, r_len
, Pmode
);
565 /* For hwasan stack tagging:
566 Store a tag into a pointer. */
568 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
570 gcc_assert (ptr_mode
== Pmode
);
571 tree g_target
= gimple_call_lhs (gc
);
572 tree g_ptr
= gimple_call_arg (gc
, 0);
573 tree g_tag
= gimple_call_arg (gc
, 1);
575 rtx ptr
= expand_normal (g_ptr
);
576 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
577 rtx target
= expand_normal (g_target
);
579 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
580 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
581 if (tagged_value
!= target
)
582 emit_move_insn (target
, tagged_value
);
585 /* This should get expanded in the sanopt pass. */
588 expand_ASAN_CHECK (internal_fn
, gcall
*)
593 /* This should get expanded in the sanopt pass. */
596 expand_ASAN_MARK (internal_fn
, gcall
*)
601 /* This should get expanded in the sanopt pass. */
604 expand_ASAN_POISON (internal_fn
, gcall
*)
609 /* This should get expanded in the sanopt pass. */
612 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
617 /* This should get expanded in the tsan pass. */
620 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
625 /* This should get expanded in the lower pass. */
628 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
630 error_at (gimple_location (call
),
631 "invalid use of attribute %<fallthrough%>");
634 /* Return minimum precision needed to represent all values
635 of ARG in SIGNed integral type. */
638 get_min_precision (tree arg
, signop sign
)
640 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
642 signop orig_sign
= sign
;
643 if (TREE_CODE (arg
) == INTEGER_CST
)
646 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
648 widest_int w
= wi::to_widest (arg
);
649 w
= wi::ext (w
, prec
, sign
);
650 p
= wi::min_precision (w
, sign
);
653 p
= wi::min_precision (wi::to_wide (arg
), sign
);
654 return MIN (p
, prec
);
656 while (CONVERT_EXPR_P (arg
)
657 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
658 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
660 arg
= TREE_OPERAND (arg
, 0);
661 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
663 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
665 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
666 return prec
+ (orig_sign
!= sign
);
667 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
670 return prec
+ (orig_sign
!= sign
);
672 if (CONVERT_EXPR_P (arg
)
673 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
674 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
676 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
677 If y_2's min precision is smaller than prec, return that. */
678 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
680 return oprec
+ (orig_sign
!= sign
);
682 if (TREE_CODE (arg
) != SSA_NAME
)
683 return prec
+ (orig_sign
!= sign
);
685 while (!get_global_range_query ()->range_of_expr (r
, arg
)
686 || r
.kind () != VR_RANGE
)
688 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
689 if (is_gimple_assign (g
)
690 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
692 tree t
= gimple_assign_rhs1 (g
);
693 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
694 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
697 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
699 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
701 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
702 return prec
+ (orig_sign
!= sign
);
703 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
706 return prec
+ (orig_sign
!= sign
);
710 return prec
+ (orig_sign
!= sign
);
712 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
714 int p1
= wi::min_precision (r
.lower_bound (), sign
);
715 int p2
= wi::min_precision (r
.upper_bound (), sign
);
717 prec
= MIN (prec
, p1
);
719 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
721 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
722 prec
= MIN (prec
, p
);
724 return prec
+ (orig_sign
!= sign
);
727 /* Helper for expand_*_overflow. Set the __imag__ part to true
728 (1 except for signed:1 type, in which case store -1). */
731 expand_arith_set_overflow (tree lhs
, rtx target
)
733 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
734 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
735 write_complex_part (target
, constm1_rtx
, true);
737 write_complex_part (target
, const1_rtx
, true);
740 /* Helper for expand_*_overflow. Store RES into the __real__ part
741 of TARGET. If RES has larger MODE than __real__ part of TARGET,
742 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
743 if LHS has smaller precision than its mode. */
746 expand_arith_overflow_result_store (tree lhs
, rtx target
,
747 scalar_int_mode mode
, rtx res
)
749 scalar_int_mode tgtmode
750 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
754 rtx_code_label
*done_label
= gen_label_rtx ();
755 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
756 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
757 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
758 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
759 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
760 profile_probability::very_likely ());
761 expand_arith_set_overflow (lhs
, target
);
762 emit_label (done_label
);
764 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
765 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
768 rtx_code_label
*done_label
= gen_label_rtx ();
769 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
774 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
776 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
777 true, OPTAB_LIB_WIDEN
);
781 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
783 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
786 do_compare_rtx_and_jump (res
, lres
,
787 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
788 profile_probability::very_likely ());
789 expand_arith_set_overflow (lhs
, target
);
790 emit_label (done_label
);
792 write_complex_part (target
, lres
, false);
795 /* Helper for expand_*_overflow. Store RES into TARGET. */
798 expand_ubsan_result_store (rtx target
, rtx res
)
800 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
801 /* If this is a scalar in a register that is stored in a wider mode
802 than the declared mode, compute the result into its declared mode
803 and then convert to the wider mode. Our value is the computed
805 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
807 emit_move_insn (target
, res
);
810 /* Add sub/add overflow checking to the statement STMT.
811 CODE says whether the operation is +, or -. */
814 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
815 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
816 bool uns1_p
, bool is_ubsan
, tree
*datap
)
818 rtx res
, target
= NULL_RTX
;
820 rtx_code_label
*done_label
= gen_label_rtx ();
821 rtx_code_label
*do_error
= gen_label_rtx ();
822 do_pending_stack_adjust ();
823 rtx op0
= expand_normal (arg0
);
824 rtx op1
= expand_normal (arg1
);
825 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
826 int prec
= GET_MODE_PRECISION (mode
);
827 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
831 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
835 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
837 write_complex_part (target
, const0_rtx
, true);
840 /* We assume both operands and result have the same precision
841 here (GET_MODE_BITSIZE (mode)), S stands for signed type
842 with that precision, U for unsigned type with that precision,
843 sgn for unsigned most significant bit in that precision.
844 s1 is signed first operand, u1 is unsigned first operand,
845 s2 is signed second operand, u2 is unsigned second operand,
846 sr is signed result, ur is unsigned result and the following
847 rules say how to compute result (which is always result of
848 the operands as if both were unsigned, cast to the right
849 signedness) and how to compute whether operation overflowed.
852 res = (S) ((U) s1 + (U) s2)
853 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
855 res = (S) ((U) s1 - (U) s2)
856 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
859 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
862 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
864 res = (S) ((U) s1 + u2)
865 ovf = ((U) res ^ sgn) < u2
870 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
872 res = (S) ((U) s1 - u2)
873 ovf = u2 > ((U) s1 ^ sgn)
876 ovf = s1 < 0 || u2 > (U) s1
879 ovf = u1 >= ((U) s2 ^ sgn)
884 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
886 res = (U) s1 + (U) s2
887 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
890 ovf = (U) res < u2 || res < 0
893 ovf = u1 >= u2 ? res < 0 : res >= 0
895 res = (U) s1 - (U) s2
896 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
898 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
900 /* PLUS_EXPR is commutative, if operand signedness differs,
901 canonicalize to the first operand being signed and second
902 unsigned to simplify following code. */
903 std::swap (op0
, op1
);
904 std::swap (arg0
, arg1
);
910 if (uns0_p
&& uns1_p
&& unsr_p
)
912 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
913 : usubv4_optab
, mode
);
914 if (icode
!= CODE_FOR_nothing
)
916 class expand_operand ops
[4];
917 rtx_insn
*last
= get_last_insn ();
919 res
= gen_reg_rtx (mode
);
920 create_output_operand (&ops
[0], res
, mode
);
921 create_input_operand (&ops
[1], op0
, mode
);
922 create_input_operand (&ops
[2], op1
, mode
);
923 create_fixed_operand (&ops
[3], do_error
);
924 if (maybe_expand_insn (icode
, 4, ops
))
926 last
= get_last_insn ();
927 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
929 && any_condjump_p (last
)
930 && !find_reg_note (last
, REG_BR_PROB
, 0))
931 add_reg_br_prob_note (last
,
932 profile_probability::very_unlikely ());
933 emit_jump (done_label
);
937 delete_insns_since (last
);
940 /* Compute the operation. On RTL level, the addition is always
942 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
943 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
945 /* For PLUS_EXPR, the operation is commutative, so we can pick
946 operand to compare against. For prec <= BITS_PER_WORD, I think
947 preferring REG operand is better over CONST_INT, because
948 the CONST_INT might enlarge the instruction or CSE would need
949 to figure out we'd already loaded it into a register before.
950 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
951 as then the multi-word comparison can be perhaps simplified. */
952 if (code
== PLUS_EXPR
953 && (prec
<= BITS_PER_WORD
954 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
955 : CONST_SCALAR_INT_P (op1
)))
957 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
958 true, mode
, NULL_RTX
, NULL
, done_label
,
959 profile_probability::very_likely ());
964 if (!uns0_p
&& uns1_p
&& !unsr_p
)
966 /* Compute the operation. On RTL level, the addition is always
968 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
969 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
970 rtx tem
= expand_binop (mode
, add_optab
,
971 code
== PLUS_EXPR
? res
: op0
, sgn
,
972 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
973 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
974 done_label
, profile_probability::very_likely ());
979 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
981 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
983 /* As we've changed op1, we have to avoid using the value range
984 for the original argument. */
985 arg1
= error_mark_node
;
991 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
993 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
995 /* As we've changed op0, we have to avoid using the value range
996 for the original argument. */
997 arg0
= error_mark_node
;
1003 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1005 /* Compute the operation. On RTL level, the addition is always
1007 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1009 int pos_neg
= get_range_pos_neg (arg0
);
1011 /* If ARG0 is known to be always negative, this is always overflow. */
1012 emit_jump (do_error
);
1013 else if (pos_neg
== 3)
1014 /* If ARG0 is not known to be always positive, check at runtime. */
1015 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1016 NULL
, do_error
, profile_probability::very_unlikely ());
1017 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1018 done_label
, profile_probability::very_likely ());
1019 goto do_error_label
;
1023 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1025 /* Compute the operation. On RTL level, the addition is always
1027 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1029 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1031 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1032 done_label
, profile_probability::very_likely ());
1033 goto do_error_label
;
1037 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1039 /* Compute the operation. On RTL level, the addition is always
1041 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1043 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1044 NULL
, do_error
, profile_probability::very_unlikely ());
1046 /* The operation is commutative, so we can pick operand to compare
1047 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1048 is better over CONST_INT, because the CONST_INT might enlarge the
1049 instruction or CSE would need to figure out we'd already loaded it
1050 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1051 might be more beneficial, as then the multi-word comparison can be
1052 perhaps simplified. */
1053 if (prec
<= BITS_PER_WORD
1054 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1055 : CONST_SCALAR_INT_P (op0
))
1057 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1058 done_label
, profile_probability::very_likely ());
1059 goto do_error_label
;
1062 /* s1 +- s2 -> ur */
1063 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1065 /* Compute the operation. On RTL level, the addition is always
1067 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1068 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1069 int pos_neg
= get_range_pos_neg (arg1
);
1070 if (code
== PLUS_EXPR
)
1072 int pos_neg0
= get_range_pos_neg (arg0
);
1073 if (pos_neg0
!= 3 && pos_neg
== 3)
1075 std::swap (op0
, op1
);
1082 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1083 ? and_optab
: ior_optab
,
1084 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1085 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1086 NULL
, done_label
, profile_probability::very_likely ());
1090 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1091 do_compare_rtx_and_jump (op1
, const0_rtx
,
1092 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1093 NULL_RTX
, NULL
, do_ior_label
,
1094 profile_probability::even ());
1095 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1097 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1098 NULL
, done_label
, profile_probability::very_likely ());
1099 emit_jump (do_error
);
1100 emit_label (do_ior_label
);
1101 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1103 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1104 NULL
, done_label
, profile_probability::very_likely ());
1106 goto do_error_label
;
1110 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1112 /* Compute the operation. On RTL level, the addition is always
1114 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1116 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1117 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1118 op0_geu_op1
, profile_probability::even ());
1119 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1120 NULL
, done_label
, profile_probability::very_likely ());
1121 emit_jump (do_error
);
1122 emit_label (op0_geu_op1
);
1123 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1124 NULL
, done_label
, profile_probability::very_likely ());
1125 goto do_error_label
;
1128 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1130 /* s1 +- s2 -> sr */
1133 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1134 : subv4_optab
, mode
);
1135 if (icode
!= CODE_FOR_nothing
)
1137 class expand_operand ops
[4];
1138 rtx_insn
*last
= get_last_insn ();
1140 res
= gen_reg_rtx (mode
);
1141 create_output_operand (&ops
[0], res
, mode
);
1142 create_input_operand (&ops
[1], op0
, mode
);
1143 create_input_operand (&ops
[2], op1
, mode
);
1144 create_fixed_operand (&ops
[3], do_error
);
1145 if (maybe_expand_insn (icode
, 4, ops
))
1147 last
= get_last_insn ();
1148 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1150 && any_condjump_p (last
)
1151 && !find_reg_note (last
, REG_BR_PROB
, 0))
1152 add_reg_br_prob_note (last
,
1153 profile_probability::very_unlikely ());
1154 emit_jump (done_label
);
1155 goto do_error_label
;
1158 delete_insns_since (last
);
1161 /* Compute the operation. On RTL level, the addition is always
1163 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1164 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1166 /* If we can prove that one of the arguments (for MINUS_EXPR only
1167 the second operand, as subtraction is not commutative) is always
1168 non-negative or always negative, we can do just one comparison
1169 and conditional jump. */
1170 int pos_neg
= get_range_pos_neg (arg1
);
1171 if (code
== PLUS_EXPR
)
1173 int pos_neg0
= get_range_pos_neg (arg0
);
1174 if (pos_neg0
!= 3 && pos_neg
== 3)
1176 std::swap (op0
, op1
);
1181 /* Addition overflows if and only if the two operands have the same sign,
1182 and the result has the opposite sign. Subtraction overflows if and
1183 only if the two operands have opposite sign, and the subtrahend has
1184 the same sign as the result. Here 0 is counted as positive. */
1187 /* Compute op0 ^ op1 (operands have opposite sign). */
1188 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1191 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1192 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1196 if (code
== PLUS_EXPR
)
1198 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1199 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1200 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1205 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1206 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1207 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1211 /* No overflow if the result has bit sign cleared. */
1212 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1213 NULL
, done_label
, profile_probability::very_likely ());
1216 /* Compare the result of the operation with the first operand.
1217 No overflow for addition if second operand is positive and result
1218 is larger or second operand is negative and result is smaller.
1219 Likewise for subtraction with sign of second operand flipped. */
1221 do_compare_rtx_and_jump (res
, op0
,
1222 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1223 false, mode
, NULL_RTX
, NULL
, done_label
,
1224 profile_probability::very_likely ());
1228 emit_label (do_error
);
1231 /* Expand the ubsan builtin call. */
1233 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1237 do_pending_stack_adjust ();
1240 expand_arith_set_overflow (lhs
, target
);
1243 emit_label (done_label
);
1248 expand_ubsan_result_store (target
, res
);
1252 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1255 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1260 /* Add negate overflow checking to the statement STMT. */
1263 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1268 rtx_code_label
*done_label
, *do_error
;
1269 rtx target
= NULL_RTX
;
1271 done_label
= gen_label_rtx ();
1272 do_error
= gen_label_rtx ();
1274 do_pending_stack_adjust ();
1275 op1
= expand_normal (arg1
);
1277 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1280 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1282 write_complex_part (target
, const0_rtx
, true);
1285 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1286 if (icode
!= CODE_FOR_nothing
)
1288 class expand_operand ops
[3];
1289 rtx_insn
*last
= get_last_insn ();
1291 res
= gen_reg_rtx (mode
);
1292 create_output_operand (&ops
[0], res
, mode
);
1293 create_input_operand (&ops
[1], op1
, mode
);
1294 create_fixed_operand (&ops
[2], do_error
);
1295 if (maybe_expand_insn (icode
, 3, ops
))
1297 last
= get_last_insn ();
1298 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1300 && any_condjump_p (last
)
1301 && !find_reg_note (last
, REG_BR_PROB
, 0))
1302 add_reg_br_prob_note (last
,
1303 profile_probability::very_unlikely ());
1304 emit_jump (done_label
);
1308 delete_insns_since (last
);
1309 icode
= CODE_FOR_nothing
;
1313 if (icode
== CODE_FOR_nothing
)
1315 /* Compute the operation. On RTL level, the addition is always
1317 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1319 /* Compare the operand with the most negative value. */
1320 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1321 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1322 done_label
, profile_probability::very_likely ());
1325 emit_label (do_error
);
1328 /* Expand the ubsan builtin call. */
1330 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1331 arg1
, NULL_TREE
, datap
);
1334 do_pending_stack_adjust ();
1337 expand_arith_set_overflow (lhs
, target
);
1340 emit_label (done_label
);
1345 expand_ubsan_result_store (target
, res
);
1347 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1351 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1352 mode MODE can be expanded without using a libcall. */
1355 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1356 rtx op0
, rtx op1
, bool uns
)
1358 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1359 != CODE_FOR_nothing
)
1362 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1363 != CODE_FOR_nothing
)
1366 rtx_insn
*last
= get_last_insn ();
1367 if (CONSTANT_P (op0
))
1368 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1370 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1371 if (CONSTANT_P (op1
))
1372 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1374 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1375 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1376 delete_insns_since (last
);
1377 return ret
!= NULL_RTX
;
1380 /* Add mul overflow checking to the statement STMT. */
1383 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1384 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1389 rtx_code_label
*done_label
, *do_error
;
1390 rtx target
= NULL_RTX
;
1392 enum insn_code icode
;
1394 done_label
= gen_label_rtx ();
1395 do_error
= gen_label_rtx ();
1397 do_pending_stack_adjust ();
1398 op0
= expand_normal (arg0
);
1399 op1
= expand_normal (arg1
);
1401 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1405 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1407 write_complex_part (target
, const0_rtx
, true);
1411 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1413 /* We assume both operands and result have the same precision
1414 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1415 with that precision, U for unsigned type with that precision,
1416 sgn for unsigned most significant bit in that precision.
1417 s1 is signed first operand, u1 is unsigned first operand,
1418 s2 is signed second operand, u2 is unsigned second operand,
1419 sr is signed result, ur is unsigned result and the following
1420 rules say how to compute result (which is always result of
1421 the operands as if both were unsigned, cast to the right
1422 signedness) and how to compute whether operation overflowed.
1423 main_ovf (false) stands for jump on signed multiplication
1424 overflow or the main algorithm with uns == false.
1425 main_ovf (true) stands for jump on unsigned multiplication
1426 overflow or the main algorithm with uns == true.
1429 res = (S) ((U) s1 * (U) s2)
1430 ovf = main_ovf (false)
1433 ovf = main_ovf (true)
1436 ovf = (s1 < 0 && u2) || main_ovf (true)
1439 ovf = res < 0 || main_ovf (true)
1441 res = (S) ((U) s1 * u2)
1442 ovf = (S) u2 >= 0 ? main_ovf (false)
1443 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1445 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1446 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1448 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1450 if (uns0_p
&& !uns1_p
)
1452 /* Multiplication is commutative, if operand signedness differs,
1453 canonicalize to the first operand being signed and second
1454 unsigned to simplify following code. */
1455 std::swap (op0
, op1
);
1456 std::swap (arg0
, arg1
);
1461 int pos_neg0
= get_range_pos_neg (arg0
);
1462 int pos_neg1
= get_range_pos_neg (arg1
);
1465 if (!uns0_p
&& uns1_p
&& unsr_p
)
1470 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1473 /* If s1 is negative, avoid the main code, just multiply and
1474 signal overflow if op1 is not 0. */
1475 struct separate_ops ops
;
1476 ops
.code
= MULT_EXPR
;
1477 ops
.type
= TREE_TYPE (arg1
);
1478 ops
.op0
= make_tree (ops
.type
, op0
);
1479 ops
.op1
= make_tree (ops
.type
, op1
);
1480 ops
.op2
= NULL_TREE
;
1482 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1483 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1484 NULL
, done_label
, profile_probability::very_likely ());
1485 goto do_error_label
;
1487 if (get_min_precision (arg1
, UNSIGNED
)
1488 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1490 /* If the first operand is sign extended from narrower type, the
1491 second operand is zero extended from narrower type and
1492 the sum of the two precisions is smaller or equal to the
1493 result precision: if the first argument is at runtime
1494 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1495 and there will be no overflow, if the first argument is
1496 negative and the second argument zero, the result will be
1497 0 and there will be no overflow, if the first argument is
1498 negative and the second argument positive, the result when
1499 treated as signed will be negative (minimum -0x7f80 or
1500 -0x7f..f80..0) there there will be always overflow. So, do
1502 ovf = (S) res < 0 */
1503 struct separate_ops ops
;
1504 ops
.code
= MULT_EXPR
;
1506 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1508 ops
.op0
= make_tree (ops
.type
, op0
);
1509 ops
.op1
= make_tree (ops
.type
, op1
);
1510 ops
.op2
= NULL_TREE
;
1512 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1513 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1514 mode
, NULL_RTX
, NULL
, done_label
,
1515 profile_probability::very_likely ());
1516 goto do_error_label
;
1518 rtx_code_label
*do_main_label
;
1519 do_main_label
= gen_label_rtx ();
1520 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1521 NULL
, do_main_label
, profile_probability::very_likely ());
1522 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1523 NULL
, do_main_label
, profile_probability::very_likely ());
1524 expand_arith_set_overflow (lhs
, target
);
1525 emit_label (do_main_label
);
1533 if (uns0_p
&& uns1_p
&& !unsr_p
)
1535 if ((pos_neg0
| pos_neg1
) == 1)
1537 /* If both arguments are zero extended from narrower types,
1538 the MSB will be clear on both and so we can pretend it is
1539 a normal s1 * s2 -> sr multiplication. */
1545 /* Rest of handling of this case after res is computed. */
1550 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1557 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1558 avoid the main code, just multiply and signal overflow
1559 unless 0 * u2 or -1 * ((U) Smin). */
1560 struct separate_ops ops
;
1561 ops
.code
= MULT_EXPR
;
1562 ops
.type
= TREE_TYPE (arg1
);
1563 ops
.op0
= make_tree (ops
.type
, op0
);
1564 ops
.op1
= make_tree (ops
.type
, op1
);
1565 ops
.op2
= NULL_TREE
;
1567 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1568 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1569 NULL
, done_label
, profile_probability::very_likely ());
1570 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1571 NULL
, do_error
, profile_probability::very_unlikely ());
1573 prec
= GET_MODE_PRECISION (mode
);
1575 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1576 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1577 NULL
, done_label
, profile_probability::very_likely ());
1578 goto do_error_label
;
1580 /* Rest of handling of this case after res is computed. */
1588 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1591 switch (pos_neg0
| pos_neg1
)
1593 case 1: /* Both operands known to be non-negative. */
1595 case 2: /* Both operands known to be negative. */
1596 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1597 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1598 /* Avoid looking at arg0/arg1 ranges, as we've changed
1600 arg0
= error_mark_node
;
1601 arg1
= error_mark_node
;
1604 if ((pos_neg0
^ pos_neg1
) == 3)
1606 /* If one operand is known to be negative and the other
1607 non-negative, this overflows always, unless the non-negative
1608 one is 0. Just do normal multiply and set overflow
1609 unless one of the operands is 0. */
1610 struct separate_ops ops
;
1611 ops
.code
= MULT_EXPR
;
1613 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1615 ops
.op0
= make_tree (ops
.type
, op0
);
1616 ops
.op1
= make_tree (ops
.type
, op1
);
1617 ops
.op2
= NULL_TREE
;
1619 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1620 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1621 true, mode
, NULL_RTX
, NULL
, done_label
,
1622 profile_probability::very_likely ());
1623 goto do_error_label
;
1625 if (get_min_precision (arg0
, SIGNED
)
1626 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1628 /* If both operands are sign extended from narrower types and
1629 the sum of the two precisions is smaller or equal to the
1630 result precision: if both arguments are at runtime
1631 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1632 and there will be no overflow, if both arguments are negative,
1633 maximum result will be 0x40..00 and there will be no overflow
1634 either, if one argument is positive and the other argument
1635 negative, the result when treated as signed will be negative
1636 and there will be always overflow, and if one argument is
1637 zero and the other negative the result will be zero and no
1640 ovf = (S) res < 0 */
1641 struct separate_ops ops
;
1642 ops
.code
= MULT_EXPR
;
1644 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1646 ops
.op0
= make_tree (ops
.type
, op0
);
1647 ops
.op1
= make_tree (ops
.type
, op1
);
1648 ops
.op2
= NULL_TREE
;
1650 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1651 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1652 mode
, NULL_RTX
, NULL
, done_label
,
1653 profile_probability::very_likely ());
1654 goto do_error_label
;
1656 /* The general case, do all the needed comparisons at runtime. */
1657 rtx_code_label
*do_main_label
, *after_negate_label
;
1659 rop0
= gen_reg_rtx (mode
);
1660 rop1
= gen_reg_rtx (mode
);
1661 emit_move_insn (rop0
, op0
);
1662 emit_move_insn (rop1
, op1
);
1665 do_main_label
= gen_label_rtx ();
1666 after_negate_label
= gen_label_rtx ();
1667 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1669 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1670 NULL
, after_negate_label
, profile_probability::very_likely ());
1671 /* Both arguments negative here, negate them and continue with
1672 normal unsigned overflow checking multiplication. */
1673 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1675 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1677 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1679 arg0
= error_mark_node
;
1680 arg1
= error_mark_node
;
1681 emit_jump (do_main_label
);
1682 emit_label (after_negate_label
);
1683 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1685 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1686 NULL
, do_main_label
,
1687 profile_probability::very_likely ());
1688 /* One argument is negative here, the other positive. This
1689 overflows always, unless one of the arguments is 0. But
1690 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1691 is, thus we can keep do_main code oring in overflow as is. */
1693 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1694 NULL
, do_main_label
,
1695 profile_probability::very_unlikely ());
1697 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1698 NULL
, do_main_label
,
1699 profile_probability::very_unlikely ());
1700 expand_arith_set_overflow (lhs
, target
);
1701 emit_label (do_main_label
);
1709 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1710 sign
= uns
? UNSIGNED
: SIGNED
;
1711 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1713 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1714 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1716 /* Optimize unsigned multiplication by power of 2 constant
1717 using 2 shifts, one for result, one to extract the shifted
1718 out bits to see if they are all zero.
1719 Don't do this if optimizing for size and we have umulv4_optab,
1720 in that case assume multiplication will be shorter.
1721 This is heuristics based on the single target that provides
1722 umulv4 right now (i?86/x86_64), if further targets add it, this
1723 might need to be revisited.
1724 Cases where both operands are constant should be folded already
1725 during GIMPLE, and cases where one operand is constant but not
1726 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1727 below can be done without multiplication, just by shifts and adds,
1728 or we'd need to divide the result (and hope it actually doesn't
1729 really divide nor multiply) and compare the result of the division
1730 with the original operand. */
1735 if (integer_pow2p (arg0
))
1737 std::swap (opn0
, opn1
);
1738 std::swap (argn0
, argn1
);
1740 int cnt
= tree_log2 (argn1
);
1741 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1743 rtx upper
= const0_rtx
;
1744 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1746 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1747 GET_MODE_PRECISION (mode
) - cnt
,
1749 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1750 NULL_RTX
, NULL
, done_label
,
1751 profile_probability::very_likely ());
1752 goto do_error_label
;
1755 if (icode
!= CODE_FOR_nothing
)
1757 class expand_operand ops
[4];
1758 rtx_insn
*last
= get_last_insn ();
1760 res
= gen_reg_rtx (mode
);
1761 create_output_operand (&ops
[0], res
, mode
);
1762 create_input_operand (&ops
[1], op0
, mode
);
1763 create_input_operand (&ops
[2], op1
, mode
);
1764 create_fixed_operand (&ops
[3], do_error
);
1765 if (maybe_expand_insn (icode
, 4, ops
))
1767 last
= get_last_insn ();
1768 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1770 && any_condjump_p (last
)
1771 && !find_reg_note (last
, REG_BR_PROB
, 0))
1772 add_reg_br_prob_note (last
,
1773 profile_probability::very_unlikely ());
1774 emit_jump (done_label
);
1778 delete_insns_since (last
);
1779 icode
= CODE_FOR_nothing
;
1783 if (icode
== CODE_FOR_nothing
)
1785 struct separate_ops ops
;
1786 int prec
= GET_MODE_PRECISION (mode
);
1787 scalar_int_mode hmode
, wmode
;
1788 ops
.op0
= make_tree (type
, op0
);
1789 ops
.op1
= make_tree (type
, op1
);
1790 ops
.op2
= NULL_TREE
;
1793 /* Optimize unsigned overflow check where we don't use the
1794 multiplication result, just whether overflow happened.
1795 If we can do MULT_HIGHPART_EXPR, that followed by
1796 comparison of the result against zero is cheapest.
1797 We'll still compute res, but it should be DCEd later. */
1803 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1804 && can_mult_highpart_p (mode
, uns
) == 1
1805 && single_imm_use (lhs
, &use
, &use_stmt
)
1806 && is_gimple_assign (use_stmt
)
1807 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1810 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1811 && targetm
.scalar_mode_supported_p (wmode
)
1812 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1815 ops
.code
= WIDEN_MULT_EXPR
;
1817 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1819 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1820 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1822 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1823 res
= convert_modes (mode
, wmode
, res
, uns
);
1825 /* For the unsigned multiplication, there was overflow if
1826 HIPART is non-zero. */
1827 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1828 NULL_RTX
, NULL
, done_label
,
1829 profile_probability::very_likely ());
1832 /* RES is used more than once, place it in a pseudo. */
1833 res
= force_reg (mode
, res
);
1835 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1837 /* RES is low half of the double width result, HIPART
1838 the high half. There was overflow if
1839 HIPART is different from RES < 0 ? -1 : 0. */
1840 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1841 NULL_RTX
, NULL
, done_label
,
1842 profile_probability::very_likely ());
1845 else if (can_mult_highpart_p (mode
, uns
) == 1)
1848 ops
.code
= MULT_HIGHPART_EXPR
;
1851 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1853 ops
.code
= MULT_EXPR
;
1854 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1856 /* For the unsigned multiplication, there was overflow if
1857 HIPART is non-zero. */
1858 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1859 NULL_RTX
, NULL
, done_label
,
1860 profile_probability::very_likely ());
1863 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1865 /* RES is low half of the double width result, HIPART
1866 the high half. There was overflow if
1867 HIPART is different from RES < 0 ? -1 : 0. */
1868 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1869 NULL_RTX
, NULL
, done_label
,
1870 profile_probability::very_likely ());
1874 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1875 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1877 rtx_code_label
*large_op0
= gen_label_rtx ();
1878 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1879 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1880 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1881 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1882 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1883 rtx_code_label
*do_overflow
= gen_label_rtx ();
1884 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1886 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1887 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1889 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1890 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1891 rtx signbit0
= const0_rtx
;
1893 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1895 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1897 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1898 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1899 rtx signbit1
= const0_rtx
;
1901 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1904 res
= gen_reg_rtx (mode
);
1906 /* True if op0 resp. op1 are known to be in the range of
1908 bool op0_small_p
= false;
1909 bool op1_small_p
= false;
1910 /* True if op0 resp. op1 are known to have all zeros or all ones
1911 in the upper half of bits, but are not known to be
1913 bool op0_medium_p
= false;
1914 bool op1_medium_p
= false;
1915 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1916 nonnegative, 1 if unknown. */
1922 else if (pos_neg0
== 2)
1926 else if (pos_neg1
== 2)
1929 unsigned int mprec0
= prec
;
1930 if (arg0
!= error_mark_node
)
1931 mprec0
= get_min_precision (arg0
, sign
);
1932 if (mprec0
<= hprec
)
1934 else if (!uns
&& mprec0
<= hprec
+ 1)
1935 op0_medium_p
= true;
1936 unsigned int mprec1
= prec
;
1937 if (arg1
!= error_mark_node
)
1938 mprec1
= get_min_precision (arg1
, sign
);
1939 if (mprec1
<= hprec
)
1941 else if (!uns
&& mprec1
<= hprec
+ 1)
1942 op1_medium_p
= true;
1944 int smaller_sign
= 1;
1945 int larger_sign
= 1;
1948 smaller_sign
= op0_sign
;
1949 larger_sign
= op1_sign
;
1951 else if (op1_small_p
)
1953 smaller_sign
= op1_sign
;
1954 larger_sign
= op0_sign
;
1956 else if (op0_sign
== op1_sign
)
1958 smaller_sign
= op0_sign
;
1959 larger_sign
= op0_sign
;
1963 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1964 NULL_RTX
, NULL
, large_op0
,
1965 profile_probability::unlikely ());
1968 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1969 NULL_RTX
, NULL
, small_op0_large_op1
,
1970 profile_probability::unlikely ());
1972 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1973 hmode to mode, the multiplication will never overflow. We can
1974 do just one hmode x hmode => mode widening multiplication. */
1975 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1976 ops
.op0
= make_tree (halfstype
, lopart0
);
1977 ops
.op1
= make_tree (halfstype
, lopart1
);
1978 ops
.code
= WIDEN_MULT_EXPR
;
1981 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1982 emit_move_insn (res
, thisres
);
1983 emit_jump (done_label
);
1985 emit_label (small_op0_large_op1
);
1987 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1988 but op1 is not, just swap the arguments and handle it as op1
1989 sign/zero extended, op0 not. */
1990 rtx larger
= gen_reg_rtx (mode
);
1991 rtx hipart
= gen_reg_rtx (hmode
);
1992 rtx lopart
= gen_reg_rtx (hmode
);
1993 emit_move_insn (larger
, op1
);
1994 emit_move_insn (hipart
, hipart1
);
1995 emit_move_insn (lopart
, lopart0
);
1996 emit_jump (one_small_one_large
);
1998 emit_label (large_op0
);
2001 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2002 NULL_RTX
, NULL
, both_ops_large
,
2003 profile_probability::unlikely ());
2005 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2006 but op0 is not, prepare larger, hipart and lopart pseudos and
2007 handle it together with small_op0_large_op1. */
2008 emit_move_insn (larger
, op0
);
2009 emit_move_insn (hipart
, hipart0
);
2010 emit_move_insn (lopart
, lopart1
);
2012 emit_label (one_small_one_large
);
2014 /* lopart is the low part of the operand that is sign extended
2015 to mode, larger is the other operand, hipart is the
2016 high part of larger and lopart0 and lopart1 are the low parts
2018 We perform lopart0 * lopart1 and lopart * hipart widening
2020 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2021 ops
.op0
= make_tree (halfutype
, lopart0
);
2022 ops
.op1
= make_tree (halfutype
, lopart1
);
2024 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2026 ops
.op0
= make_tree (halfutype
, lopart
);
2027 ops
.op1
= make_tree (halfutype
, hipart
);
2028 rtx loxhi
= gen_reg_rtx (mode
);
2029 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2030 emit_move_insn (loxhi
, tem
);
2034 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2035 if (larger_sign
== 0)
2036 emit_jump (after_hipart_neg
);
2037 else if (larger_sign
!= -1)
2038 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2039 NULL_RTX
, NULL
, after_hipart_neg
,
2040 profile_probability::even ());
2042 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2043 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2044 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2046 emit_move_insn (loxhi
, tem
);
2048 emit_label (after_hipart_neg
);
2050 /* if (lopart < 0) loxhi -= larger; */
2051 if (smaller_sign
== 0)
2052 emit_jump (after_lopart_neg
);
2053 else if (smaller_sign
!= -1)
2054 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2055 NULL_RTX
, NULL
, after_lopart_neg
,
2056 profile_probability::even ());
2058 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2060 emit_move_insn (loxhi
, tem
);
2062 emit_label (after_lopart_neg
);
2065 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2066 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2067 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2069 emit_move_insn (loxhi
, tem
);
2071 /* if (loxhi >> (bitsize / 2)
2072 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2073 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2074 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2076 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2077 rtx signbitloxhi
= const0_rtx
;
2079 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2080 convert_modes (hmode
, mode
,
2082 hprec
- 1, NULL_RTX
, 0);
2084 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2085 NULL_RTX
, NULL
, do_overflow
,
2086 profile_probability::very_unlikely ());
2088 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2089 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2091 tem
= convert_modes (mode
, hmode
,
2092 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2094 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2097 emit_move_insn (res
, tem
);
2098 emit_jump (done_label
);
2100 emit_label (both_ops_large
);
2102 /* If both operands are large (not sign (!uns) or zero (uns)
2103 extended from hmode), then perform the full multiplication
2104 which will be the result of the operation.
2105 The only cases which don't overflow are for signed multiplication
2106 some cases where both hipart0 and highpart1 are 0 or -1.
2107 For unsigned multiplication when high parts are both non-zero
2108 this overflows always. */
2109 ops
.code
= MULT_EXPR
;
2110 ops
.op0
= make_tree (type
, op0
);
2111 ops
.op1
= make_tree (type
, op1
);
2112 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2113 emit_move_insn (res
, tem
);
2119 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2120 NULL_RTX
, 1, OPTAB_WIDEN
);
2121 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2122 NULL_RTX
, NULL
, do_error
,
2123 profile_probability::very_unlikely ());
2128 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2129 NULL_RTX
, 1, OPTAB_WIDEN
);
2130 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2131 NULL_RTX
, NULL
, do_error
,
2132 profile_probability::very_unlikely ());
2135 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2136 the same, overflow happened if res is non-positive, if they
2137 are different, overflow happened if res is positive. */
2138 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2139 emit_jump (hipart_different
);
2140 else if (op0_sign
== 1 || op1_sign
== 1)
2141 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2142 NULL_RTX
, NULL
, hipart_different
,
2143 profile_probability::even ());
2145 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2146 NULL_RTX
, NULL
, do_error
,
2147 profile_probability::very_unlikely ());
2148 emit_jump (done_label
);
2150 emit_label (hipart_different
);
2152 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2153 NULL_RTX
, NULL
, do_error
,
2154 profile_probability::very_unlikely ());
2155 emit_jump (done_label
);
2158 emit_label (do_overflow
);
2160 /* Overflow, do full multiplication and fallthru into do_error. */
2161 ops
.op0
= make_tree (type
, op0
);
2162 ops
.op1
= make_tree (type
, op1
);
2163 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2164 emit_move_insn (res
, tem
);
2166 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2167 && targetm
.scalar_mode_supported_p (wmode
))
2168 /* Even emitting a libcall is better than not detecting overflow
2173 gcc_assert (!is_ubsan
);
2174 ops
.code
= MULT_EXPR
;
2176 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2177 emit_jump (done_label
);
2182 emit_label (do_error
);
2185 /* Expand the ubsan builtin call. */
2187 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2191 do_pending_stack_adjust ();
2194 expand_arith_set_overflow (lhs
, target
);
2197 emit_label (done_label
);
2200 if (uns0_p
&& uns1_p
&& !unsr_p
)
2202 rtx_code_label
*all_done_label
= gen_label_rtx ();
2203 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2204 NULL
, all_done_label
, profile_probability::very_likely ());
2205 expand_arith_set_overflow (lhs
, target
);
2206 emit_label (all_done_label
);
2210 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2212 rtx_code_label
*all_done_label
= gen_label_rtx ();
2213 rtx_code_label
*set_noovf
= gen_label_rtx ();
2214 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2215 NULL
, all_done_label
, profile_probability::very_likely ());
2216 expand_arith_set_overflow (lhs
, target
);
2217 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2218 NULL
, set_noovf
, profile_probability::very_likely ());
2219 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2220 NULL
, all_done_label
, profile_probability::very_unlikely ());
2221 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2222 all_done_label
, profile_probability::very_unlikely ());
2223 emit_label (set_noovf
);
2224 write_complex_part (target
, const0_rtx
, true);
2225 emit_label (all_done_label
);
2231 expand_ubsan_result_store (target
, res
);
2233 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2237 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2240 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2241 tree arg0
, tree arg1
)
2243 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2244 rtx_code_label
*loop_lab
= NULL
;
2245 rtx cntvar
= NULL_RTX
;
2246 tree cntv
= NULL_TREE
;
2247 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2248 tree sz
= TYPE_SIZE (eltype
);
2249 tree data
= NULL_TREE
;
2250 tree resv
= NULL_TREE
;
2251 rtx lhsr
= NULL_RTX
;
2252 rtx resvr
= NULL_RTX
;
2253 unsigned HOST_WIDE_INT const_cnt
= 0;
2254 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2259 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2260 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2261 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2262 optab_default
)) == unknown_optab
2263 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2264 == CODE_FOR_nothing
))
2267 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2270 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2271 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2277 do_pending_stack_adjust ();
2278 loop_lab
= gen_label_rtx ();
2279 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2280 cntv
= make_tree (sizetype
, cntvar
);
2281 emit_move_insn (cntvar
, const0_rtx
);
2282 emit_label (loop_lab
);
2284 if (TREE_CODE (arg0
) != VECTOR_CST
)
2286 rtx arg0r
= expand_normal (arg0
);
2287 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2289 if (TREE_CODE (arg1
) != VECTOR_CST
)
2291 rtx arg1r
= expand_normal (arg1
);
2292 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2294 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2296 tree op0
, op1
, res
= NULL_TREE
;
2299 tree atype
= build_array_type_nelts (eltype
, cnt
);
2300 op0
= uniform_vector_p (arg0
);
2301 if (op0
== NULL_TREE
)
2303 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2304 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2305 NULL_TREE
, NULL_TREE
);
2307 op1
= uniform_vector_p (arg1
);
2308 if (op1
== NULL_TREE
)
2310 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2311 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2312 NULL_TREE
, NULL_TREE
);
2316 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2317 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2318 NULL_TREE
, NULL_TREE
);
2323 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2324 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2325 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2327 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2333 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2334 false, false, false, true, &data
);
2337 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2338 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2340 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2341 false, false, false, true, &data
);
2344 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2353 struct separate_ops ops
;
2354 ops
.code
= PLUS_EXPR
;
2355 ops
.type
= TREE_TYPE (cntv
);
2357 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2358 ops
.op2
= NULL_TREE
;
2360 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2363 emit_move_insn (cntvar
, ret
);
2364 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2365 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2366 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2367 profile_probability::very_likely ());
2369 if (lhs
&& resv
== NULL_TREE
)
2371 struct separate_ops ops
;
2373 ops
.type
= TREE_TYPE (arg0
);
2376 ops
.op2
= NULL_TREE
;
2378 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2381 emit_move_insn (lhsr
, ret
);
2384 emit_move_insn (lhsr
, resvr
);
2387 /* Expand UBSAN_CHECK_ADD call STMT. */
2390 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2392 location_t loc
= gimple_location (stmt
);
2393 tree lhs
= gimple_call_lhs (stmt
);
2394 tree arg0
= gimple_call_arg (stmt
, 0);
2395 tree arg1
= gimple_call_arg (stmt
, 1);
2396 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2397 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2399 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2400 false, false, false, true, NULL
);
2403 /* Expand UBSAN_CHECK_SUB call STMT. */
2406 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2408 location_t loc
= gimple_location (stmt
);
2409 tree lhs
= gimple_call_lhs (stmt
);
2410 tree arg0
= gimple_call_arg (stmt
, 0);
2411 tree arg1
= gimple_call_arg (stmt
, 1);
2412 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2413 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2414 else if (integer_zerop (arg0
))
2415 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2417 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2418 false, false, false, true, NULL
);
2421 /* Expand UBSAN_CHECK_MUL call STMT. */
2424 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2426 location_t loc
= gimple_location (stmt
);
2427 tree lhs
= gimple_call_lhs (stmt
);
2428 tree arg0
= gimple_call_arg (stmt
, 0);
2429 tree arg1
= gimple_call_arg (stmt
, 1);
2430 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2431 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2433 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2437 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2440 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2442 tree lhs
= gimple_call_lhs (stmt
);
2443 if (lhs
== NULL_TREE
)
2445 tree arg0
= gimple_call_arg (stmt
, 0);
2446 tree arg1
= gimple_call_arg (stmt
, 1);
2447 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2448 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2449 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2450 int unsr_p
= TYPE_UNSIGNED (type
);
2451 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2452 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2453 int precres
= TYPE_PRECISION (type
);
2454 location_t loc
= gimple_location (stmt
);
2455 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2457 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2459 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2460 prec0
= MIN (prec0
, pr
);
2461 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2462 prec1
= MIN (prec1
, pr
);
2464 /* If uns0_p && uns1_p, precop is minimum needed precision
2465 of unsigned type to hold the exact result, otherwise
2466 precop is minimum needed precision of signed type to
2467 hold the exact result. */
2469 if (code
== MULT_EXPR
)
2470 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2473 if (uns0_p
== uns1_p
)
2474 precop
= MAX (prec0
, prec1
) + 1;
2476 precop
= MAX (prec0
+ 1, prec1
) + 1;
2478 precop
= MAX (prec0
, prec1
+ 1) + 1;
2480 int orig_precres
= precres
;
2484 if ((uns0_p
&& uns1_p
)
2485 ? ((precop
+ !unsr_p
) <= precres
2486 /* u1 - u2 -> ur can overflow, no matter what precision
2488 && (code
!= MINUS_EXPR
|| !unsr_p
))
2489 : (!unsr_p
&& precop
<= precres
))
2491 /* The infinity precision result will always fit into result. */
2492 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2493 write_complex_part (target
, const0_rtx
, true);
2494 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2495 struct separate_ops ops
;
2498 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2499 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2500 ops
.op2
= NULL_TREE
;
2502 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2503 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2507 /* For operations with low precision, if target doesn't have them, start
2508 with precres widening right away, otherwise do it only if the most
2509 simple cases can't be used. */
2510 const int min_precision
= targetm
.min_arithmetic_precision ();
2511 if (orig_precres
== precres
&& precres
< min_precision
)
2513 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2514 && prec1
<= precres
)
2515 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2516 && prec0
+ uns0_p
<= precres
2517 && prec1
+ uns1_p
<= precres
))
2519 arg0
= fold_convert_loc (loc
, type
, arg0
);
2520 arg1
= fold_convert_loc (loc
, type
, arg1
);
2524 if (integer_zerop (arg0
) && !unsr_p
)
2526 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2531 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2532 unsr_p
, unsr_p
, false, NULL
);
2535 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2536 unsr_p
, unsr_p
, false, NULL
);
2543 /* For sub-word operations, retry with a wider type first. */
2544 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2546 int p
= MAX (min_precision
, precop
);
2547 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2548 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2551 p
= TYPE_PRECISION (optype
);
2555 unsr_p
= TYPE_UNSIGNED (optype
);
2561 if (prec0
<= precres
&& prec1
<= precres
)
2566 types
[0] = build_nonstandard_integer_type (precres
, 0);
2572 types
[1] = build_nonstandard_integer_type (precres
, 1);
2574 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2575 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2576 if (code
!= MULT_EXPR
)
2577 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2578 uns0_p
, uns1_p
, false, NULL
);
2580 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2581 uns0_p
, uns1_p
, false, NULL
);
2585 /* Retry with a wider type. */
2586 if (orig_precres
== precres
)
2588 int p
= MAX (prec0
, prec1
);
2589 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2590 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2593 p
= TYPE_PRECISION (optype
);
2597 unsr_p
= TYPE_UNSIGNED (optype
);
2608 /* Expand ADD_OVERFLOW STMT. */
2611 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2613 expand_arith_overflow (PLUS_EXPR
, stmt
);
2616 /* Expand SUB_OVERFLOW STMT. */
2619 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2621 expand_arith_overflow (MINUS_EXPR
, stmt
);
2624 /* Expand MUL_OVERFLOW STMT. */
2627 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2629 expand_arith_overflow (MULT_EXPR
, stmt
);
2632 /* This should get folded in tree-vectorizer.c. */
2635 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2640 /* This should get folded in tree-vectorizer.c. */
2643 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2648 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2649 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2652 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2654 tree addr
= gimple_call_arg (stmt
, index
);
2655 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2656 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2657 if (TYPE_ALIGN (type
) != align
)
2658 type
= build_aligned_type (type
, align
);
2661 if (TREE_CODE (tmp
) == SSA_NAME
)
2663 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2664 if (gimple_assign_single_p (def
))
2665 tmp
= gimple_assign_rhs1 (def
);
2668 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2670 tree mem
= TREE_OPERAND (tmp
, 0);
2671 if (TREE_CODE (mem
) == TARGET_MEM_REF
2672 && types_compatible_p (TREE_TYPE (mem
), type
))
2674 tree offset
= TMR_OFFSET (mem
);
2675 if (type
!= TREE_TYPE (mem
)
2676 || alias_ptr_type
!= TREE_TYPE (offset
)
2677 || !integer_zerop (offset
))
2679 mem
= copy_node (mem
);
2680 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2681 wi::to_poly_wide (offset
));
2682 TREE_TYPE (mem
) = type
;
2688 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2691 /* Expand MASK_LOAD{,_LANES} or LEN_LOAD call STMT using optab OPTAB. */
2694 expand_partial_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2696 class expand_operand ops
[3];
2697 tree type
, lhs
, rhs
, maskt
;
2698 rtx mem
, target
, mask
;
2701 maskt
= gimple_call_arg (stmt
, 2);
2702 lhs
= gimple_call_lhs (stmt
);
2703 if (lhs
== NULL_TREE
)
2705 type
= TREE_TYPE (lhs
);
2706 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2708 if (optab
== vec_mask_load_lanes_optab
)
2709 icode
= get_multi_vector_move (type
, optab
);
2710 else if (optab
== len_load_optab
)
2711 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2713 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2714 TYPE_MODE (TREE_TYPE (maskt
)));
2716 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2717 gcc_assert (MEM_P (mem
));
2718 mask
= expand_normal (maskt
);
2719 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2720 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2721 create_fixed_operand (&ops
[1], mem
);
2722 if (optab
== len_load_optab
)
2723 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2724 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2726 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2727 expand_insn (icode
, 3, ops
);
2728 if (!rtx_equal_p (target
, ops
[0].value
))
2729 emit_move_insn (target
, ops
[0].value
);
2732 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2733 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2734 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2736 /* Expand MASK_STORE{,_LANES} or LEN_STORE call STMT using optab OPTAB. */
2739 expand_partial_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2741 class expand_operand ops
[3];
2742 tree type
, lhs
, rhs
, maskt
;
2746 maskt
= gimple_call_arg (stmt
, 2);
2747 rhs
= gimple_call_arg (stmt
, 3);
2748 type
= TREE_TYPE (rhs
);
2749 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2751 if (optab
== vec_mask_store_lanes_optab
)
2752 icode
= get_multi_vector_move (type
, optab
);
2753 else if (optab
== len_store_optab
)
2754 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
2756 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2757 TYPE_MODE (TREE_TYPE (maskt
)));
2759 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2760 gcc_assert (MEM_P (mem
));
2761 mask
= expand_normal (maskt
);
2762 reg
= expand_normal (rhs
);
2763 create_fixed_operand (&ops
[0], mem
);
2764 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2765 if (optab
== len_store_optab
)
2766 create_convert_operand_from (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)),
2767 TYPE_UNSIGNED (TREE_TYPE (maskt
)));
2769 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2770 expand_insn (icode
, 3, ops
);
2773 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
2774 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2775 #define expand_len_store_optab_fn expand_partial_store_optab_fn
2777 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
2778 The expansion of STMT happens based on OPTAB table associated. */
2781 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2783 class expand_operand ops
[6];
2785 tree lhs
= gimple_call_lhs (stmt
);
2786 tree op0a
= gimple_call_arg (stmt
, 0);
2787 tree op0b
= gimple_call_arg (stmt
, 1);
2788 tree op1
= gimple_call_arg (stmt
, 2);
2789 tree op2
= gimple_call_arg (stmt
, 3);
2790 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
2792 tree vec_cond_type
= TREE_TYPE (lhs
);
2793 tree op_mode
= TREE_TYPE (op0a
);
2794 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
2796 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2797 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
2799 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
2801 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
2802 rtx rtx_op1
= expand_normal (op1
);
2803 rtx rtx_op2
= expand_normal (op2
);
2805 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2806 create_output_operand (&ops
[0], target
, mode
);
2807 create_input_operand (&ops
[1], rtx_op1
, mode
);
2808 create_input_operand (&ops
[2], rtx_op2
, mode
);
2809 create_fixed_operand (&ops
[3], comparison
);
2810 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
2811 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
2812 expand_insn (icode
, 6, ops
);
2813 if (!rtx_equal_p (ops
[0].value
, target
))
2814 emit_move_insn (target
, ops
[0].value
);
2817 /* Expand VCOND_MASK optab internal function.
2818 The expansion of STMT happens based on OPTAB table associated. */
2821 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2823 class expand_operand ops
[4];
2825 tree lhs
= gimple_call_lhs (stmt
);
2826 tree op0
= gimple_call_arg (stmt
, 0);
2827 tree op1
= gimple_call_arg (stmt
, 1);
2828 tree op2
= gimple_call_arg (stmt
, 2);
2829 tree vec_cond_type
= TREE_TYPE (lhs
);
2831 machine_mode mode
= TYPE_MODE (vec_cond_type
);
2832 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
2833 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
2834 rtx mask
, rtx_op1
, rtx_op2
;
2836 gcc_assert (icode
!= CODE_FOR_nothing
);
2838 mask
= expand_normal (op0
);
2839 rtx_op1
= expand_normal (op1
);
2840 rtx_op2
= expand_normal (op2
);
2842 mask
= force_reg (mask_mode
, mask
);
2843 rtx_op1
= force_reg (mode
, rtx_op1
);
2845 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2846 create_output_operand (&ops
[0], target
, mode
);
2847 create_input_operand (&ops
[1], rtx_op1
, mode
);
2848 create_input_operand (&ops
[2], rtx_op2
, mode
);
2849 create_input_operand (&ops
[3], mask
, mask_mode
);
2850 expand_insn (icode
, 4, ops
);
2851 if (!rtx_equal_p (ops
[0].value
, target
))
2852 emit_move_insn (target
, ops
[0].value
);
2855 /* Expand VEC_SET internal functions. */
2858 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2860 tree lhs
= gimple_call_lhs (stmt
);
2861 tree op0
= gimple_call_arg (stmt
, 0);
2862 tree op1
= gimple_call_arg (stmt
, 1);
2863 tree op2
= gimple_call_arg (stmt
, 2);
2864 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2865 rtx src
= expand_normal (op0
);
2867 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
2868 scalar_mode innermode
= GET_MODE_INNER (outermode
);
2870 rtx value
= expand_normal (op1
);
2871 rtx pos
= expand_normal (op2
);
2873 class expand_operand ops
[3];
2874 enum insn_code icode
= optab_handler (optab
, outermode
);
2876 if (icode
!= CODE_FOR_nothing
)
2878 rtx temp
= gen_reg_rtx (outermode
);
2879 emit_move_insn (temp
, src
);
2881 create_fixed_operand (&ops
[0], temp
);
2882 create_input_operand (&ops
[1], value
, innermode
);
2883 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
2885 if (maybe_expand_insn (icode
, 3, ops
))
2887 emit_move_insn (target
, temp
);
2895 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2900 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2902 /* When guessing was done, the hints should be already stripped away. */
2903 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2906 tree lhs
= gimple_call_lhs (stmt
);
2908 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2910 target
= const0_rtx
;
2911 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2912 if (lhs
&& val
!= target
)
2913 emit_move_insn (target
, val
);
2916 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2917 should never be called. */
2920 expand_VA_ARG (internal_fn
, gcall
*)
2925 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2926 dummy function should never be called. */
2929 expand_VEC_CONVERT (internal_fn
, gcall
*)
2934 /* Expand the IFN_UNIQUE function according to its first argument. */
2937 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2939 rtx pattern
= NULL_RTX
;
2940 enum ifn_unique_kind kind
2941 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2948 case IFN_UNIQUE_UNSPEC
:
2949 if (targetm
.have_unique ())
2950 pattern
= targetm
.gen_unique ();
2953 case IFN_UNIQUE_OACC_FORK
:
2954 case IFN_UNIQUE_OACC_JOIN
:
2955 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2957 tree lhs
= gimple_call_lhs (stmt
);
2958 rtx target
= const0_rtx
;
2961 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2963 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2964 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2966 if (kind
== IFN_UNIQUE_OACC_FORK
)
2967 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2969 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2977 emit_insn (pattern
);
2980 /* The size of an OpenACC compute dimension. */
2983 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2985 tree lhs
= gimple_call_lhs (stmt
);
2990 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2991 if (targetm
.have_oacc_dim_size ())
2993 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2994 VOIDmode
, EXPAND_NORMAL
);
2995 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2998 emit_move_insn (target
, GEN_INT (1));
3001 /* The position of an OpenACC execution engine along one compute axis. */
3004 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3006 tree lhs
= gimple_call_lhs (stmt
);
3011 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3012 if (targetm
.have_oacc_dim_pos ())
3014 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3015 VOIDmode
, EXPAND_NORMAL
);
3016 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3019 emit_move_insn (target
, const0_rtx
);
3022 /* This is expanded by oacc_device_lower pass. */
3025 expand_GOACC_LOOP (internal_fn
, gcall
*)
3030 /* This is expanded by oacc_device_lower pass. */
3033 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3038 /* This is expanded by oacc_device_lower pass. */
3041 expand_GOACC_TILE (internal_fn
, gcall
*)
3046 /* Set errno to EDOM. */
3049 expand_SET_EDOM (internal_fn
, gcall
*)
3052 #ifdef GEN_ERRNO_RTX
3053 rtx errno_rtx
= GEN_ERRNO_RTX
;
3055 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3057 emit_move_insn (errno_rtx
,
3058 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3064 /* Expand atomic bit test and set. */
3067 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3069 expand_ifn_atomic_bit_test_and (call
);
3072 /* Expand atomic bit test and complement. */
3075 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3077 expand_ifn_atomic_bit_test_and (call
);
3080 /* Expand atomic bit test and reset. */
3083 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3085 expand_ifn_atomic_bit_test_and (call
);
3088 /* Expand atomic bit test and set. */
3091 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3093 expand_ifn_atomic_compare_exchange (call
);
3096 /* Expand LAUNDER to assignment, lhs = arg0. */
3099 expand_LAUNDER (internal_fn
, gcall
*call
)
3101 tree lhs
= gimple_call_lhs (call
);
3106 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3109 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3112 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3114 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3115 int rhs_index
= internal_fn_stored_value_index (ifn
);
3116 int mask_index
= internal_fn_mask_index (ifn
);
3117 tree base
= gimple_call_arg (stmt
, 0);
3118 tree offset
= gimple_call_arg (stmt
, 1);
3119 tree scale
= gimple_call_arg (stmt
, 2);
3120 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3122 rtx base_rtx
= expand_normal (base
);
3123 rtx offset_rtx
= expand_normal (offset
);
3124 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3125 rtx rhs_rtx
= expand_normal (rhs
);
3127 class expand_operand ops
[6];
3129 create_address_operand (&ops
[i
++], base_rtx
);
3130 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3131 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3132 create_integer_operand (&ops
[i
++], scale_int
);
3133 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3134 if (mask_index
>= 0)
3136 tree mask
= gimple_call_arg (stmt
, mask_index
);
3137 rtx mask_rtx
= expand_normal (mask
);
3138 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3141 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3142 TYPE_MODE (TREE_TYPE (offset
)));
3143 expand_insn (icode
, i
, ops
);
3146 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3149 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3151 tree lhs
= gimple_call_lhs (stmt
);
3152 tree base
= gimple_call_arg (stmt
, 0);
3153 tree offset
= gimple_call_arg (stmt
, 1);
3154 tree scale
= gimple_call_arg (stmt
, 2);
3156 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3157 rtx base_rtx
= expand_normal (base
);
3158 rtx offset_rtx
= expand_normal (offset
);
3159 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3162 class expand_operand ops
[6];
3163 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3164 create_address_operand (&ops
[i
++], base_rtx
);
3165 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3166 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3167 create_integer_operand (&ops
[i
++], scale_int
);
3168 if (optab
== mask_gather_load_optab
)
3170 tree mask
= gimple_call_arg (stmt
, 4);
3171 rtx mask_rtx
= expand_normal (mask
);
3172 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
3174 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3175 TYPE_MODE (TREE_TYPE (offset
)));
3176 expand_insn (icode
, i
, ops
);
3177 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3178 emit_move_insn (lhs_rtx
, ops
[0].value
);
3181 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3182 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3185 contains_call_div_mod (rtx_insn
*insn
)
3187 subrtx_iterator::array_type array
;
3188 for (; insn
; insn
= NEXT_INSN (insn
))
3191 else if (INSN_P (insn
))
3192 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3193 switch (GET_CODE (*iter
))
3207 /* Expand DIVMOD() using:
3208 a) optab handler for udivmod/sdivmod if it is available.
3209 b) If optab_handler doesn't exist, generate call to
3210 target-specific divmod libfunc. */
3213 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3215 tree lhs
= gimple_call_lhs (call_stmt
);
3216 tree arg0
= gimple_call_arg (call_stmt
, 0);
3217 tree arg1
= gimple_call_arg (call_stmt
, 1);
3219 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3220 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3221 machine_mode mode
= TYPE_MODE (type
);
3222 bool unsignedp
= TYPE_UNSIGNED (type
);
3223 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3225 rtx op0
= expand_normal (arg0
);
3226 rtx op1
= expand_normal (arg1
);
3227 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3229 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3230 rtx_insn
*insns
= NULL
;
3232 if (TREE_CODE (arg1
) == INTEGER_CST
)
3234 /* For DIVMOD by integral constants, there could be efficient code
3235 expanded inline e.g. using shifts and plus/minus. Try to expand
3236 the division and modulo and if it emits any library calls or any
3237 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3239 scalar_int_mode int_mode
;
3240 if (remainder
== NULL_RTX
3242 && CONST_INT_P (op1
)
3243 && !pow2p_hwi (INTVAL (op1
))
3244 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3245 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3246 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3247 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3248 && optimize_insn_for_speed_p ())
3250 rtx_insn
*last
= get_last_insn ();
3251 remainder
= NULL_RTX
;
3252 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3253 TYPE_UNSIGNED (type
));
3254 if (quotient
!= NULL_RTX
)
3256 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3258 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3259 set_dst_reg_note (move
, REG_EQUAL
,
3260 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3261 ? UDIV
: DIV
, int_mode
,
3262 copy_rtx (op0
), op1
),
3264 move
= emit_move_insn (remainder
, remainder
);
3265 set_dst_reg_note (move
, REG_EQUAL
,
3266 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3267 ? UMOD
: MOD
, int_mode
,
3268 copy_rtx (op0
), op1
),
3273 delete_insns_since (last
);
3276 if (remainder
== NULL_RTX
)
3278 struct separate_ops ops
;
3279 ops
.code
= TRUNC_DIV_EXPR
;
3281 ops
.op0
= make_tree (ops
.type
, op0
);
3283 ops
.op2
= NULL_TREE
;
3284 ops
.location
= gimple_location (call_stmt
);
3286 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3287 if (contains_call_div_mod (get_insns ()))
3288 quotient
= NULL_RTX
;
3291 ops
.code
= TRUNC_MOD_EXPR
;
3292 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3294 if (contains_call_div_mod (get_insns ()))
3295 remainder
= NULL_RTX
;
3298 insns
= get_insns ();
3306 /* Check if optab_handler exists for divmod_optab for given mode. */
3307 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3309 quotient
= gen_reg_rtx (mode
);
3310 remainder
= gen_reg_rtx (mode
);
3311 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3314 /* Generate call to divmod libfunc if it exists. */
3315 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3316 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3317 "ient
, &remainder
);
3322 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3323 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3324 make_tree (TREE_TYPE (arg0
), quotient
),
3325 make_tree (TREE_TYPE (arg1
), remainder
)),
3326 target
, VOIDmode
, EXPAND_NORMAL
);
3332 expand_NOP (internal_fn
, gcall
*)
3334 /* Nothing. But it shouldn't really prevail. */
3337 /* Coroutines, all should have been processed at this stage. */
3340 expand_CO_FRAME (internal_fn
, gcall
*)
3346 expand_CO_YIELD (internal_fn
, gcall
*)
3352 expand_CO_SUSPN (internal_fn
, gcall
*)
3358 expand_CO_ACTOR (internal_fn
, gcall
*)
3363 /* Expand a call to FN using the operands in STMT. FN has a single
3364 output operand and NARGS input operands. */
3367 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3370 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
3372 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3373 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3374 gcc_assert (icode
!= CODE_FOR_nothing
);
3376 tree lhs
= gimple_call_lhs (stmt
);
3377 rtx lhs_rtx
= NULL_RTX
;
3379 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3381 /* Do not assign directly to a promoted subreg, since there is no
3382 guarantee that the instruction will leave the upper bits of the
3383 register in the state required by SUBREG_PROMOTED_SIGN. */
3385 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
3388 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
3390 for (unsigned int i
= 0; i
< nargs
; ++i
)
3392 tree rhs
= gimple_call_arg (stmt
, i
);
3393 tree rhs_type
= TREE_TYPE (rhs
);
3394 rtx rhs_rtx
= expand_normal (rhs
);
3395 if (INTEGRAL_TYPE_P (rhs_type
))
3396 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
3397 TYPE_MODE (rhs_type
),
3398 TYPE_UNSIGNED (rhs_type
));
3400 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
3403 expand_insn (icode
, nargs
+ 1, ops
);
3404 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
3406 /* If the return value has an integral type, convert the instruction
3407 result to that type. This is useful for things that return an
3408 int regardless of the size of the input. If the instruction result
3409 is smaller than required, assume that it is signed.
3411 If the return value has a nonintegral type, its mode must match
3412 the instruction result. */
3413 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
3415 /* If this is a scalar in a register that is stored in a wider
3416 mode than the declared mode, compute the result into its
3417 declared mode and then convert to the wider mode. */
3418 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3419 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
3420 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
3421 SUBREG_PROMOTED_SIGN (lhs_rtx
));
3423 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
3424 emit_move_insn (lhs_rtx
, ops
[0].value
);
3427 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
3428 convert_move (lhs_rtx
, ops
[0].value
, 0);
3433 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3436 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3438 expand_operand ops
[3];
3441 tree lhs
= gimple_call_lhs (stmt
);
3442 tree lhs_type
= TREE_TYPE (lhs
);
3443 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3444 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3446 for (unsigned int i
= 0; i
< 2; ++i
)
3448 tree rhs
= gimple_call_arg (stmt
, i
);
3449 rhs_type
[i
] = TREE_TYPE (rhs
);
3450 rtx rhs_rtx
= expand_normal (rhs
);
3451 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3454 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3455 TYPE_MODE (lhs_type
));
3457 expand_insn (icode
, 3, ops
);
3458 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3459 emit_move_insn (lhs_rtx
, ops
[0].value
);
3462 /* Expanders for optabs that can use expand_direct_optab_fn. */
3464 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3465 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3467 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3468 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3470 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3471 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3473 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3474 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3476 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3477 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3479 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3480 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3482 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3483 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3485 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3486 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3488 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3489 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3491 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3492 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3494 /* RETURN_TYPE and ARGS are a return type and argument list that are
3495 in principle compatible with FN (which satisfies direct_internal_fn_p).
3496 Return the types that should be used to determine whether the
3497 target supports FN. */
3500 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3502 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3503 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3504 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3505 return tree_pair (type0
, type1
);
3508 /* CALL is a call whose return type and arguments are in principle
3509 compatible with FN (which satisfies direct_internal_fn_p). Return the
3510 types that should be used to determine whether the target supports FN. */
3513 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3515 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3516 tree op0
= (info
.type0
< 0
3517 ? gimple_call_lhs (call
)
3518 : gimple_call_arg (call
, info
.type0
));
3519 tree op1
= (info
.type1
< 0
3520 ? gimple_call_lhs (call
)
3521 : gimple_call_arg (call
, info
.type1
));
3522 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3525 /* Return true if OPTAB is supported for TYPES (whose modes should be
3526 the same) when the optimization type is OPT_TYPE. Used for simple
3530 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3531 optimization_type opt_type
)
3533 machine_mode mode
= TYPE_MODE (types
.first
);
3534 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3535 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3538 /* Return true if OPTAB is supported for TYPES, where the first type
3539 is the destination and the second type is the source. Used for
3543 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3544 optimization_type opt_type
)
3546 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3547 TYPE_MODE (types
.second
), opt_type
)
3548 != CODE_FOR_nothing
);
3551 /* Return true if load/store lanes optab OPTAB is supported for
3552 array type TYPES.first when the optimization type is OPT_TYPE. */
3555 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3556 optimization_type opt_type
)
3558 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3559 machine_mode imode
= TYPE_MODE (types
.first
);
3560 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3561 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3562 != CODE_FOR_nothing
);
3565 #define direct_unary_optab_supported_p direct_optab_supported_p
3566 #define direct_binary_optab_supported_p direct_optab_supported_p
3567 #define direct_ternary_optab_supported_p direct_optab_supported_p
3568 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3569 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3570 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3571 #define direct_mask_load_optab_supported_p convert_optab_supported_p
3572 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3573 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3574 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3575 #define direct_len_load_optab_supported_p direct_optab_supported_p
3576 #define direct_mask_store_optab_supported_p convert_optab_supported_p
3577 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3578 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3579 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
3580 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
3581 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3582 #define direct_len_store_optab_supported_p direct_optab_supported_p
3583 #define direct_while_optab_supported_p convert_optab_supported_p
3584 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3585 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3586 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3587 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3588 #define direct_vec_set_optab_supported_p direct_optab_supported_p
3590 /* Return the optab used by internal function FN. */
3593 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3597 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3598 case IFN_##CODE: break;
3599 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3600 case IFN_##CODE: return OPTAB##_optab;
3601 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3602 UNSIGNED_OPTAB, TYPE) \
3603 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3604 ? UNSIGNED_OPTAB ## _optab \
3605 : SIGNED_OPTAB ## _optab);
3606 #include "internal-fn.def"
3614 /* Return the optab used by internal function FN. */
3617 direct_internal_fn_optab (internal_fn fn
)
3621 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3622 case IFN_##CODE: break;
3623 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3624 case IFN_##CODE: return OPTAB##_optab;
3625 #include "internal-fn.def"
3633 /* Return true if FN is supported for the types in TYPES when the
3634 optimization type is OPT_TYPE. The types are those associated with
3635 the "type0" and "type1" fields of FN's direct_internal_fn_info
3639 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3640 optimization_type opt_type
)
3644 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3645 case IFN_##CODE: break;
3646 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3648 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3650 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3651 UNSIGNED_OPTAB, TYPE) \
3654 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3655 ? UNSIGNED_OPTAB ## _optab \
3656 : SIGNED_OPTAB ## _optab); \
3657 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3660 #include "internal-fn.def"
3668 /* Return true if FN is supported for type TYPE when the optimization
3669 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3670 fields of FN's direct_internal_fn_info structure are the same. */
3673 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3674 optimization_type opt_type
)
3676 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3677 gcc_checking_assert (info
.type0
== info
.type1
);
3678 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3681 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3682 given that STMT is a call to a direct internal function. */
3685 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
3687 internal_fn fn
= gimple_call_internal_fn (stmt
);
3688 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3689 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
3692 /* If FN is commutative in two consecutive arguments, return the
3693 index of the first, otherwise return -1. */
3696 first_commutative_argument (internal_fn fn
)
3731 /* Return true if IFN_SET_EDOM is supported. */
3734 set_edom_supported_p (void)
3743 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3745 expand_##CODE (internal_fn fn, gcall *stmt) \
3747 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3749 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3750 UNSIGNED_OPTAB, TYPE) \
3752 expand_##CODE (internal_fn fn, gcall *stmt) \
3754 tree_pair types = direct_internal_fn_types (fn, stmt); \
3755 optab which_optab = direct_internal_fn_optab (fn, types); \
3756 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3758 #include "internal-fn.def"
3760 /* Routines to expand each internal function, indexed by function number.
3761 Each routine has the prototype:
3763 expand_<NAME> (gcall *stmt)
3765 where STMT is the statement that performs the call. */
3766 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3767 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3768 #include "internal-fn.def"
3772 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
3774 #define FOR_EACH_CODE_MAPPING(T) \
3775 T (PLUS_EXPR, IFN_COND_ADD) \
3776 T (MINUS_EXPR, IFN_COND_SUB) \
3777 T (MULT_EXPR, IFN_COND_MUL) \
3778 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
3779 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
3780 T (RDIV_EXPR, IFN_COND_RDIV) \
3781 T (MIN_EXPR, IFN_COND_MIN) \
3782 T (MAX_EXPR, IFN_COND_MAX) \
3783 T (BIT_AND_EXPR, IFN_COND_AND) \
3784 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3785 T (BIT_XOR_EXPR, IFN_COND_XOR) \
3786 T (LSHIFT_EXPR, IFN_COND_SHL) \
3787 T (RSHIFT_EXPR, IFN_COND_SHR)
3789 /* Return a function that only performs CODE when a certain condition is met
3790 and that uses a given fallback value otherwise. For example, if CODE is
3791 a binary operation associated with conditional function FN:
3793 LHS = FN (COND, A, B, ELSE)
3795 is equivalent to the C expression:
3797 LHS = COND ? A CODE B : ELSE;
3799 operating elementwise if the operands are vectors.
3801 Return IFN_LAST if no such function exists. */
3804 get_conditional_internal_fn (tree_code code
)
3808 #define CASE(CODE, IFN) case CODE: return IFN;
3809 FOR_EACH_CODE_MAPPING(CASE
)
3816 /* If IFN implements the conditional form of a tree code, return that
3817 tree code, otherwise return ERROR_MARK. */
3820 conditional_internal_fn_code (internal_fn ifn
)
3824 #define CASE(CODE, IFN) case IFN: return CODE;
3825 FOR_EACH_CODE_MAPPING(CASE
)
3832 /* Invoke T(IFN) for each internal function IFN that also has an
3834 #define FOR_EACH_COND_FN_PAIR(T) \
3840 /* Return a function that only performs internal function FN when a
3841 certain condition is met and that uses a given fallback value otherwise.
3842 In other words, the returned function FN' is such that:
3844 LHS = FN' (COND, A1, ... An, ELSE)
3846 is equivalent to the C expression:
3848 LHS = COND ? FN (A1, ..., An) : ELSE;
3850 operating elementwise if the operands are vectors.
3852 Return IFN_LAST if no such function exists. */
3855 get_conditional_internal_fn (internal_fn fn
)
3859 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
3860 FOR_EACH_COND_FN_PAIR(CASE
)
3867 /* If IFN implements the conditional form of an unconditional internal
3868 function, return that unconditional function, otherwise return IFN_LAST. */
3871 get_unconditional_internal_fn (internal_fn ifn
)
3875 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
3876 FOR_EACH_COND_FN_PAIR(CASE
)
3883 /* Return true if STMT can be interpreted as a conditional tree code
3884 operation of the form:
3886 LHS = COND ? OP (RHS1, ...) : ELSE;
3888 operating elementwise if the operands are vectors. This includes
3889 the case of an all-true COND, so that the operation always happens.
3891 When returning true, set:
3893 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
3894 is known to be all-true
3895 - *CODE_OUT to the tree code
3896 - OPS[I] to operand I of *CODE_OUT
3897 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
3898 condition is known to be all true. */
3901 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
3902 tree_code
*code_out
,
3903 tree (&ops
)[3], tree
*else_out
)
3905 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
3907 *cond_out
= NULL_TREE
;
3908 *code_out
= gimple_assign_rhs_code (assign
);
3909 ops
[0] = gimple_assign_rhs1 (assign
);
3910 ops
[1] = gimple_assign_rhs2 (assign
);
3911 ops
[2] = gimple_assign_rhs3 (assign
);
3912 *else_out
= NULL_TREE
;
3915 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
3916 if (gimple_call_internal_p (call
))
3918 internal_fn ifn
= gimple_call_internal_fn (call
);
3919 tree_code code
= conditional_internal_fn_code (ifn
);
3920 if (code
!= ERROR_MARK
)
3922 *cond_out
= gimple_call_arg (call
, 0);
3924 unsigned int nops
= gimple_call_num_args (call
) - 2;
3925 for (unsigned int i
= 0; i
< 3; ++i
)
3926 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
3927 *else_out
= gimple_call_arg (call
, nops
+ 1);
3928 if (integer_truep (*cond_out
))
3930 *cond_out
= NULL_TREE
;
3931 *else_out
= NULL_TREE
;
3939 /* Return true if IFN is some form of load from memory. */
3942 internal_load_fn_p (internal_fn fn
)
3947 case IFN_LOAD_LANES
:
3948 case IFN_MASK_LOAD_LANES
:
3949 case IFN_GATHER_LOAD
:
3950 case IFN_MASK_GATHER_LOAD
:
3959 /* Return true if IFN is some form of store to memory. */
3962 internal_store_fn_p (internal_fn fn
)
3966 case IFN_MASK_STORE
:
3967 case IFN_STORE_LANES
:
3968 case IFN_MASK_STORE_LANES
:
3969 case IFN_SCATTER_STORE
:
3970 case IFN_MASK_SCATTER_STORE
:
3979 /* Return true if IFN is some form of gather load or scatter store. */
3982 internal_gather_scatter_fn_p (internal_fn fn
)
3986 case IFN_GATHER_LOAD
:
3987 case IFN_MASK_GATHER_LOAD
:
3988 case IFN_SCATTER_STORE
:
3989 case IFN_MASK_SCATTER_STORE
:
3997 /* If FN takes a vector mask argument, return the index of that argument,
3998 otherwise return -1. */
4001 internal_fn_mask_index (internal_fn fn
)
4006 case IFN_MASK_LOAD_LANES
:
4007 case IFN_MASK_STORE
:
4008 case IFN_MASK_STORE_LANES
:
4011 case IFN_MASK_GATHER_LOAD
:
4012 case IFN_MASK_SCATTER_STORE
:
4016 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4017 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4021 /* If FN takes a value that should be stored to memory, return the index
4022 of that argument, otherwise return -1. */
4025 internal_fn_stored_value_index (internal_fn fn
)
4029 case IFN_MASK_STORE
:
4030 case IFN_MASK_STORE_LANES
:
4031 case IFN_SCATTER_STORE
:
4032 case IFN_MASK_SCATTER_STORE
:
4041 /* Return true if the target supports gather load or scatter store function
4042 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4043 while for stores it is the vector type of the stored data argument.
4044 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4045 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4046 offset from the shared base address of each loaded or stored element.
4047 SCALE is the amount by which these offsets should be multiplied
4048 *after* they have been extended to address width. */
4051 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4052 tree memory_element_type
,
4053 tree offset_vector_type
, int scale
)
4055 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4056 TYPE_SIZE (memory_element_type
)))
4058 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4059 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4061 optab optab
= direct_internal_fn_optab (ifn
);
4062 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4063 TYPE_MODE (offset_vector_type
));
4064 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4065 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4066 return (icode
!= CODE_FOR_nothing
4067 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4068 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4071 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4072 for pointers of type TYPE when the accesses have LENGTH bytes and their
4073 common byte alignment is ALIGN. */
4076 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4077 poly_uint64 length
, unsigned int align
)
4079 machine_mode mode
= TYPE_MODE (type
);
4080 optab optab
= direct_internal_fn_optab (ifn
);
4081 insn_code icode
= direct_optab_handler (optab
, mode
);
4082 if (icode
== CODE_FOR_nothing
)
4084 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4085 return (insn_operand_matches (icode
, 3, length_rtx
)
4086 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4089 /* Expand STMT as though it were a call to internal function FN. */
4092 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4094 internal_fn_expanders
[fn
] (fn
, stmt
);
4097 /* Expand STMT, which is a call to internal function FN. */
4100 expand_internal_call (gcall
*stmt
)
4102 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4105 /* If TYPE is a vector type, return true if IFN is a direct internal
4106 function that is supported for that type. If TYPE is a scalar type,
4107 return true if IFN is a direct internal function that is supported for
4108 the target's preferred vector version of TYPE. */
4111 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4113 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4114 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4117 if (!is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4120 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4121 if (VECTOR_MODE_P (vmode
))
4123 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4124 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4128 auto_vector_modes vector_modes
;
4129 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4130 for (machine_mode base_mode
: vector_modes
)
4131 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
4133 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4134 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4142 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
4148 expand_PHI (internal_fn
, gcall
*)