2 Copyright (C) 2011-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
55 #include "fold-const-call.h"
56 #include "tree-ssa-live.h"
57 #include "tree-outof-ssa.h"
59 /* For lang_hooks.types.type_for_mode. */
60 #include "langhooks.h"
62 /* The names of each internal function, indexed by function number. */
63 const char *const internal_fn_name_array
[] = {
64 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
65 #include "internal-fn.def"
69 /* The ECF_* flags of each internal function, indexed by function number. */
70 const int internal_fn_flags_array
[] = {
71 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
72 #include "internal-fn.def"
76 /* Return the internal function called NAME, or IFN_LAST if there's
80 lookup_internal_fn (const char *name
)
82 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
83 static name_to_fn_map_type
*name_to_fn_map
;
87 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
88 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
89 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
92 internal_fn
*entry
= name_to_fn_map
->get (name
);
93 return entry
? *entry
: IFN_LAST
;
96 /* Geven an internal_fn IFN that is a widening function, return its
97 corresponding LO and HI internal_fns. */
100 lookup_hilo_internal_fn (internal_fn ifn
, internal_fn
*lo
, internal_fn
*hi
)
102 gcc_assert (widening_fn_p (ifn
));
108 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
109 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
111 *lo = internal_fn (IFN_##NAME##_LO); \
112 *hi = internal_fn (IFN_##NAME##_HI); \
114 #include "internal-fn.def"
118 /* Given an internal_fn IFN that is a widening function, return its
119 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
122 lookup_evenodd_internal_fn (internal_fn ifn
, internal_fn
*even
,
125 gcc_assert (widening_fn_p (ifn
));
131 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
132 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
134 *even = internal_fn (IFN_##NAME##_EVEN); \
135 *odd = internal_fn (IFN_##NAME##_ODD); \
137 #include "internal-fn.def"
142 /* Fnspec of each internal function, indexed by function number. */
143 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
148 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
149 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
150 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
151 #include "internal-fn.def"
152 internal_fn_fnspec_array
[IFN_LAST
] = 0;
155 /* Create static initializers for the information returned by
156 direct_internal_fn. */
157 #define not_direct { -2, -2, false }
158 #define mask_load_direct { -1, 2, false }
159 #define load_lanes_direct { -1, -1, false }
160 #define mask_load_lanes_direct { -1, -1, false }
161 #define gather_load_direct { 3, 1, false }
162 #define len_load_direct { -1, -1, false }
163 #define mask_len_load_direct { -1, 4, false }
164 #define mask_store_direct { 3, 2, false }
165 #define store_lanes_direct { 0, 0, false }
166 #define mask_store_lanes_direct { 0, 0, false }
167 #define vec_cond_mask_direct { 1, 0, false }
168 #define vec_cond_mask_len_direct { 1, 1, false }
169 #define vec_cond_direct { 2, 0, false }
170 #define scatter_store_direct { 3, 1, false }
171 #define len_store_direct { 3, 3, false }
172 #define mask_len_store_direct { 4, 5, false }
173 #define vec_set_direct { 3, 3, false }
174 #define vec_extract_direct { 0, -1, false }
175 #define unary_direct { 0, 0, true }
176 #define unary_convert_direct { -1, 0, true }
177 #define binary_direct { 0, 0, true }
178 #define ternary_direct { 0, 0, true }
179 #define cond_unary_direct { 1, 1, true }
180 #define cond_binary_direct { 1, 1, true }
181 #define cond_ternary_direct { 1, 1, true }
182 #define cond_len_unary_direct { 1, 1, true }
183 #define cond_len_binary_direct { 1, 1, true }
184 #define cond_len_ternary_direct { 1, 1, true }
185 #define while_direct { 0, 2, false }
186 #define fold_extract_direct { 2, 2, false }
187 #define fold_len_extract_direct { 2, 2, false }
188 #define fold_left_direct { 1, 1, false }
189 #define mask_fold_left_direct { 1, 1, false }
190 #define mask_len_fold_left_direct { 1, 1, false }
191 #define check_ptrs_direct { 0, 0, false }
193 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
194 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
195 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
196 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
197 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
198 #include "internal-fn.def"
202 /* Like create_output_operand, but for callers that will use
203 assign_call_lhs afterwards. */
206 create_call_lhs_operand (expand_operand
*op
, rtx lhs_rtx
, machine_mode mode
)
208 /* Do not assign directly to a promoted subreg, since there is no
209 guarantee that the instruction will leave the upper bits of the
210 register in the state required by SUBREG_PROMOTED_SIGN. */
212 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
214 create_output_operand (op
, dest
, mode
);
217 /* Move the result of an expanded instruction into the lhs of a gimple call.
218 LHS is the lhs of the call, LHS_RTX is its expanded form, and OP is the
219 result of the expanded instruction. OP should have been set up by
220 create_call_lhs_operand. */
223 assign_call_lhs (tree lhs
, rtx lhs_rtx
, expand_operand
*op
)
225 if (rtx_equal_p (lhs_rtx
, op
->value
))
228 /* If the return value has an integral type, convert the instruction
229 result to that type. This is useful for things that return an
230 int regardless of the size of the input. If the instruction result
231 is smaller than required, assume that it is signed.
233 If the return value has a nonintegral type, its mode must match
234 the instruction result. */
235 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
237 /* If this is a scalar in a register that is stored in a wider
238 mode than the declared mode, compute the result into its
239 declared mode and then convert to the wider mode. */
240 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
241 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), op
->value
, 0);
242 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
243 SUBREG_PROMOTED_SIGN (lhs_rtx
));
245 else if (GET_MODE (lhs_rtx
) == GET_MODE (op
->value
))
246 emit_move_insn (lhs_rtx
, op
->value
);
249 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
250 convert_move (lhs_rtx
, op
->value
, 0);
254 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
255 output operands and NINPUTS input operands, where NOUTPUTS is either
256 0 or 1. The output operand (if any) comes first, followed by the
257 NINPUTS input operands. */
260 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
261 unsigned int ninputs
)
263 gcc_assert (icode
!= CODE_FOR_nothing
);
265 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
266 unsigned int opno
= 0;
267 rtx lhs_rtx
= NULL_RTX
;
268 tree lhs
= gimple_call_lhs (stmt
);
272 gcc_assert (noutputs
== 1);
274 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
275 create_call_lhs_operand (&ops
[opno
], lhs_rtx
,
276 insn_data
[icode
].operand
[opno
].mode
);
282 for (unsigned int i
= 0; i
< ninputs
; ++i
)
284 tree rhs
= gimple_call_arg (stmt
, i
);
285 tree rhs_type
= TREE_TYPE (rhs
);
286 rtx rhs_rtx
= expand_normal (rhs
);
287 if (INTEGRAL_TYPE_P (rhs_type
))
288 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
289 TYPE_MODE (rhs_type
),
290 TYPE_UNSIGNED (rhs_type
));
291 else if (TREE_CODE (rhs
) == SSA_NAME
292 && SSA_NAME_IS_DEFAULT_DEF (rhs
)
293 && VAR_P (SSA_NAME_VAR (rhs
)))
294 create_undefined_input_operand (&ops
[opno
], TYPE_MODE (rhs_type
));
295 else if (VECTOR_BOOLEAN_TYPE_P (rhs_type
)
296 && SCALAR_INT_MODE_P (TYPE_MODE (rhs_type
))
297 && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (rhs_type
)),
298 TYPE_VECTOR_SUBPARTS (rhs_type
).to_constant ()))
300 /* Ensure that the vector bitmasks do not have excess bits. */
301 int nunits
= TYPE_VECTOR_SUBPARTS (rhs_type
).to_constant ();
302 rtx tmp
= expand_binop (TYPE_MODE (rhs_type
), and_optab
, rhs_rtx
,
303 GEN_INT ((HOST_WIDE_INT_1U
<< nunits
) - 1),
304 NULL_RTX
, true, OPTAB_WIDEN
);
305 create_input_operand (&ops
[opno
], tmp
, TYPE_MODE (rhs_type
));
308 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
312 gcc_assert (opno
== noutputs
+ ninputs
);
313 expand_insn (icode
, opno
, ops
);
315 assign_call_lhs (lhs
, lhs_rtx
, &ops
[0]);
318 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
319 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
321 static enum insn_code
322 get_multi_vector_move (tree array_type
, convert_optab optab
)
327 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
328 imode
= TYPE_MODE (array_type
);
329 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
331 return convert_optab_handler (optab
, imode
, vmode
);
334 /* Add mask and len arguments according to the STMT. */
337 add_mask_and_len_args (expand_operand
*ops
, unsigned int opno
, gcall
*stmt
)
339 internal_fn ifn
= gimple_call_internal_fn (stmt
);
340 int len_index
= internal_fn_len_index (ifn
);
341 /* BIAS is always consecutive next of LEN. */
342 int bias_index
= len_index
+ 1;
343 int mask_index
= internal_fn_mask_index (ifn
);
344 /* The order of arguments are always {len,bias,mask}. */
347 tree mask
= gimple_call_arg (stmt
, mask_index
);
348 rtx mask_rtx
= expand_normal (mask
);
350 tree mask_type
= TREE_TYPE (mask
);
351 if (VECTOR_BOOLEAN_TYPE_P (mask_type
)
352 && SCALAR_INT_MODE_P (TYPE_MODE (mask_type
))
353 && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (mask_type
)),
354 TYPE_VECTOR_SUBPARTS (mask_type
).to_constant ()))
356 /* Ensure that the vector bitmasks do not have excess bits. */
357 int nunits
= TYPE_VECTOR_SUBPARTS (mask_type
).to_constant ();
358 mask_rtx
= expand_binop (TYPE_MODE (mask_type
), and_optab
, mask_rtx
,
359 GEN_INT ((HOST_WIDE_INT_1U
<< nunits
) - 1),
360 NULL_RTX
, true, OPTAB_WIDEN
);
363 create_input_operand (&ops
[opno
++], mask_rtx
,
364 TYPE_MODE (TREE_TYPE (mask
)));
368 tree len
= gimple_call_arg (stmt
, len_index
);
369 rtx len_rtx
= expand_normal (len
);
370 create_convert_operand_from (&ops
[opno
++], len_rtx
,
371 TYPE_MODE (TREE_TYPE (len
)),
372 TYPE_UNSIGNED (TREE_TYPE (len
)));
373 tree biast
= gimple_call_arg (stmt
, bias_index
);
374 rtx bias
= expand_normal (biast
);
375 create_input_operand (&ops
[opno
++], bias
, QImode
);
380 /* Expand LOAD_LANES call STMT using optab OPTAB. */
383 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
385 class expand_operand ops
[2];
389 lhs
= gimple_call_lhs (stmt
);
390 rhs
= gimple_call_arg (stmt
, 0);
391 type
= TREE_TYPE (lhs
);
393 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
394 mem
= expand_normal (rhs
);
396 gcc_assert (MEM_P (mem
));
397 PUT_MODE (mem
, TYPE_MODE (type
));
399 create_call_lhs_operand (&ops
[0], target
, TYPE_MODE (type
));
400 create_fixed_operand (&ops
[1], mem
);
401 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
402 assign_call_lhs (lhs
, target
, &ops
[0]);
405 /* Expand STORE_LANES call STMT using optab OPTAB. */
408 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
410 class expand_operand ops
[2];
414 lhs
= gimple_call_lhs (stmt
);
415 rhs
= gimple_call_arg (stmt
, 0);
416 type
= TREE_TYPE (rhs
);
418 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
419 reg
= expand_normal (rhs
);
421 gcc_assert (MEM_P (target
));
422 PUT_MODE (target
, TYPE_MODE (type
));
424 create_fixed_operand (&ops
[0], target
);
425 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
426 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
430 expand_ANNOTATE (internal_fn
, gcall
*)
435 /* This should get expanded in omp_device_lower pass. */
438 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
443 /* This should get expanded in omp_device_lower pass. */
446 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
451 /* Allocate per-lane storage and begin non-uniform execution region. */
454 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
457 tree lhs
= gimple_call_lhs (stmt
);
459 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
461 target
= gen_reg_rtx (Pmode
);
462 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
463 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
464 class expand_operand ops
[3];
465 create_call_lhs_operand (&ops
[0], target
, Pmode
);
466 create_input_operand (&ops
[1], size
, Pmode
);
467 create_input_operand (&ops
[2], align
, Pmode
);
468 gcc_assert (targetm
.have_omp_simt_enter ());
469 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
470 assign_call_lhs (lhs
, target
, &ops
[0]);
473 /* Deallocate per-lane storage and leave non-uniform execution region. */
476 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
478 gcc_checking_assert (!gimple_call_lhs (stmt
));
479 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
480 class expand_operand ops
[1];
481 create_input_operand (&ops
[0], arg
, Pmode
);
482 gcc_assert (targetm
.have_omp_simt_exit ());
483 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
486 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
487 without SIMT execution this should be expanded in omp_device_lower pass. */
490 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
492 tree lhs
= gimple_call_lhs (stmt
);
496 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
497 gcc_assert (targetm
.have_omp_simt_lane ());
498 emit_insn (targetm
.gen_omp_simt_lane (target
));
501 /* This should get expanded in omp_device_lower pass. */
504 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
509 /* This should get expanded in omp_device_lower pass. */
512 expand_GOMP_TARGET_REV (internal_fn
, gcall
*)
517 /* Lane index of the first SIMT lane that supplies a non-zero argument.
518 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
519 lane that executed the last iteration for handling OpenMP lastprivate. */
522 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
524 tree lhs
= gimple_call_lhs (stmt
);
528 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
529 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
530 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
531 class expand_operand ops
[2];
532 create_call_lhs_operand (&ops
[0], target
, mode
);
533 create_input_operand (&ops
[1], cond
, mode
);
534 gcc_assert (targetm
.have_omp_simt_last_lane ());
535 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
536 assign_call_lhs (lhs
, target
, &ops
[0]);
539 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
542 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
544 tree lhs
= gimple_call_lhs (stmt
);
548 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
549 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
550 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
551 class expand_operand ops
[2];
552 create_call_lhs_operand (&ops
[0], target
, mode
);
553 create_input_operand (&ops
[1], ctr
, mode
);
554 gcc_assert (targetm
.have_omp_simt_ordered ());
555 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
556 assign_call_lhs (lhs
, target
, &ops
[0]);
559 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
560 any lane supplies a non-zero argument. */
563 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
565 tree lhs
= gimple_call_lhs (stmt
);
569 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
570 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
571 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
572 class expand_operand ops
[2];
573 create_call_lhs_operand (&ops
[0], target
, mode
);
574 create_input_operand (&ops
[1], cond
, mode
);
575 gcc_assert (targetm
.have_omp_simt_vote_any ());
576 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
577 assign_call_lhs (lhs
, target
, &ops
[0]);
580 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
581 is destination lane index XOR given offset. */
584 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
586 tree lhs
= gimple_call_lhs (stmt
);
590 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
591 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
592 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
593 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
594 class expand_operand ops
[3];
595 create_call_lhs_operand (&ops
[0], target
, mode
);
596 create_input_operand (&ops
[1], src
, mode
);
597 create_input_operand (&ops
[2], idx
, SImode
);
598 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
599 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
600 assign_call_lhs (lhs
, target
, &ops
[0]);
603 /* Exchange between SIMT lanes according to given source lane index. */
606 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
608 tree lhs
= gimple_call_lhs (stmt
);
612 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
613 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
614 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
615 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
616 class expand_operand ops
[3];
617 create_call_lhs_operand (&ops
[0], target
, mode
);
618 create_input_operand (&ops
[1], src
, mode
);
619 create_input_operand (&ops
[2], idx
, SImode
);
620 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
621 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
622 assign_call_lhs (lhs
, target
, &ops
[0]);
625 /* This should get expanded in adjust_simduid_builtins. */
628 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
633 /* This should get expanded in adjust_simduid_builtins. */
636 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
641 /* This should get expanded in adjust_simduid_builtins. */
644 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
649 /* This should get expanded in adjust_simduid_builtins. */
652 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
657 /* This should get expanded in adjust_simduid_builtins. */
660 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
665 /* This should get expanded in the sanopt pass. */
668 expand_UBSAN_NULL (internal_fn
, gcall
*)
673 /* This should get expanded in the sanopt pass. */
676 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
681 /* This should get expanded in the sanopt pass. */
684 expand_UBSAN_VPTR (internal_fn
, gcall
*)
689 /* This should get expanded in the sanopt pass. */
692 expand_UBSAN_PTR (internal_fn
, gcall
*)
697 /* This should get expanded in the sanopt pass. */
700 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
705 /* This should get expanded in the sanopt pass. */
708 expand_HWASAN_CHECK (internal_fn
, gcall
*)
713 /* For hwasan stack tagging:
714 Clear tags on the dynamically allocated space.
715 For use after an object dynamically allocated on the stack goes out of
718 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
720 gcc_assert (Pmode
== ptr_mode
);
721 tree restored_position
= gimple_call_arg (gc
, 0);
722 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
724 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
725 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
726 stack_pointer_rtx
, NULL_RTX
, 0,
728 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
729 virtual_stack_dynamic_rtx
, Pmode
,
730 HWASAN_STACK_BACKGROUND
, QImode
,
734 /* For hwasan stack tagging:
735 Return a tag to be used for a dynamic allocation. */
737 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
739 tree tag
= gimple_call_lhs (gc
);
740 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
741 machine_mode mode
= GET_MODE (target
);
742 gcc_assert (mode
== QImode
);
744 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
745 gcc_assert (base_tag
);
746 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
747 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
748 target
, /* unsignedp = */1,
750 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
752 /* Really need to put the tag into the `target` RTX. */
753 if (chosen_tag
!= target
)
755 rtx temp
= chosen_tag
;
756 gcc_assert (GET_MODE (chosen_tag
) == mode
);
757 emit_move_insn (target
, temp
);
760 hwasan_increment_frame_tag ();
763 /* For hwasan stack tagging:
764 Tag a region of space in the shadow stack according to the base pointer of
765 an object on the stack. N.b. the length provided in the internal call is
766 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
768 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
770 gcc_assert (ptr_mode
== Pmode
);
771 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
772 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
774 tree base
= gimple_call_arg (gc
, 1);
775 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
776 rtx base_rtx
= expand_normal (base
);
778 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
779 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
780 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
782 tree len
= gimple_call_arg (gc
, 2);
783 rtx r_len
= expand_normal (len
);
785 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
786 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
787 tag
, QImode
, r_len
, Pmode
);
790 /* For hwasan stack tagging:
791 Store a tag into a pointer. */
793 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
795 gcc_assert (ptr_mode
== Pmode
);
796 tree g_target
= gimple_call_lhs (gc
);
797 tree g_ptr
= gimple_call_arg (gc
, 0);
798 tree g_tag
= gimple_call_arg (gc
, 1);
800 rtx ptr
= expand_normal (g_ptr
);
801 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
802 rtx target
= expand_normal (g_target
);
804 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
805 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
806 if (tagged_value
!= target
)
807 emit_move_insn (target
, tagged_value
);
810 /* This should get expanded in the sanopt pass. */
813 expand_ASAN_CHECK (internal_fn
, gcall
*)
818 /* This should get expanded in the sanopt pass. */
821 expand_ASAN_MARK (internal_fn
, gcall
*)
826 /* This should get expanded in the sanopt pass. */
829 expand_ASAN_POISON (internal_fn
, gcall
*)
834 /* This should get expanded in the sanopt pass. */
837 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
842 /* This should get expanded in the tsan pass. */
845 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
850 /* This should get expanded in the lower pass. */
853 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
855 error_at (gimple_location (call
),
856 "invalid use of attribute %<fallthrough%>");
859 /* Return minimum precision needed to represent all values
860 of ARG in SIGNed integral type. */
863 get_min_precision (tree arg
, signop sign
)
865 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
867 signop orig_sign
= sign
;
868 if (TREE_CODE (arg
) == INTEGER_CST
)
871 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
873 widest_int w
= wi::to_widest (arg
);
874 w
= wi::ext (w
, prec
, sign
);
875 p
= wi::min_precision (w
, sign
);
878 p
= wi::min_precision (wi::to_wide (arg
), sign
);
879 return MIN (p
, prec
);
881 while (CONVERT_EXPR_P (arg
)
882 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
883 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
885 arg
= TREE_OPERAND (arg
, 0);
886 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
888 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
890 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
891 return prec
+ (orig_sign
!= sign
);
892 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
895 return prec
+ (orig_sign
!= sign
);
897 if (CONVERT_EXPR_P (arg
)
898 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
899 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
901 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
902 If y_2's min precision is smaller than prec, return that. */
903 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
905 return oprec
+ (orig_sign
!= sign
);
907 if (TREE_CODE (arg
) != SSA_NAME
)
908 return prec
+ (orig_sign
!= sign
);
910 while (!get_global_range_query ()->range_of_expr (r
, arg
)
914 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
915 if (is_gimple_assign (g
)
916 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
918 tree t
= gimple_assign_rhs1 (g
);
919 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
920 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
923 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
925 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
927 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
928 return prec
+ (orig_sign
!= sign
);
929 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
932 return prec
+ (orig_sign
!= sign
);
936 return prec
+ (orig_sign
!= sign
);
938 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
940 int p1
= wi::min_precision (r
.lower_bound (), sign
);
941 int p2
= wi::min_precision (r
.upper_bound (), sign
);
943 prec
= MIN (prec
, p1
);
945 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
947 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
948 prec
= MIN (prec
, p
);
950 return prec
+ (orig_sign
!= sign
);
953 /* Helper for expand_*_overflow. Set the __imag__ part to true
954 (1 except for signed:1 type, in which case store -1). */
957 expand_arith_set_overflow (tree lhs
, rtx target
)
959 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
960 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
961 write_complex_part (target
, constm1_rtx
, true, false);
963 write_complex_part (target
, const1_rtx
, true, false);
966 /* Helper for expand_*_overflow. Store RES into the __real__ part
967 of TARGET. If RES has larger MODE than __real__ part of TARGET,
968 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
969 if LHS has smaller precision than its mode. */
972 expand_arith_overflow_result_store (tree lhs
, rtx target
,
973 scalar_int_mode mode
, rtx res
)
975 scalar_int_mode tgtmode
976 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
980 rtx_code_label
*done_label
= gen_label_rtx ();
981 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
982 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
983 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
984 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
985 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
986 profile_probability::very_likely ());
987 expand_arith_set_overflow (lhs
, target
);
988 emit_label (done_label
);
990 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
991 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
994 rtx_code_label
*done_label
= gen_label_rtx ();
995 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
1000 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
1002 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
1003 true, OPTAB_LIB_WIDEN
);
1007 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
1009 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
1012 do_compare_rtx_and_jump (res
, lres
,
1013 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
1014 profile_probability::very_likely ());
1015 expand_arith_set_overflow (lhs
, target
);
1016 emit_label (done_label
);
1018 write_complex_part (target
, lres
, false, false);
1021 /* Helper for expand_*_overflow. Store RES into TARGET. */
1024 expand_ubsan_result_store (tree lhs
, rtx target
, scalar_int_mode mode
,
1025 rtx res
, rtx_code_label
*do_error
)
1027 if (TREE_CODE (TREE_TYPE (lhs
)) == BITINT_TYPE
1028 && TYPE_PRECISION (TREE_TYPE (lhs
)) < GET_MODE_PRECISION (mode
))
1030 int uns
= TYPE_UNSIGNED (TREE_TYPE (lhs
));
1031 int prec
= TYPE_PRECISION (TREE_TYPE (lhs
));
1032 int tgtprec
= GET_MODE_PRECISION (mode
);
1033 rtx resc
= gen_reg_rtx (mode
), lres
;
1034 emit_move_insn (resc
, res
);
1038 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
1040 lres
= expand_simple_binop (mode
, AND
, res
, mask
, NULL_RTX
,
1041 true, OPTAB_LIB_WIDEN
);
1045 lres
= expand_shift (LSHIFT_EXPR
, mode
, res
, tgtprec
- prec
,
1047 lres
= expand_shift (RSHIFT_EXPR
, mode
, lres
, tgtprec
- prec
,
1051 emit_move_insn (res
, lres
);
1052 do_compare_rtx_and_jump (res
, resc
,
1053 NE
, true, mode
, NULL_RTX
, NULL
, do_error
,
1054 profile_probability::very_unlikely ());
1056 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
1057 /* If this is a scalar in a register that is stored in a wider mode
1058 than the declared mode, compute the result into its declared mode
1059 and then convert to the wider mode. Our value is the computed
1061 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
1063 emit_move_insn (target
, res
);
1066 /* Add sub/add overflow checking to the statement STMT.
1067 CODE says whether the operation is +, or -. */
1070 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
1071 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
1072 bool uns1_p
, bool is_ubsan
, tree
*datap
)
1074 rtx res
, target
= NULL_RTX
;
1076 rtx_code_label
*done_label
= gen_label_rtx ();
1077 rtx_code_label
*do_error
= gen_label_rtx ();
1078 do_pending_stack_adjust ();
1079 rtx op0
= expand_normal (arg0
);
1080 rtx op1
= expand_normal (arg1
);
1081 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1082 int prec
= GET_MODE_PRECISION (mode
);
1083 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1084 bool do_xor
= false;
1087 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1091 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1093 write_complex_part (target
, const0_rtx
, true, false);
1096 /* We assume both operands and result have the same precision
1097 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1098 with that precision, U for unsigned type with that precision,
1099 sgn for unsigned most significant bit in that precision.
1100 s1 is signed first operand, u1 is unsigned first operand,
1101 s2 is signed second operand, u2 is unsigned second operand,
1102 sr is signed result, ur is unsigned result and the following
1103 rules say how to compute result (which is always result of
1104 the operands as if both were unsigned, cast to the right
1105 signedness) and how to compute whether operation overflowed.
1108 res = (S) ((U) s1 + (U) s2)
1109 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1111 res = (S) ((U) s1 - (U) s2)
1112 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1115 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1118 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1120 res = (S) ((U) s1 + u2)
1121 ovf = ((U) res ^ sgn) < u2
1126 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1128 res = (S) ((U) s1 - u2)
1129 ovf = u2 > ((U) s1 ^ sgn)
1132 ovf = s1 < 0 || u2 > (U) s1
1135 ovf = u1 >= ((U) s2 ^ sgn)
1140 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1142 res = (U) s1 + (U) s2
1143 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1146 ovf = (U) res < u2 || res < 0
1149 ovf = u1 >= u2 ? res < 0 : res >= 0
1151 res = (U) s1 - (U) s2
1152 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1154 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
1156 /* PLUS_EXPR is commutative, if operand signedness differs,
1157 canonicalize to the first operand being signed and second
1158 unsigned to simplify following code. */
1159 std::swap (op0
, op1
);
1160 std::swap (arg0
, arg1
);
1165 /* u1 +- u2 -> ur */
1166 if (uns0_p
&& uns1_p
&& unsr_p
)
1168 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
1169 : usubv4_optab
, mode
);
1170 if (icode
!= CODE_FOR_nothing
)
1172 class expand_operand ops
[4];
1173 rtx_insn
*last
= get_last_insn ();
1175 res
= gen_reg_rtx (mode
);
1176 create_output_operand (&ops
[0], res
, mode
);
1177 create_input_operand (&ops
[1], op0
, mode
);
1178 create_input_operand (&ops
[2], op1
, mode
);
1179 create_fixed_operand (&ops
[3], do_error
);
1180 if (maybe_expand_insn (icode
, 4, ops
))
1182 last
= get_last_insn ();
1183 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1185 && any_condjump_p (last
)
1186 && !find_reg_note (last
, REG_BR_PROB
, 0))
1187 add_reg_br_prob_note (last
,
1188 profile_probability::very_unlikely ());
1189 emit_jump (done_label
);
1190 goto do_error_label
;
1193 delete_insns_since (last
);
1196 /* Compute the operation. On RTL level, the addition is always
1198 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1199 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1201 /* For PLUS_EXPR, the operation is commutative, so we can pick
1202 operand to compare against. For prec <= BITS_PER_WORD, I think
1203 preferring REG operand is better over CONST_INT, because
1204 the CONST_INT might enlarge the instruction or CSE would need
1205 to figure out we'd already loaded it into a register before.
1206 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1207 as then the multi-word comparison can be perhaps simplified. */
1208 if (code
== PLUS_EXPR
1209 && (prec
<= BITS_PER_WORD
1210 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1211 : CONST_SCALAR_INT_P (op1
)))
1213 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1214 true, mode
, NULL_RTX
, NULL
, done_label
,
1215 profile_probability::very_likely ());
1216 goto do_error_label
;
1219 /* s1 +- u2 -> sr */
1220 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1222 /* Compute the operation. On RTL level, the addition is always
1224 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1225 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1226 rtx tem
= expand_binop (mode
, add_optab
,
1227 code
== PLUS_EXPR
? res
: op0
, sgn
,
1228 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1229 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1230 done_label
, profile_probability::very_likely ());
1231 goto do_error_label
;
1235 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1237 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1239 /* As we've changed op1, we have to avoid using the value range
1240 for the original argument. */
1241 arg1
= error_mark_node
;
1247 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1249 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1251 /* As we've changed op0, we have to avoid using the value range
1252 for the original argument. */
1253 arg0
= error_mark_node
;
1259 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1261 /* Compute the operation. On RTL level, the addition is always
1263 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1265 int pos_neg
= get_range_pos_neg (arg0
);
1267 /* If ARG0 is known to be always negative, this is always overflow. */
1268 emit_jump (do_error
);
1269 else if (pos_neg
== 3)
1270 /* If ARG0 is not known to be always positive, check at runtime. */
1271 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1272 NULL
, do_error
, profile_probability::very_unlikely ());
1273 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1274 done_label
, profile_probability::very_likely ());
1275 goto do_error_label
;
1279 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1281 /* Compute the operation. On RTL level, the addition is always
1283 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1285 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1287 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1288 done_label
, profile_probability::very_likely ());
1289 goto do_error_label
;
1293 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1295 /* Compute the operation. On RTL level, the addition is always
1297 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1299 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1300 NULL
, do_error
, profile_probability::very_unlikely ());
1302 /* The operation is commutative, so we can pick operand to compare
1303 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1304 is better over CONST_INT, because the CONST_INT might enlarge the
1305 instruction or CSE would need to figure out we'd already loaded it
1306 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1307 might be more beneficial, as then the multi-word comparison can be
1308 perhaps simplified. */
1309 if (prec
<= BITS_PER_WORD
1310 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1311 : CONST_SCALAR_INT_P (op0
))
1313 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1314 done_label
, profile_probability::very_likely ());
1315 goto do_error_label
;
1318 /* s1 +- s2 -> ur */
1319 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1321 /* Compute the operation. On RTL level, the addition is always
1323 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1324 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1325 int pos_neg
= get_range_pos_neg (arg1
);
1326 if (code
== PLUS_EXPR
)
1328 int pos_neg0
= get_range_pos_neg (arg0
);
1329 if (pos_neg0
!= 3 && pos_neg
== 3)
1331 std::swap (op0
, op1
);
1338 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1339 ? and_optab
: ior_optab
,
1340 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1341 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1342 NULL
, done_label
, profile_probability::very_likely ());
1346 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1347 do_compare_rtx_and_jump (op1
, const0_rtx
,
1348 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1349 NULL_RTX
, NULL
, do_ior_label
,
1350 profile_probability::even ());
1351 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1353 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1354 NULL
, done_label
, profile_probability::very_likely ());
1355 emit_jump (do_error
);
1356 emit_label (do_ior_label
);
1357 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1359 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1360 NULL
, done_label
, profile_probability::very_likely ());
1362 goto do_error_label
;
1366 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1368 /* Compute the operation. On RTL level, the addition is always
1370 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1372 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1373 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1374 op0_geu_op1
, profile_probability::even ());
1375 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1376 NULL
, done_label
, profile_probability::very_likely ());
1377 emit_jump (do_error
);
1378 emit_label (op0_geu_op1
);
1379 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1380 NULL
, done_label
, profile_probability::very_likely ());
1381 goto do_error_label
;
1384 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1386 /* s1 +- s2 -> sr */
1389 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1390 : subv4_optab
, mode
);
1391 if (icode
!= CODE_FOR_nothing
)
1393 class expand_operand ops
[4];
1394 rtx_insn
*last
= get_last_insn ();
1396 res
= gen_reg_rtx (mode
);
1397 create_output_operand (&ops
[0], res
, mode
);
1398 create_input_operand (&ops
[1], op0
, mode
);
1399 create_input_operand (&ops
[2], op1
, mode
);
1400 create_fixed_operand (&ops
[3], do_error
);
1401 if (maybe_expand_insn (icode
, 4, ops
))
1403 last
= get_last_insn ();
1404 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1406 && any_condjump_p (last
)
1407 && !find_reg_note (last
, REG_BR_PROB
, 0))
1408 add_reg_br_prob_note (last
,
1409 profile_probability::very_unlikely ());
1410 emit_jump (done_label
);
1411 goto do_error_label
;
1414 delete_insns_since (last
);
1417 /* Compute the operation. On RTL level, the addition is always
1419 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1420 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1422 /* If we can prove that one of the arguments (for MINUS_EXPR only
1423 the second operand, as subtraction is not commutative) is always
1424 non-negative or always negative, we can do just one comparison
1425 and conditional jump. */
1426 int pos_neg
= get_range_pos_neg (arg1
);
1427 if (code
== PLUS_EXPR
)
1429 int pos_neg0
= get_range_pos_neg (arg0
);
1430 if (pos_neg0
!= 3 && pos_neg
== 3)
1432 std::swap (op0
, op1
);
1437 /* Addition overflows if and only if the two operands have the same sign,
1438 and the result has the opposite sign. Subtraction overflows if and
1439 only if the two operands have opposite sign, and the subtrahend has
1440 the same sign as the result. Here 0 is counted as positive. */
1443 /* Compute op0 ^ op1 (operands have opposite sign). */
1444 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1447 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1448 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1452 if (code
== PLUS_EXPR
)
1454 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1455 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1456 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1461 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1462 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1463 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1467 /* No overflow if the result has bit sign cleared. */
1468 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1469 NULL
, done_label
, profile_probability::very_likely ());
1472 /* Compare the result of the operation with the first operand.
1473 No overflow for addition if second operand is positive and result
1474 is larger or second operand is negative and result is smaller.
1475 Likewise for subtraction with sign of second operand flipped. */
1477 do_compare_rtx_and_jump (res
, op0
,
1478 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1479 false, mode
, NULL_RTX
, NULL
, done_label
,
1480 profile_probability::very_likely ());
1484 emit_label (do_error
);
1487 /* Expand the ubsan builtin call. */
1489 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1493 do_pending_stack_adjust ();
1496 expand_arith_set_overflow (lhs
, target
);
1499 emit_label (done_label
);
1504 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1508 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1511 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1516 /* Add negate overflow checking to the statement STMT. */
1519 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1524 rtx_code_label
*done_label
, *do_error
;
1525 rtx target
= NULL_RTX
;
1527 done_label
= gen_label_rtx ();
1528 do_error
= gen_label_rtx ();
1530 do_pending_stack_adjust ();
1531 op1
= expand_normal (arg1
);
1533 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1536 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1538 write_complex_part (target
, const0_rtx
, true, false);
1541 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1542 if (icode
!= CODE_FOR_nothing
)
1544 class expand_operand ops
[3];
1545 rtx_insn
*last
= get_last_insn ();
1547 res
= gen_reg_rtx (mode
);
1548 create_output_operand (&ops
[0], res
, mode
);
1549 create_input_operand (&ops
[1], op1
, mode
);
1550 create_fixed_operand (&ops
[2], do_error
);
1551 if (maybe_expand_insn (icode
, 3, ops
))
1553 last
= get_last_insn ();
1554 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1556 && any_condjump_p (last
)
1557 && !find_reg_note (last
, REG_BR_PROB
, 0))
1558 add_reg_br_prob_note (last
,
1559 profile_probability::very_unlikely ());
1560 emit_jump (done_label
);
1564 delete_insns_since (last
);
1565 icode
= CODE_FOR_nothing
;
1569 if (icode
== CODE_FOR_nothing
)
1571 /* Compute the operation. On RTL level, the addition is always
1573 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1575 /* Compare the operand with the most negative value. */
1576 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1577 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1578 done_label
, profile_probability::very_likely ());
1581 emit_label (do_error
);
1584 /* Expand the ubsan builtin call. */
1586 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1587 arg1
, NULL_TREE
, datap
);
1590 do_pending_stack_adjust ();
1593 expand_arith_set_overflow (lhs
, target
);
1596 emit_label (done_label
);
1601 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1603 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1607 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1608 mode MODE can be expanded without using a libcall. */
1611 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1612 rtx op0
, rtx op1
, bool uns
)
1614 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1615 != CODE_FOR_nothing
)
1618 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1619 != CODE_FOR_nothing
)
1622 rtx_insn
*last
= get_last_insn ();
1623 if (CONSTANT_P (op0
))
1624 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1626 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1627 if (CONSTANT_P (op1
))
1628 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1630 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1631 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1632 delete_insns_since (last
);
1633 return ret
!= NULL_RTX
;
1636 /* Add mul overflow checking to the statement STMT. */
1639 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1640 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1645 rtx_code_label
*done_label
, *do_error
;
1646 rtx target
= NULL_RTX
;
1648 enum insn_code icode
;
1649 int save_flag_trapv
= flag_trapv
;
1651 /* We don't want any __mulv?i3 etc. calls from the expansion of
1652 these internal functions, so disable -ftrapv temporarily. */
1654 done_label
= gen_label_rtx ();
1655 do_error
= gen_label_rtx ();
1657 do_pending_stack_adjust ();
1658 op0
= expand_normal (arg0
);
1659 op1
= expand_normal (arg1
);
1661 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1665 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1667 write_complex_part (target
, const0_rtx
, true, false);
1671 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1673 /* We assume both operands and result have the same precision
1674 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1675 with that precision, U for unsigned type with that precision,
1676 sgn for unsigned most significant bit in that precision.
1677 s1 is signed first operand, u1 is unsigned first operand,
1678 s2 is signed second operand, u2 is unsigned second operand,
1679 sr is signed result, ur is unsigned result and the following
1680 rules say how to compute result (which is always result of
1681 the operands as if both were unsigned, cast to the right
1682 signedness) and how to compute whether operation overflowed.
1683 main_ovf (false) stands for jump on signed multiplication
1684 overflow or the main algorithm with uns == false.
1685 main_ovf (true) stands for jump on unsigned multiplication
1686 overflow or the main algorithm with uns == true.
1689 res = (S) ((U) s1 * (U) s2)
1690 ovf = main_ovf (false)
1693 ovf = main_ovf (true)
1696 ovf = (s1 < 0 && u2) || main_ovf (true)
1699 ovf = res < 0 || main_ovf (true)
1701 res = (S) ((U) s1 * u2)
1702 ovf = (S) u2 >= 0 ? main_ovf (false)
1703 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1705 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1706 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1708 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1710 if (uns0_p
&& !uns1_p
)
1712 /* Multiplication is commutative, if operand signedness differs,
1713 canonicalize to the first operand being signed and second
1714 unsigned to simplify following code. */
1715 std::swap (op0
, op1
);
1716 std::swap (arg0
, arg1
);
1721 int pos_neg0
= get_range_pos_neg (arg0
);
1722 int pos_neg1
= get_range_pos_neg (arg1
);
1723 /* Unsigned types with smaller than mode precision, even if they have most
1724 significant bit set, are still zero-extended. */
1725 if (uns0_p
&& TYPE_PRECISION (TREE_TYPE (arg0
)) < GET_MODE_PRECISION (mode
))
1727 if (uns1_p
&& TYPE_PRECISION (TREE_TYPE (arg1
)) < GET_MODE_PRECISION (mode
))
1731 if (!uns0_p
&& uns1_p
&& unsr_p
)
1736 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1739 /* If s1 is negative, avoid the main code, just multiply and
1740 signal overflow if op1 is not 0. */
1741 struct separate_ops ops
;
1742 ops
.code
= MULT_EXPR
;
1743 ops
.type
= TREE_TYPE (arg1
);
1744 ops
.op0
= make_tree (ops
.type
, op0
);
1745 ops
.op1
= make_tree (ops
.type
, op1
);
1746 ops
.op2
= NULL_TREE
;
1748 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1749 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1750 NULL
, done_label
, profile_probability::very_likely ());
1751 goto do_error_label
;
1753 if (get_min_precision (arg1
, UNSIGNED
)
1754 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1756 /* If the first operand is sign extended from narrower type, the
1757 second operand is zero extended from narrower type and
1758 the sum of the two precisions is smaller or equal to the
1759 result precision: if the first argument is at runtime
1760 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1761 and there will be no overflow, if the first argument is
1762 negative and the second argument zero, the result will be
1763 0 and there will be no overflow, if the first argument is
1764 negative and the second argument positive, the result when
1765 treated as signed will be negative (minimum -0x7f80 or
1766 -0x7f..f80..0) there will be always overflow. So, do
1768 ovf = (S) res < 0 */
1769 struct separate_ops ops
;
1770 ops
.code
= MULT_EXPR
;
1772 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1774 ops
.op0
= make_tree (ops
.type
, op0
);
1775 ops
.op1
= make_tree (ops
.type
, op1
);
1776 ops
.op2
= NULL_TREE
;
1778 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1779 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1780 mode
, NULL_RTX
, NULL
, done_label
,
1781 profile_probability::very_likely ());
1782 goto do_error_label
;
1784 rtx_code_label
*do_main_label
;
1785 do_main_label
= gen_label_rtx ();
1786 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1787 NULL
, do_main_label
, profile_probability::very_likely ());
1788 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1789 NULL
, do_main_label
, profile_probability::very_likely ());
1790 expand_arith_set_overflow (lhs
, target
);
1791 emit_label (do_main_label
);
1799 if (uns0_p
&& uns1_p
&& !unsr_p
)
1801 if ((pos_neg0
| pos_neg1
) == 1)
1803 /* If both arguments are zero extended from narrower types,
1804 the MSB will be clear on both and so we can pretend it is
1805 a normal s1 * s2 -> sr multiplication. */
1811 /* Rest of handling of this case after res is computed. */
1816 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1823 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1824 avoid the main code, just multiply and signal overflow
1825 unless 0 * u2 or -1 * ((U) Smin). */
1826 struct separate_ops ops
;
1827 ops
.code
= MULT_EXPR
;
1828 ops
.type
= TREE_TYPE (arg1
);
1829 ops
.op0
= make_tree (ops
.type
, op0
);
1830 ops
.op1
= make_tree (ops
.type
, op1
);
1831 ops
.op2
= NULL_TREE
;
1833 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1834 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1835 NULL
, done_label
, profile_probability::very_likely ());
1836 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1837 NULL
, do_error
, profile_probability::very_unlikely ());
1839 prec
= GET_MODE_PRECISION (mode
);
1841 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1842 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1843 NULL
, done_label
, profile_probability::very_likely ());
1844 goto do_error_label
;
1846 /* Rest of handling of this case after res is computed. */
1854 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1857 switch (pos_neg0
| pos_neg1
)
1859 case 1: /* Both operands known to be non-negative. */
1861 case 2: /* Both operands known to be negative. */
1862 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1863 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1864 /* Avoid looking at arg0/arg1 ranges, as we've changed
1866 arg0
= error_mark_node
;
1867 arg1
= error_mark_node
;
1870 if ((pos_neg0
^ pos_neg1
) == 3)
1872 /* If one operand is known to be negative and the other
1873 non-negative, this overflows always, unless the non-negative
1874 one is 0. Just do normal multiply and set overflow
1875 unless one of the operands is 0. */
1876 struct separate_ops ops
;
1877 ops
.code
= MULT_EXPR
;
1879 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1881 ops
.op0
= make_tree (ops
.type
, op0
);
1882 ops
.op1
= make_tree (ops
.type
, op1
);
1883 ops
.op2
= NULL_TREE
;
1885 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1886 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1887 true, mode
, NULL_RTX
, NULL
, done_label
,
1888 profile_probability::very_likely ());
1889 goto do_error_label
;
1891 if (get_min_precision (arg0
, SIGNED
)
1892 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1894 /* If both operands are sign extended from narrower types and
1895 the sum of the two precisions is smaller or equal to the
1896 result precision: if both arguments are at runtime
1897 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1898 and there will be no overflow, if both arguments are negative,
1899 maximum result will be 0x40..00 and there will be no overflow
1900 either, if one argument is positive and the other argument
1901 negative, the result when treated as signed will be negative
1902 and there will be always overflow, and if one argument is
1903 zero and the other negative the result will be zero and no
1906 ovf = (S) res < 0 */
1907 struct separate_ops ops
;
1908 ops
.code
= MULT_EXPR
;
1910 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1912 ops
.op0
= make_tree (ops
.type
, op0
);
1913 ops
.op1
= make_tree (ops
.type
, op1
);
1914 ops
.op2
= NULL_TREE
;
1916 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1917 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1918 mode
, NULL_RTX
, NULL
, done_label
,
1919 profile_probability::very_likely ());
1920 goto do_error_label
;
1922 /* The general case, do all the needed comparisons at runtime. */
1923 rtx_code_label
*do_main_label
, *after_negate_label
;
1925 rop0
= gen_reg_rtx (mode
);
1926 rop1
= gen_reg_rtx (mode
);
1927 emit_move_insn (rop0
, op0
);
1928 emit_move_insn (rop1
, op1
);
1931 do_main_label
= gen_label_rtx ();
1932 after_negate_label
= gen_label_rtx ();
1933 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1935 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1936 NULL
, after_negate_label
, profile_probability::very_likely ());
1937 /* Both arguments negative here, negate them and continue with
1938 normal unsigned overflow checking multiplication. */
1939 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1941 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1943 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1945 arg0
= error_mark_node
;
1946 arg1
= error_mark_node
;
1947 emit_jump (do_main_label
);
1948 emit_label (after_negate_label
);
1949 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1951 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1952 NULL
, do_main_label
,
1953 profile_probability::very_likely ());
1954 /* One argument is negative here, the other positive. This
1955 overflows always, unless one of the arguments is 0. But
1956 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1957 is, thus we can keep do_main code oring in overflow as is. */
1959 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1960 NULL
, do_main_label
,
1961 profile_probability::very_unlikely ());
1963 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1964 NULL
, do_main_label
,
1965 profile_probability::very_unlikely ());
1966 expand_arith_set_overflow (lhs
, target
);
1967 emit_label (do_main_label
);
1975 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1976 sign
= uns
? UNSIGNED
: SIGNED
;
1977 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1979 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1980 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1982 /* Optimize unsigned multiplication by power of 2 constant
1983 using 2 shifts, one for result, one to extract the shifted
1984 out bits to see if they are all zero.
1985 Don't do this if optimizing for size and we have umulv4_optab,
1986 in that case assume multiplication will be shorter.
1987 This is heuristics based on the single target that provides
1988 umulv4 right now (i?86/x86_64), if further targets add it, this
1989 might need to be revisited.
1990 Cases where both operands are constant should be folded already
1991 during GIMPLE, and cases where one operand is constant but not
1992 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1993 below can be done without multiplication, just by shifts and adds,
1994 or we'd need to divide the result (and hope it actually doesn't
1995 really divide nor multiply) and compare the result of the division
1996 with the original operand. */
2001 if (integer_pow2p (arg0
))
2003 std::swap (opn0
, opn1
);
2004 std::swap (argn0
, argn1
);
2006 int cnt
= tree_log2 (argn1
);
2007 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
2009 rtx upper
= const0_rtx
;
2010 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
2012 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
2013 GET_MODE_PRECISION (mode
) - cnt
,
2015 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
2016 NULL_RTX
, NULL
, done_label
,
2017 profile_probability::very_likely ());
2018 goto do_error_label
;
2021 if (icode
!= CODE_FOR_nothing
)
2023 class expand_operand ops
[4];
2024 rtx_insn
*last
= get_last_insn ();
2026 res
= gen_reg_rtx (mode
);
2027 create_output_operand (&ops
[0], res
, mode
);
2028 create_input_operand (&ops
[1], op0
, mode
);
2029 create_input_operand (&ops
[2], op1
, mode
);
2030 create_fixed_operand (&ops
[3], do_error
);
2031 if (maybe_expand_insn (icode
, 4, ops
))
2033 last
= get_last_insn ();
2034 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
2036 && any_condjump_p (last
)
2037 && !find_reg_note (last
, REG_BR_PROB
, 0))
2038 add_reg_br_prob_note (last
,
2039 profile_probability::very_unlikely ());
2040 emit_jump (done_label
);
2044 delete_insns_since (last
);
2045 icode
= CODE_FOR_nothing
;
2049 if (icode
== CODE_FOR_nothing
)
2051 struct separate_ops ops
;
2052 int prec
= GET_MODE_PRECISION (mode
);
2053 scalar_int_mode hmode
, wmode
;
2054 ops
.op0
= make_tree (type
, op0
);
2055 ops
.op1
= make_tree (type
, op1
);
2056 ops
.op2
= NULL_TREE
;
2059 /* Optimize unsigned overflow check where we don't use the
2060 multiplication result, just whether overflow happened.
2061 If we can do MULT_HIGHPART_EXPR, that followed by
2062 comparison of the result against zero is cheapest.
2063 We'll still compute res, but it should be DCEd later. */
2069 && !(uns0_p
&& uns1_p
&& !unsr_p
)
2070 && can_mult_highpart_p (mode
, uns
) == 1
2071 && single_imm_use (lhs
, &use
, &use_stmt
)
2072 && is_gimple_assign (use_stmt
)
2073 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
2076 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2077 && targetm
.scalar_mode_supported_p (wmode
)
2078 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
2081 ops
.code
= WIDEN_MULT_EXPR
;
2083 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
2085 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
2086 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
2088 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
2089 res
= convert_modes (mode
, wmode
, res
, uns
);
2091 /* For the unsigned multiplication, there was overflow if
2092 HIPART is non-zero. */
2093 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2094 NULL_RTX
, NULL
, done_label
,
2095 profile_probability::very_likely ());
2098 /* RES is used more than once, place it in a pseudo. */
2099 res
= force_reg (mode
, res
);
2101 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2103 /* RES is low half of the double width result, HIPART
2104 the high half. There was overflow if
2105 HIPART is different from RES < 0 ? -1 : 0. */
2106 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2107 NULL_RTX
, NULL
, done_label
,
2108 profile_probability::very_likely ());
2111 else if (can_mult_highpart_p (mode
, uns
) == 1)
2114 ops
.code
= MULT_HIGHPART_EXPR
;
2117 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
2119 ops
.code
= MULT_EXPR
;
2120 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2122 /* For the unsigned multiplication, there was overflow if
2123 HIPART is non-zero. */
2124 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2125 NULL_RTX
, NULL
, done_label
,
2126 profile_probability::very_likely ());
2129 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2131 /* RES is low half of the double width result, HIPART
2132 the high half. There was overflow if
2133 HIPART is different from RES < 0 ? -1 : 0. */
2134 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2135 NULL_RTX
, NULL
, done_label
,
2136 profile_probability::very_likely ());
2140 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
2141 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
2143 rtx_code_label
*large_op0
= gen_label_rtx ();
2144 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
2145 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
2146 rtx_code_label
*both_ops_large
= gen_label_rtx ();
2147 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
2148 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
2149 rtx_code_label
*do_overflow
= gen_label_rtx ();
2150 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
2152 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
2153 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
2155 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
2156 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
2157 rtx signbit0
= const0_rtx
;
2159 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
2161 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
2163 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
2164 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
2165 rtx signbit1
= const0_rtx
;
2167 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
2170 res
= gen_reg_rtx (mode
);
2172 /* True if op0 resp. op1 are known to be in the range of
2174 bool op0_small_p
= false;
2175 bool op1_small_p
= false;
2176 /* True if op0 resp. op1 are known to have all zeros or all ones
2177 in the upper half of bits, but are not known to be
2179 bool op0_medium_p
= false;
2180 bool op1_medium_p
= false;
2181 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2182 nonnegative, 1 if unknown. */
2188 else if (pos_neg0
== 2)
2192 else if (pos_neg1
== 2)
2195 unsigned int mprec0
= prec
;
2196 if (arg0
!= error_mark_node
)
2197 mprec0
= get_min_precision (arg0
, sign
);
2198 if (mprec0
<= hprec
)
2200 else if (!uns
&& mprec0
<= hprec
+ 1)
2201 op0_medium_p
= true;
2202 unsigned int mprec1
= prec
;
2203 if (arg1
!= error_mark_node
)
2204 mprec1
= get_min_precision (arg1
, sign
);
2205 if (mprec1
<= hprec
)
2207 else if (!uns
&& mprec1
<= hprec
+ 1)
2208 op1_medium_p
= true;
2210 int smaller_sign
= 1;
2211 int larger_sign
= 1;
2214 smaller_sign
= op0_sign
;
2215 larger_sign
= op1_sign
;
2217 else if (op1_small_p
)
2219 smaller_sign
= op1_sign
;
2220 larger_sign
= op0_sign
;
2222 else if (op0_sign
== op1_sign
)
2224 smaller_sign
= op0_sign
;
2225 larger_sign
= op0_sign
;
2229 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2230 NULL_RTX
, NULL
, large_op0
,
2231 profile_probability::unlikely ());
2234 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2235 NULL_RTX
, NULL
, small_op0_large_op1
,
2236 profile_probability::unlikely ());
2238 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2239 hmode to mode, the multiplication will never overflow. We can
2240 do just one hmode x hmode => mode widening multiplication. */
2241 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2242 ops
.op0
= make_tree (halfstype
, lopart0
);
2243 ops
.op1
= make_tree (halfstype
, lopart1
);
2244 ops
.code
= WIDEN_MULT_EXPR
;
2247 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2248 emit_move_insn (res
, thisres
);
2249 emit_jump (done_label
);
2251 emit_label (small_op0_large_op1
);
2253 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2254 but op1 is not, just swap the arguments and handle it as op1
2255 sign/zero extended, op0 not. */
2256 rtx larger
= gen_reg_rtx (mode
);
2257 rtx hipart
= gen_reg_rtx (hmode
);
2258 rtx lopart
= gen_reg_rtx (hmode
);
2259 emit_move_insn (larger
, op1
);
2260 emit_move_insn (hipart
, hipart1
);
2261 emit_move_insn (lopart
, lopart0
);
2262 emit_jump (one_small_one_large
);
2264 emit_label (large_op0
);
2267 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2268 NULL_RTX
, NULL
, both_ops_large
,
2269 profile_probability::unlikely ());
2271 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2272 but op0 is not, prepare larger, hipart and lopart pseudos and
2273 handle it together with small_op0_large_op1. */
2274 emit_move_insn (larger
, op0
);
2275 emit_move_insn (hipart
, hipart0
);
2276 emit_move_insn (lopart
, lopart1
);
2278 emit_label (one_small_one_large
);
2280 /* lopart is the low part of the operand that is sign extended
2281 to mode, larger is the other operand, hipart is the
2282 high part of larger and lopart0 and lopart1 are the low parts
2284 We perform lopart0 * lopart1 and lopart * hipart widening
2286 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2287 ops
.op0
= make_tree (halfutype
, lopart0
);
2288 ops
.op1
= make_tree (halfutype
, lopart1
);
2290 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2292 ops
.op0
= make_tree (halfutype
, lopart
);
2293 ops
.op1
= make_tree (halfutype
, hipart
);
2294 rtx loxhi
= gen_reg_rtx (mode
);
2295 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2296 emit_move_insn (loxhi
, tem
);
2300 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2301 if (larger_sign
== 0)
2302 emit_jump (after_hipart_neg
);
2303 else if (larger_sign
!= -1)
2304 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2305 NULL_RTX
, NULL
, after_hipart_neg
,
2306 profile_probability::even ());
2308 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2309 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2310 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2312 emit_move_insn (loxhi
, tem
);
2314 emit_label (after_hipart_neg
);
2316 /* if (lopart < 0) loxhi -= larger; */
2317 if (smaller_sign
== 0)
2318 emit_jump (after_lopart_neg
);
2319 else if (smaller_sign
!= -1)
2320 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2321 NULL_RTX
, NULL
, after_lopart_neg
,
2322 profile_probability::even ());
2324 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2326 emit_move_insn (loxhi
, tem
);
2328 emit_label (after_lopart_neg
);
2331 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2332 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2333 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2335 emit_move_insn (loxhi
, tem
);
2337 /* if (loxhi >> (bitsize / 2)
2338 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2339 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2340 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2342 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2343 rtx signbitloxhi
= const0_rtx
;
2345 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2346 convert_modes (hmode
, mode
,
2348 hprec
- 1, NULL_RTX
, 0);
2350 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2351 NULL_RTX
, NULL
, do_overflow
,
2352 profile_probability::very_unlikely ());
2354 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2355 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2357 tem
= convert_modes (mode
, hmode
,
2358 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2360 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2363 emit_move_insn (res
, tem
);
2364 emit_jump (done_label
);
2366 emit_label (both_ops_large
);
2368 /* If both operands are large (not sign (!uns) or zero (uns)
2369 extended from hmode), then perform the full multiplication
2370 which will be the result of the operation.
2371 The only cases which don't overflow are for signed multiplication
2372 some cases where both hipart0 and highpart1 are 0 or -1.
2373 For unsigned multiplication when high parts are both non-zero
2374 this overflows always. */
2375 ops
.code
= MULT_EXPR
;
2376 ops
.op0
= make_tree (type
, op0
);
2377 ops
.op1
= make_tree (type
, op1
);
2378 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2379 emit_move_insn (res
, tem
);
2385 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2386 NULL_RTX
, 1, OPTAB_WIDEN
);
2387 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2388 NULL_RTX
, NULL
, do_error
,
2389 profile_probability::very_unlikely ());
2394 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2395 NULL_RTX
, 1, OPTAB_WIDEN
);
2396 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2397 NULL_RTX
, NULL
, do_error
,
2398 profile_probability::very_unlikely ());
2401 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2402 the same, overflow happened if res is non-positive, if they
2403 are different, overflow happened if res is positive. */
2404 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2405 emit_jump (hipart_different
);
2406 else if (op0_sign
== 1 || op1_sign
== 1)
2407 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2408 NULL_RTX
, NULL
, hipart_different
,
2409 profile_probability::even ());
2411 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2412 NULL_RTX
, NULL
, do_error
,
2413 profile_probability::very_unlikely ());
2414 emit_jump (done_label
);
2416 emit_label (hipart_different
);
2418 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2419 NULL_RTX
, NULL
, do_error
,
2420 profile_probability::very_unlikely ());
2421 emit_jump (done_label
);
2424 emit_label (do_overflow
);
2426 /* Overflow, do full multiplication and fallthru into do_error. */
2427 ops
.op0
= make_tree (type
, op0
);
2428 ops
.op1
= make_tree (type
, op1
);
2429 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2430 emit_move_insn (res
, tem
);
2432 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2433 && targetm
.scalar_mode_supported_p (wmode
))
2434 /* Even emitting a libcall is better than not detecting overflow
2439 gcc_assert (!is_ubsan
);
2440 ops
.code
= MULT_EXPR
;
2442 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2443 emit_jump (done_label
);
2448 emit_label (do_error
);
2451 /* Expand the ubsan builtin call. */
2453 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2457 do_pending_stack_adjust ();
2460 expand_arith_set_overflow (lhs
, target
);
2463 emit_label (done_label
);
2466 if (uns0_p
&& uns1_p
&& !unsr_p
)
2468 rtx_code_label
*all_done_label
= gen_label_rtx ();
2469 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2470 NULL
, all_done_label
, profile_probability::very_likely ());
2471 expand_arith_set_overflow (lhs
, target
);
2472 emit_label (all_done_label
);
2476 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2478 rtx_code_label
*all_done_label
= gen_label_rtx ();
2479 rtx_code_label
*set_noovf
= gen_label_rtx ();
2480 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2481 NULL
, all_done_label
, profile_probability::very_likely ());
2482 expand_arith_set_overflow (lhs
, target
);
2483 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2484 NULL
, set_noovf
, profile_probability::very_likely ());
2485 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2486 NULL
, all_done_label
, profile_probability::very_unlikely ());
2487 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2488 all_done_label
, profile_probability::very_unlikely ());
2489 emit_label (set_noovf
);
2490 write_complex_part (target
, const0_rtx
, true, false);
2491 emit_label (all_done_label
);
2497 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
2499 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2501 flag_trapv
= save_flag_trapv
;
2504 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2507 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2508 tree arg0
, tree arg1
)
2510 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2511 rtx_code_label
*loop_lab
= NULL
;
2512 rtx cntvar
= NULL_RTX
;
2513 tree cntv
= NULL_TREE
;
2514 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2515 tree sz
= TYPE_SIZE (eltype
);
2516 tree data
= NULL_TREE
;
2517 tree resv
= NULL_TREE
;
2518 rtx lhsr
= NULL_RTX
;
2519 rtx resvr
= NULL_RTX
;
2520 unsigned HOST_WIDE_INT const_cnt
= 0;
2521 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2522 int save_flag_trapv
= flag_trapv
;
2524 /* We don't want any __mulv?i3 etc. calls from the expansion of
2525 these internal functions, so disable -ftrapv temporarily. */
2530 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2531 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2532 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2533 optab_default
)) == unknown_optab
2534 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2535 == CODE_FOR_nothing
))
2538 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2541 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2542 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2548 do_pending_stack_adjust ();
2549 loop_lab
= gen_label_rtx ();
2550 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2551 cntv
= make_tree (sizetype
, cntvar
);
2552 emit_move_insn (cntvar
, const0_rtx
);
2553 emit_label (loop_lab
);
2555 if (TREE_CODE (arg0
) != VECTOR_CST
)
2557 rtx arg0r
= expand_normal (arg0
);
2558 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2560 if (TREE_CODE (arg1
) != VECTOR_CST
)
2562 rtx arg1r
= expand_normal (arg1
);
2563 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2565 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2567 tree op0
, op1
, res
= NULL_TREE
;
2570 tree atype
= build_array_type_nelts (eltype
, cnt
);
2571 op0
= uniform_vector_p (arg0
);
2572 if (op0
== NULL_TREE
)
2574 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2575 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2576 NULL_TREE
, NULL_TREE
);
2578 op1
= uniform_vector_p (arg1
);
2579 if (op1
== NULL_TREE
)
2581 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2582 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2583 NULL_TREE
, NULL_TREE
);
2587 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2588 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2589 NULL_TREE
, NULL_TREE
);
2594 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2595 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2596 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2598 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2604 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2605 false, false, false, true, &data
);
2608 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2609 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2611 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2612 false, false, false, true, &data
);
2615 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2624 struct separate_ops ops
;
2625 ops
.code
= PLUS_EXPR
;
2626 ops
.type
= TREE_TYPE (cntv
);
2628 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2629 ops
.op2
= NULL_TREE
;
2631 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2634 emit_move_insn (cntvar
, ret
);
2635 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2636 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2637 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2638 profile_probability::very_likely ());
2640 if (lhs
&& resv
== NULL_TREE
)
2642 struct separate_ops ops
;
2644 ops
.type
= TREE_TYPE (arg0
);
2647 ops
.op2
= NULL_TREE
;
2649 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2652 emit_move_insn (lhsr
, ret
);
2655 emit_move_insn (lhsr
, resvr
);
2656 flag_trapv
= save_flag_trapv
;
2659 /* Expand UBSAN_CHECK_ADD call STMT. */
2662 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2664 location_t loc
= gimple_location (stmt
);
2665 tree lhs
= gimple_call_lhs (stmt
);
2666 tree arg0
= gimple_call_arg (stmt
, 0);
2667 tree arg1
= gimple_call_arg (stmt
, 1);
2668 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2669 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2671 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2672 false, false, false, true, NULL
);
2675 /* Expand UBSAN_CHECK_SUB call STMT. */
2678 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2680 location_t loc
= gimple_location (stmt
);
2681 tree lhs
= gimple_call_lhs (stmt
);
2682 tree arg0
= gimple_call_arg (stmt
, 0);
2683 tree arg1
= gimple_call_arg (stmt
, 1);
2684 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2685 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2686 else if (integer_zerop (arg0
))
2687 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2689 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2690 false, false, false, true, NULL
);
2693 /* Expand UBSAN_CHECK_MUL call STMT. */
2696 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2698 location_t loc
= gimple_location (stmt
);
2699 tree lhs
= gimple_call_lhs (stmt
);
2700 tree arg0
= gimple_call_arg (stmt
, 0);
2701 tree arg1
= gimple_call_arg (stmt
, 1);
2702 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2703 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2705 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2709 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2712 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2714 tree lhs
= gimple_call_lhs (stmt
);
2715 if (lhs
== NULL_TREE
)
2717 tree arg0
= gimple_call_arg (stmt
, 0);
2718 tree arg1
= gimple_call_arg (stmt
, 1);
2719 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2720 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2721 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2722 int unsr_p
= TYPE_UNSIGNED (type
);
2723 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2724 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2725 int precres
= TYPE_PRECISION (type
);
2726 location_t loc
= gimple_location (stmt
);
2727 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2729 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2731 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2732 prec0
= MIN (prec0
, pr
);
2733 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2734 prec1
= MIN (prec1
, pr
);
2735 int save_flag_trapv
= flag_trapv
;
2737 /* We don't want any __mulv?i3 etc. calls from the expansion of
2738 these internal functions, so disable -ftrapv temporarily. */
2740 /* If uns0_p && uns1_p, precop is minimum needed precision
2741 of unsigned type to hold the exact result, otherwise
2742 precop is minimum needed precision of signed type to
2743 hold the exact result. */
2745 if (code
== MULT_EXPR
)
2746 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2749 if (uns0_p
== uns1_p
)
2750 precop
= MAX (prec0
, prec1
) + 1;
2752 precop
= MAX (prec0
+ 1, prec1
) + 1;
2754 precop
= MAX (prec0
, prec1
+ 1) + 1;
2756 int orig_precres
= precres
;
2760 if ((uns0_p
&& uns1_p
)
2761 ? ((precop
+ !unsr_p
) <= precres
2762 /* u1 - u2 -> ur can overflow, no matter what precision
2764 && (code
!= MINUS_EXPR
|| !unsr_p
))
2765 : (!unsr_p
&& precop
<= precres
))
2767 /* The infinity precision result will always fit into result. */
2768 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2769 write_complex_part (target
, const0_rtx
, true, false);
2770 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2771 struct separate_ops ops
;
2774 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2775 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2776 ops
.op2
= NULL_TREE
;
2778 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2779 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2780 flag_trapv
= save_flag_trapv
;
2784 /* For operations with low precision, if target doesn't have them, start
2785 with precres widening right away, otherwise do it only if the most
2786 simple cases can't be used. */
2787 const int min_precision
= targetm
.min_arithmetic_precision ();
2788 if (orig_precres
== precres
&& precres
< min_precision
)
2790 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2791 && prec1
<= precres
)
2792 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2793 && prec0
+ uns0_p
<= precres
2794 && prec1
+ uns1_p
<= precres
))
2796 arg0
= fold_convert_loc (loc
, type
, arg0
);
2797 arg1
= fold_convert_loc (loc
, type
, arg1
);
2801 if (integer_zerop (arg0
) && !unsr_p
)
2803 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2804 flag_trapv
= save_flag_trapv
;
2809 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2810 unsr_p
, unsr_p
, false, NULL
);
2811 flag_trapv
= save_flag_trapv
;
2814 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2815 unsr_p
, unsr_p
, false, NULL
);
2816 flag_trapv
= save_flag_trapv
;
2823 /* For sub-word operations, retry with a wider type first. */
2824 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2826 int p
= MAX (min_precision
, precop
);
2827 scalar_int_mode m
= smallest_int_mode_for_size (p
).require ();
2828 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2831 p
= TYPE_PRECISION (optype
);
2835 unsr_p
= TYPE_UNSIGNED (optype
);
2841 if (prec0
<= precres
&& prec1
<= precres
)
2846 types
[0] = build_nonstandard_integer_type (precres
, 0);
2852 types
[1] = build_nonstandard_integer_type (precres
, 1);
2854 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2855 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2856 if (code
!= MULT_EXPR
)
2857 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2858 uns0_p
, uns1_p
, false, NULL
);
2860 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2861 uns0_p
, uns1_p
, false, NULL
);
2862 flag_trapv
= save_flag_trapv
;
2866 /* Retry with a wider type. */
2867 if (orig_precres
== precres
)
2869 int p
= MAX (prec0
, prec1
);
2870 scalar_int_mode m
= smallest_int_mode_for_size (p
).require ();
2871 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2874 p
= TYPE_PRECISION (optype
);
2878 unsr_p
= TYPE_UNSIGNED (optype
);
2889 /* Expand ADD_OVERFLOW STMT. */
2892 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2894 expand_arith_overflow (PLUS_EXPR
, stmt
);
2897 /* Expand SUB_OVERFLOW STMT. */
2900 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2902 expand_arith_overflow (MINUS_EXPR
, stmt
);
2905 /* Expand MUL_OVERFLOW STMT. */
2908 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2910 expand_arith_overflow (MULT_EXPR
, stmt
);
2913 /* Expand UADDC STMT. */
2916 expand_UADDC (internal_fn ifn
, gcall
*stmt
)
2918 tree lhs
= gimple_call_lhs (stmt
);
2919 tree arg1
= gimple_call_arg (stmt
, 0);
2920 tree arg2
= gimple_call_arg (stmt
, 1);
2921 tree arg3
= gimple_call_arg (stmt
, 2);
2922 tree type
= TREE_TYPE (arg1
);
2923 machine_mode mode
= TYPE_MODE (type
);
2924 insn_code icode
= optab_handler (ifn
== IFN_UADDC
2925 ? uaddc5_optab
: usubc5_optab
, mode
);
2926 rtx op1
= expand_normal (arg1
);
2927 rtx op2
= expand_normal (arg2
);
2928 rtx op3
= expand_normal (arg3
);
2929 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2930 rtx re
= gen_reg_rtx (mode
);
2931 rtx im
= gen_reg_rtx (mode
);
2932 class expand_operand ops
[5];
2933 create_output_operand (&ops
[0], re
, mode
);
2934 create_output_operand (&ops
[1], im
, mode
);
2935 create_input_operand (&ops
[2], op1
, mode
);
2936 create_input_operand (&ops
[3], op2
, mode
);
2937 create_input_operand (&ops
[4], op3
, mode
);
2938 expand_insn (icode
, 5, ops
);
2939 write_complex_part (target
, re
, false, false);
2940 write_complex_part (target
, im
, true, false);
2943 /* Expand USUBC STMT. */
2946 expand_USUBC (internal_fn ifn
, gcall
*stmt
)
2948 expand_UADDC (ifn
, stmt
);
2951 /* This should get folded in tree-vectorizer.cc. */
2954 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2959 /* This should get folded in tree-vectorizer.cc. */
2962 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2967 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2968 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2971 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2973 tree addr
= gimple_call_arg (stmt
, index
);
2974 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2975 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2976 if (TYPE_ALIGN (type
) != align
)
2977 type
= build_aligned_type (type
, align
);
2980 if (TREE_CODE (tmp
) == SSA_NAME
)
2982 gimple
*def
= get_gimple_for_ssa_name (tmp
);
2983 if (def
&& gimple_assign_single_p (def
))
2984 tmp
= gimple_assign_rhs1 (def
);
2987 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2989 tree mem
= TREE_OPERAND (tmp
, 0);
2990 if (TREE_CODE (mem
) == TARGET_MEM_REF
2991 && types_compatible_p (TREE_TYPE (mem
), type
))
2993 tree offset
= TMR_OFFSET (mem
);
2994 if (type
!= TREE_TYPE (mem
)
2995 || alias_ptr_type
!= TREE_TYPE (offset
)
2996 || !integer_zerop (offset
))
2998 mem
= copy_node (mem
);
2999 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
3000 wi::to_poly_wide (offset
));
3001 TREE_TYPE (mem
) = type
;
3007 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
3010 /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
3014 expand_partial_load_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3017 class expand_operand ops
[5];
3018 tree type
, lhs
, rhs
, maskt
;
3022 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3023 lhs
= gimple_call_lhs (stmt
);
3024 if (lhs
== NULL_TREE
)
3026 type
= TREE_TYPE (lhs
);
3027 rhs
= expand_call_mem_ref (type
, stmt
, 0);
3029 if (optab
== vec_mask_load_lanes_optab
3030 || optab
== vec_mask_len_load_lanes_optab
)
3031 icode
= get_multi_vector_move (type
, optab
);
3032 else if (optab
== len_load_optab
)
3033 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3035 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3036 TYPE_MODE (TREE_TYPE (maskt
)));
3038 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3039 gcc_assert (MEM_P (mem
));
3040 /* The built MEM_REF does not accurately reflect that the load
3041 is only partial. Clear it. */
3042 set_mem_expr (mem
, NULL_TREE
);
3043 clear_mem_offset (mem
);
3044 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3045 create_call_lhs_operand (&ops
[i
++], target
, TYPE_MODE (type
));
3046 create_fixed_operand (&ops
[i
++], mem
);
3047 i
= add_mask_and_len_args (ops
, i
, stmt
);
3048 expand_insn (icode
, i
, ops
);
3050 assign_call_lhs (lhs
, target
, &ops
[0]);
3053 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
3054 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
3055 #define expand_len_load_optab_fn expand_partial_load_optab_fn
3056 #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3058 /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3062 expand_partial_store_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3065 class expand_operand ops
[5];
3066 tree type
, lhs
, rhs
, maskt
;
3070 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3071 rhs
= gimple_call_arg (stmt
, internal_fn_stored_value_index (ifn
));
3072 type
= TREE_TYPE (rhs
);
3073 lhs
= expand_call_mem_ref (type
, stmt
, 0);
3075 if (optab
== vec_mask_store_lanes_optab
3076 || optab
== vec_mask_len_store_lanes_optab
)
3077 icode
= get_multi_vector_move (type
, optab
);
3078 else if (optab
== len_store_optab
)
3079 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3081 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3082 TYPE_MODE (TREE_TYPE (maskt
)));
3084 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3085 gcc_assert (MEM_P (mem
));
3086 /* The built MEM_REF does not accurately reflect that the store
3087 is only partial. Clear it. */
3088 set_mem_expr (mem
, NULL_TREE
);
3089 clear_mem_offset (mem
);
3090 reg
= expand_normal (rhs
);
3091 create_fixed_operand (&ops
[i
++], mem
);
3092 create_input_operand (&ops
[i
++], reg
, TYPE_MODE (type
));
3093 i
= add_mask_and_len_args (ops
, i
, stmt
);
3094 expand_insn (icode
, i
, ops
);
3097 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3098 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3099 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3100 #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3102 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3103 The expansion of STMT happens based on OPTAB table associated. */
3106 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3108 class expand_operand ops
[6];
3110 tree lhs
= gimple_call_lhs (stmt
);
3111 tree op0a
= gimple_call_arg (stmt
, 0);
3112 tree op0b
= gimple_call_arg (stmt
, 1);
3113 tree op1
= gimple_call_arg (stmt
, 2);
3114 tree op2
= gimple_call_arg (stmt
, 3);
3115 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
3117 tree vec_cond_type
= TREE_TYPE (lhs
);
3118 tree op_mode
= TREE_TYPE (op0a
);
3119 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
3121 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3122 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
3124 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
3126 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
3127 /* vector_compare_rtx legitimizes operands, preserve equality when
3128 expanding op1/op2. */
3129 rtx rtx_op1
, rtx_op2
;
3130 if (operand_equal_p (op1
, op0a
))
3131 rtx_op1
= XEXP (comparison
, 0);
3132 else if (operand_equal_p (op1
, op0b
))
3133 rtx_op1
= XEXP (comparison
, 1);
3135 rtx_op1
= expand_normal (op1
);
3136 if (operand_equal_p (op2
, op0a
))
3137 rtx_op2
= XEXP (comparison
, 0);
3138 else if (operand_equal_p (op2
, op0b
))
3139 rtx_op2
= XEXP (comparison
, 1);
3141 rtx_op2
= expand_normal (op2
);
3143 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3144 create_call_lhs_operand (&ops
[0], target
, mode
);
3145 create_input_operand (&ops
[1], rtx_op1
, mode
);
3146 create_input_operand (&ops
[2], rtx_op2
, mode
);
3147 create_fixed_operand (&ops
[3], comparison
);
3148 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
3149 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
3150 expand_insn (icode
, 6, ops
);
3151 assign_call_lhs (lhs
, target
, &ops
[0]);
3154 /* Expand VCOND_MASK optab internal function.
3155 The expansion of STMT happens based on OPTAB table associated. */
3158 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3160 class expand_operand ops
[4];
3162 tree lhs
= gimple_call_lhs (stmt
);
3163 tree op0
= gimple_call_arg (stmt
, 0);
3164 tree op1
= gimple_call_arg (stmt
, 1);
3165 tree op2
= gimple_call_arg (stmt
, 2);
3166 tree vec_cond_type
= TREE_TYPE (lhs
);
3168 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3169 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
3170 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
3171 rtx mask
, rtx_op1
, rtx_op2
;
3173 gcc_assert (icode
!= CODE_FOR_nothing
);
3175 mask
= expand_normal (op0
);
3176 rtx_op1
= expand_normal (op1
);
3177 rtx_op2
= expand_normal (op2
);
3179 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3180 create_call_lhs_operand (&ops
[0], target
, mode
);
3181 create_input_operand (&ops
[1], rtx_op1
, mode
);
3182 create_input_operand (&ops
[2], rtx_op2
, mode
);
3183 create_input_operand (&ops
[3], mask
, mask_mode
);
3184 expand_insn (icode
, 4, ops
);
3185 assign_call_lhs (lhs
, target
, &ops
[0]);
3188 /* Expand VEC_SET internal functions. */
3191 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3193 tree lhs
= gimple_call_lhs (stmt
);
3194 tree op0
= gimple_call_arg (stmt
, 0);
3195 tree op1
= gimple_call_arg (stmt
, 1);
3196 tree op2
= gimple_call_arg (stmt
, 2);
3197 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3198 rtx src
= expand_normal (op0
);
3200 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
3201 scalar_mode innermode
= GET_MODE_INNER (outermode
);
3203 rtx value
= expand_normal (op1
);
3204 rtx pos
= expand_normal (op2
);
3206 class expand_operand ops
[3];
3207 enum insn_code icode
= optab_handler (optab
, outermode
);
3209 if (icode
!= CODE_FOR_nothing
)
3211 rtx temp
= gen_reg_rtx (outermode
);
3212 emit_move_insn (temp
, src
);
3214 create_fixed_operand (&ops
[0], temp
);
3215 create_input_operand (&ops
[1], value
, innermode
);
3216 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
3218 if (maybe_expand_insn (icode
, 3, ops
))
3220 emit_move_insn (target
, temp
);
3228 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3233 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3235 /* When guessing was done, the hints should be already stripped away. */
3236 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3239 tree lhs
= gimple_call_lhs (stmt
);
3241 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3243 target
= const0_rtx
;
3244 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3245 if (lhs
&& val
!= target
)
3246 emit_move_insn (target
, val
);
3249 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3250 should never be called. */
3253 expand_VA_ARG (internal_fn
, gcall
*)
3258 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3259 dummy function should never be called. */
3262 expand_VEC_CONVERT (internal_fn
, gcall
*)
3267 /* Expand IFN_RAWMEMCHR internal function. */
3270 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3272 expand_operand ops
[3];
3274 tree lhs
= gimple_call_lhs (stmt
);
3277 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3278 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3279 create_call_lhs_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3281 tree mem
= gimple_call_arg (stmt
, 0);
3282 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3283 create_fixed_operand (&ops
[1], mem_rtx
);
3285 tree pattern
= gimple_call_arg (stmt
, 1);
3286 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3287 rtx pattern_rtx
= expand_normal (pattern
);
3288 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3290 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3292 expand_insn (icode
, 3, ops
);
3293 assign_call_lhs (lhs
, lhs_rtx
, &ops
[0]);
3296 /* Expand the IFN_UNIQUE function according to its first argument. */
3299 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3301 rtx pattern
= NULL_RTX
;
3302 enum ifn_unique_kind kind
3303 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3310 case IFN_UNIQUE_UNSPEC
:
3311 if (targetm
.have_unique ())
3312 pattern
= targetm
.gen_unique ();
3315 case IFN_UNIQUE_OACC_FORK
:
3316 case IFN_UNIQUE_OACC_JOIN
:
3317 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3319 tree lhs
= gimple_call_lhs (stmt
);
3320 rtx target
= const0_rtx
;
3323 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3325 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3326 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3328 if (kind
== IFN_UNIQUE_OACC_FORK
)
3329 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3331 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3339 emit_insn (pattern
);
3342 /* Expand the IFN_DEFERRED_INIT function:
3343 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3345 Initialize the LHS with zero/pattern according to its second argument
3347 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3348 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3350 The LHS variable is initialized including paddings.
3351 The reasons to choose 0xFE for pattern initialization are:
3352 1. It is a non-canonical virtual address on x86_64, and at the
3353 high end of the i386 kernel address space.
3354 2. It is a very large float value (-1.694739530317379e+38).
3355 3. It is also an unusual number for integers. */
3356 #define INIT_PATTERN_VALUE 0xFE
3358 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3360 tree lhs
= gimple_call_lhs (stmt
);
3361 tree var_size
= gimple_call_arg (stmt
, 0);
3362 enum auto_init_type init_type
3363 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3364 bool reg_lhs
= true;
3366 tree var_type
= TREE_TYPE (lhs
);
3367 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3369 if (TREE_CODE (lhs
) == SSA_NAME
)
3373 tree lhs_base
= lhs
;
3374 while (handled_component_p (lhs_base
))
3375 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3376 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3377 || non_mem_decl_p (lhs_base
));
3378 /* If this expands to a register and the underlying decl is wrapped in
3379 a MEM_REF that just serves as an access type change expose the decl
3380 if it is of correct size. This avoids a situation as in PR103271
3381 if the target does not support a direct move to the registers mode. */
3383 && TREE_CODE (lhs_base
) == MEM_REF
3384 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3385 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3386 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3387 && tree_fits_uhwi_p (var_size
)
3388 && tree_int_cst_equal
3390 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3392 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3393 var_type
= TREE_TYPE (lhs
);
3399 /* If the variable is not in register, expand to a memset
3400 to initialize it. */
3401 mark_addressable (lhs
);
3402 tree var_addr
= build_fold_addr_expr (lhs
);
3404 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3405 ? build_int_cst (integer_type_node
,
3407 : integer_zero_node
;
3408 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3409 3, var_addr
, value
, var_size
);
3410 /* Expand this memset call. */
3411 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3415 /* If this variable is in a register use expand_assignment.
3416 For boolean scalars force zero-init. */
3418 scalar_int_mode var_mode
;
3419 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3420 && tree_fits_uhwi_p (var_size
)
3421 && (init_type
== AUTO_INIT_PATTERN
3422 || !is_gimple_reg_type (var_type
))
3423 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3424 0).exists (&var_mode
)
3425 && have_insn_for (SET
, var_mode
))
3427 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3428 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3429 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3430 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3431 tree itype
= build_nonstandard_integer_type
3432 (total_bytes
* BITS_PER_UNIT
, 1);
3433 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3434 init
= wide_int_to_tree (itype
, w
);
3435 /* Pun the LHS to make sure its type has constant size
3436 unless it is an SSA name where that's already known. */
3437 if (TREE_CODE (lhs
) != SSA_NAME
)
3438 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3440 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3443 /* Use zero-init also for variable-length sizes. */
3444 init
= build_zero_cst (var_type
);
3446 expand_assignment (lhs
, init
, false);
3450 /* Expand the IFN_ACCESS_WITH_SIZE function:
3451 ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE, CLASS_OF_SIZE,
3452 TYPE_OF_SIZE, ACCESS_MODE)
3453 which returns the REF_TO_OBJ same as the 1st argument;
3455 1st argument REF_TO_OBJ: The reference to the object;
3456 2nd argument REF_TO_SIZE: The reference to the size of the object,
3457 3rd argument CLASS_OF_SIZE: The size referenced by the REF_TO_SIZE represents
3458 0: the number of bytes.
3459 1: the number of the elements of the object type;
3460 4th argument TYPE_OF_SIZE: A constant 0 with its TYPE being the same as the TYPE
3461 of the object referenced by REF_TO_SIZE
3462 5th argument ACCESS_MODE:
3463 -1: Unknown access semantics
3468 6th argument: A constant 0 with the pointer TYPE to the original flexible
3471 Both the return type and the type of the first argument of this
3472 function have been converted from the incomplete array type to
3473 the corresponding pointer type.
3475 For each call to a .ACCESS_WITH_SIZE, replace it with its 1st argument. */
3478 expand_ACCESS_WITH_SIZE (internal_fn
, gcall
*stmt
)
3480 tree lhs
= gimple_call_lhs (stmt
);
3481 tree ref_to_obj
= gimple_call_arg (stmt
, 0);
3483 expand_assignment (lhs
, ref_to_obj
, false);
3486 /* The size of an OpenACC compute dimension. */
3489 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3491 tree lhs
= gimple_call_lhs (stmt
);
3496 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3497 if (targetm
.have_oacc_dim_size ())
3499 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3500 VOIDmode
, EXPAND_NORMAL
);
3501 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3504 emit_move_insn (target
, GEN_INT (1));
3507 /* The position of an OpenACC execution engine along one compute axis. */
3510 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3512 tree lhs
= gimple_call_lhs (stmt
);
3517 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3518 if (targetm
.have_oacc_dim_pos ())
3520 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3521 VOIDmode
, EXPAND_NORMAL
);
3522 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3525 emit_move_insn (target
, const0_rtx
);
3528 /* This is expanded by oacc_device_lower pass. */
3531 expand_GOACC_LOOP (internal_fn
, gcall
*)
3536 /* This is expanded by oacc_device_lower pass. */
3539 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3544 /* This is expanded by oacc_device_lower pass. */
3547 expand_GOACC_TILE (internal_fn
, gcall
*)
3552 /* Set errno to EDOM. */
3555 expand_SET_EDOM (internal_fn
, gcall
*)
3558 #ifdef GEN_ERRNO_RTX
3559 rtx errno_rtx
= GEN_ERRNO_RTX
;
3561 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3563 emit_move_insn (errno_rtx
,
3564 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3570 /* Expand atomic bit test and set. */
3573 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3575 expand_ifn_atomic_bit_test_and (call
);
3578 /* Expand atomic bit test and complement. */
3581 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3583 expand_ifn_atomic_bit_test_and (call
);
3586 /* Expand atomic bit test and reset. */
3589 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3591 expand_ifn_atomic_bit_test_and (call
);
3594 /* Expand atomic bit test and set. */
3597 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3599 expand_ifn_atomic_compare_exchange (call
);
3602 /* Expand atomic add fetch and cmp with 0. */
3605 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3607 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3610 /* Expand atomic sub fetch and cmp with 0. */
3613 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3615 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3618 /* Expand atomic and fetch and cmp with 0. */
3621 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3623 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3626 /* Expand atomic or fetch and cmp with 0. */
3629 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3631 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3634 /* Expand atomic xor fetch and cmp with 0. */
3637 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3639 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3642 /* Expand LAUNDER to assignment, lhs = arg0. */
3645 expand_LAUNDER (internal_fn
, gcall
*call
)
3647 tree lhs
= gimple_call_lhs (call
);
3652 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3655 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3658 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3660 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3661 int rhs_index
= internal_fn_stored_value_index (ifn
);
3662 tree base
= gimple_call_arg (stmt
, 0);
3663 tree offset
= gimple_call_arg (stmt
, 1);
3664 tree scale
= gimple_call_arg (stmt
, 2);
3665 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3667 rtx base_rtx
= expand_normal (base
);
3668 rtx offset_rtx
= expand_normal (offset
);
3669 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3670 rtx rhs_rtx
= expand_normal (rhs
);
3672 class expand_operand ops
[8];
3674 create_address_operand (&ops
[i
++], base_rtx
);
3675 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3676 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3677 create_integer_operand (&ops
[i
++], scale_int
);
3678 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3679 i
= add_mask_and_len_args (ops
, i
, stmt
);
3681 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3682 TYPE_MODE (TREE_TYPE (offset
)));
3683 expand_insn (icode
, i
, ops
);
3686 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3689 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3691 tree lhs
= gimple_call_lhs (stmt
);
3692 tree base
= gimple_call_arg (stmt
, 0);
3693 tree offset
= gimple_call_arg (stmt
, 1);
3694 tree scale
= gimple_call_arg (stmt
, 2);
3696 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3697 rtx base_rtx
= expand_normal (base
);
3698 rtx offset_rtx
= expand_normal (offset
);
3699 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3702 class expand_operand ops
[8];
3703 create_call_lhs_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3704 create_address_operand (&ops
[i
++], base_rtx
);
3705 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3706 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3707 create_integer_operand (&ops
[i
++], scale_int
);
3708 i
= add_mask_and_len_args (ops
, i
, stmt
);
3709 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3710 TYPE_MODE (TREE_TYPE (offset
)));
3711 expand_insn (icode
, i
, ops
);
3712 assign_call_lhs (lhs
, lhs_rtx
, &ops
[0]);
3715 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3716 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3719 contains_call_div_mod (rtx_insn
*insn
)
3721 subrtx_iterator::array_type array
;
3722 for (; insn
; insn
= NEXT_INSN (insn
))
3725 else if (INSN_P (insn
))
3726 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3727 switch (GET_CODE (*iter
))
3741 /* Expand DIVMOD() using:
3742 a) optab handler for udivmod/sdivmod if it is available.
3743 b) If optab_handler doesn't exist, generate call to
3744 target-specific divmod libfunc. */
3747 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3749 tree lhs
= gimple_call_lhs (call_stmt
);
3750 tree arg0
= gimple_call_arg (call_stmt
, 0);
3751 tree arg1
= gimple_call_arg (call_stmt
, 1);
3753 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3754 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3755 machine_mode mode
= TYPE_MODE (type
);
3756 bool unsignedp
= TYPE_UNSIGNED (type
);
3757 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3759 rtx op0
= expand_normal (arg0
);
3760 rtx op1
= expand_normal (arg1
);
3761 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3763 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3764 rtx_insn
*insns
= NULL
;
3766 if (TREE_CODE (arg1
) == INTEGER_CST
)
3768 /* For DIVMOD by integral constants, there could be efficient code
3769 expanded inline e.g. using shifts and plus/minus. Try to expand
3770 the division and modulo and if it emits any library calls or any
3771 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3773 scalar_int_mode int_mode
;
3774 if (remainder
== NULL_RTX
3776 && CONST_INT_P (op1
)
3777 && !pow2p_hwi (INTVAL (op1
))
3778 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3779 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3780 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3781 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3782 && optimize_insn_for_speed_p ())
3784 rtx_insn
*last
= get_last_insn ();
3785 remainder
= NULL_RTX
;
3786 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3787 TYPE_UNSIGNED (type
));
3788 if (quotient
!= NULL_RTX
)
3790 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3792 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3793 set_dst_reg_note (move
, REG_EQUAL
,
3794 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3795 ? UDIV
: DIV
, int_mode
,
3796 copy_rtx (op0
), op1
),
3798 move
= emit_move_insn (remainder
, remainder
);
3799 set_dst_reg_note (move
, REG_EQUAL
,
3800 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3801 ? UMOD
: MOD
, int_mode
,
3802 copy_rtx (op0
), op1
),
3807 delete_insns_since (last
);
3810 if (remainder
== NULL_RTX
)
3812 struct separate_ops ops
;
3813 ops
.code
= TRUNC_DIV_EXPR
;
3815 ops
.op0
= make_tree (ops
.type
, op0
);
3817 ops
.op2
= NULL_TREE
;
3818 ops
.location
= gimple_location (call_stmt
);
3820 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3821 if (contains_call_div_mod (get_insns ()))
3822 quotient
= NULL_RTX
;
3825 ops
.code
= TRUNC_MOD_EXPR
;
3826 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3828 if (contains_call_div_mod (get_insns ()))
3829 remainder
= NULL_RTX
;
3832 insns
= get_insns ();
3840 /* Check if optab_handler exists for divmod_optab for given mode. */
3841 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3843 quotient
= gen_reg_rtx (mode
);
3844 remainder
= gen_reg_rtx (mode
);
3845 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3848 /* Generate call to divmod libfunc if it exists. */
3849 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3850 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3851 "ient
, &remainder
);
3856 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3857 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3858 make_tree (TREE_TYPE (arg0
), quotient
),
3859 make_tree (TREE_TYPE (arg1
), remainder
)),
3860 target
, VOIDmode
, EXPAND_NORMAL
);
3866 expand_NOP (internal_fn
, gcall
*)
3868 /* Nothing. But it shouldn't really prevail. */
3871 /* Coroutines, all should have been processed at this stage. */
3874 expand_CO_FRAME (internal_fn
, gcall
*)
3880 expand_CO_YIELD (internal_fn
, gcall
*)
3886 expand_CO_SUSPN (internal_fn
, gcall
*)
3892 expand_CO_ACTOR (internal_fn
, gcall
*)
3897 /* Expand a call to FN using the operands in STMT. FN has a single
3898 output operand and NARGS input operands. */
3901 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3904 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3905 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3906 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3909 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3912 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3914 expand_operand ops
[4];
3917 tree lhs
= gimple_call_lhs (stmt
);
3918 tree lhs_type
= TREE_TYPE (lhs
);
3919 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3920 create_call_lhs_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3922 for (unsigned int i
= 0; i
< 2; ++i
)
3924 tree rhs
= gimple_call_arg (stmt
, i
);
3925 rhs_type
[i
] = TREE_TYPE (rhs
);
3926 rtx rhs_rtx
= expand_normal (rhs
);
3927 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3931 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type
)))
3933 /* When the mask is an integer mode the exact vector length may not
3934 be clear to the backend, so we pass it in operand[3].
3935 Use the vector in arg2 for the most reliable intended size. */
3936 tree type
= TREE_TYPE (gimple_call_arg (stmt
, 2));
3937 create_integer_operand (&ops
[3], TYPE_VECTOR_SUBPARTS (type
));
3941 /* The mask has a vector type so the length operand is unnecessary. */
3944 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3945 TYPE_MODE (lhs_type
));
3947 expand_insn (icode
, opcnt
, ops
);
3948 assign_call_lhs (lhs
, lhs_rtx
, &ops
[0]);
3951 /* Expand a call to a convert-like optab using the operands in STMT.
3952 FN has a single output operand and NARGS input operands. */
3955 expand_convert_optab_fn (internal_fn fn
, gcall
*stmt
, convert_optab optab
,
3958 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3959 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3960 TYPE_MODE (types
.second
));
3961 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3964 /* Expanders for optabs that can use expand_direct_optab_fn. */
3966 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3967 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3969 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3970 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3972 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3973 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3975 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3976 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3978 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3979 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3981 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3982 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3984 #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3985 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3987 #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3988 expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3990 #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3991 expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3993 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3994 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3996 #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3997 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3999 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
4000 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
4002 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
4003 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
4005 #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
4006 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
4008 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
4009 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
4011 /* Expanders for optabs that can use expand_convert_optab_fn. */
4013 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
4014 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
4016 #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
4017 expand_convert_optab_fn (FN, STMT, OPTAB, 2)
4019 /* RETURN_TYPE and ARGS are a return type and argument list that are
4020 in principle compatible with FN (which satisfies direct_internal_fn_p).
4021 Return the types that should be used to determine whether the
4022 target supports FN. */
4025 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
4027 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4028 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
4029 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
4030 return tree_pair (type0
, type1
);
4033 /* CALL is a call whose return type and arguments are in principle
4034 compatible with FN (which satisfies direct_internal_fn_p). Return the
4035 types that should be used to determine whether the target supports FN. */
4038 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
4040 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4041 tree op0
= (info
.type0
< 0
4042 ? gimple_call_lhs (call
)
4043 : gimple_call_arg (call
, info
.type0
));
4044 tree op1
= (info
.type1
< 0
4045 ? gimple_call_lhs (call
)
4046 : gimple_call_arg (call
, info
.type1
));
4047 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
4050 /* Return true if OPTAB is supported for TYPES (whose modes should be
4051 the same) when the optimization type is OPT_TYPE. Used for simple
4055 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
4056 optimization_type opt_type
)
4058 machine_mode mode
= TYPE_MODE (types
.first
);
4059 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
4060 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
4063 /* Return true if OPTAB is supported for TYPES, where the first type
4064 is the destination and the second type is the source. Used for
4068 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
4069 optimization_type opt_type
)
4071 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
4072 TYPE_MODE (types
.second
), opt_type
)
4073 != CODE_FOR_nothing
);
4076 /* Return true if load/store lanes optab OPTAB is supported for
4077 array type TYPES.first when the optimization type is OPT_TYPE. */
4080 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
4081 optimization_type opt_type
)
4083 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
4084 machine_mode imode
= TYPE_MODE (types
.first
);
4085 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
4086 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
4087 != CODE_FOR_nothing
);
4090 #define direct_unary_optab_supported_p direct_optab_supported_p
4091 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
4092 #define direct_binary_optab_supported_p direct_optab_supported_p
4093 #define direct_ternary_optab_supported_p direct_optab_supported_p
4094 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
4095 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
4096 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4097 #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4098 #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4099 #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4100 #define direct_mask_load_optab_supported_p convert_optab_supported_p
4101 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4102 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4103 #define direct_gather_load_optab_supported_p convert_optab_supported_p
4104 #define direct_len_load_optab_supported_p direct_optab_supported_p
4105 #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4106 #define direct_mask_store_optab_supported_p convert_optab_supported_p
4107 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4108 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4109 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4110 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
4111 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
4112 #define direct_len_store_optab_supported_p direct_optab_supported_p
4113 #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4114 #define direct_while_optab_supported_p convert_optab_supported_p
4115 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
4116 #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4117 #define direct_fold_left_optab_supported_p direct_optab_supported_p
4118 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4119 #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4120 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4121 #define direct_vec_set_optab_supported_p direct_optab_supported_p
4122 #define direct_vec_extract_optab_supported_p convert_optab_supported_p
4124 /* Return the optab used by internal function FN. */
4127 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
4131 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4132 case IFN_##CODE: break;
4133 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4134 case IFN_##CODE: return OPTAB##_optab;
4135 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4136 UNSIGNED_OPTAB, TYPE) \
4137 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4138 ? UNSIGNED_OPTAB ## _optab \
4139 : SIGNED_OPTAB ## _optab);
4140 #include "internal-fn.def"
4148 /* Return the optab used by internal function FN. */
4151 direct_internal_fn_optab (internal_fn fn
)
4155 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4156 case IFN_##CODE: break;
4157 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4158 case IFN_##CODE: return OPTAB##_optab;
4159 #include "internal-fn.def"
4167 /* Return true if TYPE's mode has the same format as TYPE, and if there is
4168 a 1:1 correspondence between the values that the mode can store and the
4169 values that the type can store. */
4172 type_strictly_matches_mode_p (const_tree type
)
4174 /* The masked vector operations have both vector data operands and vector
4175 boolean operands. The vector data operands are expected to have a vector
4176 mode, but the vector boolean operands can be an integer mode rather than
4177 a vector mode, depending on how TARGET_VECTORIZE_GET_MASK_MODE is
4178 defined. PR116103. */
4179 if (VECTOR_BOOLEAN_TYPE_P (type
)
4180 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
4181 && TYPE_PRECISION (TREE_TYPE (type
)) == 1)
4184 if (VECTOR_TYPE_P (type
))
4185 return VECTOR_MODE_P (TYPE_MODE (type
));
4187 if (INTEGRAL_TYPE_P (type
))
4188 return type_has_mode_precision_p (type
);
4190 if (SCALAR_FLOAT_TYPE_P (type
) || COMPLEX_FLOAT_TYPE_P (type
))
4196 /* Returns true if both types of TYPE_PAIR strictly match their modes,
4197 else returns false. */
4200 type_pair_strictly_matches_mode_p (tree_pair type_pair
)
4202 return type_strictly_matches_mode_p (type_pair
.first
)
4203 && type_strictly_matches_mode_p (type_pair
.second
);
4206 /* Return true if FN is supported for the types in TYPES when the
4207 optimization type is OPT_TYPE. The types are those associated with
4208 the "type0" and "type1" fields of FN's direct_internal_fn_info
4212 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
4213 optimization_type opt_type
)
4215 if (!type_pair_strictly_matches_mode_p (types
))
4220 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4221 case IFN_##CODE: break;
4222 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4224 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4226 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4227 UNSIGNED_OPTAB, TYPE) \
4230 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4231 ? UNSIGNED_OPTAB ## _optab \
4232 : SIGNED_OPTAB ## _optab); \
4233 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4236 #include "internal-fn.def"
4244 /* Return true if FN is supported for type TYPE when the optimization
4245 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4246 fields of FN's direct_internal_fn_info structure are the same. */
4249 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
4250 optimization_type opt_type
)
4252 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4253 gcc_checking_assert (info
.type0
== info
.type1
);
4254 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
4257 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4258 given that STMT is a call to a direct internal function. */
4261 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
4263 internal_fn fn
= gimple_call_internal_fn (stmt
);
4264 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
4265 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
4268 /* Return true if FN is a binary operation and if FN is commutative. */
4271 commutative_binary_fn_p (internal_fn fn
)
4282 case IFN_COMPLEX_MUL
:
4283 case IFN_UBSAN_CHECK_ADD
:
4284 case IFN_UBSAN_CHECK_MUL
:
4285 case IFN_ADD_OVERFLOW
:
4286 case IFN_MUL_OVERFLOW
:
4288 case IFN_VEC_WIDEN_PLUS
:
4289 case IFN_VEC_WIDEN_PLUS_LO
:
4290 case IFN_VEC_WIDEN_PLUS_HI
:
4291 case IFN_VEC_WIDEN_PLUS_EVEN
:
4292 case IFN_VEC_WIDEN_PLUS_ODD
:
4300 /* Return true if FN is a ternary operation and if its first two arguments
4304 commutative_ternary_fn_p (internal_fn fn
)
4320 /* Return true if FN is an associative binary operation. */
4323 associative_binary_fn_p (internal_fn fn
)
4336 /* If FN is commutative in two consecutive arguments, return the
4337 index of the first, otherwise return -1. */
4340 first_commutative_argument (internal_fn fn
)
4357 case IFN_COND_LEN_ADD
:
4358 case IFN_COND_LEN_MUL
:
4359 case IFN_COND_LEN_MIN
:
4360 case IFN_COND_LEN_MAX
:
4361 case IFN_COND_LEN_FMIN
:
4362 case IFN_COND_LEN_FMAX
:
4363 case IFN_COND_LEN_AND
:
4364 case IFN_COND_LEN_IOR
:
4365 case IFN_COND_LEN_XOR
:
4366 case IFN_COND_LEN_FMA
:
4367 case IFN_COND_LEN_FMS
:
4368 case IFN_COND_LEN_FNMA
:
4369 case IFN_COND_LEN_FNMS
:
4373 if (commutative_binary_fn_p (fn
)
4374 || commutative_ternary_fn_p (fn
))
4380 /* Return true if this CODE describes an internal_fn that returns a vector with
4381 elements twice as wide as the element size of the input vectors. */
4384 widening_fn_p (code_helper code
)
4386 if (!code
.is_fn_code ())
4389 if (!internal_fn_p ((combined_fn
) code
))
4392 internal_fn fn
= as_internal_fn ((combined_fn
) code
);
4395 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4397 case IFN_##NAME##_HI: \
4398 case IFN_##NAME##_LO: \
4399 case IFN_##NAME##_EVEN: \
4400 case IFN_##NAME##_ODD: \
4402 #include "internal-fn.def"
4409 /* Return true if IFN_SET_EDOM is supported. */
4412 set_edom_supported_p (void)
4421 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4423 expand_##CODE (internal_fn fn, gcall *stmt) \
4425 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4427 #define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4428 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4429 UNSIGNED_OPTAB, TYPE) \
4431 expand_##CODE (internal_fn fn, gcall *stmt) \
4433 tree_pair types = direct_internal_fn_types (fn, stmt); \
4434 optab which_optab = direct_internal_fn_optab (fn, types); \
4435 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4437 #include "internal-fn.def"
4439 /* Routines to expand each internal function, indexed by function number.
4440 Each routine has the prototype:
4442 expand_<NAME> (gcall *stmt)
4444 where STMT is the statement that performs the call. */
4445 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4447 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4448 #include "internal-fn.def"
4452 /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4453 that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4454 for each such IFN_COND_##SUFFIX. */
4455 #define FOR_EACH_CODE_MAPPING(T) \
4456 T (PLUS_EXPR, ADD) \
4457 T (MINUS_EXPR, SUB) \
4458 T (MULT_EXPR, MUL) \
4459 T (TRUNC_DIV_EXPR, DIV) \
4460 T (TRUNC_MOD_EXPR, MOD) \
4461 T (RDIV_EXPR, RDIV) \
4464 T (BIT_AND_EXPR, AND) \
4465 T (BIT_IOR_EXPR, IOR) \
4466 T (BIT_XOR_EXPR, XOR) \
4467 T (LSHIFT_EXPR, SHL) \
4468 T (RSHIFT_EXPR, SHR) \
4469 T (NEGATE_EXPR, NEG)
4471 /* Return a function that only performs CODE when a certain condition is met
4472 and that uses a given fallback value otherwise. For example, if CODE is
4473 a binary operation associated with conditional function FN:
4475 LHS = FN (COND, A, B, ELSE)
4477 is equivalent to the C expression:
4479 LHS = COND ? A CODE B : ELSE;
4481 operating elementwise if the operands are vectors.
4483 Return IFN_LAST if no such function exists. */
4486 get_conditional_internal_fn (tree_code code
)
4490 #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4491 FOR_EACH_CODE_MAPPING(CASE
)
4498 /* If IFN implements the conditional form of a tree code, return that
4499 tree code, otherwise return ERROR_MARK. */
4502 conditional_internal_fn_code (internal_fn ifn
)
4506 #define CASE(CODE, IFN) \
4507 case IFN_COND_##IFN: \
4508 case IFN_COND_LEN_##IFN: \
4510 FOR_EACH_CODE_MAPPING (CASE
)
4517 /* Like get_conditional_internal_fn, but return a function that
4518 additionally restricts the operation to the leading elements
4519 of a vector. The number of elements to process is given by a length
4520 and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4521 elements are taken from the fallback ("else") argument.
4523 For example, if CODE is a binary operation associated with FN:
4525 LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4527 is equivalent to the C code:
4529 for (int i = 0; i < NUNITS; i++)
4531 if (i < LEN + BIAS && COND[i])
4532 LHS[i] = A[i] CODE B[i];
4539 get_conditional_len_internal_fn (tree_code code
)
4543 #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4544 FOR_EACH_CODE_MAPPING(CASE
)
4551 /* Invoke T(IFN) for each internal function IFN that also has an
4553 #define FOR_EACH_COND_FN_PAIR(T) \
4561 /* Return a function that only performs internal function FN when a
4562 certain condition is met and that uses a given fallback value otherwise.
4563 In other words, the returned function FN' is such that:
4565 LHS = FN' (COND, A1, ... An, ELSE)
4567 is equivalent to the C expression:
4569 LHS = COND ? FN (A1, ..., An) : ELSE;
4571 operating elementwise if the operands are vectors.
4573 Return IFN_LAST if no such function exists. */
4576 get_conditional_internal_fn (internal_fn fn
)
4580 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4581 FOR_EACH_COND_FN_PAIR(CASE
)
4588 /* If there exists an internal function like IFN that operates on vectors,
4589 but with additional length and bias parameters, return the internal_fn
4590 for that function, otherwise return IFN_LAST. */
4592 get_len_internal_fn (internal_fn fn
)
4596 #define DEF_INTERNAL_COND_FN(NAME, ...) \
4597 case IFN_COND_##NAME: \
4598 return IFN_COND_LEN_##NAME;
4599 #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4600 case IFN_COND_##NAME: \
4601 return IFN_COND_LEN_##NAME;
4602 #include "internal-fn.def"
4608 /* If IFN implements the conditional form of an unconditional internal
4609 function, return that unconditional function, otherwise return IFN_LAST. */
4612 get_unconditional_internal_fn (internal_fn ifn
)
4616 #define CASE(NAME) \
4617 case IFN_COND_##NAME: \
4618 case IFN_COND_LEN_##NAME: \
4620 FOR_EACH_COND_FN_PAIR (CASE
)
4627 /* Return true if STMT can be interpreted as a conditional tree code
4628 operation of the form:
4630 LHS = COND ? OP (RHS1, ...) : ELSE;
4632 operating elementwise if the operands are vectors. This includes
4633 the case of an all-true COND, so that the operation always happens.
4635 There is an alternative approach to interpret the STMT when the operands
4636 are vectors which is the operation predicated by both conditional mask
4637 and loop control length, the equivalent C code:
4639 for (int i = 0; i < NUNTIS; i++)
4641 if (i < LEN + BIAS && COND[i])
4642 LHS[i] = A[i] CODE B[i];
4647 When returning true, set:
4649 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4650 is known to be all-true
4651 - *CODE_OUT to the tree code
4652 - OPS[I] to operand I of *CODE_OUT
4653 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4654 condition is known to be all true.
4655 - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4656 - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4659 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4660 tree_code
*code_out
,
4661 tree (&ops
)[3], tree
*else_out
,
4662 tree
*len
, tree
*bias
)
4666 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4668 *cond_out
= NULL_TREE
;
4669 *code_out
= gimple_assign_rhs_code (assign
);
4670 ops
[0] = gimple_assign_rhs1 (assign
);
4671 ops
[1] = gimple_assign_rhs2 (assign
);
4672 ops
[2] = gimple_assign_rhs3 (assign
);
4673 *else_out
= NULL_TREE
;
4676 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4677 if (gimple_call_internal_p (call
))
4679 internal_fn ifn
= gimple_call_internal_fn (call
);
4680 tree_code code
= conditional_internal_fn_code (ifn
);
4681 int len_index
= internal_fn_len_index (ifn
);
4682 int cond_nargs
= len_index
>= 0 ? 4 : 2;
4683 if (code
!= ERROR_MARK
)
4685 *cond_out
= gimple_call_arg (call
, 0);
4687 unsigned int nops
= gimple_call_num_args (call
) - cond_nargs
;
4688 for (unsigned int i
= 0; i
< 3; ++i
)
4689 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4690 *else_out
= gimple_call_arg (call
, nops
+ 1);
4693 if (integer_truep (*cond_out
))
4695 *cond_out
= NULL_TREE
;
4696 *else_out
= NULL_TREE
;
4701 *len
= gimple_call_arg (call
, len_index
);
4702 *bias
= gimple_call_arg (call
, len_index
+ 1);
4710 /* Return true if IFN is some form of load from memory. */
4713 internal_load_fn_p (internal_fn fn
)
4718 case IFN_LOAD_LANES
:
4719 case IFN_MASK_LOAD_LANES
:
4720 case IFN_MASK_LEN_LOAD_LANES
:
4721 case IFN_GATHER_LOAD
:
4722 case IFN_MASK_GATHER_LOAD
:
4723 case IFN_MASK_LEN_GATHER_LOAD
:
4725 case IFN_MASK_LEN_LOAD
:
4733 /* Return true if IFN is some form of store to memory. */
4736 internal_store_fn_p (internal_fn fn
)
4740 case IFN_MASK_STORE
:
4741 case IFN_STORE_LANES
:
4742 case IFN_MASK_STORE_LANES
:
4743 case IFN_MASK_LEN_STORE_LANES
:
4744 case IFN_SCATTER_STORE
:
4745 case IFN_MASK_SCATTER_STORE
:
4746 case IFN_MASK_LEN_SCATTER_STORE
:
4748 case IFN_MASK_LEN_STORE
:
4756 /* Return true if IFN is some form of gather load or scatter store. */
4759 internal_gather_scatter_fn_p (internal_fn fn
)
4763 case IFN_GATHER_LOAD
:
4764 case IFN_MASK_GATHER_LOAD
:
4765 case IFN_MASK_LEN_GATHER_LOAD
:
4766 case IFN_SCATTER_STORE
:
4767 case IFN_MASK_SCATTER_STORE
:
4768 case IFN_MASK_LEN_SCATTER_STORE
:
4776 /* If FN takes a vector len argument, return the index of that argument,
4777 otherwise return -1. */
4780 internal_fn_len_index (internal_fn fn
)
4788 case IFN_MASK_LEN_GATHER_LOAD
:
4789 case IFN_MASK_LEN_SCATTER_STORE
:
4790 case IFN_COND_LEN_FMA
:
4791 case IFN_COND_LEN_FMS
:
4792 case IFN_COND_LEN_FNMA
:
4793 case IFN_COND_LEN_FNMS
:
4796 case IFN_COND_LEN_ADD
:
4797 case IFN_COND_LEN_SUB
:
4798 case IFN_COND_LEN_MUL
:
4799 case IFN_COND_LEN_DIV
:
4800 case IFN_COND_LEN_MOD
:
4801 case IFN_COND_LEN_RDIV
:
4802 case IFN_COND_LEN_MIN
:
4803 case IFN_COND_LEN_MAX
:
4804 case IFN_COND_LEN_FMIN
:
4805 case IFN_COND_LEN_FMAX
:
4806 case IFN_COND_LEN_AND
:
4807 case IFN_COND_LEN_IOR
:
4808 case IFN_COND_LEN_XOR
:
4809 case IFN_COND_LEN_SHL
:
4810 case IFN_COND_LEN_SHR
:
4813 case IFN_COND_LEN_NEG
:
4814 case IFN_MASK_LEN_LOAD
:
4815 case IFN_MASK_LEN_STORE
:
4816 case IFN_MASK_LEN_LOAD_LANES
:
4817 case IFN_MASK_LEN_STORE_LANES
:
4818 case IFN_VCOND_MASK_LEN
:
4826 /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4827 argument that is used when the condition is false. Return -1 otherwise. */
4830 internal_fn_else_index (internal_fn fn
)
4836 case IFN_COND_LEN_NEG
:
4837 case IFN_COND_LEN_NOT
:
4854 case IFN_COND_LEN_ADD
:
4855 case IFN_COND_LEN_SUB
:
4856 case IFN_COND_LEN_MUL
:
4857 case IFN_COND_LEN_DIV
:
4858 case IFN_COND_LEN_MOD
:
4859 case IFN_COND_LEN_MIN
:
4860 case IFN_COND_LEN_MAX
:
4861 case IFN_COND_LEN_FMIN
:
4862 case IFN_COND_LEN_FMAX
:
4863 case IFN_COND_LEN_AND
:
4864 case IFN_COND_LEN_IOR
:
4865 case IFN_COND_LEN_XOR
:
4866 case IFN_COND_LEN_SHL
:
4867 case IFN_COND_LEN_SHR
:
4874 case IFN_COND_LEN_FMA
:
4875 case IFN_COND_LEN_FMS
:
4876 case IFN_COND_LEN_FNMA
:
4877 case IFN_COND_LEN_FNMS
:
4887 /* If FN takes a vector mask argument, return the index of that argument,
4888 otherwise return -1. */
4891 internal_fn_mask_index (internal_fn fn
)
4896 case IFN_MASK_LOAD_LANES
:
4897 case IFN_MASK_LEN_LOAD_LANES
:
4898 case IFN_MASK_STORE
:
4899 case IFN_MASK_STORE_LANES
:
4900 case IFN_MASK_LEN_STORE_LANES
:
4901 case IFN_MASK_LEN_LOAD
:
4902 case IFN_MASK_LEN_STORE
:
4905 case IFN_MASK_GATHER_LOAD
:
4906 case IFN_MASK_SCATTER_STORE
:
4907 case IFN_MASK_LEN_GATHER_LOAD
:
4908 case IFN_MASK_LEN_SCATTER_STORE
:
4911 case IFN_VCOND_MASK_LEN
:
4915 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4916 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4920 /* If FN takes a value that should be stored to memory, return the index
4921 of that argument, otherwise return -1. */
4924 internal_fn_stored_value_index (internal_fn fn
)
4928 case IFN_MASK_STORE
:
4929 case IFN_MASK_STORE_LANES
:
4930 case IFN_SCATTER_STORE
:
4931 case IFN_MASK_SCATTER_STORE
:
4932 case IFN_MASK_LEN_SCATTER_STORE
:
4938 case IFN_MASK_LEN_STORE
:
4939 case IFN_MASK_LEN_STORE_LANES
:
4947 /* Return true if the target supports gather load or scatter store function
4948 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4949 while for stores it is the vector type of the stored data argument.
4950 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4951 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4952 offset from the shared base address of each loaded or stored element.
4953 SCALE is the amount by which these offsets should be multiplied
4954 *after* they have been extended to address width. */
4957 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4958 tree memory_element_type
,
4959 tree offset_vector_type
, int scale
)
4961 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4962 TYPE_SIZE (memory_element_type
)))
4964 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4965 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4967 optab optab
= direct_internal_fn_optab (ifn
);
4968 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4969 TYPE_MODE (offset_vector_type
));
4970 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4971 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4972 return (icode
!= CODE_FOR_nothing
4973 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4974 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4977 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4978 for pointers of type TYPE when the accesses have LENGTH bytes and their
4979 common byte alignment is ALIGN. */
4982 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4983 poly_uint64 length
, unsigned int align
)
4985 machine_mode mode
= TYPE_MODE (type
);
4986 optab optab
= direct_internal_fn_optab (ifn
);
4987 insn_code icode
= direct_optab_handler (optab
, mode
);
4988 if (icode
== CODE_FOR_nothing
)
4990 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4991 return (insn_operand_matches (icode
, 3, length_rtx
)
4992 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4995 /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4996 or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4997 -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4998 {len_,mask_len_}store). If none of the biases match what the backend
4999 provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
5002 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
5004 optab optab
= direct_internal_fn_optab (ifn
);
5005 insn_code icode
= direct_optab_handler (optab
, mode
);
5008 if (icode
== CODE_FOR_nothing
)
5010 machine_mode mask_mode
;
5011 if (!targetm
.vectorize
.get_mask_mode (mode
).exists (&mask_mode
))
5012 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
5013 if (ifn
== IFN_LEN_LOAD
)
5015 /* Try MASK_LEN_LOAD. */
5016 optab
= direct_internal_fn_optab (IFN_MASK_LEN_LOAD
);
5020 /* Try MASK_LEN_STORE. */
5021 optab
= direct_internal_fn_optab (IFN_MASK_LEN_STORE
);
5023 icode
= convert_optab_handler (optab
, mode
, mask_mode
);
5027 if (icode
!= CODE_FOR_nothing
)
5029 /* For now we only support biases of 0 or -1. Try both of them. */
5030 if (insn_operand_matches (icode
, bias_no
, GEN_INT (0)))
5032 if (insn_operand_matches (icode
, bias_no
, GEN_INT (-1)))
5036 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
5039 /* Expand STMT as though it were a call to internal function FN. */
5042 expand_internal_call (internal_fn fn
, gcall
*stmt
)
5044 internal_fn_expanders
[fn
] (fn
, stmt
);
5047 /* Expand STMT, which is a call to internal function FN. */
5050 expand_internal_call (gcall
*stmt
)
5052 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
5055 /* If TYPE is a vector type, return true if IFN is a direct internal
5056 function that is supported for that type. If TYPE is a scalar type,
5057 return true if IFN is a direct internal function that is supported for
5058 the target's preferred vector version of TYPE. */
5061 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
5063 if (VECTOR_MODE_P (TYPE_MODE (type
)))
5064 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
5067 if (VECTOR_TYPE_P (type
)
5068 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
5071 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
5072 if (VECTOR_MODE_P (vmode
))
5074 tree vectype
= build_vector_type_for_mode (type
, vmode
);
5075 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
5079 auto_vector_modes vector_modes
;
5080 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
5081 for (machine_mode base_mode
: vector_modes
)
5082 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
5084 tree vectype
= build_vector_type_for_mode (type
, vmode
);
5085 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
5093 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
5099 expand_PHI (internal_fn
, gcall
*)
5105 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
5107 tree lhs
= gimple_call_lhs (stmt
);
5108 tree rhs1
= gimple_call_arg (stmt
, 0);
5109 tree rhs2
= gimple_call_arg (stmt
, 1);
5110 tree rhs3
= gimple_call_arg (stmt
, 2);
5111 tree type
= TREE_TYPE (rhs1
);
5113 do_pending_stack_adjust ();
5115 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5116 rtx op1
= expand_normal (rhs1
);
5117 rtx op2
= expand_normal (rhs2
);
5118 rtx op3
= expand_normal (rhs3
);
5120 class expand_operand ops
[4];
5121 create_call_lhs_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
5122 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
5123 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
5124 create_input_operand (&ops
[3], op3
, TYPE_MODE (TREE_TYPE (rhs3
)));
5125 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
5126 expand_insn (icode
, 4, ops
);
5127 assign_call_lhs (lhs
, target
, &ops
[0]);
5131 expand_ASSUME (internal_fn
, gcall
*)
5136 expand_MASK_CALL (internal_fn
, gcall
*)
5138 /* This IFN should only exist between ifcvt and vect passes. */
5143 expand_MULBITINT (internal_fn
, gcall
*stmt
)
5146 for (int i
= 0; i
< 6; i
++)
5147 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5148 (i
& 1) ? SImode
: ptr_mode
);
5149 rtx fun
= init_one_libfunc ("__mulbitint3");
5150 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 6, args
);
5154 expand_DIVMODBITINT (internal_fn
, gcall
*stmt
)
5157 for (int i
= 0; i
< 8; i
++)
5158 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5159 (i
& 1) ? SImode
: ptr_mode
);
5160 rtx fun
= init_one_libfunc ("__divmodbitint4");
5161 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 8, args
);
5165 expand_FLOATTOBITINT (internal_fn
, gcall
*stmt
)
5167 machine_mode mode
= TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt
, 2)));
5168 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5169 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5170 rtx arg2
= expand_normal (gimple_call_arg (stmt
, 2));
5171 const char *mname
= GET_MODE_NAME (mode
);
5172 unsigned mname_len
= strlen (mname
);
5173 int len
= 12 + mname_len
;
5174 if (DECIMAL_FLOAT_MODE_P (mode
))
5176 char *libfunc_name
= XALLOCAVEC (char, len
);
5177 char *p
= libfunc_name
;
5179 if (DECIMAL_FLOAT_MODE_P (mode
))
5181 #if ENABLE_DECIMAL_BID_FORMAT
5182 memcpy (p
, "__bid_fix", 9);
5184 memcpy (p
, "__dpd_fix", 9);
5190 memcpy (p
, "__fix", 5);
5193 for (q
= mname
; *q
; q
++)
5194 *p
++ = TOLOWER (*q
);
5195 memcpy (p
, "bitint", 7);
5196 rtx fun
= init_one_libfunc (libfunc_name
);
5197 emit_library_call (fun
, LCT_NORMAL
, VOIDmode
, arg0
, ptr_mode
, arg1
,
5198 SImode
, arg2
, mode
);
5202 expand_BITINTTOFLOAT (internal_fn
, gcall
*stmt
)
5204 tree lhs
= gimple_call_lhs (stmt
);
5207 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
5208 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5209 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5210 const char *mname
= GET_MODE_NAME (mode
);
5211 unsigned mname_len
= strlen (mname
);
5212 int len
= 14 + mname_len
;
5213 if (DECIMAL_FLOAT_MODE_P (mode
))
5215 char *libfunc_name
= XALLOCAVEC (char, len
);
5216 char *p
= libfunc_name
;
5218 if (DECIMAL_FLOAT_MODE_P (mode
))
5220 #if ENABLE_DECIMAL_BID_FORMAT
5221 memcpy (p
, "__bid_floatbitint", 17);
5223 memcpy (p
, "__dpd_floatbitint", 17);
5229 memcpy (p
, "__floatbitint", 13);
5232 for (q
= mname
; *q
; q
++)
5233 *p
++ = TOLOWER (*q
);
5235 rtx fun
= init_one_libfunc (libfunc_name
);
5236 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5237 rtx val
= emit_library_call_value (fun
, target
, LCT_PURE
, mode
,
5238 arg0
, ptr_mode
, arg1
, SImode
);
5240 emit_move_insn (target
, val
);
5244 expand_bitquery (internal_fn fn
, gcall
*stmt
)
5246 tree lhs
= gimple_call_lhs (stmt
);
5247 if (lhs
== NULL_TREE
)
5249 tree arg
= gimple_call_arg (stmt
, 0);
5250 if (TREE_CODE (arg
) == INTEGER_CST
)
5252 tree ret
= fold_const_call (as_combined_fn (fn
), TREE_TYPE (arg
), arg
);
5253 gcc_checking_assert (ret
&& TREE_CODE (ret
) == INTEGER_CST
);
5254 expand_assignment (lhs
, ret
, false);
5261 expand_CLRSB (internal_fn fn
, gcall
*stmt
)
5263 if (expand_bitquery (fn
, stmt
))
5264 expand_unary_optab_fn (fn
, stmt
, clrsb_optab
);
5268 expand_CLZ (internal_fn fn
, gcall
*stmt
)
5270 if (expand_bitquery (fn
, stmt
))
5271 expand_unary_optab_fn (fn
, stmt
, clz_optab
);
5275 expand_CTZ (internal_fn fn
, gcall
*stmt
)
5277 if (expand_bitquery (fn
, stmt
))
5278 expand_unary_optab_fn (fn
, stmt
, ctz_optab
);
5282 expand_FFS (internal_fn fn
, gcall
*stmt
)
5284 if (expand_bitquery (fn
, stmt
))
5285 expand_unary_optab_fn (fn
, stmt
, ffs_optab
);
5289 expand_PARITY (internal_fn fn
, gcall
*stmt
)
5291 if (expand_bitquery (fn
, stmt
))
5292 expand_unary_optab_fn (fn
, stmt
, parity_optab
);
5296 expand_POPCOUNT (internal_fn fn
, gcall
*stmt
)
5298 if (!expand_bitquery (fn
, stmt
))
5300 if (gimple_call_num_args (stmt
) == 1)
5302 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5305 /* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5306 because the result is only used in an equality comparison against 1.
5307 Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5308 or (arg ^ (arg - 1)) > arg - 1 is cheaper.
5309 If .POPCOUNT second argument is 0, we additionally know that arg
5310 is non-zero, so use arg & (arg - 1) == 0 instead.
5311 If .POPCOUNT second argument is -1, the comparison was either `<= 1`
5313 bool speed_p
= optimize_insn_for_speed_p ();
5314 tree lhs
= gimple_call_lhs (stmt
);
5315 tree arg
= gimple_call_arg (stmt
, 0);
5316 bool nonzero_arg
= integer_zerop (gimple_call_arg (stmt
, 1));
5317 bool was_le
= integer_minus_onep (gimple_call_arg (stmt
, 1));
5320 tree type
= TREE_TYPE (arg
);
5321 machine_mode mode
= TYPE_MODE (type
);
5322 machine_mode lhsmode
= TYPE_MODE (TREE_TYPE (lhs
));
5323 do_pending_stack_adjust ();
5325 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5326 rtx_insn
*popcount_insns
= get_insns ();
5329 rtx plhs
= expand_normal (lhs
);
5330 rtx pcmp
= emit_store_flag (NULL_RTX
, EQ
, plhs
, const1_rtx
, lhsmode
, 0, 0);
5331 if (pcmp
== NULL_RTX
)
5335 emit_insn (popcount_insns
);
5338 rtx_insn
*popcount_cmp_insns
= get_insns ();
5341 rtx op0
= expand_normal (arg
);
5342 rtx argm1
= expand_simple_binop (mode
, PLUS
, op0
, constm1_rtx
, NULL_RTX
,
5344 if (argm1
== NULL_RTX
)
5346 rtx argxorargm1
= expand_simple_binop (mode
, nonzero_arg
? AND
: XOR
, op0
,
5347 argm1
, NULL_RTX
, 1, OPTAB_WIDEN
);
5348 if (argxorargm1
== NULL_RTX
)
5352 cmp
= emit_store_flag (NULL_RTX
, EQ
, argxorargm1
, const0_rtx
, mode
, 1, 1);
5354 cmp
= emit_store_flag (NULL_RTX
, GTU
, argxorargm1
, argm1
, mode
, 1, 1);
5355 if (cmp
== NULL_RTX
)
5357 rtx_insn
*cmp_insns
= get_insns ();
5359 unsigned popcount_cost
= (seq_cost (popcount_insns
, speed_p
)
5360 + seq_cost (popcount_cmp_insns
, speed_p
));
5361 unsigned cmp_cost
= seq_cost (cmp_insns
, speed_p
);
5363 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
5364 fprintf(dump_file
, "popcount == 1: popcount cost: %u; cmp cost: %u\n",
5365 popcount_cost
, cmp_cost
);
5367 if (popcount_cost
<= cmp_cost
)
5368 emit_insn (popcount_insns
);
5372 emit_insn (cmp_insns
);
5373 plhs
= expand_normal (lhs
);
5374 if (GET_MODE (cmp
) != GET_MODE (plhs
))
5375 cmp
= convert_to_mode (GET_MODE (plhs
), cmp
, 1);
5376 /* For `<= 1`, we need to produce `2 - cmp` or `cmp ? 1 : 2` as that
5377 then gets compared against 1 and we need the false case to be 2. */
5380 cmp
= expand_simple_binop (GET_MODE (cmp
), MINUS
, const2_rtx
,
5381 cmp
, NULL_RTX
, 1, OPTAB_WIDEN
);
5385 emit_move_insn (plhs
, cmp
);
5386 rtx_insn
*all_insns
= get_insns ();
5388 emit_insn (all_insns
);