2 Copyright (C) 2011-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
54 #include "gimple-range.h"
55 #include "fold-const-call.h"
56 #include "tree-ssa-live.h"
57 #include "tree-outof-ssa.h"
59 /* For lang_hooks.types.type_for_mode. */
60 #include "langhooks.h"
62 /* The names of each internal function, indexed by function number. */
63 const char *const internal_fn_name_array
[] = {
64 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
65 #include "internal-fn.def"
69 /* The ECF_* flags of each internal function, indexed by function number. */
70 const int internal_fn_flags_array
[] = {
71 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
72 #include "internal-fn.def"
76 /* Return the internal function called NAME, or IFN_LAST if there's
80 lookup_internal_fn (const char *name
)
82 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
83 static name_to_fn_map_type
*name_to_fn_map
;
87 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
88 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
89 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
92 internal_fn
*entry
= name_to_fn_map
->get (name
);
93 return entry
? *entry
: IFN_LAST
;
96 /* Geven an internal_fn IFN that is a widening function, return its
97 corresponding LO and HI internal_fns. */
100 lookup_hilo_internal_fn (internal_fn ifn
, internal_fn
*lo
, internal_fn
*hi
)
102 gcc_assert (widening_fn_p (ifn
));
108 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
109 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
111 *lo = internal_fn (IFN_##NAME##_LO); \
112 *hi = internal_fn (IFN_##NAME##_HI); \
114 #include "internal-fn.def"
118 /* Given an internal_fn IFN that is a widening function, return its
119 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
122 lookup_evenodd_internal_fn (internal_fn ifn
, internal_fn
*even
,
125 gcc_assert (widening_fn_p (ifn
));
131 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
132 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
134 *even = internal_fn (IFN_##NAME##_EVEN); \
135 *odd = internal_fn (IFN_##NAME##_ODD); \
137 #include "internal-fn.def"
142 /* Fnspec of each internal function, indexed by function number. */
143 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
148 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
149 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
150 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
151 #include "internal-fn.def"
152 internal_fn_fnspec_array
[IFN_LAST
] = 0;
155 /* Create static initializers for the information returned by
156 direct_internal_fn. */
157 #define not_direct { -2, -2, false }
158 #define mask_load_direct { -1, 2, false }
159 #define load_lanes_direct { -1, -1, false }
160 #define mask_load_lanes_direct { -1, -1, false }
161 #define gather_load_direct { 3, 1, false }
162 #define len_load_direct { -1, -1, false }
163 #define mask_len_load_direct { -1, 4, false }
164 #define mask_store_direct { 3, 2, false }
165 #define store_lanes_direct { 0, 0, false }
166 #define mask_store_lanes_direct { 0, 0, false }
167 #define vec_cond_mask_direct { 1, 0, false }
168 #define vec_cond_mask_len_direct { 1, 1, false }
169 #define vec_cond_direct { 2, 0, false }
170 #define scatter_store_direct { 3, 1, false }
171 #define len_store_direct { 3, 3, false }
172 #define mask_len_store_direct { 4, 5, false }
173 #define vec_set_direct { 3, 3, false }
174 #define vec_extract_direct { 0, -1, false }
175 #define unary_direct { 0, 0, true }
176 #define unary_convert_direct { -1, 0, true }
177 #define binary_direct { 0, 0, true }
178 #define ternary_direct { 0, 0, true }
179 #define cond_unary_direct { 1, 1, true }
180 #define cond_binary_direct { 1, 1, true }
181 #define cond_ternary_direct { 1, 1, true }
182 #define cond_len_unary_direct { 1, 1, true }
183 #define cond_len_binary_direct { 1, 1, true }
184 #define cond_len_ternary_direct { 1, 1, true }
185 #define while_direct { 0, 2, false }
186 #define fold_extract_direct { 2, 2, false }
187 #define fold_len_extract_direct { 2, 2, false }
188 #define fold_left_direct { 1, 1, false }
189 #define mask_fold_left_direct { 1, 1, false }
190 #define mask_len_fold_left_direct { 1, 1, false }
191 #define check_ptrs_direct { 0, 0, false }
193 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
194 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
195 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
196 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
197 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
198 #include "internal-fn.def"
202 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
203 output operands and NINPUTS input operands, where NOUTPUTS is either
204 0 or 1. The output operand (if any) comes first, followed by the
205 NINPUTS input operands. */
208 expand_fn_using_insn (gcall
*stmt
, insn_code icode
, unsigned int noutputs
,
209 unsigned int ninputs
)
211 gcc_assert (icode
!= CODE_FOR_nothing
);
213 expand_operand
*ops
= XALLOCAVEC (expand_operand
, noutputs
+ ninputs
);
214 unsigned int opno
= 0;
215 rtx lhs_rtx
= NULL_RTX
;
216 tree lhs
= gimple_call_lhs (stmt
);
220 gcc_assert (noutputs
== 1);
222 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
224 /* Do not assign directly to a promoted subreg, since there is no
225 guarantee that the instruction will leave the upper bits of the
226 register in the state required by SUBREG_PROMOTED_SIGN. */
228 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
230 create_output_operand (&ops
[opno
], dest
,
231 insn_data
[icode
].operand
[opno
].mode
);
237 for (unsigned int i
= 0; i
< ninputs
; ++i
)
239 tree rhs
= gimple_call_arg (stmt
, i
);
240 tree rhs_type
= TREE_TYPE (rhs
);
241 rtx rhs_rtx
= expand_normal (rhs
);
242 if (INTEGRAL_TYPE_P (rhs_type
))
243 create_convert_operand_from (&ops
[opno
], rhs_rtx
,
244 TYPE_MODE (rhs_type
),
245 TYPE_UNSIGNED (rhs_type
));
246 else if (TREE_CODE (rhs
) == SSA_NAME
247 && SSA_NAME_IS_DEFAULT_DEF (rhs
)
248 && VAR_P (SSA_NAME_VAR (rhs
)))
249 create_undefined_input_operand (&ops
[opno
], TYPE_MODE (rhs_type
));
250 else if (VECTOR_BOOLEAN_TYPE_P (rhs_type
)
251 && SCALAR_INT_MODE_P (TYPE_MODE (rhs_type
))
252 && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (rhs_type
)),
253 TYPE_VECTOR_SUBPARTS (rhs_type
).to_constant ()))
255 /* Ensure that the vector bitmasks do not have excess bits. */
256 int nunits
= TYPE_VECTOR_SUBPARTS (rhs_type
).to_constant ();
257 rtx tmp
= expand_binop (TYPE_MODE (rhs_type
), and_optab
, rhs_rtx
,
258 GEN_INT ((HOST_WIDE_INT_1U
<< nunits
) - 1),
259 NULL_RTX
, true, OPTAB_WIDEN
);
260 create_input_operand (&ops
[opno
], tmp
, TYPE_MODE (rhs_type
));
263 create_input_operand (&ops
[opno
], rhs_rtx
, TYPE_MODE (rhs_type
));
267 gcc_assert (opno
== noutputs
+ ninputs
);
268 expand_insn (icode
, opno
, ops
);
269 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
271 /* If the return value has an integral type, convert the instruction
272 result to that type. This is useful for things that return an
273 int regardless of the size of the input. If the instruction result
274 is smaller than required, assume that it is signed.
276 If the return value has a nonintegral type, its mode must match
277 the instruction result. */
278 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
280 /* If this is a scalar in a register that is stored in a wider
281 mode than the declared mode, compute the result into its
282 declared mode and then convert to the wider mode. */
283 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
284 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
285 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
286 SUBREG_PROMOTED_SIGN (lhs_rtx
));
288 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
289 emit_move_insn (lhs_rtx
, ops
[0].value
);
292 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
293 convert_move (lhs_rtx
, ops
[0].value
, 0);
298 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
299 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
301 static enum insn_code
302 get_multi_vector_move (tree array_type
, convert_optab optab
)
307 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
308 imode
= TYPE_MODE (array_type
);
309 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
311 return convert_optab_handler (optab
, imode
, vmode
);
314 /* Add mask and len arguments according to the STMT. */
317 add_mask_and_len_args (expand_operand
*ops
, unsigned int opno
, gcall
*stmt
)
319 internal_fn ifn
= gimple_call_internal_fn (stmt
);
320 int len_index
= internal_fn_len_index (ifn
);
321 /* BIAS is always consecutive next of LEN. */
322 int bias_index
= len_index
+ 1;
323 int mask_index
= internal_fn_mask_index (ifn
);
324 /* The order of arguments are always {len,bias,mask}. */
327 tree mask
= gimple_call_arg (stmt
, mask_index
);
328 rtx mask_rtx
= expand_normal (mask
);
330 tree mask_type
= TREE_TYPE (mask
);
331 if (VECTOR_BOOLEAN_TYPE_P (mask_type
)
332 && SCALAR_INT_MODE_P (TYPE_MODE (mask_type
))
333 && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (mask_type
)),
334 TYPE_VECTOR_SUBPARTS (mask_type
).to_constant ()))
336 /* Ensure that the vector bitmasks do not have excess bits. */
337 int nunits
= TYPE_VECTOR_SUBPARTS (mask_type
).to_constant ();
338 mask_rtx
= expand_binop (TYPE_MODE (mask_type
), and_optab
, mask_rtx
,
339 GEN_INT ((HOST_WIDE_INT_1U
<< nunits
) - 1),
340 NULL_RTX
, true, OPTAB_WIDEN
);
343 create_input_operand (&ops
[opno
++], mask_rtx
,
344 TYPE_MODE (TREE_TYPE (mask
)));
348 tree len
= gimple_call_arg (stmt
, len_index
);
349 rtx len_rtx
= expand_normal (len
);
350 create_convert_operand_from (&ops
[opno
++], len_rtx
,
351 TYPE_MODE (TREE_TYPE (len
)),
352 TYPE_UNSIGNED (TREE_TYPE (len
)));
353 tree biast
= gimple_call_arg (stmt
, bias_index
);
354 rtx bias
= expand_normal (biast
);
355 create_input_operand (&ops
[opno
++], bias
, QImode
);
360 /* Expand LOAD_LANES call STMT using optab OPTAB. */
363 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
365 class expand_operand ops
[2];
369 lhs
= gimple_call_lhs (stmt
);
370 rhs
= gimple_call_arg (stmt
, 0);
371 type
= TREE_TYPE (lhs
);
373 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
374 mem
= expand_normal (rhs
);
376 gcc_assert (MEM_P (mem
));
377 PUT_MODE (mem
, TYPE_MODE (type
));
379 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
380 create_fixed_operand (&ops
[1], mem
);
381 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
382 if (!rtx_equal_p (target
, ops
[0].value
))
383 emit_move_insn (target
, ops
[0].value
);
386 /* Expand STORE_LANES call STMT using optab OPTAB. */
389 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
391 class expand_operand ops
[2];
395 lhs
= gimple_call_lhs (stmt
);
396 rhs
= gimple_call_arg (stmt
, 0);
397 type
= TREE_TYPE (rhs
);
399 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
400 reg
= expand_normal (rhs
);
402 gcc_assert (MEM_P (target
));
403 PUT_MODE (target
, TYPE_MODE (type
));
405 create_fixed_operand (&ops
[0], target
);
406 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
407 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
411 expand_ANNOTATE (internal_fn
, gcall
*)
416 /* This should get expanded in omp_device_lower pass. */
419 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
424 /* This should get expanded in omp_device_lower pass. */
427 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
432 /* Allocate per-lane storage and begin non-uniform execution region. */
435 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
438 tree lhs
= gimple_call_lhs (stmt
);
440 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
442 target
= gen_reg_rtx (Pmode
);
443 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
444 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
445 class expand_operand ops
[3];
446 create_output_operand (&ops
[0], target
, Pmode
);
447 create_input_operand (&ops
[1], size
, Pmode
);
448 create_input_operand (&ops
[2], align
, Pmode
);
449 gcc_assert (targetm
.have_omp_simt_enter ());
450 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
451 if (!rtx_equal_p (target
, ops
[0].value
))
452 emit_move_insn (target
, ops
[0].value
);
455 /* Deallocate per-lane storage and leave non-uniform execution region. */
458 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
460 gcc_checking_assert (!gimple_call_lhs (stmt
));
461 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
462 class expand_operand ops
[1];
463 create_input_operand (&ops
[0], arg
, Pmode
);
464 gcc_assert (targetm
.have_omp_simt_exit ());
465 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
468 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
469 without SIMT execution this should be expanded in omp_device_lower pass. */
472 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
474 tree lhs
= gimple_call_lhs (stmt
);
478 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
479 gcc_assert (targetm
.have_omp_simt_lane ());
480 emit_insn (targetm
.gen_omp_simt_lane (target
));
483 /* This should get expanded in omp_device_lower pass. */
486 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
491 /* This should get expanded in omp_device_lower pass. */
494 expand_GOMP_TARGET_REV (internal_fn
, gcall
*)
499 /* Lane index of the first SIMT lane that supplies a non-zero argument.
500 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
501 lane that executed the last iteration for handling OpenMP lastprivate. */
504 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
506 tree lhs
= gimple_call_lhs (stmt
);
510 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
511 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
512 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
513 class expand_operand ops
[2];
514 create_output_operand (&ops
[0], target
, mode
);
515 create_input_operand (&ops
[1], cond
, mode
);
516 gcc_assert (targetm
.have_omp_simt_last_lane ());
517 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
518 if (!rtx_equal_p (target
, ops
[0].value
))
519 emit_move_insn (target
, ops
[0].value
);
522 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
525 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
527 tree lhs
= gimple_call_lhs (stmt
);
531 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
532 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
533 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
534 class expand_operand ops
[2];
535 create_output_operand (&ops
[0], target
, mode
);
536 create_input_operand (&ops
[1], ctr
, mode
);
537 gcc_assert (targetm
.have_omp_simt_ordered ());
538 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
539 if (!rtx_equal_p (target
, ops
[0].value
))
540 emit_move_insn (target
, ops
[0].value
);
543 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
544 any lane supplies a non-zero argument. */
547 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
549 tree lhs
= gimple_call_lhs (stmt
);
553 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
554 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
555 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
556 class expand_operand ops
[2];
557 create_output_operand (&ops
[0], target
, mode
);
558 create_input_operand (&ops
[1], cond
, mode
);
559 gcc_assert (targetm
.have_omp_simt_vote_any ());
560 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
561 if (!rtx_equal_p (target
, ops
[0].value
))
562 emit_move_insn (target
, ops
[0].value
);
565 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
566 is destination lane index XOR given offset. */
569 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
571 tree lhs
= gimple_call_lhs (stmt
);
575 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
576 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
577 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
578 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
579 class expand_operand ops
[3];
580 create_output_operand (&ops
[0], target
, mode
);
581 create_input_operand (&ops
[1], src
, mode
);
582 create_input_operand (&ops
[2], idx
, SImode
);
583 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
584 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
585 if (!rtx_equal_p (target
, ops
[0].value
))
586 emit_move_insn (target
, ops
[0].value
);
589 /* Exchange between SIMT lanes according to given source lane index. */
592 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
594 tree lhs
= gimple_call_lhs (stmt
);
598 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
599 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
600 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
601 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
602 class expand_operand ops
[3];
603 create_output_operand (&ops
[0], target
, mode
);
604 create_input_operand (&ops
[1], src
, mode
);
605 create_input_operand (&ops
[2], idx
, SImode
);
606 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
607 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
608 if (!rtx_equal_p (target
, ops
[0].value
))
609 emit_move_insn (target
, ops
[0].value
);
612 /* This should get expanded in adjust_simduid_builtins. */
615 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
620 /* This should get expanded in adjust_simduid_builtins. */
623 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
628 /* This should get expanded in adjust_simduid_builtins. */
631 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
636 /* This should get expanded in adjust_simduid_builtins. */
639 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
644 /* This should get expanded in adjust_simduid_builtins. */
647 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
652 /* This should get expanded in the sanopt pass. */
655 expand_UBSAN_NULL (internal_fn
, gcall
*)
660 /* This should get expanded in the sanopt pass. */
663 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
668 /* This should get expanded in the sanopt pass. */
671 expand_UBSAN_VPTR (internal_fn
, gcall
*)
676 /* This should get expanded in the sanopt pass. */
679 expand_UBSAN_PTR (internal_fn
, gcall
*)
684 /* This should get expanded in the sanopt pass. */
687 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
692 /* This should get expanded in the sanopt pass. */
695 expand_HWASAN_CHECK (internal_fn
, gcall
*)
700 /* For hwasan stack tagging:
701 Clear tags on the dynamically allocated space.
702 For use after an object dynamically allocated on the stack goes out of
705 expand_HWASAN_ALLOCA_UNPOISON (internal_fn
, gcall
*gc
)
707 gcc_assert (Pmode
== ptr_mode
);
708 tree restored_position
= gimple_call_arg (gc
, 0);
709 rtx restored_rtx
= expand_expr (restored_position
, NULL_RTX
, VOIDmode
,
711 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
712 rtx off
= expand_simple_binop (Pmode
, MINUS
, restored_rtx
,
713 stack_pointer_rtx
, NULL_RTX
, 0,
715 emit_library_call_value (func
, NULL_RTX
, LCT_NORMAL
, VOIDmode
,
716 virtual_stack_dynamic_rtx
, Pmode
,
717 HWASAN_STACK_BACKGROUND
, QImode
,
721 /* For hwasan stack tagging:
722 Return a tag to be used for a dynamic allocation. */
724 expand_HWASAN_CHOOSE_TAG (internal_fn
, gcall
*gc
)
726 tree tag
= gimple_call_lhs (gc
);
727 rtx target
= expand_expr (tag
, NULL_RTX
, VOIDmode
, EXPAND_NORMAL
);
728 machine_mode mode
= GET_MODE (target
);
729 gcc_assert (mode
== QImode
);
731 rtx base_tag
= targetm
.memtag
.extract_tag (hwasan_frame_base (), NULL_RTX
);
732 gcc_assert (base_tag
);
733 rtx tag_offset
= gen_int_mode (hwasan_current_frame_tag (), QImode
);
734 rtx chosen_tag
= expand_simple_binop (QImode
, PLUS
, base_tag
, tag_offset
,
735 target
, /* unsignedp = */1,
737 chosen_tag
= hwasan_truncate_to_tag_size (chosen_tag
, target
);
739 /* Really need to put the tag into the `target` RTX. */
740 if (chosen_tag
!= target
)
742 rtx temp
= chosen_tag
;
743 gcc_assert (GET_MODE (chosen_tag
) == mode
);
744 emit_move_insn (target
, temp
);
747 hwasan_increment_frame_tag ();
750 /* For hwasan stack tagging:
751 Tag a region of space in the shadow stack according to the base pointer of
752 an object on the stack. N.b. the length provided in the internal call is
753 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
755 expand_HWASAN_MARK (internal_fn
, gcall
*gc
)
757 gcc_assert (ptr_mode
== Pmode
);
758 HOST_WIDE_INT flag
= tree_to_shwi (gimple_call_arg (gc
, 0));
759 bool is_poison
= ((asan_mark_flags
)flag
) == ASAN_MARK_POISON
;
761 tree base
= gimple_call_arg (gc
, 1);
762 gcc_checking_assert (TREE_CODE (base
) == ADDR_EXPR
);
763 rtx base_rtx
= expand_normal (base
);
765 rtx tag
= is_poison
? HWASAN_STACK_BACKGROUND
766 : targetm
.memtag
.extract_tag (base_rtx
, NULL_RTX
);
767 rtx address
= targetm
.memtag
.untagged_pointer (base_rtx
, NULL_RTX
);
769 tree len
= gimple_call_arg (gc
, 2);
770 rtx r_len
= expand_normal (len
);
772 rtx func
= init_one_libfunc ("__hwasan_tag_memory");
773 emit_library_call (func
, LCT_NORMAL
, VOIDmode
, address
, Pmode
,
774 tag
, QImode
, r_len
, Pmode
);
777 /* For hwasan stack tagging:
778 Store a tag into a pointer. */
780 expand_HWASAN_SET_TAG (internal_fn
, gcall
*gc
)
782 gcc_assert (ptr_mode
== Pmode
);
783 tree g_target
= gimple_call_lhs (gc
);
784 tree g_ptr
= gimple_call_arg (gc
, 0);
785 tree g_tag
= gimple_call_arg (gc
, 1);
787 rtx ptr
= expand_normal (g_ptr
);
788 rtx tag
= expand_expr (g_tag
, NULL_RTX
, QImode
, EXPAND_NORMAL
);
789 rtx target
= expand_normal (g_target
);
791 rtx untagged
= targetm
.memtag
.untagged_pointer (ptr
, target
);
792 rtx tagged_value
= targetm
.memtag
.set_tag (untagged
, tag
, target
);
793 if (tagged_value
!= target
)
794 emit_move_insn (target
, tagged_value
);
797 /* This should get expanded in the sanopt pass. */
800 expand_ASAN_CHECK (internal_fn
, gcall
*)
805 /* This should get expanded in the sanopt pass. */
808 expand_ASAN_MARK (internal_fn
, gcall
*)
813 /* This should get expanded in the sanopt pass. */
816 expand_ASAN_POISON (internal_fn
, gcall
*)
821 /* This should get expanded in the sanopt pass. */
824 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
829 /* This should get expanded in the tsan pass. */
832 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
837 /* This should get expanded in the lower pass. */
840 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
842 error_at (gimple_location (call
),
843 "invalid use of attribute %<fallthrough%>");
846 /* Return minimum precision needed to represent all values
847 of ARG in SIGNed integral type. */
850 get_min_precision (tree arg
, signop sign
)
852 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
854 signop orig_sign
= sign
;
855 if (TREE_CODE (arg
) == INTEGER_CST
)
858 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
860 widest_int w
= wi::to_widest (arg
);
861 w
= wi::ext (w
, prec
, sign
);
862 p
= wi::min_precision (w
, sign
);
865 p
= wi::min_precision (wi::to_wide (arg
), sign
);
866 return MIN (p
, prec
);
868 while (CONVERT_EXPR_P (arg
)
869 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
870 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
872 arg
= TREE_OPERAND (arg
, 0);
873 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
875 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
877 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
878 return prec
+ (orig_sign
!= sign
);
879 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
882 return prec
+ (orig_sign
!= sign
);
884 if (CONVERT_EXPR_P (arg
)
885 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
886 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) > prec
)
888 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
889 If y_2's min precision is smaller than prec, return that. */
890 int oprec
= get_min_precision (TREE_OPERAND (arg
, 0), sign
);
892 return oprec
+ (orig_sign
!= sign
);
894 if (TREE_CODE (arg
) != SSA_NAME
)
895 return prec
+ (orig_sign
!= sign
);
897 while (!get_global_range_query ()->range_of_expr (r
, arg
)
901 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
902 if (is_gimple_assign (g
)
903 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
905 tree t
= gimple_assign_rhs1 (g
);
906 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
907 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
910 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
912 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
914 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
915 return prec
+ (orig_sign
!= sign
);
916 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
919 return prec
+ (orig_sign
!= sign
);
923 return prec
+ (orig_sign
!= sign
);
925 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
927 int p1
= wi::min_precision (r
.lower_bound (), sign
);
928 int p2
= wi::min_precision (r
.upper_bound (), sign
);
930 prec
= MIN (prec
, p1
);
932 else if (sign
== UNSIGNED
&& !wi::neg_p (r
.lower_bound (), SIGNED
))
934 int p
= wi::min_precision (r
.upper_bound (), UNSIGNED
);
935 prec
= MIN (prec
, p
);
937 return prec
+ (orig_sign
!= sign
);
940 /* Helper for expand_*_overflow. Set the __imag__ part to true
941 (1 except for signed:1 type, in which case store -1). */
944 expand_arith_set_overflow (tree lhs
, rtx target
)
946 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
947 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
948 write_complex_part (target
, constm1_rtx
, true, false);
950 write_complex_part (target
, const1_rtx
, true, false);
953 /* Helper for expand_*_overflow. Store RES into the __real__ part
954 of TARGET. If RES has larger MODE than __real__ part of TARGET,
955 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
956 if LHS has smaller precision than its mode. */
959 expand_arith_overflow_result_store (tree lhs
, rtx target
,
960 scalar_int_mode mode
, rtx res
)
962 scalar_int_mode tgtmode
963 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
967 rtx_code_label
*done_label
= gen_label_rtx ();
968 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
969 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
970 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
971 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
972 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
973 profile_probability::very_likely ());
974 expand_arith_set_overflow (lhs
, target
);
975 emit_label (done_label
);
977 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
978 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
981 rtx_code_label
*done_label
= gen_label_rtx ();
982 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
987 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
989 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
990 true, OPTAB_LIB_WIDEN
);
994 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
996 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
999 do_compare_rtx_and_jump (res
, lres
,
1000 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
1001 profile_probability::very_likely ());
1002 expand_arith_set_overflow (lhs
, target
);
1003 emit_label (done_label
);
1005 write_complex_part (target
, lres
, false, false);
1008 /* Helper for expand_*_overflow. Store RES into TARGET. */
1011 expand_ubsan_result_store (tree lhs
, rtx target
, scalar_int_mode mode
,
1012 rtx res
, rtx_code_label
*do_error
)
1014 if (TREE_CODE (TREE_TYPE (lhs
)) == BITINT_TYPE
1015 && TYPE_PRECISION (TREE_TYPE (lhs
)) < GET_MODE_PRECISION (mode
))
1017 int uns
= TYPE_UNSIGNED (TREE_TYPE (lhs
));
1018 int prec
= TYPE_PRECISION (TREE_TYPE (lhs
));
1019 int tgtprec
= GET_MODE_PRECISION (mode
);
1020 rtx resc
= gen_reg_rtx (mode
), lres
;
1021 emit_move_insn (resc
, res
);
1025 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
1027 lres
= expand_simple_binop (mode
, AND
, res
, mask
, NULL_RTX
,
1028 true, OPTAB_LIB_WIDEN
);
1032 lres
= expand_shift (LSHIFT_EXPR
, mode
, res
, tgtprec
- prec
,
1034 lres
= expand_shift (RSHIFT_EXPR
, mode
, lres
, tgtprec
- prec
,
1038 emit_move_insn (res
, lres
);
1039 do_compare_rtx_and_jump (res
, resc
,
1040 NE
, true, mode
, NULL_RTX
, NULL
, do_error
,
1041 profile_probability::very_unlikely ());
1043 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
1044 /* If this is a scalar in a register that is stored in a wider mode
1045 than the declared mode, compute the result into its declared mode
1046 and then convert to the wider mode. Our value is the computed
1048 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
1050 emit_move_insn (target
, res
);
1053 /* Add sub/add overflow checking to the statement STMT.
1054 CODE says whether the operation is +, or -. */
1057 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
1058 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
1059 bool uns1_p
, bool is_ubsan
, tree
*datap
)
1061 rtx res
, target
= NULL_RTX
;
1063 rtx_code_label
*done_label
= gen_label_rtx ();
1064 rtx_code_label
*do_error
= gen_label_rtx ();
1065 do_pending_stack_adjust ();
1066 rtx op0
= expand_normal (arg0
);
1067 rtx op1
= expand_normal (arg1
);
1068 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1069 int prec
= GET_MODE_PRECISION (mode
);
1070 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1071 bool do_xor
= false;
1074 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1078 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1080 write_complex_part (target
, const0_rtx
, true, false);
1083 /* We assume both operands and result have the same precision
1084 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1085 with that precision, U for unsigned type with that precision,
1086 sgn for unsigned most significant bit in that precision.
1087 s1 is signed first operand, u1 is unsigned first operand,
1088 s2 is signed second operand, u2 is unsigned second operand,
1089 sr is signed result, ur is unsigned result and the following
1090 rules say how to compute result (which is always result of
1091 the operands as if both were unsigned, cast to the right
1092 signedness) and how to compute whether operation overflowed.
1095 res = (S) ((U) s1 + (U) s2)
1096 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1098 res = (S) ((U) s1 - (U) s2)
1099 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1102 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1105 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1107 res = (S) ((U) s1 + u2)
1108 ovf = ((U) res ^ sgn) < u2
1113 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1115 res = (S) ((U) s1 - u2)
1116 ovf = u2 > ((U) s1 ^ sgn)
1119 ovf = s1 < 0 || u2 > (U) s1
1122 ovf = u1 >= ((U) s2 ^ sgn)
1127 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1129 res = (U) s1 + (U) s2
1130 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1133 ovf = (U) res < u2 || res < 0
1136 ovf = u1 >= u2 ? res < 0 : res >= 0
1138 res = (U) s1 - (U) s2
1139 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1141 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
1143 /* PLUS_EXPR is commutative, if operand signedness differs,
1144 canonicalize to the first operand being signed and second
1145 unsigned to simplify following code. */
1146 std::swap (op0
, op1
);
1147 std::swap (arg0
, arg1
);
1152 /* u1 +- u2 -> ur */
1153 if (uns0_p
&& uns1_p
&& unsr_p
)
1155 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
1156 : usubv4_optab
, mode
);
1157 if (icode
!= CODE_FOR_nothing
)
1159 class expand_operand ops
[4];
1160 rtx_insn
*last
= get_last_insn ();
1162 res
= gen_reg_rtx (mode
);
1163 create_output_operand (&ops
[0], res
, mode
);
1164 create_input_operand (&ops
[1], op0
, mode
);
1165 create_input_operand (&ops
[2], op1
, mode
);
1166 create_fixed_operand (&ops
[3], do_error
);
1167 if (maybe_expand_insn (icode
, 4, ops
))
1169 last
= get_last_insn ();
1170 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1172 && any_condjump_p (last
)
1173 && !find_reg_note (last
, REG_BR_PROB
, 0))
1174 add_reg_br_prob_note (last
,
1175 profile_probability::very_unlikely ());
1176 emit_jump (done_label
);
1177 goto do_error_label
;
1180 delete_insns_since (last
);
1183 /* Compute the operation. On RTL level, the addition is always
1185 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1186 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1188 /* For PLUS_EXPR, the operation is commutative, so we can pick
1189 operand to compare against. For prec <= BITS_PER_WORD, I think
1190 preferring REG operand is better over CONST_INT, because
1191 the CONST_INT might enlarge the instruction or CSE would need
1192 to figure out we'd already loaded it into a register before.
1193 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1194 as then the multi-word comparison can be perhaps simplified. */
1195 if (code
== PLUS_EXPR
1196 && (prec
<= BITS_PER_WORD
1197 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
1198 : CONST_SCALAR_INT_P (op1
)))
1200 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
1201 true, mode
, NULL_RTX
, NULL
, done_label
,
1202 profile_probability::very_likely ());
1203 goto do_error_label
;
1206 /* s1 +- u2 -> sr */
1207 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1209 /* Compute the operation. On RTL level, the addition is always
1211 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1212 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1213 rtx tem
= expand_binop (mode
, add_optab
,
1214 code
== PLUS_EXPR
? res
: op0
, sgn
,
1215 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1216 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1217 done_label
, profile_probability::very_likely ());
1218 goto do_error_label
;
1222 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1224 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1226 /* As we've changed op1, we have to avoid using the value range
1227 for the original argument. */
1228 arg1
= error_mark_node
;
1234 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
1236 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
1238 /* As we've changed op0, we have to avoid using the value range
1239 for the original argument. */
1240 arg0
= error_mark_node
;
1246 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
1248 /* Compute the operation. On RTL level, the addition is always
1250 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1252 int pos_neg
= get_range_pos_neg (arg0
);
1254 /* If ARG0 is known to be always negative, this is always overflow. */
1255 emit_jump (do_error
);
1256 else if (pos_neg
== 3)
1257 /* If ARG0 is not known to be always positive, check at runtime. */
1258 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1259 NULL
, do_error
, profile_probability::very_unlikely ());
1260 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
1261 done_label
, profile_probability::very_likely ());
1262 goto do_error_label
;
1266 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
1268 /* Compute the operation. On RTL level, the addition is always
1270 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1272 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
1274 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
1275 done_label
, profile_probability::very_likely ());
1276 goto do_error_label
;
1280 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1282 /* Compute the operation. On RTL level, the addition is always
1284 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
1286 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1287 NULL
, do_error
, profile_probability::very_unlikely ());
1289 /* The operation is commutative, so we can pick operand to compare
1290 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1291 is better over CONST_INT, because the CONST_INT might enlarge the
1292 instruction or CSE would need to figure out we'd already loaded it
1293 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1294 might be more beneficial, as then the multi-word comparison can be
1295 perhaps simplified. */
1296 if (prec
<= BITS_PER_WORD
1297 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
1298 : CONST_SCALAR_INT_P (op0
))
1300 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
1301 done_label
, profile_probability::very_likely ());
1302 goto do_error_label
;
1305 /* s1 +- s2 -> ur */
1306 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1308 /* Compute the operation. On RTL level, the addition is always
1310 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1311 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1312 int pos_neg
= get_range_pos_neg (arg1
);
1313 if (code
== PLUS_EXPR
)
1315 int pos_neg0
= get_range_pos_neg (arg0
);
1316 if (pos_neg0
!= 3 && pos_neg
== 3)
1318 std::swap (op0
, op1
);
1325 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
1326 ? and_optab
: ior_optab
,
1327 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1328 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
1329 NULL
, done_label
, profile_probability::very_likely ());
1333 rtx_code_label
*do_ior_label
= gen_label_rtx ();
1334 do_compare_rtx_and_jump (op1
, const0_rtx
,
1335 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
1336 NULL_RTX
, NULL
, do_ior_label
,
1337 profile_probability::even ());
1338 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
1340 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1341 NULL
, done_label
, profile_probability::very_likely ());
1342 emit_jump (do_error
);
1343 emit_label (do_ior_label
);
1344 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
1346 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1347 NULL
, done_label
, profile_probability::very_likely ());
1349 goto do_error_label
;
1353 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
1355 /* Compute the operation. On RTL level, the addition is always
1357 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
1359 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
1360 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
1361 op0_geu_op1
, profile_probability::even ());
1362 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
1363 NULL
, done_label
, profile_probability::very_likely ());
1364 emit_jump (do_error
);
1365 emit_label (op0_geu_op1
);
1366 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1367 NULL
, done_label
, profile_probability::very_likely ());
1368 goto do_error_label
;
1371 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
1373 /* s1 +- s2 -> sr */
1376 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
1377 : subv4_optab
, mode
);
1378 if (icode
!= CODE_FOR_nothing
)
1380 class expand_operand ops
[4];
1381 rtx_insn
*last
= get_last_insn ();
1383 res
= gen_reg_rtx (mode
);
1384 create_output_operand (&ops
[0], res
, mode
);
1385 create_input_operand (&ops
[1], op0
, mode
);
1386 create_input_operand (&ops
[2], op1
, mode
);
1387 create_fixed_operand (&ops
[3], do_error
);
1388 if (maybe_expand_insn (icode
, 4, ops
))
1390 last
= get_last_insn ();
1391 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1393 && any_condjump_p (last
)
1394 && !find_reg_note (last
, REG_BR_PROB
, 0))
1395 add_reg_br_prob_note (last
,
1396 profile_probability::very_unlikely ());
1397 emit_jump (done_label
);
1398 goto do_error_label
;
1401 delete_insns_since (last
);
1404 /* Compute the operation. On RTL level, the addition is always
1406 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1407 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1409 /* If we can prove that one of the arguments (for MINUS_EXPR only
1410 the second operand, as subtraction is not commutative) is always
1411 non-negative or always negative, we can do just one comparison
1412 and conditional jump. */
1413 int pos_neg
= get_range_pos_neg (arg1
);
1414 if (code
== PLUS_EXPR
)
1416 int pos_neg0
= get_range_pos_neg (arg0
);
1417 if (pos_neg0
!= 3 && pos_neg
== 3)
1419 std::swap (op0
, op1
);
1424 /* Addition overflows if and only if the two operands have the same sign,
1425 and the result has the opposite sign. Subtraction overflows if and
1426 only if the two operands have opposite sign, and the subtrahend has
1427 the same sign as the result. Here 0 is counted as positive. */
1430 /* Compute op0 ^ op1 (operands have opposite sign). */
1431 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1434 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1435 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1439 if (code
== PLUS_EXPR
)
1441 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1442 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1443 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1448 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1449 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1450 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1454 /* No overflow if the result has bit sign cleared. */
1455 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1456 NULL
, done_label
, profile_probability::very_likely ());
1459 /* Compare the result of the operation with the first operand.
1460 No overflow for addition if second operand is positive and result
1461 is larger or second operand is negative and result is smaller.
1462 Likewise for subtraction with sign of second operand flipped. */
1464 do_compare_rtx_and_jump (res
, op0
,
1465 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1466 false, mode
, NULL_RTX
, NULL
, done_label
,
1467 profile_probability::very_likely ());
1471 emit_label (do_error
);
1474 /* Expand the ubsan builtin call. */
1476 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1480 do_pending_stack_adjust ();
1483 expand_arith_set_overflow (lhs
, target
);
1486 emit_label (done_label
);
1491 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1495 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1498 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1503 /* Add negate overflow checking to the statement STMT. */
1506 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1511 rtx_code_label
*done_label
, *do_error
;
1512 rtx target
= NULL_RTX
;
1514 done_label
= gen_label_rtx ();
1515 do_error
= gen_label_rtx ();
1517 do_pending_stack_adjust ();
1518 op1
= expand_normal (arg1
);
1520 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1523 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1525 write_complex_part (target
, const0_rtx
, true, false);
1528 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1529 if (icode
!= CODE_FOR_nothing
)
1531 class expand_operand ops
[3];
1532 rtx_insn
*last
= get_last_insn ();
1534 res
= gen_reg_rtx (mode
);
1535 create_output_operand (&ops
[0], res
, mode
);
1536 create_input_operand (&ops
[1], op1
, mode
);
1537 create_fixed_operand (&ops
[2], do_error
);
1538 if (maybe_expand_insn (icode
, 3, ops
))
1540 last
= get_last_insn ();
1541 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1543 && any_condjump_p (last
)
1544 && !find_reg_note (last
, REG_BR_PROB
, 0))
1545 add_reg_br_prob_note (last
,
1546 profile_probability::very_unlikely ());
1547 emit_jump (done_label
);
1551 delete_insns_since (last
);
1552 icode
= CODE_FOR_nothing
;
1556 if (icode
== CODE_FOR_nothing
)
1558 /* Compute the operation. On RTL level, the addition is always
1560 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1562 /* Compare the operand with the most negative value. */
1563 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1564 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1565 done_label
, profile_probability::very_likely ());
1568 emit_label (do_error
);
1571 /* Expand the ubsan builtin call. */
1573 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1574 arg1
, NULL_TREE
, datap
);
1577 do_pending_stack_adjust ();
1580 expand_arith_set_overflow (lhs
, target
);
1583 emit_label (done_label
);
1588 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
1590 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1594 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1595 mode MODE can be expanded without using a libcall. */
1598 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1599 rtx op0
, rtx op1
, bool uns
)
1601 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1602 != CODE_FOR_nothing
)
1605 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1606 != CODE_FOR_nothing
)
1609 rtx_insn
*last
= get_last_insn ();
1610 if (CONSTANT_P (op0
))
1611 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1613 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1614 if (CONSTANT_P (op1
))
1615 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1617 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1618 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1619 delete_insns_since (last
);
1620 return ret
!= NULL_RTX
;
1623 /* Add mul overflow checking to the statement STMT. */
1626 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1627 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1632 rtx_code_label
*done_label
, *do_error
;
1633 rtx target
= NULL_RTX
;
1635 enum insn_code icode
;
1636 int save_flag_trapv
= flag_trapv
;
1638 /* We don't want any __mulv?i3 etc. calls from the expansion of
1639 these internal functions, so disable -ftrapv temporarily. */
1641 done_label
= gen_label_rtx ();
1642 do_error
= gen_label_rtx ();
1644 do_pending_stack_adjust ();
1645 op0
= expand_normal (arg0
);
1646 op1
= expand_normal (arg1
);
1648 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1652 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1654 write_complex_part (target
, const0_rtx
, true, false);
1658 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1660 /* We assume both operands and result have the same precision
1661 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1662 with that precision, U for unsigned type with that precision,
1663 sgn for unsigned most significant bit in that precision.
1664 s1 is signed first operand, u1 is unsigned first operand,
1665 s2 is signed second operand, u2 is unsigned second operand,
1666 sr is signed result, ur is unsigned result and the following
1667 rules say how to compute result (which is always result of
1668 the operands as if both were unsigned, cast to the right
1669 signedness) and how to compute whether operation overflowed.
1670 main_ovf (false) stands for jump on signed multiplication
1671 overflow or the main algorithm with uns == false.
1672 main_ovf (true) stands for jump on unsigned multiplication
1673 overflow or the main algorithm with uns == true.
1676 res = (S) ((U) s1 * (U) s2)
1677 ovf = main_ovf (false)
1680 ovf = main_ovf (true)
1683 ovf = (s1 < 0 && u2) || main_ovf (true)
1686 ovf = res < 0 || main_ovf (true)
1688 res = (S) ((U) s1 * u2)
1689 ovf = (S) u2 >= 0 ? main_ovf (false)
1690 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1692 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1693 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1695 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1697 if (uns0_p
&& !uns1_p
)
1699 /* Multiplication is commutative, if operand signedness differs,
1700 canonicalize to the first operand being signed and second
1701 unsigned to simplify following code. */
1702 std::swap (op0
, op1
);
1703 std::swap (arg0
, arg1
);
1708 int pos_neg0
= get_range_pos_neg (arg0
);
1709 int pos_neg1
= get_range_pos_neg (arg1
);
1710 /* Unsigned types with smaller than mode precision, even if they have most
1711 significant bit set, are still zero-extended. */
1712 if (uns0_p
&& TYPE_PRECISION (TREE_TYPE (arg0
)) < GET_MODE_PRECISION (mode
))
1714 if (uns1_p
&& TYPE_PRECISION (TREE_TYPE (arg1
)) < GET_MODE_PRECISION (mode
))
1718 if (!uns0_p
&& uns1_p
&& unsr_p
)
1723 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1726 /* If s1 is negative, avoid the main code, just multiply and
1727 signal overflow if op1 is not 0. */
1728 struct separate_ops ops
;
1729 ops
.code
= MULT_EXPR
;
1730 ops
.type
= TREE_TYPE (arg1
);
1731 ops
.op0
= make_tree (ops
.type
, op0
);
1732 ops
.op1
= make_tree (ops
.type
, op1
);
1733 ops
.op2
= NULL_TREE
;
1735 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1736 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1737 NULL
, done_label
, profile_probability::very_likely ());
1738 goto do_error_label
;
1740 if (get_min_precision (arg1
, UNSIGNED
)
1741 + get_min_precision (arg0
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1743 /* If the first operand is sign extended from narrower type, the
1744 second operand is zero extended from narrower type and
1745 the sum of the two precisions is smaller or equal to the
1746 result precision: if the first argument is at runtime
1747 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1748 and there will be no overflow, if the first argument is
1749 negative and the second argument zero, the result will be
1750 0 and there will be no overflow, if the first argument is
1751 negative and the second argument positive, the result when
1752 treated as signed will be negative (minimum -0x7f80 or
1753 -0x7f..f80..0) there will be always overflow. So, do
1755 ovf = (S) res < 0 */
1756 struct separate_ops ops
;
1757 ops
.code
= MULT_EXPR
;
1759 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1761 ops
.op0
= make_tree (ops
.type
, op0
);
1762 ops
.op1
= make_tree (ops
.type
, op1
);
1763 ops
.op2
= NULL_TREE
;
1765 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1766 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1767 mode
, NULL_RTX
, NULL
, done_label
,
1768 profile_probability::very_likely ());
1769 goto do_error_label
;
1771 rtx_code_label
*do_main_label
;
1772 do_main_label
= gen_label_rtx ();
1773 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1774 NULL
, do_main_label
, profile_probability::very_likely ());
1775 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1776 NULL
, do_main_label
, profile_probability::very_likely ());
1777 expand_arith_set_overflow (lhs
, target
);
1778 emit_label (do_main_label
);
1786 if (uns0_p
&& uns1_p
&& !unsr_p
)
1788 if ((pos_neg0
| pos_neg1
) == 1)
1790 /* If both arguments are zero extended from narrower types,
1791 the MSB will be clear on both and so we can pretend it is
1792 a normal s1 * s2 -> sr multiplication. */
1798 /* Rest of handling of this case after res is computed. */
1803 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1810 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1811 avoid the main code, just multiply and signal overflow
1812 unless 0 * u2 or -1 * ((U) Smin). */
1813 struct separate_ops ops
;
1814 ops
.code
= MULT_EXPR
;
1815 ops
.type
= TREE_TYPE (arg1
);
1816 ops
.op0
= make_tree (ops
.type
, op0
);
1817 ops
.op1
= make_tree (ops
.type
, op1
);
1818 ops
.op2
= NULL_TREE
;
1820 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1821 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1822 NULL
, done_label
, profile_probability::very_likely ());
1823 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1824 NULL
, do_error
, profile_probability::very_unlikely ());
1826 prec
= GET_MODE_PRECISION (mode
);
1828 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1829 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1830 NULL
, done_label
, profile_probability::very_likely ());
1831 goto do_error_label
;
1833 /* Rest of handling of this case after res is computed. */
1841 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1844 switch (pos_neg0
| pos_neg1
)
1846 case 1: /* Both operands known to be non-negative. */
1848 case 2: /* Both operands known to be negative. */
1849 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1850 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1851 /* Avoid looking at arg0/arg1 ranges, as we've changed
1853 arg0
= error_mark_node
;
1854 arg1
= error_mark_node
;
1857 if ((pos_neg0
^ pos_neg1
) == 3)
1859 /* If one operand is known to be negative and the other
1860 non-negative, this overflows always, unless the non-negative
1861 one is 0. Just do normal multiply and set overflow
1862 unless one of the operands is 0. */
1863 struct separate_ops ops
;
1864 ops
.code
= MULT_EXPR
;
1866 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1868 ops
.op0
= make_tree (ops
.type
, op0
);
1869 ops
.op1
= make_tree (ops
.type
, op1
);
1870 ops
.op2
= NULL_TREE
;
1872 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1873 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1874 true, mode
, NULL_RTX
, NULL
, done_label
,
1875 profile_probability::very_likely ());
1876 goto do_error_label
;
1878 if (get_min_precision (arg0
, SIGNED
)
1879 + get_min_precision (arg1
, SIGNED
) <= GET_MODE_PRECISION (mode
))
1881 /* If both operands are sign extended from narrower types and
1882 the sum of the two precisions is smaller or equal to the
1883 result precision: if both arguments are at runtime
1884 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1885 and there will be no overflow, if both arguments are negative,
1886 maximum result will be 0x40..00 and there will be no overflow
1887 either, if one argument is positive and the other argument
1888 negative, the result when treated as signed will be negative
1889 and there will be always overflow, and if one argument is
1890 zero and the other negative the result will be zero and no
1893 ovf = (S) res < 0 */
1894 struct separate_ops ops
;
1895 ops
.code
= MULT_EXPR
;
1897 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1899 ops
.op0
= make_tree (ops
.type
, op0
);
1900 ops
.op1
= make_tree (ops
.type
, op1
);
1901 ops
.op2
= NULL_TREE
;
1903 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1904 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false,
1905 mode
, NULL_RTX
, NULL
, done_label
,
1906 profile_probability::very_likely ());
1907 goto do_error_label
;
1909 /* The general case, do all the needed comparisons at runtime. */
1910 rtx_code_label
*do_main_label
, *after_negate_label
;
1912 rop0
= gen_reg_rtx (mode
);
1913 rop1
= gen_reg_rtx (mode
);
1914 emit_move_insn (rop0
, op0
);
1915 emit_move_insn (rop1
, op1
);
1918 do_main_label
= gen_label_rtx ();
1919 after_negate_label
= gen_label_rtx ();
1920 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1922 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1923 NULL
, after_negate_label
, profile_probability::very_likely ());
1924 /* Both arguments negative here, negate them and continue with
1925 normal unsigned overflow checking multiplication. */
1926 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1928 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1930 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1932 arg0
= error_mark_node
;
1933 arg1
= error_mark_node
;
1934 emit_jump (do_main_label
);
1935 emit_label (after_negate_label
);
1936 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1938 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1939 NULL
, do_main_label
,
1940 profile_probability::very_likely ());
1941 /* One argument is negative here, the other positive. This
1942 overflows always, unless one of the arguments is 0. But
1943 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1944 is, thus we can keep do_main code oring in overflow as is. */
1946 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1947 NULL
, do_main_label
,
1948 profile_probability::very_unlikely ());
1950 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1951 NULL
, do_main_label
,
1952 profile_probability::very_unlikely ());
1953 expand_arith_set_overflow (lhs
, target
);
1954 emit_label (do_main_label
);
1962 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1963 sign
= uns
? UNSIGNED
: SIGNED
;
1964 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1966 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1967 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1969 /* Optimize unsigned multiplication by power of 2 constant
1970 using 2 shifts, one for result, one to extract the shifted
1971 out bits to see if they are all zero.
1972 Don't do this if optimizing for size and we have umulv4_optab,
1973 in that case assume multiplication will be shorter.
1974 This is heuristics based on the single target that provides
1975 umulv4 right now (i?86/x86_64), if further targets add it, this
1976 might need to be revisited.
1977 Cases where both operands are constant should be folded already
1978 during GIMPLE, and cases where one operand is constant but not
1979 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1980 below can be done without multiplication, just by shifts and adds,
1981 or we'd need to divide the result (and hope it actually doesn't
1982 really divide nor multiply) and compare the result of the division
1983 with the original operand. */
1988 if (integer_pow2p (arg0
))
1990 std::swap (opn0
, opn1
);
1991 std::swap (argn0
, argn1
);
1993 int cnt
= tree_log2 (argn1
);
1994 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1996 rtx upper
= const0_rtx
;
1997 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1999 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
2000 GET_MODE_PRECISION (mode
) - cnt
,
2002 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
2003 NULL_RTX
, NULL
, done_label
,
2004 profile_probability::very_likely ());
2005 goto do_error_label
;
2008 if (icode
!= CODE_FOR_nothing
)
2010 class expand_operand ops
[4];
2011 rtx_insn
*last
= get_last_insn ();
2013 res
= gen_reg_rtx (mode
);
2014 create_output_operand (&ops
[0], res
, mode
);
2015 create_input_operand (&ops
[1], op0
, mode
);
2016 create_input_operand (&ops
[2], op1
, mode
);
2017 create_fixed_operand (&ops
[3], do_error
);
2018 if (maybe_expand_insn (icode
, 4, ops
))
2020 last
= get_last_insn ();
2021 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
2023 && any_condjump_p (last
)
2024 && !find_reg_note (last
, REG_BR_PROB
, 0))
2025 add_reg_br_prob_note (last
,
2026 profile_probability::very_unlikely ());
2027 emit_jump (done_label
);
2031 delete_insns_since (last
);
2032 icode
= CODE_FOR_nothing
;
2036 if (icode
== CODE_FOR_nothing
)
2038 struct separate_ops ops
;
2039 int prec
= GET_MODE_PRECISION (mode
);
2040 scalar_int_mode hmode
, wmode
;
2041 ops
.op0
= make_tree (type
, op0
);
2042 ops
.op1
= make_tree (type
, op1
);
2043 ops
.op2
= NULL_TREE
;
2046 /* Optimize unsigned overflow check where we don't use the
2047 multiplication result, just whether overflow happened.
2048 If we can do MULT_HIGHPART_EXPR, that followed by
2049 comparison of the result against zero is cheapest.
2050 We'll still compute res, but it should be DCEd later. */
2056 && !(uns0_p
&& uns1_p
&& !unsr_p
)
2057 && can_mult_highpart_p (mode
, uns
) == 1
2058 && single_imm_use (lhs
, &use
, &use_stmt
)
2059 && is_gimple_assign (use_stmt
)
2060 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
2063 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2064 && targetm
.scalar_mode_supported_p (wmode
)
2065 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
2068 ops
.code
= WIDEN_MULT_EXPR
;
2070 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
2072 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
2073 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
2075 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
2076 res
= convert_modes (mode
, wmode
, res
, uns
);
2078 /* For the unsigned multiplication, there was overflow if
2079 HIPART is non-zero. */
2080 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2081 NULL_RTX
, NULL
, done_label
,
2082 profile_probability::very_likely ());
2085 /* RES is used more than once, place it in a pseudo. */
2086 res
= force_reg (mode
, res
);
2088 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2090 /* RES is low half of the double width result, HIPART
2091 the high half. There was overflow if
2092 HIPART is different from RES < 0 ? -1 : 0. */
2093 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2094 NULL_RTX
, NULL
, done_label
,
2095 profile_probability::very_likely ());
2098 else if (can_mult_highpart_p (mode
, uns
) == 1)
2101 ops
.code
= MULT_HIGHPART_EXPR
;
2104 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
2106 ops
.code
= MULT_EXPR
;
2107 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2109 /* For the unsigned multiplication, there was overflow if
2110 HIPART is non-zero. */
2111 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
2112 NULL_RTX
, NULL
, done_label
,
2113 profile_probability::very_likely ());
2116 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
2118 /* RES is low half of the double width result, HIPART
2119 the high half. There was overflow if
2120 HIPART is different from RES < 0 ? -1 : 0. */
2121 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
2122 NULL_RTX
, NULL
, done_label
,
2123 profile_probability::very_likely ());
2127 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
2128 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
2130 rtx_code_label
*large_op0
= gen_label_rtx ();
2131 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
2132 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
2133 rtx_code_label
*both_ops_large
= gen_label_rtx ();
2134 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
2135 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
2136 rtx_code_label
*do_overflow
= gen_label_rtx ();
2137 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
2139 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
2140 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
2142 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
2143 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
2144 rtx signbit0
= const0_rtx
;
2146 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
2148 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
2150 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
2151 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
2152 rtx signbit1
= const0_rtx
;
2154 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
2157 res
= gen_reg_rtx (mode
);
2159 /* True if op0 resp. op1 are known to be in the range of
2161 bool op0_small_p
= false;
2162 bool op1_small_p
= false;
2163 /* True if op0 resp. op1 are known to have all zeros or all ones
2164 in the upper half of bits, but are not known to be
2166 bool op0_medium_p
= false;
2167 bool op1_medium_p
= false;
2168 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2169 nonnegative, 1 if unknown. */
2175 else if (pos_neg0
== 2)
2179 else if (pos_neg1
== 2)
2182 unsigned int mprec0
= prec
;
2183 if (arg0
!= error_mark_node
)
2184 mprec0
= get_min_precision (arg0
, sign
);
2185 if (mprec0
<= hprec
)
2187 else if (!uns
&& mprec0
<= hprec
+ 1)
2188 op0_medium_p
= true;
2189 unsigned int mprec1
= prec
;
2190 if (arg1
!= error_mark_node
)
2191 mprec1
= get_min_precision (arg1
, sign
);
2192 if (mprec1
<= hprec
)
2194 else if (!uns
&& mprec1
<= hprec
+ 1)
2195 op1_medium_p
= true;
2197 int smaller_sign
= 1;
2198 int larger_sign
= 1;
2201 smaller_sign
= op0_sign
;
2202 larger_sign
= op1_sign
;
2204 else if (op1_small_p
)
2206 smaller_sign
= op1_sign
;
2207 larger_sign
= op0_sign
;
2209 else if (op0_sign
== op1_sign
)
2211 smaller_sign
= op0_sign
;
2212 larger_sign
= op0_sign
;
2216 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
2217 NULL_RTX
, NULL
, large_op0
,
2218 profile_probability::unlikely ());
2221 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2222 NULL_RTX
, NULL
, small_op0_large_op1
,
2223 profile_probability::unlikely ());
2225 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2226 hmode to mode, the multiplication will never overflow. We can
2227 do just one hmode x hmode => mode widening multiplication. */
2228 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
2229 ops
.op0
= make_tree (halfstype
, lopart0
);
2230 ops
.op1
= make_tree (halfstype
, lopart1
);
2231 ops
.code
= WIDEN_MULT_EXPR
;
2234 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2235 emit_move_insn (res
, thisres
);
2236 emit_jump (done_label
);
2238 emit_label (small_op0_large_op1
);
2240 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2241 but op1 is not, just swap the arguments and handle it as op1
2242 sign/zero extended, op0 not. */
2243 rtx larger
= gen_reg_rtx (mode
);
2244 rtx hipart
= gen_reg_rtx (hmode
);
2245 rtx lopart
= gen_reg_rtx (hmode
);
2246 emit_move_insn (larger
, op1
);
2247 emit_move_insn (hipart
, hipart1
);
2248 emit_move_insn (lopart
, lopart0
);
2249 emit_jump (one_small_one_large
);
2251 emit_label (large_op0
);
2254 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
2255 NULL_RTX
, NULL
, both_ops_large
,
2256 profile_probability::unlikely ());
2258 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2259 but op0 is not, prepare larger, hipart and lopart pseudos and
2260 handle it together with small_op0_large_op1. */
2261 emit_move_insn (larger
, op0
);
2262 emit_move_insn (hipart
, hipart0
);
2263 emit_move_insn (lopart
, lopart1
);
2265 emit_label (one_small_one_large
);
2267 /* lopart is the low part of the operand that is sign extended
2268 to mode, larger is the other operand, hipart is the
2269 high part of larger and lopart0 and lopart1 are the low parts
2271 We perform lopart0 * lopart1 and lopart * hipart widening
2273 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
2274 ops
.op0
= make_tree (halfutype
, lopart0
);
2275 ops
.op1
= make_tree (halfutype
, lopart1
);
2277 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2279 ops
.op0
= make_tree (halfutype
, lopart
);
2280 ops
.op1
= make_tree (halfutype
, hipart
);
2281 rtx loxhi
= gen_reg_rtx (mode
);
2282 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2283 emit_move_insn (loxhi
, tem
);
2287 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2288 if (larger_sign
== 0)
2289 emit_jump (after_hipart_neg
);
2290 else if (larger_sign
!= -1)
2291 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
2292 NULL_RTX
, NULL
, after_hipart_neg
,
2293 profile_probability::even ());
2295 tem
= convert_modes (mode
, hmode
, lopart
, 1);
2296 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
2297 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
2299 emit_move_insn (loxhi
, tem
);
2301 emit_label (after_hipart_neg
);
2303 /* if (lopart < 0) loxhi -= larger; */
2304 if (smaller_sign
== 0)
2305 emit_jump (after_lopart_neg
);
2306 else if (smaller_sign
!= -1)
2307 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
2308 NULL_RTX
, NULL
, after_lopart_neg
,
2309 profile_probability::even ());
2311 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
2313 emit_move_insn (loxhi
, tem
);
2315 emit_label (after_lopart_neg
);
2318 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2319 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
2320 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
2322 emit_move_insn (loxhi
, tem
);
2324 /* if (loxhi >> (bitsize / 2)
2325 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2326 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2327 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
2329 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
2330 rtx signbitloxhi
= const0_rtx
;
2332 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
2333 convert_modes (hmode
, mode
,
2335 hprec
- 1, NULL_RTX
, 0);
2337 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
2338 NULL_RTX
, NULL
, do_overflow
,
2339 profile_probability::very_unlikely ());
2341 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2342 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
2344 tem
= convert_modes (mode
, hmode
,
2345 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
2347 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
2350 emit_move_insn (res
, tem
);
2351 emit_jump (done_label
);
2353 emit_label (both_ops_large
);
2355 /* If both operands are large (not sign (!uns) or zero (uns)
2356 extended from hmode), then perform the full multiplication
2357 which will be the result of the operation.
2358 The only cases which don't overflow are for signed multiplication
2359 some cases where both hipart0 and highpart1 are 0 or -1.
2360 For unsigned multiplication when high parts are both non-zero
2361 this overflows always. */
2362 ops
.code
= MULT_EXPR
;
2363 ops
.op0
= make_tree (type
, op0
);
2364 ops
.op1
= make_tree (type
, op1
);
2365 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2366 emit_move_insn (res
, tem
);
2372 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
2373 NULL_RTX
, 1, OPTAB_WIDEN
);
2374 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2375 NULL_RTX
, NULL
, do_error
,
2376 profile_probability::very_unlikely ());
2381 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
2382 NULL_RTX
, 1, OPTAB_WIDEN
);
2383 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
2384 NULL_RTX
, NULL
, do_error
,
2385 profile_probability::very_unlikely ());
2388 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2389 the same, overflow happened if res is non-positive, if they
2390 are different, overflow happened if res is positive. */
2391 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
2392 emit_jump (hipart_different
);
2393 else if (op0_sign
== 1 || op1_sign
== 1)
2394 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
2395 NULL_RTX
, NULL
, hipart_different
,
2396 profile_probability::even ());
2398 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
2399 NULL_RTX
, NULL
, do_error
,
2400 profile_probability::very_unlikely ());
2401 emit_jump (done_label
);
2403 emit_label (hipart_different
);
2405 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
2406 NULL_RTX
, NULL
, do_error
,
2407 profile_probability::very_unlikely ());
2408 emit_jump (done_label
);
2411 emit_label (do_overflow
);
2413 /* Overflow, do full multiplication and fallthru into do_error. */
2414 ops
.op0
= make_tree (type
, op0
);
2415 ops
.op1
= make_tree (type
, op1
);
2416 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2417 emit_move_insn (res
, tem
);
2419 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
2420 && targetm
.scalar_mode_supported_p (wmode
))
2421 /* Even emitting a libcall is better than not detecting overflow
2426 gcc_assert (!is_ubsan
);
2427 ops
.code
= MULT_EXPR
;
2429 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2430 emit_jump (done_label
);
2435 emit_label (do_error
);
2438 /* Expand the ubsan builtin call. */
2440 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
2444 do_pending_stack_adjust ();
2447 expand_arith_set_overflow (lhs
, target
);
2450 emit_label (done_label
);
2453 if (uns0_p
&& uns1_p
&& !unsr_p
)
2455 rtx_code_label
*all_done_label
= gen_label_rtx ();
2456 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2457 NULL
, all_done_label
, profile_probability::very_likely ());
2458 expand_arith_set_overflow (lhs
, target
);
2459 emit_label (all_done_label
);
2463 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2465 rtx_code_label
*all_done_label
= gen_label_rtx ();
2466 rtx_code_label
*set_noovf
= gen_label_rtx ();
2467 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2468 NULL
, all_done_label
, profile_probability::very_likely ());
2469 expand_arith_set_overflow (lhs
, target
);
2470 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2471 NULL
, set_noovf
, profile_probability::very_likely ());
2472 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2473 NULL
, all_done_label
, profile_probability::very_unlikely ());
2474 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2475 all_done_label
, profile_probability::very_unlikely ());
2476 emit_label (set_noovf
);
2477 write_complex_part (target
, const0_rtx
, true, false);
2478 emit_label (all_done_label
);
2484 expand_ubsan_result_store (lhs
, target
, mode
, res
, do_error
);
2486 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2488 flag_trapv
= save_flag_trapv
;
2491 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2494 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2495 tree arg0
, tree arg1
)
2497 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2498 rtx_code_label
*loop_lab
= NULL
;
2499 rtx cntvar
= NULL_RTX
;
2500 tree cntv
= NULL_TREE
;
2501 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2502 tree sz
= TYPE_SIZE (eltype
);
2503 tree data
= NULL_TREE
;
2504 tree resv
= NULL_TREE
;
2505 rtx lhsr
= NULL_RTX
;
2506 rtx resvr
= NULL_RTX
;
2507 unsigned HOST_WIDE_INT const_cnt
= 0;
2508 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2509 int save_flag_trapv
= flag_trapv
;
2511 /* We don't want any __mulv?i3 etc. calls from the expansion of
2512 these internal functions, so disable -ftrapv temporarily. */
2517 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2518 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2519 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2520 optab_default
)) == unknown_optab
2521 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2522 == CODE_FOR_nothing
))
2525 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2528 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2529 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2535 do_pending_stack_adjust ();
2536 loop_lab
= gen_label_rtx ();
2537 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2538 cntv
= make_tree (sizetype
, cntvar
);
2539 emit_move_insn (cntvar
, const0_rtx
);
2540 emit_label (loop_lab
);
2542 if (TREE_CODE (arg0
) != VECTOR_CST
)
2544 rtx arg0r
= expand_normal (arg0
);
2545 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2547 if (TREE_CODE (arg1
) != VECTOR_CST
)
2549 rtx arg1r
= expand_normal (arg1
);
2550 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2552 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2554 tree op0
, op1
, res
= NULL_TREE
;
2557 tree atype
= build_array_type_nelts (eltype
, cnt
);
2558 op0
= uniform_vector_p (arg0
);
2559 if (op0
== NULL_TREE
)
2561 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2562 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2563 NULL_TREE
, NULL_TREE
);
2565 op1
= uniform_vector_p (arg1
);
2566 if (op1
== NULL_TREE
)
2568 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2569 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2570 NULL_TREE
, NULL_TREE
);
2574 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2575 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2576 NULL_TREE
, NULL_TREE
);
2581 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2582 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2583 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2585 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2591 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2592 false, false, false, true, &data
);
2595 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2596 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2598 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2599 false, false, false, true, &data
);
2602 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2611 struct separate_ops ops
;
2612 ops
.code
= PLUS_EXPR
;
2613 ops
.type
= TREE_TYPE (cntv
);
2615 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2616 ops
.op2
= NULL_TREE
;
2618 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2621 emit_move_insn (cntvar
, ret
);
2622 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2623 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2624 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2625 profile_probability::very_likely ());
2627 if (lhs
&& resv
== NULL_TREE
)
2629 struct separate_ops ops
;
2631 ops
.type
= TREE_TYPE (arg0
);
2634 ops
.op2
= NULL_TREE
;
2636 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2639 emit_move_insn (lhsr
, ret
);
2642 emit_move_insn (lhsr
, resvr
);
2643 flag_trapv
= save_flag_trapv
;
2646 /* Expand UBSAN_CHECK_ADD call STMT. */
2649 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2651 location_t loc
= gimple_location (stmt
);
2652 tree lhs
= gimple_call_lhs (stmt
);
2653 tree arg0
= gimple_call_arg (stmt
, 0);
2654 tree arg1
= gimple_call_arg (stmt
, 1);
2655 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2656 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2658 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2659 false, false, false, true, NULL
);
2662 /* Expand UBSAN_CHECK_SUB call STMT. */
2665 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2667 location_t loc
= gimple_location (stmt
);
2668 tree lhs
= gimple_call_lhs (stmt
);
2669 tree arg0
= gimple_call_arg (stmt
, 0);
2670 tree arg1
= gimple_call_arg (stmt
, 1);
2671 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2672 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2673 else if (integer_zerop (arg0
))
2674 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2676 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2677 false, false, false, true, NULL
);
2680 /* Expand UBSAN_CHECK_MUL call STMT. */
2683 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2685 location_t loc
= gimple_location (stmt
);
2686 tree lhs
= gimple_call_lhs (stmt
);
2687 tree arg0
= gimple_call_arg (stmt
, 0);
2688 tree arg1
= gimple_call_arg (stmt
, 1);
2689 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2690 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2692 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2696 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2699 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2701 tree lhs
= gimple_call_lhs (stmt
);
2702 if (lhs
== NULL_TREE
)
2704 tree arg0
= gimple_call_arg (stmt
, 0);
2705 tree arg1
= gimple_call_arg (stmt
, 1);
2706 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2707 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2708 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2709 int unsr_p
= TYPE_UNSIGNED (type
);
2710 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2711 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2712 int precres
= TYPE_PRECISION (type
);
2713 location_t loc
= gimple_location (stmt
);
2714 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2716 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2718 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2719 prec0
= MIN (prec0
, pr
);
2720 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2721 prec1
= MIN (prec1
, pr
);
2722 int save_flag_trapv
= flag_trapv
;
2724 /* We don't want any __mulv?i3 etc. calls from the expansion of
2725 these internal functions, so disable -ftrapv temporarily. */
2727 /* If uns0_p && uns1_p, precop is minimum needed precision
2728 of unsigned type to hold the exact result, otherwise
2729 precop is minimum needed precision of signed type to
2730 hold the exact result. */
2732 if (code
== MULT_EXPR
)
2733 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2736 if (uns0_p
== uns1_p
)
2737 precop
= MAX (prec0
, prec1
) + 1;
2739 precop
= MAX (prec0
+ 1, prec1
) + 1;
2741 precop
= MAX (prec0
, prec1
+ 1) + 1;
2743 int orig_precres
= precres
;
2747 if ((uns0_p
&& uns1_p
)
2748 ? ((precop
+ !unsr_p
) <= precres
2749 /* u1 - u2 -> ur can overflow, no matter what precision
2751 && (code
!= MINUS_EXPR
|| !unsr_p
))
2752 : (!unsr_p
&& precop
<= precres
))
2754 /* The infinity precision result will always fit into result. */
2755 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2756 write_complex_part (target
, const0_rtx
, true, false);
2757 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2758 struct separate_ops ops
;
2761 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2762 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2763 ops
.op2
= NULL_TREE
;
2765 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2766 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2767 flag_trapv
= save_flag_trapv
;
2771 /* For operations with low precision, if target doesn't have them, start
2772 with precres widening right away, otherwise do it only if the most
2773 simple cases can't be used. */
2774 const int min_precision
= targetm
.min_arithmetic_precision ();
2775 if (orig_precres
== precres
&& precres
< min_precision
)
2777 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2778 && prec1
<= precres
)
2779 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2780 && prec0
+ uns0_p
<= precres
2781 && prec1
+ uns1_p
<= precres
))
2783 arg0
= fold_convert_loc (loc
, type
, arg0
);
2784 arg1
= fold_convert_loc (loc
, type
, arg1
);
2788 if (integer_zerop (arg0
) && !unsr_p
)
2790 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2791 flag_trapv
= save_flag_trapv
;
2796 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2797 unsr_p
, unsr_p
, false, NULL
);
2798 flag_trapv
= save_flag_trapv
;
2801 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2802 unsr_p
, unsr_p
, false, NULL
);
2803 flag_trapv
= save_flag_trapv
;
2810 /* For sub-word operations, retry with a wider type first. */
2811 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2813 int p
= MAX (min_precision
, precop
);
2814 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2815 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2818 p
= TYPE_PRECISION (optype
);
2822 unsr_p
= TYPE_UNSIGNED (optype
);
2828 if (prec0
<= precres
&& prec1
<= precres
)
2833 types
[0] = build_nonstandard_integer_type (precres
, 0);
2839 types
[1] = build_nonstandard_integer_type (precres
, 1);
2841 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2842 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2843 if (code
!= MULT_EXPR
)
2844 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2845 uns0_p
, uns1_p
, false, NULL
);
2847 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2848 uns0_p
, uns1_p
, false, NULL
);
2849 flag_trapv
= save_flag_trapv
;
2853 /* Retry with a wider type. */
2854 if (orig_precres
== precres
)
2856 int p
= MAX (prec0
, prec1
);
2857 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2858 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2861 p
= TYPE_PRECISION (optype
);
2865 unsr_p
= TYPE_UNSIGNED (optype
);
2876 /* Expand ADD_OVERFLOW STMT. */
2879 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2881 expand_arith_overflow (PLUS_EXPR
, stmt
);
2884 /* Expand SUB_OVERFLOW STMT. */
2887 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2889 expand_arith_overflow (MINUS_EXPR
, stmt
);
2892 /* Expand MUL_OVERFLOW STMT. */
2895 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2897 expand_arith_overflow (MULT_EXPR
, stmt
);
2900 /* Expand UADDC STMT. */
2903 expand_UADDC (internal_fn ifn
, gcall
*stmt
)
2905 tree lhs
= gimple_call_lhs (stmt
);
2906 tree arg1
= gimple_call_arg (stmt
, 0);
2907 tree arg2
= gimple_call_arg (stmt
, 1);
2908 tree arg3
= gimple_call_arg (stmt
, 2);
2909 tree type
= TREE_TYPE (arg1
);
2910 machine_mode mode
= TYPE_MODE (type
);
2911 insn_code icode
= optab_handler (ifn
== IFN_UADDC
2912 ? uaddc5_optab
: usubc5_optab
, mode
);
2913 rtx op1
= expand_normal (arg1
);
2914 rtx op2
= expand_normal (arg2
);
2915 rtx op3
= expand_normal (arg3
);
2916 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2917 rtx re
= gen_reg_rtx (mode
);
2918 rtx im
= gen_reg_rtx (mode
);
2919 class expand_operand ops
[5];
2920 create_output_operand (&ops
[0], re
, mode
);
2921 create_output_operand (&ops
[1], im
, mode
);
2922 create_input_operand (&ops
[2], op1
, mode
);
2923 create_input_operand (&ops
[3], op2
, mode
);
2924 create_input_operand (&ops
[4], op3
, mode
);
2925 expand_insn (icode
, 5, ops
);
2926 write_complex_part (target
, re
, false, false);
2927 write_complex_part (target
, im
, true, false);
2930 /* Expand USUBC STMT. */
2933 expand_USUBC (internal_fn ifn
, gcall
*stmt
)
2935 expand_UADDC (ifn
, stmt
);
2938 /* This should get folded in tree-vectorizer.cc. */
2941 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2946 /* This should get folded in tree-vectorizer.cc. */
2949 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2954 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2955 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2958 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2960 tree addr
= gimple_call_arg (stmt
, index
);
2961 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2962 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2963 if (TYPE_ALIGN (type
) != align
)
2964 type
= build_aligned_type (type
, align
);
2967 if (TREE_CODE (tmp
) == SSA_NAME
)
2969 gimple
*def
= get_gimple_for_ssa_name (tmp
);
2970 if (def
&& gimple_assign_single_p (def
))
2971 tmp
= gimple_assign_rhs1 (def
);
2974 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2976 tree mem
= TREE_OPERAND (tmp
, 0);
2977 if (TREE_CODE (mem
) == TARGET_MEM_REF
2978 && types_compatible_p (TREE_TYPE (mem
), type
))
2980 tree offset
= TMR_OFFSET (mem
);
2981 if (type
!= TREE_TYPE (mem
)
2982 || alias_ptr_type
!= TREE_TYPE (offset
)
2983 || !integer_zerop (offset
))
2985 mem
= copy_node (mem
);
2986 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2987 wi::to_poly_wide (offset
));
2988 TREE_TYPE (mem
) = type
;
2994 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2997 /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
3001 expand_partial_load_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3004 class expand_operand ops
[5];
3005 tree type
, lhs
, rhs
, maskt
;
3009 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3010 lhs
= gimple_call_lhs (stmt
);
3011 if (lhs
== NULL_TREE
)
3013 type
= TREE_TYPE (lhs
);
3014 rhs
= expand_call_mem_ref (type
, stmt
, 0);
3016 if (optab
== vec_mask_load_lanes_optab
3017 || optab
== vec_mask_len_load_lanes_optab
)
3018 icode
= get_multi_vector_move (type
, optab
);
3019 else if (optab
== len_load_optab
)
3020 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3022 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3023 TYPE_MODE (TREE_TYPE (maskt
)));
3025 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3026 gcc_assert (MEM_P (mem
));
3027 /* The built MEM_REF does not accurately reflect that the load
3028 is only partial. Clear it. */
3029 set_mem_expr (mem
, NULL_TREE
);
3030 clear_mem_offset (mem
);
3031 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3032 create_output_operand (&ops
[i
++], target
, TYPE_MODE (type
));
3033 create_fixed_operand (&ops
[i
++], mem
);
3034 i
= add_mask_and_len_args (ops
, i
, stmt
);
3035 expand_insn (icode
, i
, ops
);
3037 if (!rtx_equal_p (target
, ops
[0].value
))
3038 emit_move_insn (target
, ops
[0].value
);
3041 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
3042 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
3043 #define expand_len_load_optab_fn expand_partial_load_optab_fn
3044 #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
3046 /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
3050 expand_partial_store_optab_fn (internal_fn ifn
, gcall
*stmt
, convert_optab optab
)
3053 class expand_operand ops
[5];
3054 tree type
, lhs
, rhs
, maskt
;
3058 maskt
= gimple_call_arg (stmt
, internal_fn_mask_index (ifn
));
3059 rhs
= gimple_call_arg (stmt
, internal_fn_stored_value_index (ifn
));
3060 type
= TREE_TYPE (rhs
);
3061 lhs
= expand_call_mem_ref (type
, stmt
, 0);
3063 if (optab
== vec_mask_store_lanes_optab
3064 || optab
== vec_mask_len_store_lanes_optab
)
3065 icode
= get_multi_vector_move (type
, optab
);
3066 else if (optab
== len_store_optab
)
3067 icode
= direct_optab_handler (optab
, TYPE_MODE (type
));
3069 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
3070 TYPE_MODE (TREE_TYPE (maskt
)));
3072 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3073 gcc_assert (MEM_P (mem
));
3074 /* The built MEM_REF does not accurately reflect that the store
3075 is only partial. Clear it. */
3076 set_mem_expr (mem
, NULL_TREE
);
3077 clear_mem_offset (mem
);
3078 reg
= expand_normal (rhs
);
3079 create_fixed_operand (&ops
[i
++], mem
);
3080 create_input_operand (&ops
[i
++], reg
, TYPE_MODE (type
));
3081 i
= add_mask_and_len_args (ops
, i
, stmt
);
3082 expand_insn (icode
, i
, ops
);
3085 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3086 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3087 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3088 #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3090 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3091 The expansion of STMT happens based on OPTAB table associated. */
3094 expand_vec_cond_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3096 class expand_operand ops
[6];
3098 tree lhs
= gimple_call_lhs (stmt
);
3099 tree op0a
= gimple_call_arg (stmt
, 0);
3100 tree op0b
= gimple_call_arg (stmt
, 1);
3101 tree op1
= gimple_call_arg (stmt
, 2);
3102 tree op2
= gimple_call_arg (stmt
, 3);
3103 enum tree_code tcode
= (tree_code
) int_cst_value (gimple_call_arg (stmt
, 4));
3105 tree vec_cond_type
= TREE_TYPE (lhs
);
3106 tree op_mode
= TREE_TYPE (op0a
);
3107 bool unsignedp
= TYPE_UNSIGNED (op_mode
);
3109 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3110 machine_mode cmp_op_mode
= TYPE_MODE (op_mode
);
3112 icode
= convert_optab_handler (optab
, mode
, cmp_op_mode
);
3114 = vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
, icode
, 4);
3115 /* vector_compare_rtx legitimizes operands, preserve equality when
3116 expanding op1/op2. */
3117 rtx rtx_op1
, rtx_op2
;
3118 if (operand_equal_p (op1
, op0a
))
3119 rtx_op1
= XEXP (comparison
, 0);
3120 else if (operand_equal_p (op1
, op0b
))
3121 rtx_op1
= XEXP (comparison
, 1);
3123 rtx_op1
= expand_normal (op1
);
3124 if (operand_equal_p (op2
, op0a
))
3125 rtx_op2
= XEXP (comparison
, 0);
3126 else if (operand_equal_p (op2
, op0b
))
3127 rtx_op2
= XEXP (comparison
, 1);
3129 rtx_op2
= expand_normal (op2
);
3131 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3132 create_output_operand (&ops
[0], target
, mode
);
3133 create_input_operand (&ops
[1], rtx_op1
, mode
);
3134 create_input_operand (&ops
[2], rtx_op2
, mode
);
3135 create_fixed_operand (&ops
[3], comparison
);
3136 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
3137 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
3138 expand_insn (icode
, 6, ops
);
3139 if (!rtx_equal_p (ops
[0].value
, target
))
3140 emit_move_insn (target
, ops
[0].value
);
3143 /* Expand VCOND_MASK optab internal function.
3144 The expansion of STMT happens based on OPTAB table associated. */
3147 expand_vec_cond_mask_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3149 class expand_operand ops
[4];
3151 tree lhs
= gimple_call_lhs (stmt
);
3152 tree op0
= gimple_call_arg (stmt
, 0);
3153 tree op1
= gimple_call_arg (stmt
, 1);
3154 tree op2
= gimple_call_arg (stmt
, 2);
3155 tree vec_cond_type
= TREE_TYPE (lhs
);
3157 machine_mode mode
= TYPE_MODE (vec_cond_type
);
3158 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
3159 enum insn_code icode
= convert_optab_handler (optab
, mode
, mask_mode
);
3160 rtx mask
, rtx_op1
, rtx_op2
;
3162 gcc_assert (icode
!= CODE_FOR_nothing
);
3164 mask
= expand_normal (op0
);
3165 rtx_op1
= expand_normal (op1
);
3166 rtx_op2
= expand_normal (op2
);
3168 mask
= force_reg (mask_mode
, mask
);
3169 rtx_op1
= force_reg (mode
, rtx_op1
);
3171 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3172 create_output_operand (&ops
[0], target
, mode
);
3173 create_input_operand (&ops
[1], rtx_op1
, mode
);
3174 create_input_operand (&ops
[2], rtx_op2
, mode
);
3175 create_input_operand (&ops
[3], mask
, mask_mode
);
3176 expand_insn (icode
, 4, ops
);
3177 if (!rtx_equal_p (ops
[0].value
, target
))
3178 emit_move_insn (target
, ops
[0].value
);
3181 /* Expand VEC_SET internal functions. */
3184 expand_vec_set_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3186 tree lhs
= gimple_call_lhs (stmt
);
3187 tree op0
= gimple_call_arg (stmt
, 0);
3188 tree op1
= gimple_call_arg (stmt
, 1);
3189 tree op2
= gimple_call_arg (stmt
, 2);
3190 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3191 rtx src
= expand_normal (op0
);
3193 machine_mode outermode
= TYPE_MODE (TREE_TYPE (op0
));
3194 scalar_mode innermode
= GET_MODE_INNER (outermode
);
3196 rtx value
= expand_normal (op1
);
3197 rtx pos
= expand_normal (op2
);
3199 class expand_operand ops
[3];
3200 enum insn_code icode
= optab_handler (optab
, outermode
);
3202 if (icode
!= CODE_FOR_nothing
)
3204 rtx temp
= gen_reg_rtx (outermode
);
3205 emit_move_insn (temp
, src
);
3207 create_fixed_operand (&ops
[0], temp
);
3208 create_input_operand (&ops
[1], value
, innermode
);
3209 create_convert_operand_from (&ops
[2], pos
, TYPE_MODE (TREE_TYPE (op2
)),
3211 if (maybe_expand_insn (icode
, 3, ops
))
3213 emit_move_insn (target
, temp
);
3221 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
3226 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
3228 /* When guessing was done, the hints should be already stripped away. */
3229 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
3232 tree lhs
= gimple_call_lhs (stmt
);
3234 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3236 target
= const0_rtx
;
3237 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
3238 if (lhs
&& val
!= target
)
3239 emit_move_insn (target
, val
);
3242 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3243 should never be called. */
3246 expand_VA_ARG (internal_fn
, gcall
*)
3251 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3252 dummy function should never be called. */
3255 expand_VEC_CONVERT (internal_fn
, gcall
*)
3260 /* Expand IFN_RAWMEMCHR internal function. */
3263 expand_RAWMEMCHR (internal_fn
, gcall
*stmt
)
3265 expand_operand ops
[3];
3267 tree lhs
= gimple_call_lhs (stmt
);
3270 machine_mode lhs_mode
= TYPE_MODE (TREE_TYPE (lhs
));
3271 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3272 create_output_operand (&ops
[0], lhs_rtx
, lhs_mode
);
3274 tree mem
= gimple_call_arg (stmt
, 0);
3275 rtx mem_rtx
= get_memory_rtx (mem
, NULL
);
3276 create_fixed_operand (&ops
[1], mem_rtx
);
3278 tree pattern
= gimple_call_arg (stmt
, 1);
3279 machine_mode mode
= TYPE_MODE (TREE_TYPE (pattern
));
3280 rtx pattern_rtx
= expand_normal (pattern
);
3281 create_input_operand (&ops
[2], pattern_rtx
, mode
);
3283 insn_code icode
= direct_optab_handler (rawmemchr_optab
, mode
);
3285 expand_insn (icode
, 3, ops
);
3286 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3287 emit_move_insn (lhs_rtx
, ops
[0].value
);
3290 /* Expand the IFN_UNIQUE function according to its first argument. */
3293 expand_UNIQUE (internal_fn
, gcall
*stmt
)
3295 rtx pattern
= NULL_RTX
;
3296 enum ifn_unique_kind kind
3297 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
3304 case IFN_UNIQUE_UNSPEC
:
3305 if (targetm
.have_unique ())
3306 pattern
= targetm
.gen_unique ();
3309 case IFN_UNIQUE_OACC_FORK
:
3310 case IFN_UNIQUE_OACC_JOIN
:
3311 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
3313 tree lhs
= gimple_call_lhs (stmt
);
3314 rtx target
= const0_rtx
;
3317 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3319 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
3320 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
3322 if (kind
== IFN_UNIQUE_OACC_FORK
)
3323 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
3325 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
3333 emit_insn (pattern
);
3336 /* Expand the IFN_DEFERRED_INIT function:
3337 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3339 Initialize the LHS with zero/pattern according to its second argument
3341 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3342 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3344 The LHS variable is initialized including paddings.
3345 The reasons to choose 0xFE for pattern initialization are:
3346 1. It is a non-canonical virtual address on x86_64, and at the
3347 high end of the i386 kernel address space.
3348 2. It is a very large float value (-1.694739530317379e+38).
3349 3. It is also an unusual number for integers. */
3350 #define INIT_PATTERN_VALUE 0xFE
3352 expand_DEFERRED_INIT (internal_fn
, gcall
*stmt
)
3354 tree lhs
= gimple_call_lhs (stmt
);
3355 tree var_size
= gimple_call_arg (stmt
, 0);
3356 enum auto_init_type init_type
3357 = (enum auto_init_type
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 1));
3358 bool reg_lhs
= true;
3360 tree var_type
= TREE_TYPE (lhs
);
3361 gcc_assert (init_type
> AUTO_INIT_UNINITIALIZED
);
3363 if (TREE_CODE (lhs
) == SSA_NAME
)
3367 tree lhs_base
= lhs
;
3368 while (handled_component_p (lhs_base
))
3369 lhs_base
= TREE_OPERAND (lhs_base
, 0);
3370 reg_lhs
= (mem_ref_refers_to_non_mem_p (lhs_base
)
3371 || non_mem_decl_p (lhs_base
));
3372 /* If this expands to a register and the underlying decl is wrapped in
3373 a MEM_REF that just serves as an access type change expose the decl
3374 if it is of correct size. This avoids a situation as in PR103271
3375 if the target does not support a direct move to the registers mode. */
3377 && TREE_CODE (lhs_base
) == MEM_REF
3378 && TREE_CODE (TREE_OPERAND (lhs_base
, 0)) == ADDR_EXPR
3379 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))
3380 && integer_zerop (TREE_OPERAND (lhs_base
, 1))
3381 && tree_fits_uhwi_p (var_size
)
3382 && tree_int_cst_equal
3384 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0))))
3386 lhs
= TREE_OPERAND (TREE_OPERAND (lhs_base
, 0), 0);
3387 var_type
= TREE_TYPE (lhs
);
3393 /* If the variable is not in register, expand to a memset
3394 to initialize it. */
3395 mark_addressable (lhs
);
3396 tree var_addr
= build_fold_addr_expr (lhs
);
3398 tree value
= (init_type
== AUTO_INIT_PATTERN
)
3399 ? build_int_cst (integer_type_node
,
3401 : integer_zero_node
;
3402 tree m_call
= build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET
),
3403 3, var_addr
, value
, var_size
);
3404 /* Expand this memset call. */
3405 expand_builtin_memset (m_call
, NULL_RTX
, TYPE_MODE (var_type
));
3409 /* If this variable is in a register use expand_assignment.
3410 For boolean scalars force zero-init. */
3412 scalar_int_mode var_mode
;
3413 if (TREE_CODE (TREE_TYPE (lhs
)) != BOOLEAN_TYPE
3414 && tree_fits_uhwi_p (var_size
)
3415 && (init_type
== AUTO_INIT_PATTERN
3416 || !is_gimple_reg_type (var_type
))
3417 && int_mode_for_size (tree_to_uhwi (var_size
) * BITS_PER_UNIT
,
3418 0).exists (&var_mode
)
3419 && have_insn_for (SET
, var_mode
))
3421 unsigned HOST_WIDE_INT total_bytes
= tree_to_uhwi (var_size
);
3422 unsigned char *buf
= XALLOCAVEC (unsigned char, total_bytes
);
3423 memset (buf
, (init_type
== AUTO_INIT_PATTERN
3424 ? INIT_PATTERN_VALUE
: 0), total_bytes
);
3425 tree itype
= build_nonstandard_integer_type
3426 (total_bytes
* BITS_PER_UNIT
, 1);
3427 wide_int w
= wi::from_buffer (buf
, total_bytes
);
3428 init
= wide_int_to_tree (itype
, w
);
3429 /* Pun the LHS to make sure its type has constant size
3430 unless it is an SSA name where that's already known. */
3431 if (TREE_CODE (lhs
) != SSA_NAME
)
3432 lhs
= build1 (VIEW_CONVERT_EXPR
, itype
, lhs
);
3434 init
= fold_build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
), init
);
3437 /* Use zero-init also for variable-length sizes. */
3438 init
= build_zero_cst (var_type
);
3440 expand_assignment (lhs
, init
, false);
3444 /* The size of an OpenACC compute dimension. */
3447 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
3449 tree lhs
= gimple_call_lhs (stmt
);
3454 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3455 if (targetm
.have_oacc_dim_size ())
3457 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3458 VOIDmode
, EXPAND_NORMAL
);
3459 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
3462 emit_move_insn (target
, GEN_INT (1));
3465 /* The position of an OpenACC execution engine along one compute axis. */
3468 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
3470 tree lhs
= gimple_call_lhs (stmt
);
3475 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3476 if (targetm
.have_oacc_dim_pos ())
3478 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
3479 VOIDmode
, EXPAND_NORMAL
);
3480 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
3483 emit_move_insn (target
, const0_rtx
);
3486 /* This is expanded by oacc_device_lower pass. */
3489 expand_GOACC_LOOP (internal_fn
, gcall
*)
3494 /* This is expanded by oacc_device_lower pass. */
3497 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
3502 /* This is expanded by oacc_device_lower pass. */
3505 expand_GOACC_TILE (internal_fn
, gcall
*)
3510 /* Set errno to EDOM. */
3513 expand_SET_EDOM (internal_fn
, gcall
*)
3516 #ifdef GEN_ERRNO_RTX
3517 rtx errno_rtx
= GEN_ERRNO_RTX
;
3519 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
3521 emit_move_insn (errno_rtx
,
3522 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
3528 /* Expand atomic bit test and set. */
3531 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
3533 expand_ifn_atomic_bit_test_and (call
);
3536 /* Expand atomic bit test and complement. */
3539 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
3541 expand_ifn_atomic_bit_test_and (call
);
3544 /* Expand atomic bit test and reset. */
3547 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
3549 expand_ifn_atomic_bit_test_and (call
);
3552 /* Expand atomic bit test and set. */
3555 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
3557 expand_ifn_atomic_compare_exchange (call
);
3560 /* Expand atomic add fetch and cmp with 0. */
3563 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3565 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3568 /* Expand atomic sub fetch and cmp with 0. */
3571 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3573 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3576 /* Expand atomic and fetch and cmp with 0. */
3579 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3581 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3584 /* Expand atomic or fetch and cmp with 0. */
3587 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3589 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3592 /* Expand atomic xor fetch and cmp with 0. */
3595 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn
, gcall
*call
)
3597 expand_ifn_atomic_op_fetch_cmp_0 (call
);
3600 /* Expand LAUNDER to assignment, lhs = arg0. */
3603 expand_LAUNDER (internal_fn
, gcall
*call
)
3605 tree lhs
= gimple_call_lhs (call
);
3610 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
3613 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3616 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3618 internal_fn ifn
= gimple_call_internal_fn (stmt
);
3619 int rhs_index
= internal_fn_stored_value_index (ifn
);
3620 tree base
= gimple_call_arg (stmt
, 0);
3621 tree offset
= gimple_call_arg (stmt
, 1);
3622 tree scale
= gimple_call_arg (stmt
, 2);
3623 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
3625 rtx base_rtx
= expand_normal (base
);
3626 rtx offset_rtx
= expand_normal (offset
);
3627 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3628 rtx rhs_rtx
= expand_normal (rhs
);
3630 class expand_operand ops
[8];
3632 create_address_operand (&ops
[i
++], base_rtx
);
3633 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3634 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3635 create_integer_operand (&ops
[i
++], scale_int
);
3636 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
3637 i
= add_mask_and_len_args (ops
, i
, stmt
);
3639 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
3640 TYPE_MODE (TREE_TYPE (offset
)));
3641 expand_insn (icode
, i
, ops
);
3644 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3647 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
3649 tree lhs
= gimple_call_lhs (stmt
);
3650 tree base
= gimple_call_arg (stmt
, 0);
3651 tree offset
= gimple_call_arg (stmt
, 1);
3652 tree scale
= gimple_call_arg (stmt
, 2);
3654 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3655 rtx base_rtx
= expand_normal (base
);
3656 rtx offset_rtx
= expand_normal (offset
);
3657 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
3660 class expand_operand ops
[8];
3661 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
3662 create_address_operand (&ops
[i
++], base_rtx
);
3663 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
3664 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
3665 create_integer_operand (&ops
[i
++], scale_int
);
3666 i
= add_mask_and_len_args (ops
, i
, stmt
);
3667 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
3668 TYPE_MODE (TREE_TYPE (offset
)));
3669 expand_insn (icode
, i
, ops
);
3670 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3671 emit_move_insn (lhs_rtx
, ops
[0].value
);
3674 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3675 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3678 contains_call_div_mod (rtx_insn
*insn
)
3680 subrtx_iterator::array_type array
;
3681 for (; insn
; insn
= NEXT_INSN (insn
))
3684 else if (INSN_P (insn
))
3685 FOR_EACH_SUBRTX (iter
, array
, PATTERN (insn
), NONCONST
)
3686 switch (GET_CODE (*iter
))
3700 /* Expand DIVMOD() using:
3701 a) optab handler for udivmod/sdivmod if it is available.
3702 b) If optab_handler doesn't exist, generate call to
3703 target-specific divmod libfunc. */
3706 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
3708 tree lhs
= gimple_call_lhs (call_stmt
);
3709 tree arg0
= gimple_call_arg (call_stmt
, 0);
3710 tree arg1
= gimple_call_arg (call_stmt
, 1);
3712 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
3713 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
3714 machine_mode mode
= TYPE_MODE (type
);
3715 bool unsignedp
= TYPE_UNSIGNED (type
);
3716 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
3718 rtx op0
= expand_normal (arg0
);
3719 rtx op1
= expand_normal (arg1
);
3720 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3722 rtx quotient
= NULL_RTX
, remainder
= NULL_RTX
;
3723 rtx_insn
*insns
= NULL
;
3725 if (TREE_CODE (arg1
) == INTEGER_CST
)
3727 /* For DIVMOD by integral constants, there could be efficient code
3728 expanded inline e.g. using shifts and plus/minus. Try to expand
3729 the division and modulo and if it emits any library calls or any
3730 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3732 scalar_int_mode int_mode
;
3733 if (remainder
== NULL_RTX
3735 && CONST_INT_P (op1
)
3736 && !pow2p_hwi (INTVAL (op1
))
3737 && is_int_mode (TYPE_MODE (type
), &int_mode
)
3738 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3739 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
3740 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
3741 && optimize_insn_for_speed_p ())
3743 rtx_insn
*last
= get_last_insn ();
3744 remainder
= NULL_RTX
;
3745 quotient
= expand_doubleword_divmod (int_mode
, op0
, op1
, &remainder
,
3746 TYPE_UNSIGNED (type
));
3747 if (quotient
!= NULL_RTX
)
3749 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
3751 rtx_insn
*move
= emit_move_insn (quotient
, quotient
);
3752 set_dst_reg_note (move
, REG_EQUAL
,
3753 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3754 ? UDIV
: DIV
, int_mode
,
3755 copy_rtx (op0
), op1
),
3757 move
= emit_move_insn (remainder
, remainder
);
3758 set_dst_reg_note (move
, REG_EQUAL
,
3759 gen_rtx_fmt_ee (TYPE_UNSIGNED (type
)
3760 ? UMOD
: MOD
, int_mode
,
3761 copy_rtx (op0
), op1
),
3766 delete_insns_since (last
);
3769 if (remainder
== NULL_RTX
)
3771 struct separate_ops ops
;
3772 ops
.code
= TRUNC_DIV_EXPR
;
3774 ops
.op0
= make_tree (ops
.type
, op0
);
3776 ops
.op2
= NULL_TREE
;
3777 ops
.location
= gimple_location (call_stmt
);
3779 quotient
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
3780 if (contains_call_div_mod (get_insns ()))
3781 quotient
= NULL_RTX
;
3784 ops
.code
= TRUNC_MOD_EXPR
;
3785 remainder
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
3787 if (contains_call_div_mod (get_insns ()))
3788 remainder
= NULL_RTX
;
3791 insns
= get_insns ();
3799 /* Check if optab_handler exists for divmod_optab for given mode. */
3800 else if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
3802 quotient
= gen_reg_rtx (mode
);
3803 remainder
= gen_reg_rtx (mode
);
3804 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
3807 /* Generate call to divmod libfunc if it exists. */
3808 else if (rtx libfunc
= optab_libfunc (tab
, mode
))
3809 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
3810 "ient
, &remainder
);
3815 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3816 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
3817 make_tree (TREE_TYPE (arg0
), quotient
),
3818 make_tree (TREE_TYPE (arg1
), remainder
)),
3819 target
, VOIDmode
, EXPAND_NORMAL
);
3825 expand_NOP (internal_fn
, gcall
*)
3827 /* Nothing. But it shouldn't really prevail. */
3830 /* Coroutines, all should have been processed at this stage. */
3833 expand_CO_FRAME (internal_fn
, gcall
*)
3839 expand_CO_YIELD (internal_fn
, gcall
*)
3845 expand_CO_SUSPN (internal_fn
, gcall
*)
3851 expand_CO_ACTOR (internal_fn
, gcall
*)
3856 /* Expand a call to FN using the operands in STMT. FN has a single
3857 output operand and NARGS input operands. */
3860 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
3863 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3864 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
3865 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3868 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3871 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
3873 expand_operand ops
[4];
3876 tree lhs
= gimple_call_lhs (stmt
);
3877 tree lhs_type
= TREE_TYPE (lhs
);
3878 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
3879 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
3881 for (unsigned int i
= 0; i
< 2; ++i
)
3883 tree rhs
= gimple_call_arg (stmt
, i
);
3884 rhs_type
[i
] = TREE_TYPE (rhs
);
3885 rtx rhs_rtx
= expand_normal (rhs
);
3886 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3890 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type
)))
3892 /* When the mask is an integer mode the exact vector length may not
3893 be clear to the backend, so we pass it in operand[3].
3894 Use the vector in arg2 for the most reliable intended size. */
3895 tree type
= TREE_TYPE (gimple_call_arg (stmt
, 2));
3896 create_integer_operand (&ops
[3], TYPE_VECTOR_SUBPARTS (type
));
3900 /* The mask has a vector type so the length operand is unnecessary. */
3903 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3904 TYPE_MODE (lhs_type
));
3906 expand_insn (icode
, opcnt
, ops
);
3907 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3908 emit_move_insn (lhs_rtx
, ops
[0].value
);
3911 /* Expand a call to a convert-like optab using the operands in STMT.
3912 FN has a single output operand and NARGS input operands. */
3915 expand_convert_optab_fn (internal_fn fn
, gcall
*stmt
, convert_optab optab
,
3918 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3919 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3920 TYPE_MODE (types
.second
));
3921 expand_fn_using_insn (stmt
, icode
, 1, nargs
);
3924 /* Expanders for optabs that can use expand_direct_optab_fn. */
3926 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3927 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3929 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3930 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3932 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3933 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3935 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3936 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3938 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3939 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3941 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3942 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3944 #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3945 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3947 #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3948 expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3950 #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3951 expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3953 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3954 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3956 #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3957 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3959 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3960 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3962 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3963 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3965 #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3966 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3968 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3969 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3971 /* Expanders for optabs that can use expand_convert_optab_fn. */
3973 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3974 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3976 #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3977 expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3979 /* RETURN_TYPE and ARGS are a return type and argument list that are
3980 in principle compatible with FN (which satisfies direct_internal_fn_p).
3981 Return the types that should be used to determine whether the
3982 target supports FN. */
3985 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3987 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3988 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3989 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3990 return tree_pair (type0
, type1
);
3993 /* CALL is a call whose return type and arguments are in principle
3994 compatible with FN (which satisfies direct_internal_fn_p). Return the
3995 types that should be used to determine whether the target supports FN. */
3998 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
4000 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4001 tree op0
= (info
.type0
< 0
4002 ? gimple_call_lhs (call
)
4003 : gimple_call_arg (call
, info
.type0
));
4004 tree op1
= (info
.type1
< 0
4005 ? gimple_call_lhs (call
)
4006 : gimple_call_arg (call
, info
.type1
));
4007 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
4010 /* Return true if OPTAB is supported for TYPES (whose modes should be
4011 the same) when the optimization type is OPT_TYPE. Used for simple
4015 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
4016 optimization_type opt_type
)
4018 machine_mode mode
= TYPE_MODE (types
.first
);
4019 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
4020 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
4023 /* Return true if OPTAB is supported for TYPES, where the first type
4024 is the destination and the second type is the source. Used for
4028 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
4029 optimization_type opt_type
)
4031 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
4032 TYPE_MODE (types
.second
), opt_type
)
4033 != CODE_FOR_nothing
);
4036 /* Return true if load/store lanes optab OPTAB is supported for
4037 array type TYPES.first when the optimization type is OPT_TYPE. */
4040 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
4041 optimization_type opt_type
)
4043 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
4044 machine_mode imode
= TYPE_MODE (types
.first
);
4045 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
4046 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
4047 != CODE_FOR_nothing
);
4050 #define direct_unary_optab_supported_p direct_optab_supported_p
4051 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
4052 #define direct_binary_optab_supported_p direct_optab_supported_p
4053 #define direct_ternary_optab_supported_p direct_optab_supported_p
4054 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
4055 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
4056 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4057 #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4058 #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4059 #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4060 #define direct_mask_load_optab_supported_p convert_optab_supported_p
4061 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4062 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4063 #define direct_gather_load_optab_supported_p convert_optab_supported_p
4064 #define direct_len_load_optab_supported_p direct_optab_supported_p
4065 #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4066 #define direct_mask_store_optab_supported_p convert_optab_supported_p
4067 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4068 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4069 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4070 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
4071 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
4072 #define direct_len_store_optab_supported_p direct_optab_supported_p
4073 #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4074 #define direct_while_optab_supported_p convert_optab_supported_p
4075 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
4076 #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4077 #define direct_fold_left_optab_supported_p direct_optab_supported_p
4078 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4079 #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4080 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4081 #define direct_vec_set_optab_supported_p direct_optab_supported_p
4082 #define direct_vec_extract_optab_supported_p convert_optab_supported_p
4084 /* Return the optab used by internal function FN. */
4087 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
4091 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4092 case IFN_##CODE: break;
4093 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4094 case IFN_##CODE: return OPTAB##_optab;
4095 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4096 UNSIGNED_OPTAB, TYPE) \
4097 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4098 ? UNSIGNED_OPTAB ## _optab \
4099 : SIGNED_OPTAB ## _optab);
4100 #include "internal-fn.def"
4108 /* Return the optab used by internal function FN. */
4111 direct_internal_fn_optab (internal_fn fn
)
4115 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4116 case IFN_##CODE: break;
4117 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4118 case IFN_##CODE: return OPTAB##_optab;
4119 #include "internal-fn.def"
4127 /* Return true if FN is supported for the types in TYPES when the
4128 optimization type is OPT_TYPE. The types are those associated with
4129 the "type0" and "type1" fields of FN's direct_internal_fn_info
4133 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
4134 optimization_type opt_type
)
4138 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4139 case IFN_##CODE: break;
4140 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4142 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4144 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4145 UNSIGNED_OPTAB, TYPE) \
4148 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4149 ? UNSIGNED_OPTAB ## _optab \
4150 : SIGNED_OPTAB ## _optab); \
4151 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4154 #include "internal-fn.def"
4162 /* Return true if FN is supported for type TYPE when the optimization
4163 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4164 fields of FN's direct_internal_fn_info structure are the same. */
4167 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
4168 optimization_type opt_type
)
4170 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
4171 gcc_checking_assert (info
.type0
== info
.type1
);
4172 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
4175 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4176 given that STMT is a call to a direct internal function. */
4179 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
4181 internal_fn fn
= gimple_call_internal_fn (stmt
);
4182 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
4183 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
4186 /* Return true if FN is a binary operation and if FN is commutative. */
4189 commutative_binary_fn_p (internal_fn fn
)
4200 case IFN_COMPLEX_MUL
:
4201 case IFN_UBSAN_CHECK_ADD
:
4202 case IFN_UBSAN_CHECK_MUL
:
4203 case IFN_ADD_OVERFLOW
:
4204 case IFN_MUL_OVERFLOW
:
4205 case IFN_VEC_WIDEN_PLUS
:
4206 case IFN_VEC_WIDEN_PLUS_LO
:
4207 case IFN_VEC_WIDEN_PLUS_HI
:
4208 case IFN_VEC_WIDEN_PLUS_EVEN
:
4209 case IFN_VEC_WIDEN_PLUS_ODD
:
4217 /* Return true if FN is a ternary operation and if its first two arguments
4221 commutative_ternary_fn_p (internal_fn fn
)
4237 /* Return true if FN is an associative binary operation. */
4240 associative_binary_fn_p (internal_fn fn
)
4253 /* If FN is commutative in two consecutive arguments, return the
4254 index of the first, otherwise return -1. */
4257 first_commutative_argument (internal_fn fn
)
4274 case IFN_COND_LEN_ADD
:
4275 case IFN_COND_LEN_MUL
:
4276 case IFN_COND_LEN_MIN
:
4277 case IFN_COND_LEN_MAX
:
4278 case IFN_COND_LEN_FMIN
:
4279 case IFN_COND_LEN_FMAX
:
4280 case IFN_COND_LEN_AND
:
4281 case IFN_COND_LEN_IOR
:
4282 case IFN_COND_LEN_XOR
:
4283 case IFN_COND_LEN_FMA
:
4284 case IFN_COND_LEN_FMS
:
4285 case IFN_COND_LEN_FNMA
:
4286 case IFN_COND_LEN_FNMS
:
4290 if (commutative_binary_fn_p (fn
)
4291 || commutative_ternary_fn_p (fn
))
4297 /* Return true if this CODE describes an internal_fn that returns a vector with
4298 elements twice as wide as the element size of the input vectors. */
4301 widening_fn_p (code_helper code
)
4303 if (!code
.is_fn_code ())
4306 if (!internal_fn_p ((combined_fn
) code
))
4309 internal_fn fn
= as_internal_fn ((combined_fn
) code
);
4312 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4314 case IFN_##NAME##_HI: \
4315 case IFN_##NAME##_LO: \
4316 case IFN_##NAME##_EVEN: \
4317 case IFN_##NAME##_ODD: \
4319 #include "internal-fn.def"
4326 /* Return true if IFN_SET_EDOM is supported. */
4329 set_edom_supported_p (void)
4338 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4340 expand_##CODE (internal_fn fn, gcall *stmt) \
4342 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4344 #define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4345 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4346 UNSIGNED_OPTAB, TYPE) \
4348 expand_##CODE (internal_fn fn, gcall *stmt) \
4350 tree_pair types = direct_internal_fn_types (fn, stmt); \
4351 optab which_optab = direct_internal_fn_optab (fn, types); \
4352 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4354 #include "internal-fn.def"
4356 /* Routines to expand each internal function, indexed by function number.
4357 Each routine has the prototype:
4359 expand_<NAME> (gcall *stmt)
4361 where STMT is the statement that performs the call. */
4362 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
4364 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4365 #include "internal-fn.def"
4369 /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4370 that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4371 for each such IFN_COND_##SUFFIX. */
4372 #define FOR_EACH_CODE_MAPPING(T) \
4373 T (PLUS_EXPR, ADD) \
4374 T (MINUS_EXPR, SUB) \
4375 T (MULT_EXPR, MUL) \
4376 T (TRUNC_DIV_EXPR, DIV) \
4377 T (TRUNC_MOD_EXPR, MOD) \
4378 T (RDIV_EXPR, RDIV) \
4381 T (BIT_AND_EXPR, AND) \
4382 T (BIT_IOR_EXPR, IOR) \
4383 T (BIT_XOR_EXPR, XOR) \
4384 T (LSHIFT_EXPR, SHL) \
4385 T (RSHIFT_EXPR, SHR) \
4386 T (NEGATE_EXPR, NEG)
4388 /* Return a function that only performs CODE when a certain condition is met
4389 and that uses a given fallback value otherwise. For example, if CODE is
4390 a binary operation associated with conditional function FN:
4392 LHS = FN (COND, A, B, ELSE)
4394 is equivalent to the C expression:
4396 LHS = COND ? A CODE B : ELSE;
4398 operating elementwise if the operands are vectors.
4400 Return IFN_LAST if no such function exists. */
4403 get_conditional_internal_fn (tree_code code
)
4407 #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4408 FOR_EACH_CODE_MAPPING(CASE
)
4415 /* If IFN implements the conditional form of a tree code, return that
4416 tree code, otherwise return ERROR_MARK. */
4419 conditional_internal_fn_code (internal_fn ifn
)
4423 #define CASE(CODE, IFN) \
4424 case IFN_COND_##IFN: \
4425 case IFN_COND_LEN_##IFN: \
4427 FOR_EACH_CODE_MAPPING (CASE
)
4434 /* Like get_conditional_internal_fn, but return a function that
4435 additionally restricts the operation to the leading elements
4436 of a vector. The number of elements to process is given by a length
4437 and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4438 elements are taken from the fallback ("else") argument.
4440 For example, if CODE is a binary operation associated with FN:
4442 LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4444 is equivalent to the C code:
4446 for (int i = 0; i < NUNITS; i++)
4448 if (i < LEN + BIAS && COND[i])
4449 LHS[i] = A[i] CODE B[i];
4456 get_conditional_len_internal_fn (tree_code code
)
4460 #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4461 FOR_EACH_CODE_MAPPING(CASE
)
4468 /* Invoke T(IFN) for each internal function IFN that also has an
4470 #define FOR_EACH_COND_FN_PAIR(T) \
4478 /* Return a function that only performs internal function FN when a
4479 certain condition is met and that uses a given fallback value otherwise.
4480 In other words, the returned function FN' is such that:
4482 LHS = FN' (COND, A1, ... An, ELSE)
4484 is equivalent to the C expression:
4486 LHS = COND ? FN (A1, ..., An) : ELSE;
4488 operating elementwise if the operands are vectors.
4490 Return IFN_LAST if no such function exists. */
4493 get_conditional_internal_fn (internal_fn fn
)
4497 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4498 FOR_EACH_COND_FN_PAIR(CASE
)
4505 /* If there exists an internal function like IFN that operates on vectors,
4506 but with additional length and bias parameters, return the internal_fn
4507 for that function, otherwise return IFN_LAST. */
4509 get_len_internal_fn (internal_fn fn
)
4513 #define DEF_INTERNAL_COND_FN(NAME, ...) \
4514 case IFN_COND_##NAME: \
4515 return IFN_COND_LEN_##NAME;
4516 #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4517 case IFN_COND_##NAME: \
4518 return IFN_COND_LEN_##NAME;
4519 #include "internal-fn.def"
4525 /* If IFN implements the conditional form of an unconditional internal
4526 function, return that unconditional function, otherwise return IFN_LAST. */
4529 get_unconditional_internal_fn (internal_fn ifn
)
4533 #define CASE(NAME) \
4534 case IFN_COND_##NAME: \
4535 case IFN_COND_LEN_##NAME: \
4537 FOR_EACH_COND_FN_PAIR (CASE
)
4544 /* Return true if STMT can be interpreted as a conditional tree code
4545 operation of the form:
4547 LHS = COND ? OP (RHS1, ...) : ELSE;
4549 operating elementwise if the operands are vectors. This includes
4550 the case of an all-true COND, so that the operation always happens.
4552 There is an alternative approach to interpret the STMT when the operands
4553 are vectors which is the operation predicated by both conditional mask
4554 and loop control length, the equivalent C code:
4556 for (int i = 0; i < NUNTIS; i++)
4558 if (i < LEN + BIAS && COND[i])
4559 LHS[i] = A[i] CODE B[i];
4564 When returning true, set:
4566 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4567 is known to be all-true
4568 - *CODE_OUT to the tree code
4569 - OPS[I] to operand I of *CODE_OUT
4570 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4571 condition is known to be all true.
4572 - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4573 - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4576 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
4577 tree_code
*code_out
,
4578 tree (&ops
)[3], tree
*else_out
,
4579 tree
*len
, tree
*bias
)
4583 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4585 *cond_out
= NULL_TREE
;
4586 *code_out
= gimple_assign_rhs_code (assign
);
4587 ops
[0] = gimple_assign_rhs1 (assign
);
4588 ops
[1] = gimple_assign_rhs2 (assign
);
4589 ops
[2] = gimple_assign_rhs3 (assign
);
4590 *else_out
= NULL_TREE
;
4593 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
4594 if (gimple_call_internal_p (call
))
4596 internal_fn ifn
= gimple_call_internal_fn (call
);
4597 tree_code code
= conditional_internal_fn_code (ifn
);
4598 int len_index
= internal_fn_len_index (ifn
);
4599 int cond_nargs
= len_index
>= 0 ? 4 : 2;
4600 if (code
!= ERROR_MARK
)
4602 *cond_out
= gimple_call_arg (call
, 0);
4604 unsigned int nops
= gimple_call_num_args (call
) - cond_nargs
;
4605 for (unsigned int i
= 0; i
< 3; ++i
)
4606 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
4607 *else_out
= gimple_call_arg (call
, nops
+ 1);
4610 if (integer_truep (*cond_out
))
4612 *cond_out
= NULL_TREE
;
4613 *else_out
= NULL_TREE
;
4618 *len
= gimple_call_arg (call
, len_index
);
4619 *bias
= gimple_call_arg (call
, len_index
+ 1);
4627 /* Return true if IFN is some form of load from memory. */
4630 internal_load_fn_p (internal_fn fn
)
4635 case IFN_LOAD_LANES
:
4636 case IFN_MASK_LOAD_LANES
:
4637 case IFN_MASK_LEN_LOAD_LANES
:
4638 case IFN_GATHER_LOAD
:
4639 case IFN_MASK_GATHER_LOAD
:
4640 case IFN_MASK_LEN_GATHER_LOAD
:
4642 case IFN_MASK_LEN_LOAD
:
4650 /* Return true if IFN is some form of store to memory. */
4653 internal_store_fn_p (internal_fn fn
)
4657 case IFN_MASK_STORE
:
4658 case IFN_STORE_LANES
:
4659 case IFN_MASK_STORE_LANES
:
4660 case IFN_MASK_LEN_STORE_LANES
:
4661 case IFN_SCATTER_STORE
:
4662 case IFN_MASK_SCATTER_STORE
:
4663 case IFN_MASK_LEN_SCATTER_STORE
:
4665 case IFN_MASK_LEN_STORE
:
4673 /* Return true if IFN is some form of gather load or scatter store. */
4676 internal_gather_scatter_fn_p (internal_fn fn
)
4680 case IFN_GATHER_LOAD
:
4681 case IFN_MASK_GATHER_LOAD
:
4682 case IFN_MASK_LEN_GATHER_LOAD
:
4683 case IFN_SCATTER_STORE
:
4684 case IFN_MASK_SCATTER_STORE
:
4685 case IFN_MASK_LEN_SCATTER_STORE
:
4693 /* If FN takes a vector len argument, return the index of that argument,
4694 otherwise return -1. */
4697 internal_fn_len_index (internal_fn fn
)
4705 case IFN_MASK_LEN_GATHER_LOAD
:
4706 case IFN_MASK_LEN_SCATTER_STORE
:
4707 case IFN_COND_LEN_FMA
:
4708 case IFN_COND_LEN_FMS
:
4709 case IFN_COND_LEN_FNMA
:
4710 case IFN_COND_LEN_FNMS
:
4713 case IFN_COND_LEN_ADD
:
4714 case IFN_COND_LEN_SUB
:
4715 case IFN_COND_LEN_MUL
:
4716 case IFN_COND_LEN_DIV
:
4717 case IFN_COND_LEN_MOD
:
4718 case IFN_COND_LEN_RDIV
:
4719 case IFN_COND_LEN_MIN
:
4720 case IFN_COND_LEN_MAX
:
4721 case IFN_COND_LEN_FMIN
:
4722 case IFN_COND_LEN_FMAX
:
4723 case IFN_COND_LEN_AND
:
4724 case IFN_COND_LEN_IOR
:
4725 case IFN_COND_LEN_XOR
:
4726 case IFN_COND_LEN_SHL
:
4727 case IFN_COND_LEN_SHR
:
4730 case IFN_COND_LEN_NEG
:
4731 case IFN_MASK_LEN_LOAD
:
4732 case IFN_MASK_LEN_STORE
:
4733 case IFN_MASK_LEN_LOAD_LANES
:
4734 case IFN_MASK_LEN_STORE_LANES
:
4735 case IFN_VCOND_MASK_LEN
:
4743 /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4744 argument that is used when the condition is false. Return -1 otherwise. */
4747 internal_fn_else_index (internal_fn fn
)
4753 case IFN_COND_LEN_NEG
:
4754 case IFN_COND_LEN_NOT
:
4771 case IFN_COND_LEN_ADD
:
4772 case IFN_COND_LEN_SUB
:
4773 case IFN_COND_LEN_MUL
:
4774 case IFN_COND_LEN_DIV
:
4775 case IFN_COND_LEN_MOD
:
4776 case IFN_COND_LEN_MIN
:
4777 case IFN_COND_LEN_MAX
:
4778 case IFN_COND_LEN_FMIN
:
4779 case IFN_COND_LEN_FMAX
:
4780 case IFN_COND_LEN_AND
:
4781 case IFN_COND_LEN_IOR
:
4782 case IFN_COND_LEN_XOR
:
4783 case IFN_COND_LEN_SHL
:
4784 case IFN_COND_LEN_SHR
:
4791 case IFN_COND_LEN_FMA
:
4792 case IFN_COND_LEN_FMS
:
4793 case IFN_COND_LEN_FNMA
:
4794 case IFN_COND_LEN_FNMS
:
4804 /* If FN takes a vector mask argument, return the index of that argument,
4805 otherwise return -1. */
4808 internal_fn_mask_index (internal_fn fn
)
4813 case IFN_MASK_LOAD_LANES
:
4814 case IFN_MASK_LEN_LOAD_LANES
:
4815 case IFN_MASK_STORE
:
4816 case IFN_MASK_STORE_LANES
:
4817 case IFN_MASK_LEN_STORE_LANES
:
4818 case IFN_MASK_LEN_LOAD
:
4819 case IFN_MASK_LEN_STORE
:
4822 case IFN_MASK_GATHER_LOAD
:
4823 case IFN_MASK_SCATTER_STORE
:
4824 case IFN_MASK_LEN_GATHER_LOAD
:
4825 case IFN_MASK_LEN_SCATTER_STORE
:
4828 case IFN_VCOND_MASK_LEN
:
4832 return (conditional_internal_fn_code (fn
) != ERROR_MARK
4833 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
4837 /* If FN takes a value that should be stored to memory, return the index
4838 of that argument, otherwise return -1. */
4841 internal_fn_stored_value_index (internal_fn fn
)
4845 case IFN_MASK_STORE
:
4846 case IFN_MASK_STORE_LANES
:
4847 case IFN_SCATTER_STORE
:
4848 case IFN_MASK_SCATTER_STORE
:
4849 case IFN_MASK_LEN_SCATTER_STORE
:
4855 case IFN_MASK_LEN_STORE
:
4856 case IFN_MASK_LEN_STORE_LANES
:
4864 /* Return true if the target supports gather load or scatter store function
4865 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4866 while for stores it is the vector type of the stored data argument.
4867 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4868 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4869 offset from the shared base address of each loaded or stored element.
4870 SCALE is the amount by which these offsets should be multiplied
4871 *after* they have been extended to address width. */
4874 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
4875 tree memory_element_type
,
4876 tree offset_vector_type
, int scale
)
4878 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
4879 TYPE_SIZE (memory_element_type
)))
4881 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
4882 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
4884 optab optab
= direct_internal_fn_optab (ifn
);
4885 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
4886 TYPE_MODE (offset_vector_type
));
4887 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
4888 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
4889 return (icode
!= CODE_FOR_nothing
4890 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
4891 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
4894 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4895 for pointers of type TYPE when the accesses have LENGTH bytes and their
4896 common byte alignment is ALIGN. */
4899 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
4900 poly_uint64 length
, unsigned int align
)
4902 machine_mode mode
= TYPE_MODE (type
);
4903 optab optab
= direct_internal_fn_optab (ifn
);
4904 insn_code icode
= direct_optab_handler (optab
, mode
);
4905 if (icode
== CODE_FOR_nothing
)
4907 rtx length_rtx
= immed_wide_int_const (length
, mode
);
4908 return (insn_operand_matches (icode
, 3, length_rtx
)
4909 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
4912 /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4913 or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4914 -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4915 {len_,mask_len_}store). If none of the biases match what the backend
4916 provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4919 internal_len_load_store_bias (internal_fn ifn
, machine_mode mode
)
4921 optab optab
= direct_internal_fn_optab (ifn
);
4922 insn_code icode
= direct_optab_handler (optab
, mode
);
4925 if (icode
== CODE_FOR_nothing
)
4927 machine_mode mask_mode
;
4928 if (!targetm
.vectorize
.get_mask_mode (mode
).exists (&mask_mode
))
4929 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4930 if (ifn
== IFN_LEN_LOAD
)
4932 /* Try MASK_LEN_LOAD. */
4933 optab
= direct_internal_fn_optab (IFN_MASK_LEN_LOAD
);
4937 /* Try MASK_LEN_STORE. */
4938 optab
= direct_internal_fn_optab (IFN_MASK_LEN_STORE
);
4940 icode
= convert_optab_handler (optab
, mode
, mask_mode
);
4944 if (icode
!= CODE_FOR_nothing
)
4946 /* For now we only support biases of 0 or -1. Try both of them. */
4947 if (insn_operand_matches (icode
, bias_no
, GEN_INT (0)))
4949 if (insn_operand_matches (icode
, bias_no
, GEN_INT (-1)))
4953 return VECT_PARTIAL_BIAS_UNSUPPORTED
;
4956 /* Expand STMT as though it were a call to internal function FN. */
4959 expand_internal_call (internal_fn fn
, gcall
*stmt
)
4961 internal_fn_expanders
[fn
] (fn
, stmt
);
4964 /* Expand STMT, which is a call to internal function FN. */
4967 expand_internal_call (gcall
*stmt
)
4969 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
4972 /* If TYPE is a vector type, return true if IFN is a direct internal
4973 function that is supported for that type. If TYPE is a scalar type,
4974 return true if IFN is a direct internal function that is supported for
4975 the target's preferred vector version of TYPE. */
4978 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
4980 if (VECTOR_MODE_P (TYPE_MODE (type
)))
4981 return direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
);
4984 if (VECTOR_TYPE_P (type
)
4985 || !is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
4988 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
4989 if (VECTOR_MODE_P (vmode
))
4991 tree vectype
= build_vector_type_for_mode (type
, vmode
);
4992 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
4996 auto_vector_modes vector_modes
;
4997 targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
, true);
4998 for (machine_mode base_mode
: vector_modes
)
4999 if (related_vector_mode (base_mode
, smode
).exists (&vmode
))
5001 tree vectype
= build_vector_type_for_mode (type
, vmode
);
5002 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
5010 expand_SHUFFLEVECTOR (internal_fn
, gcall
*)
5016 expand_PHI (internal_fn
, gcall
*)
5022 expand_SPACESHIP (internal_fn
, gcall
*stmt
)
5024 tree lhs
= gimple_call_lhs (stmt
);
5025 tree rhs1
= gimple_call_arg (stmt
, 0);
5026 tree rhs2
= gimple_call_arg (stmt
, 1);
5027 tree type
= TREE_TYPE (rhs1
);
5029 do_pending_stack_adjust ();
5031 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5032 rtx op1
= expand_normal (rhs1
);
5033 rtx op2
= expand_normal (rhs2
);
5035 class expand_operand ops
[3];
5036 create_output_operand (&ops
[0], target
, TYPE_MODE (TREE_TYPE (lhs
)));
5037 create_input_operand (&ops
[1], op1
, TYPE_MODE (type
));
5038 create_input_operand (&ops
[2], op2
, TYPE_MODE (type
));
5039 insn_code icode
= optab_handler (spaceship_optab
, TYPE_MODE (type
));
5040 expand_insn (icode
, 3, ops
);
5041 if (!rtx_equal_p (target
, ops
[0].value
))
5042 emit_move_insn (target
, ops
[0].value
);
5046 expand_ASSUME (internal_fn
, gcall
*)
5051 expand_MASK_CALL (internal_fn
, gcall
*)
5053 /* This IFN should only exist between ifcvt and vect passes. */
5058 expand_MULBITINT (internal_fn
, gcall
*stmt
)
5061 for (int i
= 0; i
< 6; i
++)
5062 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5063 (i
& 1) ? SImode
: ptr_mode
);
5064 rtx fun
= init_one_libfunc ("__mulbitint3");
5065 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 6, args
);
5069 expand_DIVMODBITINT (internal_fn
, gcall
*stmt
)
5072 for (int i
= 0; i
< 8; i
++)
5073 args
[i
] = rtx_mode_t (expand_normal (gimple_call_arg (stmt
, i
)),
5074 (i
& 1) ? SImode
: ptr_mode
);
5075 rtx fun
= init_one_libfunc ("__divmodbitint4");
5076 emit_library_call_value_1 (0, fun
, NULL_RTX
, LCT_NORMAL
, VOIDmode
, 8, args
);
5080 expand_FLOATTOBITINT (internal_fn
, gcall
*stmt
)
5082 machine_mode mode
= TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt
, 2)));
5083 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5084 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5085 rtx arg2
= expand_normal (gimple_call_arg (stmt
, 2));
5086 const char *mname
= GET_MODE_NAME (mode
);
5087 unsigned mname_len
= strlen (mname
);
5088 int len
= 12 + mname_len
;
5089 if (DECIMAL_FLOAT_MODE_P (mode
))
5091 char *libfunc_name
= XALLOCAVEC (char, len
);
5092 char *p
= libfunc_name
;
5094 if (DECIMAL_FLOAT_MODE_P (mode
))
5096 #if ENABLE_DECIMAL_BID_FORMAT
5097 memcpy (p
, "__bid_fix", 9);
5099 memcpy (p
, "__dpd_fix", 9);
5105 memcpy (p
, "__fix", 5);
5108 for (q
= mname
; *q
; q
++)
5109 *p
++ = TOLOWER (*q
);
5110 memcpy (p
, "bitint", 7);
5111 rtx fun
= init_one_libfunc (libfunc_name
);
5112 emit_library_call (fun
, LCT_NORMAL
, VOIDmode
, arg0
, ptr_mode
, arg1
,
5113 SImode
, arg2
, mode
);
5117 expand_BITINTTOFLOAT (internal_fn
, gcall
*stmt
)
5119 tree lhs
= gimple_call_lhs (stmt
);
5122 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
5123 rtx arg0
= expand_normal (gimple_call_arg (stmt
, 0));
5124 rtx arg1
= expand_normal (gimple_call_arg (stmt
, 1));
5125 const char *mname
= GET_MODE_NAME (mode
);
5126 unsigned mname_len
= strlen (mname
);
5127 int len
= 14 + mname_len
;
5128 if (DECIMAL_FLOAT_MODE_P (mode
))
5130 char *libfunc_name
= XALLOCAVEC (char, len
);
5131 char *p
= libfunc_name
;
5133 if (DECIMAL_FLOAT_MODE_P (mode
))
5135 #if ENABLE_DECIMAL_BID_FORMAT
5136 memcpy (p
, "__bid_floatbitint", 17);
5138 memcpy (p
, "__dpd_floatbitint", 17);
5144 memcpy (p
, "__floatbitint", 13);
5147 for (q
= mname
; *q
; q
++)
5148 *p
++ = TOLOWER (*q
);
5150 rtx fun
= init_one_libfunc (libfunc_name
);
5151 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
5152 rtx val
= emit_library_call_value (fun
, target
, LCT_PURE
, mode
,
5153 arg0
, ptr_mode
, arg1
, SImode
);
5155 emit_move_insn (target
, val
);
5159 expand_bitquery (internal_fn fn
, gcall
*stmt
)
5161 tree lhs
= gimple_call_lhs (stmt
);
5162 if (lhs
== NULL_TREE
)
5164 tree arg
= gimple_call_arg (stmt
, 0);
5165 if (TREE_CODE (arg
) == INTEGER_CST
)
5167 tree ret
= fold_const_call (as_combined_fn (fn
), TREE_TYPE (arg
), arg
);
5168 gcc_checking_assert (ret
&& TREE_CODE (ret
) == INTEGER_CST
);
5169 expand_assignment (lhs
, ret
, false);
5176 expand_CLRSB (internal_fn fn
, gcall
*stmt
)
5178 if (expand_bitquery (fn
, stmt
))
5179 expand_unary_optab_fn (fn
, stmt
, clrsb_optab
);
5183 expand_CLZ (internal_fn fn
, gcall
*stmt
)
5185 if (expand_bitquery (fn
, stmt
))
5186 expand_unary_optab_fn (fn
, stmt
, clz_optab
);
5190 expand_CTZ (internal_fn fn
, gcall
*stmt
)
5192 if (expand_bitquery (fn
, stmt
))
5193 expand_unary_optab_fn (fn
, stmt
, ctz_optab
);
5197 expand_FFS (internal_fn fn
, gcall
*stmt
)
5199 if (expand_bitquery (fn
, stmt
))
5200 expand_unary_optab_fn (fn
, stmt
, ffs_optab
);
5204 expand_PARITY (internal_fn fn
, gcall
*stmt
)
5206 if (expand_bitquery (fn
, stmt
))
5207 expand_unary_optab_fn (fn
, stmt
, parity_optab
);
5211 expand_POPCOUNT (internal_fn fn
, gcall
*stmt
)
5213 if (!expand_bitquery (fn
, stmt
))
5215 if (gimple_call_num_args (stmt
) == 1)
5217 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5220 /* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5221 because the result is only used in an equality comparison against 1.
5222 Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5223 or (arg ^ (arg - 1)) > arg - 1 is cheaper.
5224 If .POPCOUNT second argument is 0, we additionally know that arg
5225 is non-zero, so use arg & (arg - 1) == 0 instead. */
5226 bool speed_p
= optimize_insn_for_speed_p ();
5227 tree lhs
= gimple_call_lhs (stmt
);
5228 tree arg
= gimple_call_arg (stmt
, 0);
5229 bool nonzero_arg
= integer_zerop (gimple_call_arg (stmt
, 1));
5230 tree type
= TREE_TYPE (arg
);
5231 machine_mode mode
= TYPE_MODE (type
);
5232 do_pending_stack_adjust ();
5234 expand_unary_optab_fn (fn
, stmt
, popcount_optab
);
5235 rtx_insn
*popcount_insns
= get_insns ();
5238 rtx plhs
= expand_normal (lhs
);
5239 rtx pcmp
= emit_store_flag (NULL_RTX
, EQ
, plhs
, const1_rtx
, mode
, 0, 0);
5240 if (pcmp
== NULL_RTX
)
5244 emit_insn (popcount_insns
);
5247 rtx_insn
*popcount_cmp_insns
= get_insns ();
5250 rtx op0
= expand_normal (arg
);
5251 rtx argm1
= expand_simple_binop (mode
, PLUS
, op0
, constm1_rtx
, NULL_RTX
,
5253 if (argm1
== NULL_RTX
)
5255 rtx argxorargm1
= expand_simple_binop (mode
, nonzero_arg
? AND
: XOR
, op0
,
5256 argm1
, NULL_RTX
, 1, OPTAB_DIRECT
);
5257 if (argxorargm1
== NULL_RTX
)
5261 cmp
= emit_store_flag (NULL_RTX
, EQ
, argxorargm1
, const0_rtx
, mode
, 1, 1);
5263 cmp
= emit_store_flag (NULL_RTX
, GTU
, argxorargm1
, argm1
, mode
, 1, 1);
5264 if (cmp
== NULL_RTX
)
5266 rtx_insn
*cmp_insns
= get_insns ();
5268 unsigned popcount_cost
= (seq_cost (popcount_insns
, speed_p
)
5269 + seq_cost (popcount_cmp_insns
, speed_p
));
5270 unsigned cmp_cost
= seq_cost (cmp_insns
, speed_p
);
5271 if (popcount_cost
<= cmp_cost
)
5272 emit_insn (popcount_insns
);
5275 emit_insn (cmp_insns
);
5276 plhs
= expand_normal (lhs
);
5277 if (GET_MODE (cmp
) != GET_MODE (plhs
))
5278 cmp
= convert_to_mode (GET_MODE (plhs
), cmp
, 1);
5279 emit_move_insn (plhs
, cmp
);