hppa: Export main in pr104869.C on hpux
[official-gcc.git] / gcc / internal-fn.cc
blobcce73b3b2e085d57fcb4996c78962fa73e070213
1 /* Internal functions.
2 Copyright (C) 2011-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-vrp.h"
31 #include "tree-ssanames.h"
32 #include "expmed.h"
33 #include "memmodel.h"
34 #include "optabs.h"
35 #include "emit-rtl.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
40 #include "dojump.h"
41 #include "expr.h"
42 #include "stringpool.h"
43 #include "attribs.h"
44 #include "asan.h"
45 #include "ubsan.h"
46 #include "recog.h"
47 #include "builtins.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52 #include "explow.h"
53 #include "rtl-iter.h"
54 #include "gimple-range.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* The names of each internal function, indexed by function number. */
60 const char *const internal_fn_name_array[] = {
61 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
62 #include "internal-fn.def"
63 "<invalid-fn>"
66 /* The ECF_* flags of each internal function, indexed by function number. */
67 const int internal_fn_flags_array[] = {
68 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
69 #include "internal-fn.def"
73 /* Return the internal function called NAME, or IFN_LAST if there's
74 no such function. */
76 internal_fn
77 lookup_internal_fn (const char *name)
79 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
80 static name_to_fn_map_type *name_to_fn_map;
82 if (!name_to_fn_map)
84 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
85 for (unsigned int i = 0; i < IFN_LAST; ++i)
86 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
87 internal_fn (i));
89 internal_fn *entry = name_to_fn_map->get (name);
90 return entry ? *entry : IFN_LAST;
93 /* Geven an internal_fn IFN that is a widening function, return its
94 corresponding LO and HI internal_fns. */
96 extern void
97 lookup_hilo_internal_fn (internal_fn ifn, internal_fn *lo, internal_fn *hi)
99 gcc_assert (widening_fn_p (ifn));
101 switch (ifn)
103 default:
104 gcc_unreachable ();
105 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
106 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
107 case IFN_##NAME: \
108 *lo = internal_fn (IFN_##NAME##_LO); \
109 *hi = internal_fn (IFN_##NAME##_HI); \
110 break;
111 #include "internal-fn.def"
115 /* Given an internal_fn IFN that is a widening function, return its
116 corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD. */
118 extern void
119 lookup_evenodd_internal_fn (internal_fn ifn, internal_fn *even,
120 internal_fn *odd)
122 gcc_assert (widening_fn_p (ifn));
124 switch (ifn)
126 default:
127 gcc_unreachable ();
128 #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
129 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
130 case IFN_##NAME: \
131 *even = internal_fn (IFN_##NAME##_EVEN); \
132 *odd = internal_fn (IFN_##NAME##_ODD); \
133 break;
134 #include "internal-fn.def"
139 /* Fnspec of each internal function, indexed by function number. */
140 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
142 void
143 init_internal_fns ()
145 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
146 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
147 build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
148 #include "internal-fn.def"
149 internal_fn_fnspec_array[IFN_LAST] = 0;
152 /* Create static initializers for the information returned by
153 direct_internal_fn. */
154 #define not_direct { -2, -2, false }
155 #define mask_load_direct { -1, 2, false }
156 #define load_lanes_direct { -1, -1, false }
157 #define mask_load_lanes_direct { -1, -1, false }
158 #define gather_load_direct { 3, 1, false }
159 #define len_load_direct { -1, -1, false }
160 #define mask_len_load_direct { -1, 4, false }
161 #define mask_store_direct { 3, 2, false }
162 #define store_lanes_direct { 0, 0, false }
163 #define mask_store_lanes_direct { 0, 0, false }
164 #define vec_cond_mask_direct { 1, 0, false }
165 #define vec_cond_mask_len_direct { 1, 1, false }
166 #define vec_cond_direct { 2, 0, false }
167 #define scatter_store_direct { 3, 1, false }
168 #define len_store_direct { 3, 3, false }
169 #define mask_len_store_direct { 4, 5, false }
170 #define vec_set_direct { 3, 3, false }
171 #define vec_extract_direct { 0, -1, false }
172 #define unary_direct { 0, 0, true }
173 #define unary_convert_direct { -1, 0, true }
174 #define binary_direct { 0, 0, true }
175 #define ternary_direct { 0, 0, true }
176 #define cond_unary_direct { 1, 1, true }
177 #define cond_binary_direct { 1, 1, true }
178 #define cond_ternary_direct { 1, 1, true }
179 #define cond_len_unary_direct { 1, 1, true }
180 #define cond_len_binary_direct { 1, 1, true }
181 #define cond_len_ternary_direct { 1, 1, true }
182 #define while_direct { 0, 2, false }
183 #define fold_extract_direct { 2, 2, false }
184 #define fold_len_extract_direct { 2, 2, false }
185 #define fold_left_direct { 1, 1, false }
186 #define mask_fold_left_direct { 1, 1, false }
187 #define mask_len_fold_left_direct { 1, 1, false }
188 #define check_ptrs_direct { 0, 0, false }
190 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
191 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
192 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
193 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
194 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
195 #include "internal-fn.def"
196 not_direct
199 /* Expand STMT using instruction ICODE. The instruction has NOUTPUTS
200 output operands and NINPUTS input operands, where NOUTPUTS is either
201 0 or 1. The output operand (if any) comes first, followed by the
202 NINPUTS input operands. */
204 static void
205 expand_fn_using_insn (gcall *stmt, insn_code icode, unsigned int noutputs,
206 unsigned int ninputs)
208 gcc_assert (icode != CODE_FOR_nothing);
210 expand_operand *ops = XALLOCAVEC (expand_operand, noutputs + ninputs);
211 unsigned int opno = 0;
212 rtx lhs_rtx = NULL_RTX;
213 tree lhs = gimple_call_lhs (stmt);
215 if (noutputs)
217 gcc_assert (noutputs == 1);
218 if (lhs)
219 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
221 /* Do not assign directly to a promoted subreg, since there is no
222 guarantee that the instruction will leave the upper bits of the
223 register in the state required by SUBREG_PROMOTED_SIGN. */
224 rtx dest = lhs_rtx;
225 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
226 dest = NULL_RTX;
227 create_output_operand (&ops[opno], dest,
228 insn_data[icode].operand[opno].mode);
229 opno += 1;
231 else
232 gcc_assert (!lhs);
234 for (unsigned int i = 0; i < ninputs; ++i)
236 tree rhs = gimple_call_arg (stmt, i);
237 tree rhs_type = TREE_TYPE (rhs);
238 rtx rhs_rtx = expand_normal (rhs);
239 if (INTEGRAL_TYPE_P (rhs_type))
240 create_convert_operand_from (&ops[opno], rhs_rtx,
241 TYPE_MODE (rhs_type),
242 TYPE_UNSIGNED (rhs_type));
243 else if (TREE_CODE (rhs) == SSA_NAME
244 && SSA_NAME_IS_DEFAULT_DEF (rhs)
245 && VAR_P (SSA_NAME_VAR (rhs)))
246 create_undefined_input_operand (&ops[opno], TYPE_MODE (rhs_type));
247 else
248 create_input_operand (&ops[opno], rhs_rtx, TYPE_MODE (rhs_type));
249 opno += 1;
252 gcc_assert (opno == noutputs + ninputs);
253 expand_insn (icode, opno, ops);
254 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
256 /* If the return value has an integral type, convert the instruction
257 result to that type. This is useful for things that return an
258 int regardless of the size of the input. If the instruction result
259 is smaller than required, assume that it is signed.
261 If the return value has a nonintegral type, its mode must match
262 the instruction result. */
263 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
265 /* If this is a scalar in a register that is stored in a wider
266 mode than the declared mode, compute the result into its
267 declared mode and then convert to the wider mode. */
268 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
269 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
270 convert_move (SUBREG_REG (lhs_rtx), tmp,
271 SUBREG_PROMOTED_SIGN (lhs_rtx));
273 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
274 emit_move_insn (lhs_rtx, ops[0].value);
275 else
277 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
278 convert_move (lhs_rtx, ops[0].value, 0);
283 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
284 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
286 static enum insn_code
287 get_multi_vector_move (tree array_type, convert_optab optab)
289 machine_mode imode;
290 machine_mode vmode;
292 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
293 imode = TYPE_MODE (array_type);
294 vmode = TYPE_MODE (TREE_TYPE (array_type));
296 return convert_optab_handler (optab, imode, vmode);
299 /* Add mask and len arguments according to the STMT. */
301 static unsigned int
302 add_mask_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
304 internal_fn ifn = gimple_call_internal_fn (stmt);
305 int len_index = internal_fn_len_index (ifn);
306 /* BIAS is always consecutive next of LEN. */
307 int bias_index = len_index + 1;
308 int mask_index = internal_fn_mask_index (ifn);
309 /* The order of arguments are always {len,bias,mask}. */
310 if (mask_index >= 0)
312 tree mask = gimple_call_arg (stmt, mask_index);
313 rtx mask_rtx = expand_normal (mask);
314 create_input_operand (&ops[opno++], mask_rtx,
315 TYPE_MODE (TREE_TYPE (mask)));
317 if (len_index >= 0)
319 tree len = gimple_call_arg (stmt, len_index);
320 rtx len_rtx = expand_normal (len);
321 create_convert_operand_from (&ops[opno++], len_rtx,
322 TYPE_MODE (TREE_TYPE (len)),
323 TYPE_UNSIGNED (TREE_TYPE (len)));
324 tree biast = gimple_call_arg (stmt, bias_index);
325 rtx bias = expand_normal (biast);
326 create_input_operand (&ops[opno++], bias, QImode);
328 return opno;
331 /* Expand LOAD_LANES call STMT using optab OPTAB. */
333 static void
334 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
336 class expand_operand ops[2];
337 tree type, lhs, rhs;
338 rtx target, mem;
340 lhs = gimple_call_lhs (stmt);
341 rhs = gimple_call_arg (stmt, 0);
342 type = TREE_TYPE (lhs);
344 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
345 mem = expand_normal (rhs);
347 gcc_assert (MEM_P (mem));
348 PUT_MODE (mem, TYPE_MODE (type));
350 create_output_operand (&ops[0], target, TYPE_MODE (type));
351 create_fixed_operand (&ops[1], mem);
352 expand_insn (get_multi_vector_move (type, optab), 2, ops);
353 if (!rtx_equal_p (target, ops[0].value))
354 emit_move_insn (target, ops[0].value);
357 /* Expand STORE_LANES call STMT using optab OPTAB. */
359 static void
360 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
362 class expand_operand ops[2];
363 tree type, lhs, rhs;
364 rtx target, reg;
366 lhs = gimple_call_lhs (stmt);
367 rhs = gimple_call_arg (stmt, 0);
368 type = TREE_TYPE (rhs);
370 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
371 reg = expand_normal (rhs);
373 gcc_assert (MEM_P (target));
374 PUT_MODE (target, TYPE_MODE (type));
376 create_fixed_operand (&ops[0], target);
377 create_input_operand (&ops[1], reg, TYPE_MODE (type));
378 expand_insn (get_multi_vector_move (type, optab), 2, ops);
381 static void
382 expand_ANNOTATE (internal_fn, gcall *)
384 gcc_unreachable ();
387 /* This should get expanded in omp_device_lower pass. */
389 static void
390 expand_GOMP_USE_SIMT (internal_fn, gcall *)
392 gcc_unreachable ();
395 /* This should get expanded in omp_device_lower pass. */
397 static void
398 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
400 gcc_unreachable ();
403 /* Allocate per-lane storage and begin non-uniform execution region. */
405 static void
406 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
408 rtx target;
409 tree lhs = gimple_call_lhs (stmt);
410 if (lhs)
411 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
412 else
413 target = gen_reg_rtx (Pmode);
414 rtx size = expand_normal (gimple_call_arg (stmt, 0));
415 rtx align = expand_normal (gimple_call_arg (stmt, 1));
416 class expand_operand ops[3];
417 create_output_operand (&ops[0], target, Pmode);
418 create_input_operand (&ops[1], size, Pmode);
419 create_input_operand (&ops[2], align, Pmode);
420 gcc_assert (targetm.have_omp_simt_enter ());
421 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
422 if (!rtx_equal_p (target, ops[0].value))
423 emit_move_insn (target, ops[0].value);
426 /* Deallocate per-lane storage and leave non-uniform execution region. */
428 static void
429 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
431 gcc_checking_assert (!gimple_call_lhs (stmt));
432 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
433 class expand_operand ops[1];
434 create_input_operand (&ops[0], arg, Pmode);
435 gcc_assert (targetm.have_omp_simt_exit ());
436 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
439 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
440 without SIMT execution this should be expanded in omp_device_lower pass. */
442 static void
443 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
445 tree lhs = gimple_call_lhs (stmt);
446 if (!lhs)
447 return;
449 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
450 gcc_assert (targetm.have_omp_simt_lane ());
451 emit_insn (targetm.gen_omp_simt_lane (target));
454 /* This should get expanded in omp_device_lower pass. */
456 static void
457 expand_GOMP_SIMT_VF (internal_fn, gcall *)
459 gcc_unreachable ();
462 /* This should get expanded in omp_device_lower pass. */
464 static void
465 expand_GOMP_TARGET_REV (internal_fn, gcall *)
467 gcc_unreachable ();
470 /* Lane index of the first SIMT lane that supplies a non-zero argument.
471 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
472 lane that executed the last iteration for handling OpenMP lastprivate. */
474 static void
475 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
477 tree lhs = gimple_call_lhs (stmt);
478 if (!lhs)
479 return;
481 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
482 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
483 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
484 class expand_operand ops[2];
485 create_output_operand (&ops[0], target, mode);
486 create_input_operand (&ops[1], cond, mode);
487 gcc_assert (targetm.have_omp_simt_last_lane ());
488 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
489 if (!rtx_equal_p (target, ops[0].value))
490 emit_move_insn (target, ops[0].value);
493 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
495 static void
496 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
498 tree lhs = gimple_call_lhs (stmt);
499 if (!lhs)
500 return;
502 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
503 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
504 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
505 class expand_operand ops[2];
506 create_output_operand (&ops[0], target, mode);
507 create_input_operand (&ops[1], ctr, mode);
508 gcc_assert (targetm.have_omp_simt_ordered ());
509 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
510 if (!rtx_equal_p (target, ops[0].value))
511 emit_move_insn (target, ops[0].value);
514 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
515 any lane supplies a non-zero argument. */
517 static void
518 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
520 tree lhs = gimple_call_lhs (stmt);
521 if (!lhs)
522 return;
524 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
525 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
526 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
527 class expand_operand ops[2];
528 create_output_operand (&ops[0], target, mode);
529 create_input_operand (&ops[1], cond, mode);
530 gcc_assert (targetm.have_omp_simt_vote_any ());
531 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
532 if (!rtx_equal_p (target, ops[0].value))
533 emit_move_insn (target, ops[0].value);
536 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
537 is destination lane index XOR given offset. */
539 static void
540 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
542 tree lhs = gimple_call_lhs (stmt);
543 if (!lhs)
544 return;
546 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
547 rtx src = expand_normal (gimple_call_arg (stmt, 0));
548 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
549 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
550 class expand_operand ops[3];
551 create_output_operand (&ops[0], target, mode);
552 create_input_operand (&ops[1], src, mode);
553 create_input_operand (&ops[2], idx, SImode);
554 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
555 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
556 if (!rtx_equal_p (target, ops[0].value))
557 emit_move_insn (target, ops[0].value);
560 /* Exchange between SIMT lanes according to given source lane index. */
562 static void
563 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
565 tree lhs = gimple_call_lhs (stmt);
566 if (!lhs)
567 return;
569 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
570 rtx src = expand_normal (gimple_call_arg (stmt, 0));
571 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
572 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
573 class expand_operand ops[3];
574 create_output_operand (&ops[0], target, mode);
575 create_input_operand (&ops[1], src, mode);
576 create_input_operand (&ops[2], idx, SImode);
577 gcc_assert (targetm.have_omp_simt_xchg_idx ());
578 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
579 if (!rtx_equal_p (target, ops[0].value))
580 emit_move_insn (target, ops[0].value);
583 /* This should get expanded in adjust_simduid_builtins. */
585 static void
586 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
588 gcc_unreachable ();
591 /* This should get expanded in adjust_simduid_builtins. */
593 static void
594 expand_GOMP_SIMD_VF (internal_fn, gcall *)
596 gcc_unreachable ();
599 /* This should get expanded in adjust_simduid_builtins. */
601 static void
602 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
604 gcc_unreachable ();
607 /* This should get expanded in adjust_simduid_builtins. */
609 static void
610 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
612 gcc_unreachable ();
615 /* This should get expanded in adjust_simduid_builtins. */
617 static void
618 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
620 gcc_unreachable ();
623 /* This should get expanded in the sanopt pass. */
625 static void
626 expand_UBSAN_NULL (internal_fn, gcall *)
628 gcc_unreachable ();
631 /* This should get expanded in the sanopt pass. */
633 static void
634 expand_UBSAN_BOUNDS (internal_fn, gcall *)
636 gcc_unreachable ();
639 /* This should get expanded in the sanopt pass. */
641 static void
642 expand_UBSAN_VPTR (internal_fn, gcall *)
644 gcc_unreachable ();
647 /* This should get expanded in the sanopt pass. */
649 static void
650 expand_UBSAN_PTR (internal_fn, gcall *)
652 gcc_unreachable ();
655 /* This should get expanded in the sanopt pass. */
657 static void
658 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
660 gcc_unreachable ();
663 /* This should get expanded in the sanopt pass. */
665 static void
666 expand_HWASAN_CHECK (internal_fn, gcall *)
668 gcc_unreachable ();
671 /* For hwasan stack tagging:
672 Clear tags on the dynamically allocated space.
673 For use after an object dynamically allocated on the stack goes out of
674 scope. */
675 static void
676 expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
678 gcc_assert (Pmode == ptr_mode);
679 tree restored_position = gimple_call_arg (gc, 0);
680 rtx restored_rtx = expand_expr (restored_position, NULL_RTX, VOIDmode,
681 EXPAND_NORMAL);
682 rtx func = init_one_libfunc ("__hwasan_tag_memory");
683 rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
684 stack_pointer_rtx, NULL_RTX, 0,
685 OPTAB_WIDEN);
686 emit_library_call_value (func, NULL_RTX, LCT_NORMAL, VOIDmode,
687 virtual_stack_dynamic_rtx, Pmode,
688 HWASAN_STACK_BACKGROUND, QImode,
689 off, Pmode);
692 /* For hwasan stack tagging:
693 Return a tag to be used for a dynamic allocation. */
694 static void
695 expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
697 tree tag = gimple_call_lhs (gc);
698 rtx target = expand_expr (tag, NULL_RTX, VOIDmode, EXPAND_NORMAL);
699 machine_mode mode = GET_MODE (target);
700 gcc_assert (mode == QImode);
702 rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
703 gcc_assert (base_tag);
704 rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
705 rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
706 target, /* unsignedp = */1,
707 OPTAB_WIDEN);
708 chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
710 /* Really need to put the tag into the `target` RTX. */
711 if (chosen_tag != target)
713 rtx temp = chosen_tag;
714 gcc_assert (GET_MODE (chosen_tag) == mode);
715 emit_move_insn (target, temp);
718 hwasan_increment_frame_tag ();
721 /* For hwasan stack tagging:
722 Tag a region of space in the shadow stack according to the base pointer of
723 an object on the stack. N.b. the length provided in the internal call is
724 required to be aligned to HWASAN_TAG_GRANULE_SIZE. */
725 static void
726 expand_HWASAN_MARK (internal_fn, gcall *gc)
728 gcc_assert (ptr_mode == Pmode);
729 HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
730 bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
732 tree base = gimple_call_arg (gc, 1);
733 gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
734 rtx base_rtx = expand_normal (base);
736 rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
737 : targetm.memtag.extract_tag (base_rtx, NULL_RTX);
738 rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
740 tree len = gimple_call_arg (gc, 2);
741 rtx r_len = expand_normal (len);
743 rtx func = init_one_libfunc ("__hwasan_tag_memory");
744 emit_library_call (func, LCT_NORMAL, VOIDmode, address, Pmode,
745 tag, QImode, r_len, Pmode);
748 /* For hwasan stack tagging:
749 Store a tag into a pointer. */
750 static void
751 expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
753 gcc_assert (ptr_mode == Pmode);
754 tree g_target = gimple_call_lhs (gc);
755 tree g_ptr = gimple_call_arg (gc, 0);
756 tree g_tag = gimple_call_arg (gc, 1);
758 rtx ptr = expand_normal (g_ptr);
759 rtx tag = expand_expr (g_tag, NULL_RTX, QImode, EXPAND_NORMAL);
760 rtx target = expand_normal (g_target);
762 rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
763 rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
764 if (tagged_value != target)
765 emit_move_insn (target, tagged_value);
768 /* This should get expanded in the sanopt pass. */
770 static void
771 expand_ASAN_CHECK (internal_fn, gcall *)
773 gcc_unreachable ();
776 /* This should get expanded in the sanopt pass. */
778 static void
779 expand_ASAN_MARK (internal_fn, gcall *)
781 gcc_unreachable ();
784 /* This should get expanded in the sanopt pass. */
786 static void
787 expand_ASAN_POISON (internal_fn, gcall *)
789 gcc_unreachable ();
792 /* This should get expanded in the sanopt pass. */
794 static void
795 expand_ASAN_POISON_USE (internal_fn, gcall *)
797 gcc_unreachable ();
800 /* This should get expanded in the tsan pass. */
802 static void
803 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
805 gcc_unreachable ();
808 /* This should get expanded in the lower pass. */
810 static void
811 expand_FALLTHROUGH (internal_fn, gcall *call)
813 error_at (gimple_location (call),
814 "invalid use of attribute %<fallthrough%>");
817 /* Return minimum precision needed to represent all values
818 of ARG in SIGNed integral type. */
820 static int
821 get_min_precision (tree arg, signop sign)
823 int prec = TYPE_PRECISION (TREE_TYPE (arg));
824 int cnt = 0;
825 signop orig_sign = sign;
826 if (TREE_CODE (arg) == INTEGER_CST)
828 int p;
829 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
831 widest_int w = wi::to_widest (arg);
832 w = wi::ext (w, prec, sign);
833 p = wi::min_precision (w, sign);
835 else
836 p = wi::min_precision (wi::to_wide (arg), sign);
837 return MIN (p, prec);
839 while (CONVERT_EXPR_P (arg)
840 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
841 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
843 arg = TREE_OPERAND (arg, 0);
844 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
846 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
847 sign = UNSIGNED;
848 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
849 return prec + (orig_sign != sign);
850 prec = TYPE_PRECISION (TREE_TYPE (arg));
852 if (++cnt > 30)
853 return prec + (orig_sign != sign);
855 if (CONVERT_EXPR_P (arg)
856 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
857 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
859 /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
860 If y_2's min precision is smaller than prec, return that. */
861 int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
862 if (oprec < prec)
863 return oprec + (orig_sign != sign);
865 if (TREE_CODE (arg) != SSA_NAME)
866 return prec + (orig_sign != sign);
867 value_range r;
868 while (!get_global_range_query ()->range_of_expr (r, arg)
869 || r.varying_p ()
870 || r.undefined_p ())
872 gimple *g = SSA_NAME_DEF_STMT (arg);
873 if (is_gimple_assign (g)
874 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
876 tree t = gimple_assign_rhs1 (g);
877 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
878 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
880 arg = t;
881 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
883 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
884 sign = UNSIGNED;
885 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
886 return prec + (orig_sign != sign);
887 prec = TYPE_PRECISION (TREE_TYPE (arg));
889 if (++cnt > 30)
890 return prec + (orig_sign != sign);
891 continue;
894 return prec + (orig_sign != sign);
896 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
898 int p1 = wi::min_precision (r.lower_bound (), sign);
899 int p2 = wi::min_precision (r.upper_bound (), sign);
900 p1 = MAX (p1, p2);
901 prec = MIN (prec, p1);
903 else if (sign == UNSIGNED && !wi::neg_p (r.lower_bound (), SIGNED))
905 int p = wi::min_precision (r.upper_bound (), UNSIGNED);
906 prec = MIN (prec, p);
908 return prec + (orig_sign != sign);
911 /* Helper for expand_*_overflow. Set the __imag__ part to true
912 (1 except for signed:1 type, in which case store -1). */
914 static void
915 expand_arith_set_overflow (tree lhs, rtx target)
917 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
918 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
919 write_complex_part (target, constm1_rtx, true, false);
920 else
921 write_complex_part (target, const1_rtx, true, false);
924 /* Helper for expand_*_overflow. Store RES into the __real__ part
925 of TARGET. If RES has larger MODE than __real__ part of TARGET,
926 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
927 if LHS has smaller precision than its mode. */
929 static void
930 expand_arith_overflow_result_store (tree lhs, rtx target,
931 scalar_int_mode mode, rtx res)
933 scalar_int_mode tgtmode
934 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
935 rtx lres = res;
936 if (tgtmode != mode)
938 rtx_code_label *done_label = gen_label_rtx ();
939 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
940 lres = convert_modes (tgtmode, mode, res, uns);
941 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
942 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
943 EQ, true, mode, NULL_RTX, NULL, done_label,
944 profile_probability::very_likely ());
945 expand_arith_set_overflow (lhs, target);
946 emit_label (done_label);
948 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
949 int tgtprec = GET_MODE_PRECISION (tgtmode);
950 if (prec < tgtprec)
952 rtx_code_label *done_label = gen_label_rtx ();
953 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
954 res = lres;
955 if (uns)
957 rtx mask
958 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
959 tgtmode);
960 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
961 true, OPTAB_LIB_WIDEN);
963 else
965 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
966 NULL_RTX, 1);
967 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
968 NULL_RTX, 0);
970 do_compare_rtx_and_jump (res, lres,
971 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
972 profile_probability::very_likely ());
973 expand_arith_set_overflow (lhs, target);
974 emit_label (done_label);
976 write_complex_part (target, lres, false, false);
979 /* Helper for expand_*_overflow. Store RES into TARGET. */
981 static void
982 expand_ubsan_result_store (tree lhs, rtx target, scalar_int_mode mode,
983 rtx res, rtx_code_label *do_error)
985 if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE
986 && TYPE_PRECISION (TREE_TYPE (lhs)) < GET_MODE_PRECISION (mode))
988 int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
989 int prec = TYPE_PRECISION (TREE_TYPE (lhs));
990 int tgtprec = GET_MODE_PRECISION (mode);
991 rtx resc = gen_reg_rtx (mode), lres;
992 emit_move_insn (resc, res);
993 if (uns)
995 rtx mask
996 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
997 mode);
998 lres = expand_simple_binop (mode, AND, res, mask, NULL_RTX,
999 true, OPTAB_LIB_WIDEN);
1001 else
1003 lres = expand_shift (LSHIFT_EXPR, mode, res, tgtprec - prec,
1004 NULL_RTX, 1);
1005 lres = expand_shift (RSHIFT_EXPR, mode, lres, tgtprec - prec,
1006 NULL_RTX, 0);
1008 if (lres != res)
1009 emit_move_insn (res, lres);
1010 do_compare_rtx_and_jump (res, resc,
1011 NE, true, mode, NULL_RTX, NULL, do_error,
1012 profile_probability::very_unlikely ());
1014 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
1015 /* If this is a scalar in a register that is stored in a wider mode
1016 than the declared mode, compute the result into its declared mode
1017 and then convert to the wider mode. Our value is the computed
1018 expression. */
1019 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
1020 else
1021 emit_move_insn (target, res);
1024 /* Add sub/add overflow checking to the statement STMT.
1025 CODE says whether the operation is +, or -. */
1027 void
1028 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
1029 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
1030 bool uns1_p, bool is_ubsan, tree *datap)
1032 rtx res, target = NULL_RTX;
1033 tree fn;
1034 rtx_code_label *done_label = gen_label_rtx ();
1035 rtx_code_label *do_error = gen_label_rtx ();
1036 do_pending_stack_adjust ();
1037 rtx op0 = expand_normal (arg0);
1038 rtx op1 = expand_normal (arg1);
1039 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1040 int prec = GET_MODE_PRECISION (mode);
1041 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1042 bool do_xor = false;
1044 if (is_ubsan)
1045 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1047 if (lhs)
1049 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1050 if (!is_ubsan)
1051 write_complex_part (target, const0_rtx, true, false);
1054 /* We assume both operands and result have the same precision
1055 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1056 with that precision, U for unsigned type with that precision,
1057 sgn for unsigned most significant bit in that precision.
1058 s1 is signed first operand, u1 is unsigned first operand,
1059 s2 is signed second operand, u2 is unsigned second operand,
1060 sr is signed result, ur is unsigned result and the following
1061 rules say how to compute result (which is always result of
1062 the operands as if both were unsigned, cast to the right
1063 signedness) and how to compute whether operation overflowed.
1065 s1 + s2 -> sr
1066 res = (S) ((U) s1 + (U) s2)
1067 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
1068 s1 - s2 -> sr
1069 res = (S) ((U) s1 - (U) s2)
1070 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
1071 u1 + u2 -> ur
1072 res = u1 + u2
1073 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
1074 u1 - u2 -> ur
1075 res = u1 - u2
1076 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
1077 s1 + u2 -> sr
1078 res = (S) ((U) s1 + u2)
1079 ovf = ((U) res ^ sgn) < u2
1080 s1 + u2 -> ur
1081 t1 = (S) (u2 ^ sgn)
1082 t2 = s1 + t1
1083 res = (U) t2 ^ sgn
1084 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
1085 s1 - u2 -> sr
1086 res = (S) ((U) s1 - u2)
1087 ovf = u2 > ((U) s1 ^ sgn)
1088 s1 - u2 -> ur
1089 res = (U) s1 - u2
1090 ovf = s1 < 0 || u2 > (U) s1
1091 u1 - s2 -> sr
1092 res = u1 - (U) s2
1093 ovf = u1 >= ((U) s2 ^ sgn)
1094 u1 - s2 -> ur
1095 t1 = u1 ^ sgn
1096 t2 = t1 - (U) s2
1097 res = t2 ^ sgn
1098 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
1099 s1 + s2 -> ur
1100 res = (U) s1 + (U) s2
1101 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
1102 u1 + u2 -> sr
1103 res = (S) (u1 + u2)
1104 ovf = (U) res < u2 || res < 0
1105 u1 - u2 -> sr
1106 res = (S) (u1 - u2)
1107 ovf = u1 >= u2 ? res < 0 : res >= 0
1108 s1 - s2 -> ur
1109 res = (U) s1 - (U) s2
1110 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
1112 if (code == PLUS_EXPR && uns0_p && !uns1_p)
1114 /* PLUS_EXPR is commutative, if operand signedness differs,
1115 canonicalize to the first operand being signed and second
1116 unsigned to simplify following code. */
1117 std::swap (op0, op1);
1118 std::swap (arg0, arg1);
1119 uns0_p = false;
1120 uns1_p = true;
1123 /* u1 +- u2 -> ur */
1124 if (uns0_p && uns1_p && unsr_p)
1126 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
1127 : usubv4_optab, mode);
1128 if (icode != CODE_FOR_nothing)
1130 class expand_operand ops[4];
1131 rtx_insn *last = get_last_insn ();
1133 res = gen_reg_rtx (mode);
1134 create_output_operand (&ops[0], res, mode);
1135 create_input_operand (&ops[1], op0, mode);
1136 create_input_operand (&ops[2], op1, mode);
1137 create_fixed_operand (&ops[3], do_error);
1138 if (maybe_expand_insn (icode, 4, ops))
1140 last = get_last_insn ();
1141 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1142 && JUMP_P (last)
1143 && any_condjump_p (last)
1144 && !find_reg_note (last, REG_BR_PROB, 0))
1145 add_reg_br_prob_note (last,
1146 profile_probability::very_unlikely ());
1147 emit_jump (done_label);
1148 goto do_error_label;
1151 delete_insns_since (last);
1154 /* Compute the operation. On RTL level, the addition is always
1155 unsigned. */
1156 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1157 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1158 rtx tem = op0;
1159 /* For PLUS_EXPR, the operation is commutative, so we can pick
1160 operand to compare against. For prec <= BITS_PER_WORD, I think
1161 preferring REG operand is better over CONST_INT, because
1162 the CONST_INT might enlarge the instruction or CSE would need
1163 to figure out we'd already loaded it into a register before.
1164 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
1165 as then the multi-word comparison can be perhaps simplified. */
1166 if (code == PLUS_EXPR
1167 && (prec <= BITS_PER_WORD
1168 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
1169 : CONST_SCALAR_INT_P (op1)))
1170 tem = op1;
1171 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
1172 true, mode, NULL_RTX, NULL, done_label,
1173 profile_probability::very_likely ());
1174 goto do_error_label;
1177 /* s1 +- u2 -> sr */
1178 if (!uns0_p && uns1_p && !unsr_p)
1180 /* Compute the operation. On RTL level, the addition is always
1181 unsigned. */
1182 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1183 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1184 rtx tem = expand_binop (mode, add_optab,
1185 code == PLUS_EXPR ? res : op0, sgn,
1186 NULL_RTX, false, OPTAB_LIB_WIDEN);
1187 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
1188 done_label, profile_probability::very_likely ());
1189 goto do_error_label;
1192 /* s1 + u2 -> ur */
1193 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
1195 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1196 OPTAB_LIB_WIDEN);
1197 /* As we've changed op1, we have to avoid using the value range
1198 for the original argument. */
1199 arg1 = error_mark_node;
1200 do_xor = true;
1201 goto do_signed;
1204 /* u1 - s2 -> ur */
1205 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
1207 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
1208 OPTAB_LIB_WIDEN);
1209 /* As we've changed op0, we have to avoid using the value range
1210 for the original argument. */
1211 arg0 = error_mark_node;
1212 do_xor = true;
1213 goto do_signed;
1216 /* s1 - u2 -> ur */
1217 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
1219 /* Compute the operation. On RTL level, the addition is always
1220 unsigned. */
1221 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1222 OPTAB_LIB_WIDEN);
1223 int pos_neg = get_range_pos_neg (arg0);
1224 if (pos_neg == 2)
1225 /* If ARG0 is known to be always negative, this is always overflow. */
1226 emit_jump (do_error);
1227 else if (pos_neg == 3)
1228 /* If ARG0 is not known to be always positive, check at runtime. */
1229 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
1230 NULL, do_error, profile_probability::very_unlikely ());
1231 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
1232 done_label, profile_probability::very_likely ());
1233 goto do_error_label;
1236 /* u1 - s2 -> sr */
1237 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
1239 /* Compute the operation. On RTL level, the addition is always
1240 unsigned. */
1241 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1242 OPTAB_LIB_WIDEN);
1243 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
1244 OPTAB_LIB_WIDEN);
1245 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
1246 done_label, profile_probability::very_likely ());
1247 goto do_error_label;
1250 /* u1 + u2 -> sr */
1251 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
1253 /* Compute the operation. On RTL level, the addition is always
1254 unsigned. */
1255 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
1256 OPTAB_LIB_WIDEN);
1257 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1258 NULL, do_error, profile_probability::very_unlikely ());
1259 rtx tem = op1;
1260 /* The operation is commutative, so we can pick operand to compare
1261 against. For prec <= BITS_PER_WORD, I think preferring REG operand
1262 is better over CONST_INT, because the CONST_INT might enlarge the
1263 instruction or CSE would need to figure out we'd already loaded it
1264 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
1265 might be more beneficial, as then the multi-word comparison can be
1266 perhaps simplified. */
1267 if (prec <= BITS_PER_WORD
1268 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
1269 : CONST_SCALAR_INT_P (op0))
1270 tem = op0;
1271 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
1272 done_label, profile_probability::very_likely ());
1273 goto do_error_label;
1276 /* s1 +- s2 -> ur */
1277 if (!uns0_p && !uns1_p && unsr_p)
1279 /* Compute the operation. On RTL level, the addition is always
1280 unsigned. */
1281 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1282 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1283 int pos_neg = get_range_pos_neg (arg1);
1284 if (code == PLUS_EXPR)
1286 int pos_neg0 = get_range_pos_neg (arg0);
1287 if (pos_neg0 != 3 && pos_neg == 3)
1289 std::swap (op0, op1);
1290 pos_neg = pos_neg0;
1293 rtx tem;
1294 if (pos_neg != 3)
1296 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
1297 ? and_optab : ior_optab,
1298 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
1299 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
1300 NULL, done_label, profile_probability::very_likely ());
1302 else
1304 rtx_code_label *do_ior_label = gen_label_rtx ();
1305 do_compare_rtx_and_jump (op1, const0_rtx,
1306 code == MINUS_EXPR ? GE : LT, false, mode,
1307 NULL_RTX, NULL, do_ior_label,
1308 profile_probability::even ());
1309 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
1310 OPTAB_LIB_WIDEN);
1311 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1312 NULL, done_label, profile_probability::very_likely ());
1313 emit_jump (do_error);
1314 emit_label (do_ior_label);
1315 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
1316 OPTAB_LIB_WIDEN);
1317 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1318 NULL, done_label, profile_probability::very_likely ());
1320 goto do_error_label;
1323 /* u1 - u2 -> sr */
1324 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
1326 /* Compute the operation. On RTL level, the addition is always
1327 unsigned. */
1328 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
1329 OPTAB_LIB_WIDEN);
1330 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
1331 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
1332 op0_geu_op1, profile_probability::even ());
1333 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
1334 NULL, done_label, profile_probability::very_likely ());
1335 emit_jump (do_error);
1336 emit_label (op0_geu_op1);
1337 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1338 NULL, done_label, profile_probability::very_likely ());
1339 goto do_error_label;
1342 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
1344 /* s1 +- s2 -> sr */
1345 do_signed:
1347 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
1348 : subv4_optab, mode);
1349 if (icode != CODE_FOR_nothing)
1351 class expand_operand ops[4];
1352 rtx_insn *last = get_last_insn ();
1354 res = gen_reg_rtx (mode);
1355 create_output_operand (&ops[0], res, mode);
1356 create_input_operand (&ops[1], op0, mode);
1357 create_input_operand (&ops[2], op1, mode);
1358 create_fixed_operand (&ops[3], do_error);
1359 if (maybe_expand_insn (icode, 4, ops))
1361 last = get_last_insn ();
1362 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1363 && JUMP_P (last)
1364 && any_condjump_p (last)
1365 && !find_reg_note (last, REG_BR_PROB, 0))
1366 add_reg_br_prob_note (last,
1367 profile_probability::very_unlikely ());
1368 emit_jump (done_label);
1369 goto do_error_label;
1372 delete_insns_since (last);
1375 /* Compute the operation. On RTL level, the addition is always
1376 unsigned. */
1377 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1378 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1380 /* If we can prove that one of the arguments (for MINUS_EXPR only
1381 the second operand, as subtraction is not commutative) is always
1382 non-negative or always negative, we can do just one comparison
1383 and conditional jump. */
1384 int pos_neg = get_range_pos_neg (arg1);
1385 if (code == PLUS_EXPR)
1387 int pos_neg0 = get_range_pos_neg (arg0);
1388 if (pos_neg0 != 3 && pos_neg == 3)
1390 std::swap (op0, op1);
1391 pos_neg = pos_neg0;
1395 /* Addition overflows if and only if the two operands have the same sign,
1396 and the result has the opposite sign. Subtraction overflows if and
1397 only if the two operands have opposite sign, and the subtrahend has
1398 the same sign as the result. Here 0 is counted as positive. */
1399 if (pos_neg == 3)
1401 /* Compute op0 ^ op1 (operands have opposite sign). */
1402 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1403 OPTAB_LIB_WIDEN);
1405 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1406 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1407 OPTAB_LIB_WIDEN);
1409 rtx tem;
1410 if (code == PLUS_EXPR)
1412 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1413 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1414 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1415 OPTAB_LIB_WIDEN);
1417 else
1419 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1420 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1421 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1422 OPTAB_LIB_WIDEN);
1425 /* No overflow if the result has bit sign cleared. */
1426 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1427 NULL, done_label, profile_probability::very_likely ());
1430 /* Compare the result of the operation with the first operand.
1431 No overflow for addition if second operand is positive and result
1432 is larger or second operand is negative and result is smaller.
1433 Likewise for subtraction with sign of second operand flipped. */
1434 else
1435 do_compare_rtx_and_jump (res, op0,
1436 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1437 false, mode, NULL_RTX, NULL, done_label,
1438 profile_probability::very_likely ());
1441 do_error_label:
1442 emit_label (do_error);
1443 if (is_ubsan)
1445 /* Expand the ubsan builtin call. */
1446 push_temp_slots ();
1447 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1448 arg0, arg1, datap);
1449 expand_normal (fn);
1450 pop_temp_slots ();
1451 do_pending_stack_adjust ();
1453 else if (lhs)
1454 expand_arith_set_overflow (lhs, target);
1456 /* We're done. */
1457 emit_label (done_label);
1459 if (lhs)
1461 if (is_ubsan)
1462 expand_ubsan_result_store (lhs, target, mode, res, do_error);
1463 else
1465 if (do_xor)
1466 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1467 OPTAB_LIB_WIDEN);
1469 expand_arith_overflow_result_store (lhs, target, mode, res);
1474 /* Add negate overflow checking to the statement STMT. */
1476 static void
1477 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1478 tree *datap)
1480 rtx res, op1;
1481 tree fn;
1482 rtx_code_label *done_label, *do_error;
1483 rtx target = NULL_RTX;
1485 done_label = gen_label_rtx ();
1486 do_error = gen_label_rtx ();
1488 do_pending_stack_adjust ();
1489 op1 = expand_normal (arg1);
1491 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1492 if (lhs)
1494 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1495 if (!is_ubsan)
1496 write_complex_part (target, const0_rtx, true, false);
1499 enum insn_code icode = optab_handler (negv3_optab, mode);
1500 if (icode != CODE_FOR_nothing)
1502 class expand_operand ops[3];
1503 rtx_insn *last = get_last_insn ();
1505 res = gen_reg_rtx (mode);
1506 create_output_operand (&ops[0], res, mode);
1507 create_input_operand (&ops[1], op1, mode);
1508 create_fixed_operand (&ops[2], do_error);
1509 if (maybe_expand_insn (icode, 3, ops))
1511 last = get_last_insn ();
1512 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1513 && JUMP_P (last)
1514 && any_condjump_p (last)
1515 && !find_reg_note (last, REG_BR_PROB, 0))
1516 add_reg_br_prob_note (last,
1517 profile_probability::very_unlikely ());
1518 emit_jump (done_label);
1520 else
1522 delete_insns_since (last);
1523 icode = CODE_FOR_nothing;
1527 if (icode == CODE_FOR_nothing)
1529 /* Compute the operation. On RTL level, the addition is always
1530 unsigned. */
1531 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1533 /* Compare the operand with the most negative value. */
1534 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1535 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1536 done_label, profile_probability::very_likely ());
1539 emit_label (do_error);
1540 if (is_ubsan)
1542 /* Expand the ubsan builtin call. */
1543 push_temp_slots ();
1544 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1545 arg1, NULL_TREE, datap);
1546 expand_normal (fn);
1547 pop_temp_slots ();
1548 do_pending_stack_adjust ();
1550 else if (lhs)
1551 expand_arith_set_overflow (lhs, target);
1553 /* We're done. */
1554 emit_label (done_label);
1556 if (lhs)
1558 if (is_ubsan)
1559 expand_ubsan_result_store (lhs, target, mode, res, do_error);
1560 else
1561 expand_arith_overflow_result_store (lhs, target, mode, res);
1565 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1566 mode MODE can be expanded without using a libcall. */
1568 static bool
1569 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1570 rtx op0, rtx op1, bool uns)
1572 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1573 != CODE_FOR_nothing)
1574 return true;
1576 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1577 != CODE_FOR_nothing)
1578 return true;
1580 rtx_insn *last = get_last_insn ();
1581 if (CONSTANT_P (op0))
1582 op0 = convert_modes (wmode, mode, op0, uns);
1583 else
1584 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1585 if (CONSTANT_P (op1))
1586 op1 = convert_modes (wmode, mode, op1, uns);
1587 else
1588 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1589 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1590 delete_insns_since (last);
1591 return ret != NULL_RTX;
1594 /* Add mul overflow checking to the statement STMT. */
1596 static void
1597 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1598 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1599 tree *datap)
1601 rtx res, op0, op1;
1602 tree fn, type;
1603 rtx_code_label *done_label, *do_error;
1604 rtx target = NULL_RTX;
1605 signop sign;
1606 enum insn_code icode;
1608 done_label = gen_label_rtx ();
1609 do_error = gen_label_rtx ();
1611 do_pending_stack_adjust ();
1612 op0 = expand_normal (arg0);
1613 op1 = expand_normal (arg1);
1615 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1616 bool uns = unsr_p;
1617 if (lhs)
1619 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1620 if (!is_ubsan)
1621 write_complex_part (target, const0_rtx, true, false);
1624 if (is_ubsan)
1625 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1627 /* We assume both operands and result have the same precision
1628 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1629 with that precision, U for unsigned type with that precision,
1630 sgn for unsigned most significant bit in that precision.
1631 s1 is signed first operand, u1 is unsigned first operand,
1632 s2 is signed second operand, u2 is unsigned second operand,
1633 sr is signed result, ur is unsigned result and the following
1634 rules say how to compute result (which is always result of
1635 the operands as if both were unsigned, cast to the right
1636 signedness) and how to compute whether operation overflowed.
1637 main_ovf (false) stands for jump on signed multiplication
1638 overflow or the main algorithm with uns == false.
1639 main_ovf (true) stands for jump on unsigned multiplication
1640 overflow or the main algorithm with uns == true.
1642 s1 * s2 -> sr
1643 res = (S) ((U) s1 * (U) s2)
1644 ovf = main_ovf (false)
1645 u1 * u2 -> ur
1646 res = u1 * u2
1647 ovf = main_ovf (true)
1648 s1 * u2 -> ur
1649 res = (U) s1 * u2
1650 ovf = (s1 < 0 && u2) || main_ovf (true)
1651 u1 * u2 -> sr
1652 res = (S) (u1 * u2)
1653 ovf = res < 0 || main_ovf (true)
1654 s1 * u2 -> sr
1655 res = (S) ((U) s1 * u2)
1656 ovf = (S) u2 >= 0 ? main_ovf (false)
1657 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1658 s1 * s2 -> ur
1659 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1660 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1661 res = t1 * t2
1662 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1664 if (uns0_p && !uns1_p)
1666 /* Multiplication is commutative, if operand signedness differs,
1667 canonicalize to the first operand being signed and second
1668 unsigned to simplify following code. */
1669 std::swap (op0, op1);
1670 std::swap (arg0, arg1);
1671 uns0_p = false;
1672 uns1_p = true;
1675 int pos_neg0 = get_range_pos_neg (arg0);
1676 int pos_neg1 = get_range_pos_neg (arg1);
1677 /* Unsigned types with smaller than mode precision, even if they have most
1678 significant bit set, are still zero-extended. */
1679 if (uns0_p && TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
1680 pos_neg0 = 1;
1681 if (uns1_p && TYPE_PRECISION (TREE_TYPE (arg1)) < GET_MODE_PRECISION (mode))
1682 pos_neg1 = 1;
1684 /* s1 * u2 -> ur */
1685 if (!uns0_p && uns1_p && unsr_p)
1687 switch (pos_neg0)
1689 case 1:
1690 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1691 goto do_main;
1692 case 2:
1693 /* If s1 is negative, avoid the main code, just multiply and
1694 signal overflow if op1 is not 0. */
1695 struct separate_ops ops;
1696 ops.code = MULT_EXPR;
1697 ops.type = TREE_TYPE (arg1);
1698 ops.op0 = make_tree (ops.type, op0);
1699 ops.op1 = make_tree (ops.type, op1);
1700 ops.op2 = NULL_TREE;
1701 ops.location = loc;
1702 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1703 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1704 NULL, done_label, profile_probability::very_likely ());
1705 goto do_error_label;
1706 case 3:
1707 if (get_min_precision (arg1, UNSIGNED)
1708 + get_min_precision (arg0, SIGNED) <= GET_MODE_PRECISION (mode))
1710 /* If the first operand is sign extended from narrower type, the
1711 second operand is zero extended from narrower type and
1712 the sum of the two precisions is smaller or equal to the
1713 result precision: if the first argument is at runtime
1714 non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
1715 and there will be no overflow, if the first argument is
1716 negative and the second argument zero, the result will be
1717 0 and there will be no overflow, if the first argument is
1718 negative and the second argument positive, the result when
1719 treated as signed will be negative (minimum -0x7f80 or
1720 -0x7f..f80..0) there will be always overflow. So, do
1721 res = (U) (s1 * u2)
1722 ovf = (S) res < 0 */
1723 struct separate_ops ops;
1724 ops.code = MULT_EXPR;
1725 ops.type
1726 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1728 ops.op0 = make_tree (ops.type, op0);
1729 ops.op1 = make_tree (ops.type, op1);
1730 ops.op2 = NULL_TREE;
1731 ops.location = loc;
1732 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1733 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1734 mode, NULL_RTX, NULL, done_label,
1735 profile_probability::very_likely ());
1736 goto do_error_label;
1738 rtx_code_label *do_main_label;
1739 do_main_label = gen_label_rtx ();
1740 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1741 NULL, do_main_label, profile_probability::very_likely ());
1742 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1743 NULL, do_main_label, profile_probability::very_likely ());
1744 expand_arith_set_overflow (lhs, target);
1745 emit_label (do_main_label);
1746 goto do_main;
1747 default:
1748 gcc_unreachable ();
1752 /* u1 * u2 -> sr */
1753 if (uns0_p && uns1_p && !unsr_p)
1755 if ((pos_neg0 | pos_neg1) == 1)
1757 /* If both arguments are zero extended from narrower types,
1758 the MSB will be clear on both and so we can pretend it is
1759 a normal s1 * s2 -> sr multiplication. */
1760 uns0_p = false;
1761 uns1_p = false;
1763 else
1764 uns = true;
1765 /* Rest of handling of this case after res is computed. */
1766 goto do_main;
1769 /* s1 * u2 -> sr */
1770 if (!uns0_p && uns1_p && !unsr_p)
1772 switch (pos_neg1)
1774 case 1:
1775 goto do_main;
1776 case 2:
1777 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1778 avoid the main code, just multiply and signal overflow
1779 unless 0 * u2 or -1 * ((U) Smin). */
1780 struct separate_ops ops;
1781 ops.code = MULT_EXPR;
1782 ops.type = TREE_TYPE (arg1);
1783 ops.op0 = make_tree (ops.type, op0);
1784 ops.op1 = make_tree (ops.type, op1);
1785 ops.op2 = NULL_TREE;
1786 ops.location = loc;
1787 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1788 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1789 NULL, done_label, profile_probability::very_likely ());
1790 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1791 NULL, do_error, profile_probability::very_unlikely ());
1792 int prec;
1793 prec = GET_MODE_PRECISION (mode);
1794 rtx sgn;
1795 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1796 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1797 NULL, done_label, profile_probability::very_likely ());
1798 goto do_error_label;
1799 case 3:
1800 /* Rest of handling of this case after res is computed. */
1801 goto do_main;
1802 default:
1803 gcc_unreachable ();
1807 /* s1 * s2 -> ur */
1808 if (!uns0_p && !uns1_p && unsr_p)
1810 rtx tem;
1811 switch (pos_neg0 | pos_neg1)
1813 case 1: /* Both operands known to be non-negative. */
1814 goto do_main;
1815 case 2: /* Both operands known to be negative. */
1816 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1817 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1818 /* Avoid looking at arg0/arg1 ranges, as we've changed
1819 the arguments. */
1820 arg0 = error_mark_node;
1821 arg1 = error_mark_node;
1822 goto do_main;
1823 case 3:
1824 if ((pos_neg0 ^ pos_neg1) == 3)
1826 /* If one operand is known to be negative and the other
1827 non-negative, this overflows always, unless the non-negative
1828 one is 0. Just do normal multiply and set overflow
1829 unless one of the operands is 0. */
1830 struct separate_ops ops;
1831 ops.code = MULT_EXPR;
1832 ops.type
1833 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1835 ops.op0 = make_tree (ops.type, op0);
1836 ops.op1 = make_tree (ops.type, op1);
1837 ops.op2 = NULL_TREE;
1838 ops.location = loc;
1839 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1840 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1841 true, mode, NULL_RTX, NULL, done_label,
1842 profile_probability::very_likely ());
1843 goto do_error_label;
1845 if (get_min_precision (arg0, SIGNED)
1846 + get_min_precision (arg1, SIGNED) <= GET_MODE_PRECISION (mode))
1848 /* If both operands are sign extended from narrower types and
1849 the sum of the two precisions is smaller or equal to the
1850 result precision: if both arguments are at runtime
1851 non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
1852 and there will be no overflow, if both arguments are negative,
1853 maximum result will be 0x40..00 and there will be no overflow
1854 either, if one argument is positive and the other argument
1855 negative, the result when treated as signed will be negative
1856 and there will be always overflow, and if one argument is
1857 zero and the other negative the result will be zero and no
1858 overflow. So, do
1859 res = (U) (s1 * s2)
1860 ovf = (S) res < 0 */
1861 struct separate_ops ops;
1862 ops.code = MULT_EXPR;
1863 ops.type
1864 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1866 ops.op0 = make_tree (ops.type, op0);
1867 ops.op1 = make_tree (ops.type, op1);
1868 ops.op2 = NULL_TREE;
1869 ops.location = loc;
1870 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1871 do_compare_rtx_and_jump (res, const0_rtx, GE, false,
1872 mode, NULL_RTX, NULL, done_label,
1873 profile_probability::very_likely ());
1874 goto do_error_label;
1876 /* The general case, do all the needed comparisons at runtime. */
1877 rtx_code_label *do_main_label, *after_negate_label;
1878 rtx rop0, rop1;
1879 rop0 = gen_reg_rtx (mode);
1880 rop1 = gen_reg_rtx (mode);
1881 emit_move_insn (rop0, op0);
1882 emit_move_insn (rop1, op1);
1883 op0 = rop0;
1884 op1 = rop1;
1885 do_main_label = gen_label_rtx ();
1886 after_negate_label = gen_label_rtx ();
1887 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1888 OPTAB_LIB_WIDEN);
1889 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1890 NULL, after_negate_label, profile_probability::very_likely ());
1891 /* Both arguments negative here, negate them and continue with
1892 normal unsigned overflow checking multiplication. */
1893 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1894 NULL_RTX, false));
1895 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1896 NULL_RTX, false));
1897 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1898 the arguments. */
1899 arg0 = error_mark_node;
1900 arg1 = error_mark_node;
1901 emit_jump (do_main_label);
1902 emit_label (after_negate_label);
1903 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1904 OPTAB_LIB_WIDEN);
1905 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1906 NULL, do_main_label,
1907 profile_probability::very_likely ());
1908 /* One argument is negative here, the other positive. This
1909 overflows always, unless one of the arguments is 0. But
1910 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1911 is, thus we can keep do_main code oring in overflow as is. */
1912 if (pos_neg0 != 2)
1913 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1914 NULL, do_main_label,
1915 profile_probability::very_unlikely ());
1916 if (pos_neg1 != 2)
1917 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1918 NULL, do_main_label,
1919 profile_probability::very_unlikely ());
1920 expand_arith_set_overflow (lhs, target);
1921 emit_label (do_main_label);
1922 goto do_main;
1923 default:
1924 gcc_unreachable ();
1928 do_main:
1929 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1930 sign = uns ? UNSIGNED : SIGNED;
1931 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1932 if (uns
1933 && (integer_pow2p (arg0) || integer_pow2p (arg1))
1934 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1936 /* Optimize unsigned multiplication by power of 2 constant
1937 using 2 shifts, one for result, one to extract the shifted
1938 out bits to see if they are all zero.
1939 Don't do this if optimizing for size and we have umulv4_optab,
1940 in that case assume multiplication will be shorter.
1941 This is heuristics based on the single target that provides
1942 umulv4 right now (i?86/x86_64), if further targets add it, this
1943 might need to be revisited.
1944 Cases where both operands are constant should be folded already
1945 during GIMPLE, and cases where one operand is constant but not
1946 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1947 below can be done without multiplication, just by shifts and adds,
1948 or we'd need to divide the result (and hope it actually doesn't
1949 really divide nor multiply) and compare the result of the division
1950 with the original operand. */
1951 rtx opn0 = op0;
1952 rtx opn1 = op1;
1953 tree argn0 = arg0;
1954 tree argn1 = arg1;
1955 if (integer_pow2p (arg0))
1957 std::swap (opn0, opn1);
1958 std::swap (argn0, argn1);
1960 int cnt = tree_log2 (argn1);
1961 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1963 rtx upper = const0_rtx;
1964 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1965 if (cnt != 0)
1966 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1967 GET_MODE_PRECISION (mode) - cnt,
1968 NULL_RTX, uns);
1969 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1970 NULL_RTX, NULL, done_label,
1971 profile_probability::very_likely ());
1972 goto do_error_label;
1975 if (icode != CODE_FOR_nothing)
1977 class expand_operand ops[4];
1978 rtx_insn *last = get_last_insn ();
1980 res = gen_reg_rtx (mode);
1981 create_output_operand (&ops[0], res, mode);
1982 create_input_operand (&ops[1], op0, mode);
1983 create_input_operand (&ops[2], op1, mode);
1984 create_fixed_operand (&ops[3], do_error);
1985 if (maybe_expand_insn (icode, 4, ops))
1987 last = get_last_insn ();
1988 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1989 && JUMP_P (last)
1990 && any_condjump_p (last)
1991 && !find_reg_note (last, REG_BR_PROB, 0))
1992 add_reg_br_prob_note (last,
1993 profile_probability::very_unlikely ());
1994 emit_jump (done_label);
1996 else
1998 delete_insns_since (last);
1999 icode = CODE_FOR_nothing;
2003 if (icode == CODE_FOR_nothing)
2005 struct separate_ops ops;
2006 int prec = GET_MODE_PRECISION (mode);
2007 scalar_int_mode hmode, wmode;
2008 ops.op0 = make_tree (type, op0);
2009 ops.op1 = make_tree (type, op1);
2010 ops.op2 = NULL_TREE;
2011 ops.location = loc;
2013 /* Optimize unsigned overflow check where we don't use the
2014 multiplication result, just whether overflow happened.
2015 If we can do MULT_HIGHPART_EXPR, that followed by
2016 comparison of the result against zero is cheapest.
2017 We'll still compute res, but it should be DCEd later. */
2018 use_operand_p use;
2019 gimple *use_stmt;
2020 if (!is_ubsan
2021 && lhs
2022 && uns
2023 && !(uns0_p && uns1_p && !unsr_p)
2024 && can_mult_highpart_p (mode, uns) == 1
2025 && single_imm_use (lhs, &use, &use_stmt)
2026 && is_gimple_assign (use_stmt)
2027 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
2028 goto highpart;
2030 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
2031 && targetm.scalar_mode_supported_p (wmode)
2032 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
2034 twoxwider:
2035 ops.code = WIDEN_MULT_EXPR;
2036 ops.type
2037 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
2039 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
2040 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
2041 NULL_RTX, uns);
2042 hipart = convert_modes (mode, wmode, hipart, uns);
2043 res = convert_modes (mode, wmode, res, uns);
2044 if (uns)
2045 /* For the unsigned multiplication, there was overflow if
2046 HIPART is non-zero. */
2047 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2048 NULL_RTX, NULL, done_label,
2049 profile_probability::very_likely ());
2050 else
2052 /* RES is used more than once, place it in a pseudo. */
2053 res = force_reg (mode, res);
2055 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2056 NULL_RTX, 0);
2057 /* RES is low half of the double width result, HIPART
2058 the high half. There was overflow if
2059 HIPART is different from RES < 0 ? -1 : 0. */
2060 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2061 NULL_RTX, NULL, done_label,
2062 profile_probability::very_likely ());
2065 else if (can_mult_highpart_p (mode, uns) == 1)
2067 highpart:
2068 ops.code = MULT_HIGHPART_EXPR;
2069 ops.type = type;
2071 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
2072 EXPAND_NORMAL);
2073 ops.code = MULT_EXPR;
2074 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2075 if (uns)
2076 /* For the unsigned multiplication, there was overflow if
2077 HIPART is non-zero. */
2078 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
2079 NULL_RTX, NULL, done_label,
2080 profile_probability::very_likely ());
2081 else
2083 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
2084 NULL_RTX, 0);
2085 /* RES is low half of the double width result, HIPART
2086 the high half. There was overflow if
2087 HIPART is different from RES < 0 ? -1 : 0. */
2088 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
2089 NULL_RTX, NULL, done_label,
2090 profile_probability::very_likely ());
2094 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
2095 && 2 * GET_MODE_PRECISION (hmode) == prec)
2097 rtx_code_label *large_op0 = gen_label_rtx ();
2098 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
2099 rtx_code_label *one_small_one_large = gen_label_rtx ();
2100 rtx_code_label *both_ops_large = gen_label_rtx ();
2101 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
2102 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
2103 rtx_code_label *do_overflow = gen_label_rtx ();
2104 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
2106 unsigned int hprec = GET_MODE_PRECISION (hmode);
2107 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
2108 NULL_RTX, uns);
2109 hipart0 = convert_modes (hmode, mode, hipart0, uns);
2110 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
2111 rtx signbit0 = const0_rtx;
2112 if (!uns)
2113 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
2114 NULL_RTX, 0);
2115 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
2116 NULL_RTX, uns);
2117 hipart1 = convert_modes (hmode, mode, hipart1, uns);
2118 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
2119 rtx signbit1 = const0_rtx;
2120 if (!uns)
2121 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
2122 NULL_RTX, 0);
2124 res = gen_reg_rtx (mode);
2126 /* True if op0 resp. op1 are known to be in the range of
2127 halfstype. */
2128 bool op0_small_p = false;
2129 bool op1_small_p = false;
2130 /* True if op0 resp. op1 are known to have all zeros or all ones
2131 in the upper half of bits, but are not known to be
2132 op{0,1}_small_p. */
2133 bool op0_medium_p = false;
2134 bool op1_medium_p = false;
2135 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
2136 nonnegative, 1 if unknown. */
2137 int op0_sign = 1;
2138 int op1_sign = 1;
2140 if (pos_neg0 == 1)
2141 op0_sign = 0;
2142 else if (pos_neg0 == 2)
2143 op0_sign = -1;
2144 if (pos_neg1 == 1)
2145 op1_sign = 0;
2146 else if (pos_neg1 == 2)
2147 op1_sign = -1;
2149 unsigned int mprec0 = prec;
2150 if (arg0 != error_mark_node)
2151 mprec0 = get_min_precision (arg0, sign);
2152 if (mprec0 <= hprec)
2153 op0_small_p = true;
2154 else if (!uns && mprec0 <= hprec + 1)
2155 op0_medium_p = true;
2156 unsigned int mprec1 = prec;
2157 if (arg1 != error_mark_node)
2158 mprec1 = get_min_precision (arg1, sign);
2159 if (mprec1 <= hprec)
2160 op1_small_p = true;
2161 else if (!uns && mprec1 <= hprec + 1)
2162 op1_medium_p = true;
2164 int smaller_sign = 1;
2165 int larger_sign = 1;
2166 if (op0_small_p)
2168 smaller_sign = op0_sign;
2169 larger_sign = op1_sign;
2171 else if (op1_small_p)
2173 smaller_sign = op1_sign;
2174 larger_sign = op0_sign;
2176 else if (op0_sign == op1_sign)
2178 smaller_sign = op0_sign;
2179 larger_sign = op0_sign;
2182 if (!op0_small_p)
2183 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
2184 NULL_RTX, NULL, large_op0,
2185 profile_probability::unlikely ());
2187 if (!op1_small_p)
2188 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2189 NULL_RTX, NULL, small_op0_large_op1,
2190 profile_probability::unlikely ());
2192 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
2193 hmode to mode, the multiplication will never overflow. We can
2194 do just one hmode x hmode => mode widening multiplication. */
2195 tree halfstype = build_nonstandard_integer_type (hprec, uns);
2196 ops.op0 = make_tree (halfstype, lopart0);
2197 ops.op1 = make_tree (halfstype, lopart1);
2198 ops.code = WIDEN_MULT_EXPR;
2199 ops.type = type;
2200 rtx thisres
2201 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2202 emit_move_insn (res, thisres);
2203 emit_jump (done_label);
2205 emit_label (small_op0_large_op1);
2207 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
2208 but op1 is not, just swap the arguments and handle it as op1
2209 sign/zero extended, op0 not. */
2210 rtx larger = gen_reg_rtx (mode);
2211 rtx hipart = gen_reg_rtx (hmode);
2212 rtx lopart = gen_reg_rtx (hmode);
2213 emit_move_insn (larger, op1);
2214 emit_move_insn (hipart, hipart1);
2215 emit_move_insn (lopart, lopart0);
2216 emit_jump (one_small_one_large);
2218 emit_label (large_op0);
2220 if (!op1_small_p)
2221 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
2222 NULL_RTX, NULL, both_ops_large,
2223 profile_probability::unlikely ());
2225 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
2226 but op0 is not, prepare larger, hipart and lopart pseudos and
2227 handle it together with small_op0_large_op1. */
2228 emit_move_insn (larger, op0);
2229 emit_move_insn (hipart, hipart0);
2230 emit_move_insn (lopart, lopart1);
2232 emit_label (one_small_one_large);
2234 /* lopart is the low part of the operand that is sign extended
2235 to mode, larger is the other operand, hipart is the
2236 high part of larger and lopart0 and lopart1 are the low parts
2237 of both operands.
2238 We perform lopart0 * lopart1 and lopart * hipart widening
2239 multiplications. */
2240 tree halfutype = build_nonstandard_integer_type (hprec, 1);
2241 ops.op0 = make_tree (halfutype, lopart0);
2242 ops.op1 = make_tree (halfutype, lopart1);
2243 rtx lo0xlo1
2244 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2246 ops.op0 = make_tree (halfutype, lopart);
2247 ops.op1 = make_tree (halfutype, hipart);
2248 rtx loxhi = gen_reg_rtx (mode);
2249 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2250 emit_move_insn (loxhi, tem);
2252 if (!uns)
2254 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
2255 if (larger_sign == 0)
2256 emit_jump (after_hipart_neg);
2257 else if (larger_sign != -1)
2258 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
2259 NULL_RTX, NULL, after_hipart_neg,
2260 profile_probability::even ());
2262 tem = convert_modes (mode, hmode, lopart, 1);
2263 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
2264 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
2265 1, OPTAB_WIDEN);
2266 emit_move_insn (loxhi, tem);
2268 emit_label (after_hipart_neg);
2270 /* if (lopart < 0) loxhi -= larger; */
2271 if (smaller_sign == 0)
2272 emit_jump (after_lopart_neg);
2273 else if (smaller_sign != -1)
2274 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
2275 NULL_RTX, NULL, after_lopart_neg,
2276 profile_probability::even ());
2278 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
2279 1, OPTAB_WIDEN);
2280 emit_move_insn (loxhi, tem);
2282 emit_label (after_lopart_neg);
2285 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
2286 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
2287 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
2288 1, OPTAB_WIDEN);
2289 emit_move_insn (loxhi, tem);
2291 /* if (loxhi >> (bitsize / 2)
2292 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
2293 if (loxhi >> (bitsize / 2) == 0 (if uns). */
2294 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
2295 NULL_RTX, 0);
2296 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
2297 rtx signbitloxhi = const0_rtx;
2298 if (!uns)
2299 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
2300 convert_modes (hmode, mode,
2301 loxhi, 0),
2302 hprec - 1, NULL_RTX, 0);
2304 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
2305 NULL_RTX, NULL, do_overflow,
2306 profile_probability::very_unlikely ());
2308 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
2309 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
2310 NULL_RTX, 1);
2311 tem = convert_modes (mode, hmode,
2312 convert_modes (hmode, mode, lo0xlo1, 1), 1);
2314 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
2315 1, OPTAB_WIDEN);
2316 if (tem != res)
2317 emit_move_insn (res, tem);
2318 emit_jump (done_label);
2320 emit_label (both_ops_large);
2322 /* If both operands are large (not sign (!uns) or zero (uns)
2323 extended from hmode), then perform the full multiplication
2324 which will be the result of the operation.
2325 The only cases which don't overflow are for signed multiplication
2326 some cases where both hipart0 and highpart1 are 0 or -1.
2327 For unsigned multiplication when high parts are both non-zero
2328 this overflows always. */
2329 ops.code = MULT_EXPR;
2330 ops.op0 = make_tree (type, op0);
2331 ops.op1 = make_tree (type, op1);
2332 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2333 emit_move_insn (res, tem);
2335 if (!uns)
2337 if (!op0_medium_p)
2339 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
2340 NULL_RTX, 1, OPTAB_WIDEN);
2341 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2342 NULL_RTX, NULL, do_error,
2343 profile_probability::very_unlikely ());
2346 if (!op1_medium_p)
2348 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
2349 NULL_RTX, 1, OPTAB_WIDEN);
2350 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
2351 NULL_RTX, NULL, do_error,
2352 profile_probability::very_unlikely ());
2355 /* At this point hipart{0,1} are both in [-1, 0]. If they are
2356 the same, overflow happened if res is non-positive, if they
2357 are different, overflow happened if res is positive. */
2358 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
2359 emit_jump (hipart_different);
2360 else if (op0_sign == 1 || op1_sign == 1)
2361 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
2362 NULL_RTX, NULL, hipart_different,
2363 profile_probability::even ());
2365 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
2366 NULL_RTX, NULL, do_error,
2367 profile_probability::very_unlikely ());
2368 emit_jump (done_label);
2370 emit_label (hipart_different);
2372 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
2373 NULL_RTX, NULL, do_error,
2374 profile_probability::very_unlikely ());
2375 emit_jump (done_label);
2378 emit_label (do_overflow);
2380 /* Overflow, do full multiplication and fallthru into do_error. */
2381 ops.op0 = make_tree (type, op0);
2382 ops.op1 = make_tree (type, op1);
2383 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2384 emit_move_insn (res, tem);
2386 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
2387 && targetm.scalar_mode_supported_p (wmode))
2388 /* Even emitting a libcall is better than not detecting overflow
2389 at all. */
2390 goto twoxwider;
2391 else
2393 gcc_assert (!is_ubsan);
2394 ops.code = MULT_EXPR;
2395 ops.type = type;
2396 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2397 emit_jump (done_label);
2401 do_error_label:
2402 emit_label (do_error);
2403 if (is_ubsan)
2405 /* Expand the ubsan builtin call. */
2406 push_temp_slots ();
2407 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
2408 arg0, arg1, datap);
2409 expand_normal (fn);
2410 pop_temp_slots ();
2411 do_pending_stack_adjust ();
2413 else if (lhs)
2414 expand_arith_set_overflow (lhs, target);
2416 /* We're done. */
2417 emit_label (done_label);
2419 /* u1 * u2 -> sr */
2420 if (uns0_p && uns1_p && !unsr_p)
2422 rtx_code_label *all_done_label = gen_label_rtx ();
2423 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
2424 NULL, all_done_label, profile_probability::very_likely ());
2425 expand_arith_set_overflow (lhs, target);
2426 emit_label (all_done_label);
2429 /* s1 * u2 -> sr */
2430 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
2432 rtx_code_label *all_done_label = gen_label_rtx ();
2433 rtx_code_label *set_noovf = gen_label_rtx ();
2434 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2435 NULL, all_done_label, profile_probability::very_likely ());
2436 expand_arith_set_overflow (lhs, target);
2437 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2438 NULL, set_noovf, profile_probability::very_likely ());
2439 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2440 NULL, all_done_label, profile_probability::very_unlikely ());
2441 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2442 all_done_label, profile_probability::very_unlikely ());
2443 emit_label (set_noovf);
2444 write_complex_part (target, const0_rtx, true, false);
2445 emit_label (all_done_label);
2448 if (lhs)
2450 if (is_ubsan)
2451 expand_ubsan_result_store (lhs, target, mode, res, do_error);
2452 else
2453 expand_arith_overflow_result_store (lhs, target, mode, res);
2457 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2459 static void
2460 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2461 tree arg0, tree arg1)
2463 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2464 rtx_code_label *loop_lab = NULL;
2465 rtx cntvar = NULL_RTX;
2466 tree cntv = NULL_TREE;
2467 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2468 tree sz = TYPE_SIZE (eltype);
2469 tree data = NULL_TREE;
2470 tree resv = NULL_TREE;
2471 rtx lhsr = NULL_RTX;
2472 rtx resvr = NULL_RTX;
2473 unsigned HOST_WIDE_INT const_cnt = 0;
2474 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
2476 if (lhs)
2478 optab op;
2479 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2480 if (!VECTOR_MODE_P (GET_MODE (lhsr))
2481 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2482 optab_default)) == unknown_optab
2483 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2484 == CODE_FOR_nothing))
2486 if (MEM_P (lhsr))
2487 resv = make_tree (TREE_TYPE (lhs), lhsr);
2488 else
2490 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2491 resv = make_tree (TREE_TYPE (lhs), resvr);
2495 if (use_loop_p)
2497 do_pending_stack_adjust ();
2498 loop_lab = gen_label_rtx ();
2499 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2500 cntv = make_tree (sizetype, cntvar);
2501 emit_move_insn (cntvar, const0_rtx);
2502 emit_label (loop_lab);
2504 if (TREE_CODE (arg0) != VECTOR_CST)
2506 rtx arg0r = expand_normal (arg0);
2507 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2509 if (TREE_CODE (arg1) != VECTOR_CST)
2511 rtx arg1r = expand_normal (arg1);
2512 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2514 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2516 tree op0, op1, res = NULL_TREE;
2517 if (use_loop_p)
2519 tree atype = build_array_type_nelts (eltype, cnt);
2520 op0 = uniform_vector_p (arg0);
2521 if (op0 == NULL_TREE)
2523 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2524 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
2525 NULL_TREE, NULL_TREE);
2527 op1 = uniform_vector_p (arg1);
2528 if (op1 == NULL_TREE)
2530 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2531 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
2532 NULL_TREE, NULL_TREE);
2534 if (resv)
2536 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2537 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
2538 NULL_TREE, NULL_TREE);
2541 else
2543 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2544 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2545 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2546 if (resv)
2547 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2548 bitpos);
2550 switch (code)
2552 case PLUS_EXPR:
2553 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
2554 false, false, false, true, &data);
2555 break;
2556 case MINUS_EXPR:
2557 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2558 expand_neg_overflow (loc, res, op1, true, &data);
2559 else
2560 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
2561 false, false, false, true, &data);
2562 break;
2563 case MULT_EXPR:
2564 expand_mul_overflow (loc, res, op0, op1, false, false, false,
2565 true, &data);
2566 break;
2567 default:
2568 gcc_unreachable ();
2571 if (use_loop_p)
2573 struct separate_ops ops;
2574 ops.code = PLUS_EXPR;
2575 ops.type = TREE_TYPE (cntv);
2576 ops.op0 = cntv;
2577 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2578 ops.op2 = NULL_TREE;
2579 ops.location = loc;
2580 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2581 EXPAND_NORMAL);
2582 if (ret != cntvar)
2583 emit_move_insn (cntvar, ret);
2584 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2585 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2586 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2587 profile_probability::very_likely ());
2589 if (lhs && resv == NULL_TREE)
2591 struct separate_ops ops;
2592 ops.code = code;
2593 ops.type = TREE_TYPE (arg0);
2594 ops.op0 = arg0;
2595 ops.op1 = arg1;
2596 ops.op2 = NULL_TREE;
2597 ops.location = loc;
2598 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2599 EXPAND_NORMAL);
2600 if (ret != lhsr)
2601 emit_move_insn (lhsr, ret);
2603 else if (resvr)
2604 emit_move_insn (lhsr, resvr);
2607 /* Expand UBSAN_CHECK_ADD call STMT. */
2609 static void
2610 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2612 location_t loc = gimple_location (stmt);
2613 tree lhs = gimple_call_lhs (stmt);
2614 tree arg0 = gimple_call_arg (stmt, 0);
2615 tree arg1 = gimple_call_arg (stmt, 1);
2616 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2617 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
2618 else
2619 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
2620 false, false, false, true, NULL);
2623 /* Expand UBSAN_CHECK_SUB call STMT. */
2625 static void
2626 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2628 location_t loc = gimple_location (stmt);
2629 tree lhs = gimple_call_lhs (stmt);
2630 tree arg0 = gimple_call_arg (stmt, 0);
2631 tree arg1 = gimple_call_arg (stmt, 1);
2632 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2633 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
2634 else if (integer_zerop (arg0))
2635 expand_neg_overflow (loc, lhs, arg1, true, NULL);
2636 else
2637 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
2638 false, false, false, true, NULL);
2641 /* Expand UBSAN_CHECK_MUL call STMT. */
2643 static void
2644 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2646 location_t loc = gimple_location (stmt);
2647 tree lhs = gimple_call_lhs (stmt);
2648 tree arg0 = gimple_call_arg (stmt, 0);
2649 tree arg1 = gimple_call_arg (stmt, 1);
2650 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2651 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
2652 else
2653 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
2654 NULL);
2657 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2659 static void
2660 expand_arith_overflow (enum tree_code code, gimple *stmt)
2662 tree lhs = gimple_call_lhs (stmt);
2663 if (lhs == NULL_TREE)
2664 return;
2665 tree arg0 = gimple_call_arg (stmt, 0);
2666 tree arg1 = gimple_call_arg (stmt, 1);
2667 tree type = TREE_TYPE (TREE_TYPE (lhs));
2668 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2669 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2670 int unsr_p = TYPE_UNSIGNED (type);
2671 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2672 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2673 int precres = TYPE_PRECISION (type);
2674 location_t loc = gimple_location (stmt);
2675 if (!uns0_p && get_range_pos_neg (arg0) == 1)
2676 uns0_p = true;
2677 if (!uns1_p && get_range_pos_neg (arg1) == 1)
2678 uns1_p = true;
2679 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
2680 prec0 = MIN (prec0, pr);
2681 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
2682 prec1 = MIN (prec1, pr);
2684 /* If uns0_p && uns1_p, precop is minimum needed precision
2685 of unsigned type to hold the exact result, otherwise
2686 precop is minimum needed precision of signed type to
2687 hold the exact result. */
2688 int precop;
2689 if (code == MULT_EXPR)
2690 precop = prec0 + prec1 + (uns0_p != uns1_p);
2691 else
2693 if (uns0_p == uns1_p)
2694 precop = MAX (prec0, prec1) + 1;
2695 else if (uns0_p)
2696 precop = MAX (prec0 + 1, prec1) + 1;
2697 else
2698 precop = MAX (prec0, prec1 + 1) + 1;
2700 int orig_precres = precres;
2704 if ((uns0_p && uns1_p)
2705 ? ((precop + !unsr_p) <= precres
2706 /* u1 - u2 -> ur can overflow, no matter what precision
2707 the result has. */
2708 && (code != MINUS_EXPR || !unsr_p))
2709 : (!unsr_p && precop <= precres))
2711 /* The infinity precision result will always fit into result. */
2712 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2713 write_complex_part (target, const0_rtx, true, false);
2714 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2715 struct separate_ops ops;
2716 ops.code = code;
2717 ops.type = type;
2718 ops.op0 = fold_convert_loc (loc, type, arg0);
2719 ops.op1 = fold_convert_loc (loc, type, arg1);
2720 ops.op2 = NULL_TREE;
2721 ops.location = loc;
2722 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2723 expand_arith_overflow_result_store (lhs, target, mode, tem);
2724 return;
2727 /* For operations with low precision, if target doesn't have them, start
2728 with precres widening right away, otherwise do it only if the most
2729 simple cases can't be used. */
2730 const int min_precision = targetm.min_arithmetic_precision ();
2731 if (orig_precres == precres && precres < min_precision)
2733 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2734 && prec1 <= precres)
2735 || ((!uns0_p || !uns1_p) && !unsr_p
2736 && prec0 + uns0_p <= precres
2737 && prec1 + uns1_p <= precres))
2739 arg0 = fold_convert_loc (loc, type, arg0);
2740 arg1 = fold_convert_loc (loc, type, arg1);
2741 switch (code)
2743 case MINUS_EXPR:
2744 if (integer_zerop (arg0) && !unsr_p)
2746 expand_neg_overflow (loc, lhs, arg1, false, NULL);
2747 return;
2749 /* FALLTHRU */
2750 case PLUS_EXPR:
2751 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2752 unsr_p, unsr_p, false, NULL);
2753 return;
2754 case MULT_EXPR:
2755 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2756 unsr_p, unsr_p, false, NULL);
2757 return;
2758 default:
2759 gcc_unreachable ();
2763 /* For sub-word operations, retry with a wider type first. */
2764 if (orig_precres == precres && precop <= BITS_PER_WORD)
2766 int p = MAX (min_precision, precop);
2767 scalar_int_mode m = smallest_int_mode_for_size (p);
2768 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2769 uns0_p && uns1_p
2770 && unsr_p);
2771 p = TYPE_PRECISION (optype);
2772 if (p > precres)
2774 precres = p;
2775 unsr_p = TYPE_UNSIGNED (optype);
2776 type = optype;
2777 continue;
2781 if (prec0 <= precres && prec1 <= precres)
2783 tree types[2];
2784 if (unsr_p)
2786 types[0] = build_nonstandard_integer_type (precres, 0);
2787 types[1] = type;
2789 else
2791 types[0] = type;
2792 types[1] = build_nonstandard_integer_type (precres, 1);
2794 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2795 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2796 if (code != MULT_EXPR)
2797 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2798 uns0_p, uns1_p, false, NULL);
2799 else
2800 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2801 uns0_p, uns1_p, false, NULL);
2802 return;
2805 /* Retry with a wider type. */
2806 if (orig_precres == precres)
2808 int p = MAX (prec0, prec1);
2809 scalar_int_mode m = smallest_int_mode_for_size (p);
2810 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2811 uns0_p && uns1_p
2812 && unsr_p);
2813 p = TYPE_PRECISION (optype);
2814 if (p > precres)
2816 precres = p;
2817 unsr_p = TYPE_UNSIGNED (optype);
2818 type = optype;
2819 continue;
2823 gcc_unreachable ();
2825 while (1);
2828 /* Expand ADD_OVERFLOW STMT. */
2830 static void
2831 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2833 expand_arith_overflow (PLUS_EXPR, stmt);
2836 /* Expand SUB_OVERFLOW STMT. */
2838 static void
2839 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2841 expand_arith_overflow (MINUS_EXPR, stmt);
2844 /* Expand MUL_OVERFLOW STMT. */
2846 static void
2847 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2849 expand_arith_overflow (MULT_EXPR, stmt);
2852 /* Expand UADDC STMT. */
2854 static void
2855 expand_UADDC (internal_fn ifn, gcall *stmt)
2857 tree lhs = gimple_call_lhs (stmt);
2858 tree arg1 = gimple_call_arg (stmt, 0);
2859 tree arg2 = gimple_call_arg (stmt, 1);
2860 tree arg3 = gimple_call_arg (stmt, 2);
2861 tree type = TREE_TYPE (arg1);
2862 machine_mode mode = TYPE_MODE (type);
2863 insn_code icode = optab_handler (ifn == IFN_UADDC
2864 ? uaddc5_optab : usubc5_optab, mode);
2865 rtx op1 = expand_normal (arg1);
2866 rtx op2 = expand_normal (arg2);
2867 rtx op3 = expand_normal (arg3);
2868 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2869 rtx re = gen_reg_rtx (mode);
2870 rtx im = gen_reg_rtx (mode);
2871 class expand_operand ops[5];
2872 create_output_operand (&ops[0], re, mode);
2873 create_output_operand (&ops[1], im, mode);
2874 create_input_operand (&ops[2], op1, mode);
2875 create_input_operand (&ops[3], op2, mode);
2876 create_input_operand (&ops[4], op3, mode);
2877 expand_insn (icode, 5, ops);
2878 write_complex_part (target, re, false, false);
2879 write_complex_part (target, im, true, false);
2882 /* Expand USUBC STMT. */
2884 static void
2885 expand_USUBC (internal_fn ifn, gcall *stmt)
2887 expand_UADDC (ifn, stmt);
2890 /* This should get folded in tree-vectorizer.cc. */
2892 static void
2893 expand_LOOP_VECTORIZED (internal_fn, gcall *)
2895 gcc_unreachable ();
2898 /* This should get folded in tree-vectorizer.cc. */
2900 static void
2901 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2903 gcc_unreachable ();
2906 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2907 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2909 static tree
2910 expand_call_mem_ref (tree type, gcall *stmt, int index)
2912 tree addr = gimple_call_arg (stmt, index);
2913 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2914 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
2915 if (TYPE_ALIGN (type) != align)
2916 type = build_aligned_type (type, align);
2918 tree tmp = addr;
2919 if (TREE_CODE (tmp) == SSA_NAME)
2921 gimple *def = SSA_NAME_DEF_STMT (tmp);
2922 if (gimple_assign_single_p (def))
2923 tmp = gimple_assign_rhs1 (def);
2926 if (TREE_CODE (tmp) == ADDR_EXPR)
2928 tree mem = TREE_OPERAND (tmp, 0);
2929 if (TREE_CODE (mem) == TARGET_MEM_REF
2930 && types_compatible_p (TREE_TYPE (mem), type))
2932 tree offset = TMR_OFFSET (mem);
2933 if (type != TREE_TYPE (mem)
2934 || alias_ptr_type != TREE_TYPE (offset)
2935 || !integer_zerop (offset))
2937 mem = copy_node (mem);
2938 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
2939 wi::to_poly_wide (offset));
2940 TREE_TYPE (mem) = type;
2942 return mem;
2946 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2949 /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
2950 * OPTAB. */
2952 static void
2953 expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
2955 int i = 0;
2956 class expand_operand ops[5];
2957 tree type, lhs, rhs, maskt;
2958 rtx mem, target;
2959 insn_code icode;
2961 maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
2962 lhs = gimple_call_lhs (stmt);
2963 if (lhs == NULL_TREE)
2964 return;
2965 type = TREE_TYPE (lhs);
2966 rhs = expand_call_mem_ref (type, stmt, 0);
2968 if (optab == vec_mask_load_lanes_optab
2969 || optab == vec_mask_len_load_lanes_optab)
2970 icode = get_multi_vector_move (type, optab);
2971 else if (optab == len_load_optab)
2972 icode = direct_optab_handler (optab, TYPE_MODE (type));
2973 else
2974 icode = convert_optab_handler (optab, TYPE_MODE (type),
2975 TYPE_MODE (TREE_TYPE (maskt)));
2977 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2978 gcc_assert (MEM_P (mem));
2979 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2980 create_output_operand (&ops[i++], target, TYPE_MODE (type));
2981 create_fixed_operand (&ops[i++], mem);
2982 i = add_mask_and_len_args (ops, i, stmt);
2983 expand_insn (icode, i, ops);
2985 if (!rtx_equal_p (target, ops[0].value))
2986 emit_move_insn (target, ops[0].value);
2989 #define expand_mask_load_optab_fn expand_partial_load_optab_fn
2990 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2991 #define expand_len_load_optab_fn expand_partial_load_optab_fn
2992 #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
2994 /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
2995 * OPTAB. */
2997 static void
2998 expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
3000 int i = 0;
3001 class expand_operand ops[5];
3002 tree type, lhs, rhs, maskt;
3003 rtx mem, reg;
3004 insn_code icode;
3006 maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
3007 rhs = gimple_call_arg (stmt, internal_fn_stored_value_index (ifn));
3008 type = TREE_TYPE (rhs);
3009 lhs = expand_call_mem_ref (type, stmt, 0);
3011 if (optab == vec_mask_store_lanes_optab
3012 || optab == vec_mask_len_store_lanes_optab)
3013 icode = get_multi_vector_move (type, optab);
3014 else if (optab == len_store_optab)
3015 icode = direct_optab_handler (optab, TYPE_MODE (type));
3016 else
3017 icode = convert_optab_handler (optab, TYPE_MODE (type),
3018 TYPE_MODE (TREE_TYPE (maskt)));
3020 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3021 gcc_assert (MEM_P (mem));
3022 reg = expand_normal (rhs);
3023 create_fixed_operand (&ops[i++], mem);
3024 create_input_operand (&ops[i++], reg, TYPE_MODE (type));
3025 i = add_mask_and_len_args (ops, i, stmt);
3026 expand_insn (icode, i, ops);
3029 #define expand_mask_store_optab_fn expand_partial_store_optab_fn
3030 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
3031 #define expand_len_store_optab_fn expand_partial_store_optab_fn
3032 #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
3034 /* Expand VCOND, VCONDU and VCONDEQ optab internal functions.
3035 The expansion of STMT happens based on OPTAB table associated. */
3037 static void
3038 expand_vec_cond_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3040 class expand_operand ops[6];
3041 insn_code icode;
3042 tree lhs = gimple_call_lhs (stmt);
3043 tree op0a = gimple_call_arg (stmt, 0);
3044 tree op0b = gimple_call_arg (stmt, 1);
3045 tree op1 = gimple_call_arg (stmt, 2);
3046 tree op2 = gimple_call_arg (stmt, 3);
3047 enum tree_code tcode = (tree_code) int_cst_value (gimple_call_arg (stmt, 4));
3049 tree vec_cond_type = TREE_TYPE (lhs);
3050 tree op_mode = TREE_TYPE (op0a);
3051 bool unsignedp = TYPE_UNSIGNED (op_mode);
3053 machine_mode mode = TYPE_MODE (vec_cond_type);
3054 machine_mode cmp_op_mode = TYPE_MODE (op_mode);
3056 icode = convert_optab_handler (optab, mode, cmp_op_mode);
3057 rtx comparison
3058 = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, icode, 4);
3059 /* vector_compare_rtx legitimizes operands, preserve equality when
3060 expanding op1/op2. */
3061 rtx rtx_op1, rtx_op2;
3062 if (operand_equal_p (op1, op0a))
3063 rtx_op1 = XEXP (comparison, 0);
3064 else if (operand_equal_p (op1, op0b))
3065 rtx_op1 = XEXP (comparison, 1);
3066 else
3067 rtx_op1 = expand_normal (op1);
3068 if (operand_equal_p (op2, op0a))
3069 rtx_op2 = XEXP (comparison, 0);
3070 else if (operand_equal_p (op2, op0b))
3071 rtx_op2 = XEXP (comparison, 1);
3072 else
3073 rtx_op2 = expand_normal (op2);
3075 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3076 create_output_operand (&ops[0], target, mode);
3077 create_input_operand (&ops[1], rtx_op1, mode);
3078 create_input_operand (&ops[2], rtx_op2, mode);
3079 create_fixed_operand (&ops[3], comparison);
3080 create_fixed_operand (&ops[4], XEXP (comparison, 0));
3081 create_fixed_operand (&ops[5], XEXP (comparison, 1));
3082 expand_insn (icode, 6, ops);
3083 if (!rtx_equal_p (ops[0].value, target))
3084 emit_move_insn (target, ops[0].value);
3087 /* Expand VCOND_MASK optab internal function.
3088 The expansion of STMT happens based on OPTAB table associated. */
3090 static void
3091 expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3093 class expand_operand ops[4];
3095 tree lhs = gimple_call_lhs (stmt);
3096 tree op0 = gimple_call_arg (stmt, 0);
3097 tree op1 = gimple_call_arg (stmt, 1);
3098 tree op2 = gimple_call_arg (stmt, 2);
3099 tree vec_cond_type = TREE_TYPE (lhs);
3101 machine_mode mode = TYPE_MODE (vec_cond_type);
3102 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
3103 enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
3104 rtx mask, rtx_op1, rtx_op2;
3106 gcc_assert (icode != CODE_FOR_nothing);
3108 mask = expand_normal (op0);
3109 rtx_op1 = expand_normal (op1);
3110 rtx_op2 = expand_normal (op2);
3112 mask = force_reg (mask_mode, mask);
3113 rtx_op1 = force_reg (mode, rtx_op1);
3115 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3116 create_output_operand (&ops[0], target, mode);
3117 create_input_operand (&ops[1], rtx_op1, mode);
3118 create_input_operand (&ops[2], rtx_op2, mode);
3119 create_input_operand (&ops[3], mask, mask_mode);
3120 expand_insn (icode, 4, ops);
3121 if (!rtx_equal_p (ops[0].value, target))
3122 emit_move_insn (target, ops[0].value);
3125 /* Expand VEC_SET internal functions. */
3127 static void
3128 expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3130 tree lhs = gimple_call_lhs (stmt);
3131 tree op0 = gimple_call_arg (stmt, 0);
3132 tree op1 = gimple_call_arg (stmt, 1);
3133 tree op2 = gimple_call_arg (stmt, 2);
3134 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3135 rtx src = expand_normal (op0);
3137 machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
3138 scalar_mode innermode = GET_MODE_INNER (outermode);
3140 rtx value = expand_normal (op1);
3141 rtx pos = expand_normal (op2);
3143 class expand_operand ops[3];
3144 enum insn_code icode = optab_handler (optab, outermode);
3146 if (icode != CODE_FOR_nothing)
3148 rtx temp = gen_reg_rtx (outermode);
3149 emit_move_insn (temp, src);
3151 create_fixed_operand (&ops[0], temp);
3152 create_input_operand (&ops[1], value, innermode);
3153 create_convert_operand_from (&ops[2], pos, TYPE_MODE (TREE_TYPE (op2)),
3154 true);
3155 if (maybe_expand_insn (icode, 3, ops))
3157 emit_move_insn (target, temp);
3158 return;
3161 gcc_unreachable ();
3164 static void
3165 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
3169 static void
3170 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
3172 /* When guessing was done, the hints should be already stripped away. */
3173 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
3175 rtx target;
3176 tree lhs = gimple_call_lhs (stmt);
3177 if (lhs)
3178 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3179 else
3180 target = const0_rtx;
3181 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
3182 if (lhs && val != target)
3183 emit_move_insn (target, val);
3186 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
3187 should never be called. */
3189 static void
3190 expand_VA_ARG (internal_fn, gcall *)
3192 gcc_unreachable ();
3195 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
3196 dummy function should never be called. */
3198 static void
3199 expand_VEC_CONVERT (internal_fn, gcall *)
3201 gcc_unreachable ();
3204 /* Expand IFN_RAWMEMCHR internal function. */
3206 void
3207 expand_RAWMEMCHR (internal_fn, gcall *stmt)
3209 expand_operand ops[3];
3211 tree lhs = gimple_call_lhs (stmt);
3212 if (!lhs)
3213 return;
3214 machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
3215 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3216 create_output_operand (&ops[0], lhs_rtx, lhs_mode);
3218 tree mem = gimple_call_arg (stmt, 0);
3219 rtx mem_rtx = get_memory_rtx (mem, NULL);
3220 create_fixed_operand (&ops[1], mem_rtx);
3222 tree pattern = gimple_call_arg (stmt, 1);
3223 machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
3224 rtx pattern_rtx = expand_normal (pattern);
3225 create_input_operand (&ops[2], pattern_rtx, mode);
3227 insn_code icode = direct_optab_handler (rawmemchr_optab, mode);
3229 expand_insn (icode, 3, ops);
3230 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3231 emit_move_insn (lhs_rtx, ops[0].value);
3234 /* Expand the IFN_UNIQUE function according to its first argument. */
3236 static void
3237 expand_UNIQUE (internal_fn, gcall *stmt)
3239 rtx pattern = NULL_RTX;
3240 enum ifn_unique_kind kind
3241 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
3243 switch (kind)
3245 default:
3246 gcc_unreachable ();
3248 case IFN_UNIQUE_UNSPEC:
3249 if (targetm.have_unique ())
3250 pattern = targetm.gen_unique ();
3251 break;
3253 case IFN_UNIQUE_OACC_FORK:
3254 case IFN_UNIQUE_OACC_JOIN:
3255 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
3257 tree lhs = gimple_call_lhs (stmt);
3258 rtx target = const0_rtx;
3260 if (lhs)
3261 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3263 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
3264 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
3266 if (kind == IFN_UNIQUE_OACC_FORK)
3267 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
3268 else
3269 pattern = targetm.gen_oacc_join (target, data_dep, axis);
3271 else
3272 gcc_unreachable ();
3273 break;
3276 if (pattern)
3277 emit_insn (pattern);
3280 /* Expand the IFN_DEFERRED_INIT function:
3281 LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
3283 Initialize the LHS with zero/pattern according to its second argument
3284 INIT_TYPE:
3285 if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
3286 if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
3287 to initialize;
3288 The LHS variable is initialized including paddings.
3289 The reasons to choose 0xFE for pattern initialization are:
3290 1. It is a non-canonical virtual address on x86_64, and at the
3291 high end of the i386 kernel address space.
3292 2. It is a very large float value (-1.694739530317379e+38).
3293 3. It is also an unusual number for integers. */
3294 #define INIT_PATTERN_VALUE 0xFE
3295 static void
3296 expand_DEFERRED_INIT (internal_fn, gcall *stmt)
3298 tree lhs = gimple_call_lhs (stmt);
3299 tree var_size = gimple_call_arg (stmt, 0);
3300 enum auto_init_type init_type
3301 = (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
3302 bool reg_lhs = true;
3304 tree var_type = TREE_TYPE (lhs);
3305 gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
3307 if (TREE_CODE (lhs) == SSA_NAME)
3308 reg_lhs = true;
3309 else
3311 tree lhs_base = lhs;
3312 while (handled_component_p (lhs_base))
3313 lhs_base = TREE_OPERAND (lhs_base, 0);
3314 reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
3315 || non_mem_decl_p (lhs_base));
3316 /* If this expands to a register and the underlying decl is wrapped in
3317 a MEM_REF that just serves as an access type change expose the decl
3318 if it is of correct size. This avoids a situation as in PR103271
3319 if the target does not support a direct move to the registers mode. */
3320 if (reg_lhs
3321 && TREE_CODE (lhs_base) == MEM_REF
3322 && TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
3323 && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
3324 && integer_zerop (TREE_OPERAND (lhs_base, 1))
3325 && tree_fits_uhwi_p (var_size)
3326 && tree_int_cst_equal
3327 (var_size,
3328 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
3330 lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
3331 var_type = TREE_TYPE (lhs);
3335 if (!reg_lhs)
3337 /* If the variable is not in register, expand to a memset
3338 to initialize it. */
3339 mark_addressable (lhs);
3340 tree var_addr = build_fold_addr_expr (lhs);
3342 tree value = (init_type == AUTO_INIT_PATTERN)
3343 ? build_int_cst (integer_type_node,
3344 INIT_PATTERN_VALUE)
3345 : integer_zero_node;
3346 tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
3347 3, var_addr, value, var_size);
3348 /* Expand this memset call. */
3349 expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
3351 else
3353 /* If this variable is in a register use expand_assignment.
3354 For boolean scalars force zero-init. */
3355 tree init;
3356 scalar_int_mode var_mode;
3357 if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
3358 && tree_fits_uhwi_p (var_size)
3359 && (init_type == AUTO_INIT_PATTERN
3360 || !is_gimple_reg_type (var_type))
3361 && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
3362 0).exists (&var_mode)
3363 && have_insn_for (SET, var_mode))
3365 unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
3366 unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
3367 memset (buf, (init_type == AUTO_INIT_PATTERN
3368 ? INIT_PATTERN_VALUE : 0), total_bytes);
3369 tree itype = build_nonstandard_integer_type
3370 (total_bytes * BITS_PER_UNIT, 1);
3371 wide_int w = wi::from_buffer (buf, total_bytes);
3372 init = wide_int_to_tree (itype, w);
3373 /* Pun the LHS to make sure its type has constant size
3374 unless it is an SSA name where that's already known. */
3375 if (TREE_CODE (lhs) != SSA_NAME)
3376 lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
3377 else
3378 init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
3380 else
3381 /* Use zero-init also for variable-length sizes. */
3382 init = build_zero_cst (var_type);
3384 expand_assignment (lhs, init, false);
3388 /* The size of an OpenACC compute dimension. */
3390 static void
3391 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
3393 tree lhs = gimple_call_lhs (stmt);
3395 if (!lhs)
3396 return;
3398 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3399 if (targetm.have_oacc_dim_size ())
3401 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3402 VOIDmode, EXPAND_NORMAL);
3403 emit_insn (targetm.gen_oacc_dim_size (target, dim));
3405 else
3406 emit_move_insn (target, GEN_INT (1));
3409 /* The position of an OpenACC execution engine along one compute axis. */
3411 static void
3412 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
3414 tree lhs = gimple_call_lhs (stmt);
3416 if (!lhs)
3417 return;
3419 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3420 if (targetm.have_oacc_dim_pos ())
3422 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
3423 VOIDmode, EXPAND_NORMAL);
3424 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
3426 else
3427 emit_move_insn (target, const0_rtx);
3430 /* This is expanded by oacc_device_lower pass. */
3432 static void
3433 expand_GOACC_LOOP (internal_fn, gcall *)
3435 gcc_unreachable ();
3438 /* This is expanded by oacc_device_lower pass. */
3440 static void
3441 expand_GOACC_REDUCTION (internal_fn, gcall *)
3443 gcc_unreachable ();
3446 /* This is expanded by oacc_device_lower pass. */
3448 static void
3449 expand_GOACC_TILE (internal_fn, gcall *)
3451 gcc_unreachable ();
3454 /* Set errno to EDOM. */
3456 static void
3457 expand_SET_EDOM (internal_fn, gcall *)
3459 #ifdef TARGET_EDOM
3460 #ifdef GEN_ERRNO_RTX
3461 rtx errno_rtx = GEN_ERRNO_RTX;
3462 #else
3463 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
3464 #endif
3465 emit_move_insn (errno_rtx,
3466 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
3467 #else
3468 gcc_unreachable ();
3469 #endif
3472 /* Expand atomic bit test and set. */
3474 static void
3475 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
3477 expand_ifn_atomic_bit_test_and (call);
3480 /* Expand atomic bit test and complement. */
3482 static void
3483 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
3485 expand_ifn_atomic_bit_test_and (call);
3488 /* Expand atomic bit test and reset. */
3490 static void
3491 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
3493 expand_ifn_atomic_bit_test_and (call);
3496 /* Expand atomic bit test and set. */
3498 static void
3499 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
3501 expand_ifn_atomic_compare_exchange (call);
3504 /* Expand atomic add fetch and cmp with 0. */
3506 static void
3507 expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
3509 expand_ifn_atomic_op_fetch_cmp_0 (call);
3512 /* Expand atomic sub fetch and cmp with 0. */
3514 static void
3515 expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
3517 expand_ifn_atomic_op_fetch_cmp_0 (call);
3520 /* Expand atomic and fetch and cmp with 0. */
3522 static void
3523 expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
3525 expand_ifn_atomic_op_fetch_cmp_0 (call);
3528 /* Expand atomic or fetch and cmp with 0. */
3530 static void
3531 expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
3533 expand_ifn_atomic_op_fetch_cmp_0 (call);
3536 /* Expand atomic xor fetch and cmp with 0. */
3538 static void
3539 expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
3541 expand_ifn_atomic_op_fetch_cmp_0 (call);
3544 /* Expand LAUNDER to assignment, lhs = arg0. */
3546 static void
3547 expand_LAUNDER (internal_fn, gcall *call)
3549 tree lhs = gimple_call_lhs (call);
3551 if (!lhs)
3552 return;
3554 expand_assignment (lhs, gimple_call_arg (call, 0), false);
3557 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
3559 static void
3560 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3562 internal_fn ifn = gimple_call_internal_fn (stmt);
3563 int rhs_index = internal_fn_stored_value_index (ifn);
3564 tree base = gimple_call_arg (stmt, 0);
3565 tree offset = gimple_call_arg (stmt, 1);
3566 tree scale = gimple_call_arg (stmt, 2);
3567 tree rhs = gimple_call_arg (stmt, rhs_index);
3569 rtx base_rtx = expand_normal (base);
3570 rtx offset_rtx = expand_normal (offset);
3571 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3572 rtx rhs_rtx = expand_normal (rhs);
3574 class expand_operand ops[8];
3575 int i = 0;
3576 create_address_operand (&ops[i++], base_rtx);
3577 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3578 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3579 create_integer_operand (&ops[i++], scale_int);
3580 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
3581 i = add_mask_and_len_args (ops, i, stmt);
3583 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
3584 TYPE_MODE (TREE_TYPE (offset)));
3585 expand_insn (icode, i, ops);
3588 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
3590 static void
3591 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
3593 tree lhs = gimple_call_lhs (stmt);
3594 tree base = gimple_call_arg (stmt, 0);
3595 tree offset = gimple_call_arg (stmt, 1);
3596 tree scale = gimple_call_arg (stmt, 2);
3598 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3599 rtx base_rtx = expand_normal (base);
3600 rtx offset_rtx = expand_normal (offset);
3601 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
3603 int i = 0;
3604 class expand_operand ops[8];
3605 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
3606 create_address_operand (&ops[i++], base_rtx);
3607 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
3608 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
3609 create_integer_operand (&ops[i++], scale_int);
3610 i = add_mask_and_len_args (ops, i, stmt);
3611 insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
3612 TYPE_MODE (TREE_TYPE (offset)));
3613 expand_insn (icode, i, ops);
3614 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3615 emit_move_insn (lhs_rtx, ops[0].value);
3618 /* Helper for expand_DIVMOD. Return true if the sequence starting with
3619 INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes. */
3621 static bool
3622 contains_call_div_mod (rtx_insn *insn)
3624 subrtx_iterator::array_type array;
3625 for (; insn; insn = NEXT_INSN (insn))
3626 if (CALL_P (insn))
3627 return true;
3628 else if (INSN_P (insn))
3629 FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
3630 switch (GET_CODE (*iter))
3632 case CALL:
3633 case DIV:
3634 case UDIV:
3635 case MOD:
3636 case UMOD:
3637 return true;
3638 default:
3639 break;
3641 return false;
3644 /* Expand DIVMOD() using:
3645 a) optab handler for udivmod/sdivmod if it is available.
3646 b) If optab_handler doesn't exist, generate call to
3647 target-specific divmod libfunc. */
3649 static void
3650 expand_DIVMOD (internal_fn, gcall *call_stmt)
3652 tree lhs = gimple_call_lhs (call_stmt);
3653 tree arg0 = gimple_call_arg (call_stmt, 0);
3654 tree arg1 = gimple_call_arg (call_stmt, 1);
3656 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
3657 tree type = TREE_TYPE (TREE_TYPE (lhs));
3658 machine_mode mode = TYPE_MODE (type);
3659 bool unsignedp = TYPE_UNSIGNED (type);
3660 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
3662 rtx op0 = expand_normal (arg0);
3663 rtx op1 = expand_normal (arg1);
3664 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3666 rtx quotient = NULL_RTX, remainder = NULL_RTX;
3667 rtx_insn *insns = NULL;
3669 if (TREE_CODE (arg1) == INTEGER_CST)
3671 /* For DIVMOD by integral constants, there could be efficient code
3672 expanded inline e.g. using shifts and plus/minus. Try to expand
3673 the division and modulo and if it emits any library calls or any
3674 {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
3675 divmod libcall. */
3676 scalar_int_mode int_mode;
3677 if (remainder == NULL_RTX
3678 && optimize
3679 && CONST_INT_P (op1)
3680 && !pow2p_hwi (INTVAL (op1))
3681 && is_int_mode (TYPE_MODE (type), &int_mode)
3682 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3683 && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
3684 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
3685 && optimize_insn_for_speed_p ())
3687 rtx_insn *last = get_last_insn ();
3688 remainder = NULL_RTX;
3689 quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
3690 TYPE_UNSIGNED (type));
3691 if (quotient != NULL_RTX)
3693 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
3695 rtx_insn *move = emit_move_insn (quotient, quotient);
3696 set_dst_reg_note (move, REG_EQUAL,
3697 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3698 ? UDIV : DIV, int_mode,
3699 copy_rtx (op0), op1),
3700 quotient);
3701 move = emit_move_insn (remainder, remainder);
3702 set_dst_reg_note (move, REG_EQUAL,
3703 gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
3704 ? UMOD : MOD, int_mode,
3705 copy_rtx (op0), op1),
3706 quotient);
3709 else
3710 delete_insns_since (last);
3713 if (remainder == NULL_RTX)
3715 struct separate_ops ops;
3716 ops.code = TRUNC_DIV_EXPR;
3717 ops.type = type;
3718 ops.op0 = make_tree (ops.type, op0);
3719 ops.op1 = arg1;
3720 ops.op2 = NULL_TREE;
3721 ops.location = gimple_location (call_stmt);
3722 start_sequence ();
3723 quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
3724 if (contains_call_div_mod (get_insns ()))
3725 quotient = NULL_RTX;
3726 else
3728 ops.code = TRUNC_MOD_EXPR;
3729 remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
3730 EXPAND_NORMAL);
3731 if (contains_call_div_mod (get_insns ()))
3732 remainder = NULL_RTX;
3734 if (remainder)
3735 insns = get_insns ();
3736 end_sequence ();
3740 if (remainder)
3741 emit_insn (insns);
3743 /* Check if optab_handler exists for divmod_optab for given mode. */
3744 else if (optab_handler (tab, mode) != CODE_FOR_nothing)
3746 quotient = gen_reg_rtx (mode);
3747 remainder = gen_reg_rtx (mode);
3748 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
3751 /* Generate call to divmod libfunc if it exists. */
3752 else if (rtx libfunc = optab_libfunc (tab, mode))
3753 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
3754 &quotient, &remainder);
3756 else
3757 gcc_unreachable ();
3759 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
3760 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
3761 make_tree (TREE_TYPE (arg0), quotient),
3762 make_tree (TREE_TYPE (arg1), remainder)),
3763 target, VOIDmode, EXPAND_NORMAL);
3766 /* Expand a NOP. */
3768 static void
3769 expand_NOP (internal_fn, gcall *)
3771 /* Nothing. But it shouldn't really prevail. */
3774 /* Coroutines, all should have been processed at this stage. */
3776 static void
3777 expand_CO_FRAME (internal_fn, gcall *)
3779 gcc_unreachable ();
3782 static void
3783 expand_CO_YIELD (internal_fn, gcall *)
3785 gcc_unreachable ();
3788 static void
3789 expand_CO_SUSPN (internal_fn, gcall *)
3791 gcc_unreachable ();
3794 static void
3795 expand_CO_ACTOR (internal_fn, gcall *)
3797 gcc_unreachable ();
3800 /* Expand a call to FN using the operands in STMT. FN has a single
3801 output operand and NARGS input operands. */
3803 static void
3804 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
3805 unsigned int nargs)
3807 tree_pair types = direct_internal_fn_types (fn, stmt);
3808 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
3809 expand_fn_using_insn (stmt, icode, 1, nargs);
3812 /* Expand WHILE_ULT call STMT using optab OPTAB. */
3814 static void
3815 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
3817 expand_operand ops[4];
3818 tree rhs_type[2];
3820 tree lhs = gimple_call_lhs (stmt);
3821 tree lhs_type = TREE_TYPE (lhs);
3822 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
3823 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
3825 for (unsigned int i = 0; i < 2; ++i)
3827 tree rhs = gimple_call_arg (stmt, i);
3828 rhs_type[i] = TREE_TYPE (rhs);
3829 rtx rhs_rtx = expand_normal (rhs);
3830 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
3833 int opcnt;
3834 if (!VECTOR_MODE_P (TYPE_MODE (lhs_type)))
3836 /* When the mask is an integer mode the exact vector length may not
3837 be clear to the backend, so we pass it in operand[3].
3838 Use the vector in arg2 for the most reliable intended size. */
3839 tree type = TREE_TYPE (gimple_call_arg (stmt, 2));
3840 create_integer_operand (&ops[3], TYPE_VECTOR_SUBPARTS (type));
3841 opcnt = 4;
3843 else
3844 /* The mask has a vector type so the length operand is unnecessary. */
3845 opcnt = 3;
3847 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
3848 TYPE_MODE (lhs_type));
3850 expand_insn (icode, opcnt, ops);
3851 if (!rtx_equal_p (lhs_rtx, ops[0].value))
3852 emit_move_insn (lhs_rtx, ops[0].value);
3855 /* Expand a call to a convert-like optab using the operands in STMT.
3856 FN has a single output operand and NARGS input operands. */
3858 static void
3859 expand_convert_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab,
3860 unsigned int nargs)
3862 tree_pair types = direct_internal_fn_types (fn, stmt);
3863 insn_code icode = convert_optab_handler (optab, TYPE_MODE (types.first),
3864 TYPE_MODE (types.second));
3865 expand_fn_using_insn (stmt, icode, 1, nargs);
3868 /* Expanders for optabs that can use expand_direct_optab_fn. */
3870 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3871 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3873 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3874 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3876 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3877 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3879 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3880 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3882 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3883 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3885 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3886 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3888 #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
3889 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3891 #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
3892 expand_direct_optab_fn (FN, STMT, OPTAB, 6)
3894 #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
3895 expand_direct_optab_fn (FN, STMT, OPTAB, 7)
3897 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3898 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3900 #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
3901 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3903 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3904 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3906 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3907 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3909 #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
3910 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3912 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3913 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3915 /* Expanders for optabs that can use expand_convert_optab_fn. */
3917 #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
3918 expand_convert_optab_fn (FN, STMT, OPTAB, 1)
3920 #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
3921 expand_convert_optab_fn (FN, STMT, OPTAB, 2)
3923 /* RETURN_TYPE and ARGS are a return type and argument list that are
3924 in principle compatible with FN (which satisfies direct_internal_fn_p).
3925 Return the types that should be used to determine whether the
3926 target supports FN. */
3928 tree_pair
3929 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3931 const direct_internal_fn_info &info = direct_internal_fn (fn);
3932 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3933 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3934 return tree_pair (type0, type1);
3937 /* CALL is a call whose return type and arguments are in principle
3938 compatible with FN (which satisfies direct_internal_fn_p). Return the
3939 types that should be used to determine whether the target supports FN. */
3941 tree_pair
3942 direct_internal_fn_types (internal_fn fn, gcall *call)
3944 const direct_internal_fn_info &info = direct_internal_fn (fn);
3945 tree op0 = (info.type0 < 0
3946 ? gimple_call_lhs (call)
3947 : gimple_call_arg (call, info.type0));
3948 tree op1 = (info.type1 < 0
3949 ? gimple_call_lhs (call)
3950 : gimple_call_arg (call, info.type1));
3951 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3954 /* Return true if OPTAB is supported for TYPES (whose modes should be
3955 the same) when the optimization type is OPT_TYPE. Used for simple
3956 direct optabs. */
3958 static bool
3959 direct_optab_supported_p (direct_optab optab, tree_pair types,
3960 optimization_type opt_type)
3962 machine_mode mode = TYPE_MODE (types.first);
3963 gcc_checking_assert (mode == TYPE_MODE (types.second));
3964 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3967 /* Return true if OPTAB is supported for TYPES, where the first type
3968 is the destination and the second type is the source. Used for
3969 convert optabs. */
3971 static bool
3972 convert_optab_supported_p (convert_optab optab, tree_pair types,
3973 optimization_type opt_type)
3975 return (convert_optab_handler (optab, TYPE_MODE (types.first),
3976 TYPE_MODE (types.second), opt_type)
3977 != CODE_FOR_nothing);
3980 /* Return true if load/store lanes optab OPTAB is supported for
3981 array type TYPES.first when the optimization type is OPT_TYPE. */
3983 static bool
3984 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3985 optimization_type opt_type)
3987 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3988 machine_mode imode = TYPE_MODE (types.first);
3989 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3990 return (convert_optab_handler (optab, imode, vmode, opt_type)
3991 != CODE_FOR_nothing);
3994 #define direct_unary_optab_supported_p direct_optab_supported_p
3995 #define direct_unary_convert_optab_supported_p convert_optab_supported_p
3996 #define direct_binary_optab_supported_p direct_optab_supported_p
3997 #define direct_ternary_optab_supported_p direct_optab_supported_p
3998 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3999 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
4000 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
4001 #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
4002 #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
4003 #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
4004 #define direct_mask_load_optab_supported_p convert_optab_supported_p
4005 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
4006 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
4007 #define direct_gather_load_optab_supported_p convert_optab_supported_p
4008 #define direct_len_load_optab_supported_p direct_optab_supported_p
4009 #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
4010 #define direct_mask_store_optab_supported_p convert_optab_supported_p
4011 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
4012 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
4013 #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
4014 #define direct_vec_cond_optab_supported_p convert_optab_supported_p
4015 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
4016 #define direct_len_store_optab_supported_p direct_optab_supported_p
4017 #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
4018 #define direct_while_optab_supported_p convert_optab_supported_p
4019 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
4020 #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
4021 #define direct_fold_left_optab_supported_p direct_optab_supported_p
4022 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
4023 #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
4024 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
4025 #define direct_vec_set_optab_supported_p direct_optab_supported_p
4026 #define direct_vec_extract_optab_supported_p convert_optab_supported_p
4028 /* Return the optab used by internal function FN. */
4030 optab
4031 direct_internal_fn_optab (internal_fn fn, tree_pair types)
4033 switch (fn)
4035 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4036 case IFN_##CODE: break;
4037 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4038 case IFN_##CODE: return OPTAB##_optab;
4039 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4040 UNSIGNED_OPTAB, TYPE) \
4041 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
4042 ? UNSIGNED_OPTAB ## _optab \
4043 : SIGNED_OPTAB ## _optab);
4044 #include "internal-fn.def"
4046 case IFN_LAST:
4047 break;
4049 gcc_unreachable ();
4052 /* Return the optab used by internal function FN. */
4054 static optab
4055 direct_internal_fn_optab (internal_fn fn)
4057 switch (fn)
4059 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4060 case IFN_##CODE: break;
4061 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4062 case IFN_##CODE: return OPTAB##_optab;
4063 #include "internal-fn.def"
4065 case IFN_LAST:
4066 break;
4068 gcc_unreachable ();
4071 /* Return true if FN is supported for the types in TYPES when the
4072 optimization type is OPT_TYPE. The types are those associated with
4073 the "type0" and "type1" fields of FN's direct_internal_fn_info
4074 structure. */
4076 bool
4077 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
4078 optimization_type opt_type)
4080 switch (fn)
4082 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
4083 case IFN_##CODE: break;
4084 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4085 case IFN_##CODE: \
4086 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
4087 opt_type);
4088 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4089 UNSIGNED_OPTAB, TYPE) \
4090 case IFN_##CODE: \
4092 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
4093 ? UNSIGNED_OPTAB ## _optab \
4094 : SIGNED_OPTAB ## _optab); \
4095 return direct_##TYPE##_optab_supported_p (which_optab, types, \
4096 opt_type); \
4098 #include "internal-fn.def"
4100 case IFN_LAST:
4101 break;
4103 gcc_unreachable ();
4106 /* Return true if FN is supported for type TYPE when the optimization
4107 type is OPT_TYPE. The caller knows that the "type0" and "type1"
4108 fields of FN's direct_internal_fn_info structure are the same. */
4110 bool
4111 direct_internal_fn_supported_p (internal_fn fn, tree type,
4112 optimization_type opt_type)
4114 const direct_internal_fn_info &info = direct_internal_fn (fn);
4115 gcc_checking_assert (info.type0 == info.type1);
4116 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
4119 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
4120 given that STMT is a call to a direct internal function. */
4122 bool
4123 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
4125 internal_fn fn = gimple_call_internal_fn (stmt);
4126 tree_pair types = direct_internal_fn_types (fn, stmt);
4127 return direct_internal_fn_supported_p (fn, types, opt_type);
4130 /* Return true if FN is a binary operation and if FN is commutative. */
4132 bool
4133 commutative_binary_fn_p (internal_fn fn)
4135 switch (fn)
4137 case IFN_AVG_FLOOR:
4138 case IFN_AVG_CEIL:
4139 case IFN_MULH:
4140 case IFN_MULHS:
4141 case IFN_MULHRS:
4142 case IFN_FMIN:
4143 case IFN_FMAX:
4144 case IFN_COMPLEX_MUL:
4145 case IFN_UBSAN_CHECK_ADD:
4146 case IFN_UBSAN_CHECK_MUL:
4147 case IFN_ADD_OVERFLOW:
4148 case IFN_MUL_OVERFLOW:
4149 case IFN_VEC_WIDEN_PLUS:
4150 case IFN_VEC_WIDEN_PLUS_LO:
4151 case IFN_VEC_WIDEN_PLUS_HI:
4152 case IFN_VEC_WIDEN_PLUS_EVEN:
4153 case IFN_VEC_WIDEN_PLUS_ODD:
4154 return true;
4156 default:
4157 return false;
4161 /* Return true if FN is a ternary operation and if its first two arguments
4162 are commutative. */
4164 bool
4165 commutative_ternary_fn_p (internal_fn fn)
4167 switch (fn)
4169 case IFN_FMA:
4170 case IFN_FMS:
4171 case IFN_FNMA:
4172 case IFN_FNMS:
4173 case IFN_UADDC:
4174 return true;
4176 default:
4177 return false;
4181 /* Return true if FN is an associative binary operation. */
4183 bool
4184 associative_binary_fn_p (internal_fn fn)
4186 switch (fn)
4188 case IFN_FMIN:
4189 case IFN_FMAX:
4190 return true;
4192 default:
4193 return false;
4197 /* If FN is commutative in two consecutive arguments, return the
4198 index of the first, otherwise return -1. */
4201 first_commutative_argument (internal_fn fn)
4203 switch (fn)
4205 case IFN_COND_ADD:
4206 case IFN_COND_MUL:
4207 case IFN_COND_MIN:
4208 case IFN_COND_MAX:
4209 case IFN_COND_FMIN:
4210 case IFN_COND_FMAX:
4211 case IFN_COND_AND:
4212 case IFN_COND_IOR:
4213 case IFN_COND_XOR:
4214 case IFN_COND_FMA:
4215 case IFN_COND_FMS:
4216 case IFN_COND_FNMA:
4217 case IFN_COND_FNMS:
4218 case IFN_COND_LEN_ADD:
4219 case IFN_COND_LEN_MUL:
4220 case IFN_COND_LEN_MIN:
4221 case IFN_COND_LEN_MAX:
4222 case IFN_COND_LEN_FMIN:
4223 case IFN_COND_LEN_FMAX:
4224 case IFN_COND_LEN_AND:
4225 case IFN_COND_LEN_IOR:
4226 case IFN_COND_LEN_XOR:
4227 case IFN_COND_LEN_FMA:
4228 case IFN_COND_LEN_FMS:
4229 case IFN_COND_LEN_FNMA:
4230 case IFN_COND_LEN_FNMS:
4231 return 1;
4233 default:
4234 if (commutative_binary_fn_p (fn)
4235 || commutative_ternary_fn_p (fn))
4236 return 0;
4237 return -1;
4241 /* Return true if this CODE describes an internal_fn that returns a vector with
4242 elements twice as wide as the element size of the input vectors. */
4244 bool
4245 widening_fn_p (code_helper code)
4247 if (!code.is_fn_code ())
4248 return false;
4250 if (!internal_fn_p ((combined_fn) code))
4251 return false;
4253 internal_fn fn = as_internal_fn ((combined_fn) code);
4254 switch (fn)
4256 #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
4257 case IFN_##NAME: \
4258 case IFN_##NAME##_HI: \
4259 case IFN_##NAME##_LO: \
4260 case IFN_##NAME##_EVEN: \
4261 case IFN_##NAME##_ODD: \
4262 return true;
4263 #include "internal-fn.def"
4265 default:
4266 return false;
4270 /* Return true if IFN_SET_EDOM is supported. */
4272 bool
4273 set_edom_supported_p (void)
4275 #ifdef TARGET_EDOM
4276 return true;
4277 #else
4278 return false;
4279 #endif
4282 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
4283 static void \
4284 expand_##CODE (internal_fn fn, gcall *stmt) \
4286 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
4288 #define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
4289 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
4290 UNSIGNED_OPTAB, TYPE) \
4291 static void \
4292 expand_##CODE (internal_fn fn, gcall *stmt) \
4294 tree_pair types = direct_internal_fn_types (fn, stmt); \
4295 optab which_optab = direct_internal_fn_optab (fn, types); \
4296 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
4298 #include "internal-fn.def"
4300 /* Routines to expand each internal function, indexed by function number.
4301 Each routine has the prototype:
4303 expand_<NAME> (gcall *stmt)
4305 where STMT is the statement that performs the call. */
4306 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
4308 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
4309 #include "internal-fn.def"
4313 /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
4314 that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX
4315 for each such IFN_COND_##SUFFIX. */
4316 #define FOR_EACH_CODE_MAPPING(T) \
4317 T (PLUS_EXPR, ADD) \
4318 T (MINUS_EXPR, SUB) \
4319 T (MULT_EXPR, MUL) \
4320 T (TRUNC_DIV_EXPR, DIV) \
4321 T (TRUNC_MOD_EXPR, MOD) \
4322 T (RDIV_EXPR, RDIV) \
4323 T (MIN_EXPR, MIN) \
4324 T (MAX_EXPR, MAX) \
4325 T (BIT_AND_EXPR, AND) \
4326 T (BIT_IOR_EXPR, IOR) \
4327 T (BIT_XOR_EXPR, XOR) \
4328 T (LSHIFT_EXPR, SHL) \
4329 T (RSHIFT_EXPR, SHR) \
4330 T (NEGATE_EXPR, NEG)
4332 /* Return a function that only performs CODE when a certain condition is met
4333 and that uses a given fallback value otherwise. For example, if CODE is
4334 a binary operation associated with conditional function FN:
4336 LHS = FN (COND, A, B, ELSE)
4338 is equivalent to the C expression:
4340 LHS = COND ? A CODE B : ELSE;
4342 operating elementwise if the operands are vectors.
4344 Return IFN_LAST if no such function exists. */
4346 internal_fn
4347 get_conditional_internal_fn (tree_code code)
4349 switch (code)
4351 #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
4352 FOR_EACH_CODE_MAPPING(CASE)
4353 #undef CASE
4354 default:
4355 return IFN_LAST;
4359 /* If IFN implements the conditional form of a tree code, return that
4360 tree code, otherwise return ERROR_MARK. */
4362 tree_code
4363 conditional_internal_fn_code (internal_fn ifn)
4365 switch (ifn)
4367 #define CASE(CODE, IFN) \
4368 case IFN_COND_##IFN: \
4369 case IFN_COND_LEN_##IFN: \
4370 return CODE;
4371 FOR_EACH_CODE_MAPPING (CASE)
4372 #undef CASE
4373 default:
4374 return ERROR_MARK;
4378 /* Like get_conditional_internal_fn, but return a function that
4379 additionally restricts the operation to the leading elements
4380 of a vector. The number of elements to process is given by a length
4381 and bias pair, as for IFN_LOAD_LEN. The values of the remaining
4382 elements are taken from the fallback ("else") argument.
4384 For example, if CODE is a binary operation associated with FN:
4386 LHS = FN (COND, A, B, ELSE, LEN, BIAS)
4388 is equivalent to the C code:
4390 for (int i = 0; i < NUNITS; i++)
4392 if (i < LEN + BIAS && COND[i])
4393 LHS[i] = A[i] CODE B[i];
4394 else
4395 LHS[i] = ELSE[i];
4399 internal_fn
4400 get_conditional_len_internal_fn (tree_code code)
4402 switch (code)
4404 #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
4405 FOR_EACH_CODE_MAPPING(CASE)
4406 #undef CASE
4407 default:
4408 return IFN_LAST;
4412 /* Invoke T(IFN) for each internal function IFN that also has an
4413 IFN_COND_* form. */
4414 #define FOR_EACH_COND_FN_PAIR(T) \
4415 T (FMAX) \
4416 T (FMIN) \
4417 T (FMA) \
4418 T (FMS) \
4419 T (FNMA) \
4420 T (FNMS)
4422 /* Return a function that only performs internal function FN when a
4423 certain condition is met and that uses a given fallback value otherwise.
4424 In other words, the returned function FN' is such that:
4426 LHS = FN' (COND, A1, ... An, ELSE)
4428 is equivalent to the C expression:
4430 LHS = COND ? FN (A1, ..., An) : ELSE;
4432 operating elementwise if the operands are vectors.
4434 Return IFN_LAST if no such function exists. */
4436 internal_fn
4437 get_conditional_internal_fn (internal_fn fn)
4439 switch (fn)
4441 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
4442 FOR_EACH_COND_FN_PAIR(CASE)
4443 #undef CASE
4444 default:
4445 return IFN_LAST;
4449 /* If there exists an internal function like IFN that operates on vectors,
4450 but with additional length and bias parameters, return the internal_fn
4451 for that function, otherwise return IFN_LAST. */
4452 internal_fn
4453 get_len_internal_fn (internal_fn fn)
4455 switch (fn)
4457 #define DEF_INTERNAL_COND_FN(NAME, ...) \
4458 case IFN_COND_##NAME: \
4459 return IFN_COND_LEN_##NAME;
4460 #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...) \
4461 case IFN_COND_##NAME: \
4462 return IFN_COND_LEN_##NAME;
4463 #include "internal-fn.def"
4464 default:
4465 return IFN_LAST;
4469 /* If IFN implements the conditional form of an unconditional internal
4470 function, return that unconditional function, otherwise return IFN_LAST. */
4472 internal_fn
4473 get_unconditional_internal_fn (internal_fn ifn)
4475 switch (ifn)
4477 #define CASE(NAME) \
4478 case IFN_COND_##NAME: \
4479 case IFN_COND_LEN_##NAME: \
4480 return IFN_##NAME;
4481 FOR_EACH_COND_FN_PAIR (CASE)
4482 #undef CASE
4483 default:
4484 return IFN_LAST;
4488 /* Return true if STMT can be interpreted as a conditional tree code
4489 operation of the form:
4491 LHS = COND ? OP (RHS1, ...) : ELSE;
4493 operating elementwise if the operands are vectors. This includes
4494 the case of an all-true COND, so that the operation always happens.
4496 There is an alternative approach to interpret the STMT when the operands
4497 are vectors which is the operation predicated by both conditional mask
4498 and loop control length, the equivalent C code:
4500 for (int i = 0; i < NUNTIS; i++)
4502 if (i < LEN + BIAS && COND[i])
4503 LHS[i] = A[i] CODE B[i];
4504 else
4505 LHS[i] = ELSE[i];
4508 When returning true, set:
4510 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
4511 is known to be all-true
4512 - *CODE_OUT to the tree code
4513 - OPS[I] to operand I of *CODE_OUT
4514 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
4515 condition is known to be all true.
4516 - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
4517 - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE. */
4519 bool
4520 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
4521 tree_code *code_out,
4522 tree (&ops)[3], tree *else_out,
4523 tree *len, tree *bias)
4525 *len = NULL_TREE;
4526 *bias = NULL_TREE;
4527 if (gassign *assign = dyn_cast <gassign *> (stmt))
4529 *cond_out = NULL_TREE;
4530 *code_out = gimple_assign_rhs_code (assign);
4531 ops[0] = gimple_assign_rhs1 (assign);
4532 ops[1] = gimple_assign_rhs2 (assign);
4533 ops[2] = gimple_assign_rhs3 (assign);
4534 *else_out = NULL_TREE;
4535 return true;
4537 if (gcall *call = dyn_cast <gcall *> (stmt))
4538 if (gimple_call_internal_p (call))
4540 internal_fn ifn = gimple_call_internal_fn (call);
4541 tree_code code = conditional_internal_fn_code (ifn);
4542 int len_index = internal_fn_len_index (ifn);
4543 int cond_nargs = len_index >= 0 ? 4 : 2;
4544 if (code != ERROR_MARK)
4546 *cond_out = gimple_call_arg (call, 0);
4547 *code_out = code;
4548 unsigned int nops = gimple_call_num_args (call) - cond_nargs;
4549 for (unsigned int i = 0; i < 3; ++i)
4550 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
4551 *else_out = gimple_call_arg (call, nops + 1);
4552 if (len_index < 0)
4554 if (integer_truep (*cond_out))
4556 *cond_out = NULL_TREE;
4557 *else_out = NULL_TREE;
4560 else
4562 *len = gimple_call_arg (call, len_index);
4563 *bias = gimple_call_arg (call, len_index + 1);
4565 return true;
4568 return false;
4571 /* Return true if IFN is some form of load from memory. */
4573 bool
4574 internal_load_fn_p (internal_fn fn)
4576 switch (fn)
4578 case IFN_MASK_LOAD:
4579 case IFN_LOAD_LANES:
4580 case IFN_MASK_LOAD_LANES:
4581 case IFN_MASK_LEN_LOAD_LANES:
4582 case IFN_GATHER_LOAD:
4583 case IFN_MASK_GATHER_LOAD:
4584 case IFN_MASK_LEN_GATHER_LOAD:
4585 case IFN_LEN_LOAD:
4586 case IFN_MASK_LEN_LOAD:
4587 return true;
4589 default:
4590 return false;
4594 /* Return true if IFN is some form of store to memory. */
4596 bool
4597 internal_store_fn_p (internal_fn fn)
4599 switch (fn)
4601 case IFN_MASK_STORE:
4602 case IFN_STORE_LANES:
4603 case IFN_MASK_STORE_LANES:
4604 case IFN_MASK_LEN_STORE_LANES:
4605 case IFN_SCATTER_STORE:
4606 case IFN_MASK_SCATTER_STORE:
4607 case IFN_MASK_LEN_SCATTER_STORE:
4608 case IFN_LEN_STORE:
4609 case IFN_MASK_LEN_STORE:
4610 return true;
4612 default:
4613 return false;
4617 /* Return true if IFN is some form of gather load or scatter store. */
4619 bool
4620 internal_gather_scatter_fn_p (internal_fn fn)
4622 switch (fn)
4624 case IFN_GATHER_LOAD:
4625 case IFN_MASK_GATHER_LOAD:
4626 case IFN_MASK_LEN_GATHER_LOAD:
4627 case IFN_SCATTER_STORE:
4628 case IFN_MASK_SCATTER_STORE:
4629 case IFN_MASK_LEN_SCATTER_STORE:
4630 return true;
4632 default:
4633 return false;
4637 /* If FN takes a vector len argument, return the index of that argument,
4638 otherwise return -1. */
4641 internal_fn_len_index (internal_fn fn)
4643 switch (fn)
4645 case IFN_LEN_LOAD:
4646 case IFN_LEN_STORE:
4647 return 2;
4649 case IFN_MASK_LEN_GATHER_LOAD:
4650 case IFN_MASK_LEN_SCATTER_STORE:
4651 case IFN_COND_LEN_FMA:
4652 case IFN_COND_LEN_FMS:
4653 case IFN_COND_LEN_FNMA:
4654 case IFN_COND_LEN_FNMS:
4655 return 5;
4657 case IFN_COND_LEN_ADD:
4658 case IFN_COND_LEN_SUB:
4659 case IFN_COND_LEN_MUL:
4660 case IFN_COND_LEN_DIV:
4661 case IFN_COND_LEN_MOD:
4662 case IFN_COND_LEN_RDIV:
4663 case IFN_COND_LEN_MIN:
4664 case IFN_COND_LEN_MAX:
4665 case IFN_COND_LEN_FMIN:
4666 case IFN_COND_LEN_FMAX:
4667 case IFN_COND_LEN_AND:
4668 case IFN_COND_LEN_IOR:
4669 case IFN_COND_LEN_XOR:
4670 case IFN_COND_LEN_SHL:
4671 case IFN_COND_LEN_SHR:
4672 return 4;
4674 case IFN_COND_LEN_NEG:
4675 case IFN_MASK_LEN_LOAD:
4676 case IFN_MASK_LEN_STORE:
4677 case IFN_MASK_LEN_LOAD_LANES:
4678 case IFN_MASK_LEN_STORE_LANES:
4679 case IFN_VCOND_MASK_LEN:
4680 return 3;
4682 default:
4683 return -1;
4687 /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
4688 argument that is used when the condition is false. Return -1 otherwise. */
4691 internal_fn_else_index (internal_fn fn)
4693 switch (fn)
4695 case IFN_COND_NEG:
4696 case IFN_COND_NOT:
4697 case IFN_COND_LEN_NEG:
4698 case IFN_COND_LEN_NOT:
4699 return 2;
4701 case IFN_COND_ADD:
4702 case IFN_COND_SUB:
4703 case IFN_COND_MUL:
4704 case IFN_COND_DIV:
4705 case IFN_COND_MOD:
4706 case IFN_COND_MIN:
4707 case IFN_COND_MAX:
4708 case IFN_COND_FMIN:
4709 case IFN_COND_FMAX:
4710 case IFN_COND_AND:
4711 case IFN_COND_IOR:
4712 case IFN_COND_XOR:
4713 case IFN_COND_SHL:
4714 case IFN_COND_SHR:
4715 case IFN_COND_LEN_ADD:
4716 case IFN_COND_LEN_SUB:
4717 case IFN_COND_LEN_MUL:
4718 case IFN_COND_LEN_DIV:
4719 case IFN_COND_LEN_MOD:
4720 case IFN_COND_LEN_MIN:
4721 case IFN_COND_LEN_MAX:
4722 case IFN_COND_LEN_FMIN:
4723 case IFN_COND_LEN_FMAX:
4724 case IFN_COND_LEN_AND:
4725 case IFN_COND_LEN_IOR:
4726 case IFN_COND_LEN_XOR:
4727 case IFN_COND_LEN_SHL:
4728 case IFN_COND_LEN_SHR:
4729 return 3;
4731 case IFN_COND_FMA:
4732 case IFN_COND_FMS:
4733 case IFN_COND_FNMA:
4734 case IFN_COND_FNMS:
4735 case IFN_COND_LEN_FMA:
4736 case IFN_COND_LEN_FMS:
4737 case IFN_COND_LEN_FNMA:
4738 case IFN_COND_LEN_FNMS:
4739 return 4;
4741 default:
4742 return -1;
4745 return -1;
4748 /* If FN takes a vector mask argument, return the index of that argument,
4749 otherwise return -1. */
4752 internal_fn_mask_index (internal_fn fn)
4754 switch (fn)
4756 case IFN_MASK_LOAD:
4757 case IFN_MASK_LOAD_LANES:
4758 case IFN_MASK_LEN_LOAD_LANES:
4759 case IFN_MASK_STORE:
4760 case IFN_MASK_STORE_LANES:
4761 case IFN_MASK_LEN_STORE_LANES:
4762 case IFN_MASK_LEN_LOAD:
4763 case IFN_MASK_LEN_STORE:
4764 return 2;
4766 case IFN_MASK_GATHER_LOAD:
4767 case IFN_MASK_SCATTER_STORE:
4768 case IFN_MASK_LEN_GATHER_LOAD:
4769 case IFN_MASK_LEN_SCATTER_STORE:
4770 return 4;
4772 case IFN_VCOND_MASK_LEN:
4773 return 0;
4775 default:
4776 return (conditional_internal_fn_code (fn) != ERROR_MARK
4777 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
4781 /* If FN takes a value that should be stored to memory, return the index
4782 of that argument, otherwise return -1. */
4785 internal_fn_stored_value_index (internal_fn fn)
4787 switch (fn)
4789 case IFN_MASK_STORE:
4790 case IFN_MASK_STORE_LANES:
4791 case IFN_SCATTER_STORE:
4792 case IFN_MASK_SCATTER_STORE:
4793 case IFN_MASK_LEN_SCATTER_STORE:
4794 return 3;
4796 case IFN_LEN_STORE:
4797 return 4;
4799 case IFN_MASK_LEN_STORE:
4800 case IFN_MASK_LEN_STORE_LANES:
4801 return 5;
4803 default:
4804 return -1;
4808 /* Return true if the target supports gather load or scatter store function
4809 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
4810 while for stores it is the vector type of the stored data argument.
4811 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
4812 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
4813 offset from the shared base address of each loaded or stored element.
4814 SCALE is the amount by which these offsets should be multiplied
4815 *after* they have been extended to address width. */
4817 bool
4818 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
4819 tree memory_element_type,
4820 tree offset_vector_type, int scale)
4822 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
4823 TYPE_SIZE (memory_element_type)))
4824 return false;
4825 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
4826 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
4827 return false;
4828 optab optab = direct_internal_fn_optab (ifn);
4829 insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
4830 TYPE_MODE (offset_vector_type));
4831 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
4832 bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
4833 return (icode != CODE_FOR_nothing
4834 && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
4835 && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale)));
4838 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
4839 for pointers of type TYPE when the accesses have LENGTH bytes and their
4840 common byte alignment is ALIGN. */
4842 bool
4843 internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
4844 poly_uint64 length, unsigned int align)
4846 machine_mode mode = TYPE_MODE (type);
4847 optab optab = direct_internal_fn_optab (ifn);
4848 insn_code icode = direct_optab_handler (optab, mode);
4849 if (icode == CODE_FOR_nothing)
4850 return false;
4851 rtx length_rtx = immed_wide_int_const (length, mode);
4852 return (insn_operand_matches (icode, 3, length_rtx)
4853 && insn_operand_matches (icode, 4, GEN_INT (align)));
4856 /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
4857 or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and
4858 -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
4859 {len_,mask_len_}store). If none of the biases match what the backend
4860 provides, return VECT_PARTIAL_BIAS_UNSUPPORTED. */
4862 signed char
4863 internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
4865 optab optab = direct_internal_fn_optab (ifn);
4866 insn_code icode = direct_optab_handler (optab, mode);
4867 int bias_no = 3;
4869 if (icode == CODE_FOR_nothing)
4871 machine_mode mask_mode;
4872 if (!targetm.vectorize.get_mask_mode (mode).exists (&mask_mode))
4873 return VECT_PARTIAL_BIAS_UNSUPPORTED;
4874 if (ifn == IFN_LEN_LOAD)
4876 /* Try MASK_LEN_LOAD. */
4877 optab = direct_internal_fn_optab (IFN_MASK_LEN_LOAD);
4879 else
4881 /* Try MASK_LEN_STORE. */
4882 optab = direct_internal_fn_optab (IFN_MASK_LEN_STORE);
4884 icode = convert_optab_handler (optab, mode, mask_mode);
4885 bias_no = 4;
4888 if (icode != CODE_FOR_nothing)
4890 /* For now we only support biases of 0 or -1. Try both of them. */
4891 if (insn_operand_matches (icode, bias_no, GEN_INT (0)))
4892 return 0;
4893 if (insn_operand_matches (icode, bias_no, GEN_INT (-1)))
4894 return -1;
4897 return VECT_PARTIAL_BIAS_UNSUPPORTED;
4900 /* Expand STMT as though it were a call to internal function FN. */
4902 void
4903 expand_internal_call (internal_fn fn, gcall *stmt)
4905 internal_fn_expanders[fn] (fn, stmt);
4908 /* Expand STMT, which is a call to internal function FN. */
4910 void
4911 expand_internal_call (gcall *stmt)
4913 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
4916 /* If TYPE is a vector type, return true if IFN is a direct internal
4917 function that is supported for that type. If TYPE is a scalar type,
4918 return true if IFN is a direct internal function that is supported for
4919 the target's preferred vector version of TYPE. */
4921 bool
4922 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
4924 if (VECTOR_MODE_P (TYPE_MODE (type)))
4925 return direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED);
4927 scalar_mode smode;
4928 if (VECTOR_TYPE_P (type)
4929 || !is_a <scalar_mode> (TYPE_MODE (type), &smode))
4930 return false;
4932 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
4933 if (VECTOR_MODE_P (vmode))
4935 tree vectype = build_vector_type_for_mode (type, vmode);
4936 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4937 return true;
4940 auto_vector_modes vector_modes;
4941 targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
4942 for (machine_mode base_mode : vector_modes)
4943 if (related_vector_mode (base_mode, smode).exists (&vmode))
4945 tree vectype = build_vector_type_for_mode (type, vmode);
4946 if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
4947 return true;
4950 return false;
4953 void
4954 expand_SHUFFLEVECTOR (internal_fn, gcall *)
4956 gcc_unreachable ();
4959 void
4960 expand_PHI (internal_fn, gcall *)
4962 gcc_unreachable ();
4965 void
4966 expand_SPACESHIP (internal_fn, gcall *stmt)
4968 tree lhs = gimple_call_lhs (stmt);
4969 tree rhs1 = gimple_call_arg (stmt, 0);
4970 tree rhs2 = gimple_call_arg (stmt, 1);
4971 tree type = TREE_TYPE (rhs1);
4973 do_pending_stack_adjust ();
4975 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4976 rtx op1 = expand_normal (rhs1);
4977 rtx op2 = expand_normal (rhs2);
4979 class expand_operand ops[3];
4980 create_output_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (lhs)));
4981 create_input_operand (&ops[1], op1, TYPE_MODE (type));
4982 create_input_operand (&ops[2], op2, TYPE_MODE (type));
4983 insn_code icode = optab_handler (spaceship_optab, TYPE_MODE (type));
4984 expand_insn (icode, 3, ops);
4985 if (!rtx_equal_p (target, ops[0].value))
4986 emit_move_insn (target, ops[0].value);
4989 void
4990 expand_ASSUME (internal_fn, gcall *)
4994 void
4995 expand_MASK_CALL (internal_fn, gcall *)
4997 /* This IFN should only exist between ifcvt and vect passes. */
4998 gcc_unreachable ();
5001 void
5002 expand_MULBITINT (internal_fn, gcall *stmt)
5004 rtx_mode_t args[6];
5005 for (int i = 0; i < 6; i++)
5006 args[i] = rtx_mode_t (expand_normal (gimple_call_arg (stmt, i)),
5007 (i & 1) ? SImode : ptr_mode);
5008 rtx fun = init_one_libfunc ("__mulbitint3");
5009 emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 6, args);
5012 void
5013 expand_DIVMODBITINT (internal_fn, gcall *stmt)
5015 rtx_mode_t args[8];
5016 for (int i = 0; i < 8; i++)
5017 args[i] = rtx_mode_t (expand_normal (gimple_call_arg (stmt, i)),
5018 (i & 1) ? SImode : ptr_mode);
5019 rtx fun = init_one_libfunc ("__divmodbitint4");
5020 emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 8, args);
5023 void
5024 expand_FLOATTOBITINT (internal_fn, gcall *stmt)
5026 machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt, 2)));
5027 rtx arg0 = expand_normal (gimple_call_arg (stmt, 0));
5028 rtx arg1 = expand_normal (gimple_call_arg (stmt, 1));
5029 rtx arg2 = expand_normal (gimple_call_arg (stmt, 2));
5030 const char *mname = GET_MODE_NAME (mode);
5031 unsigned mname_len = strlen (mname);
5032 int len = 12 + mname_len;
5033 if (DECIMAL_FLOAT_MODE_P (mode))
5034 len += 4;
5035 char *libfunc_name = XALLOCAVEC (char, len);
5036 char *p = libfunc_name;
5037 const char *q;
5038 if (DECIMAL_FLOAT_MODE_P (mode))
5040 #if ENABLE_DECIMAL_BID_FORMAT
5041 memcpy (p, "__bid_fix", 9);
5042 #else
5043 memcpy (p, "__dpd_fix", 9);
5044 #endif
5045 p += 9;
5047 else
5049 memcpy (p, "__fix", 5);
5050 p += 5;
5052 for (q = mname; *q; q++)
5053 *p++ = TOLOWER (*q);
5054 memcpy (p, "bitint", 7);
5055 rtx fun = init_one_libfunc (libfunc_name);
5056 emit_library_call (fun, LCT_NORMAL, VOIDmode, arg0, ptr_mode, arg1,
5057 SImode, arg2, mode);
5060 void
5061 expand_BITINTTOFLOAT (internal_fn, gcall *stmt)
5063 tree lhs = gimple_call_lhs (stmt);
5064 if (!lhs)
5065 return;
5066 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
5067 rtx arg0 = expand_normal (gimple_call_arg (stmt, 0));
5068 rtx arg1 = expand_normal (gimple_call_arg (stmt, 1));
5069 const char *mname = GET_MODE_NAME (mode);
5070 unsigned mname_len = strlen (mname);
5071 int len = 14 + mname_len;
5072 if (DECIMAL_FLOAT_MODE_P (mode))
5073 len += 4;
5074 char *libfunc_name = XALLOCAVEC (char, len);
5075 char *p = libfunc_name;
5076 const char *q;
5077 if (DECIMAL_FLOAT_MODE_P (mode))
5079 #if ENABLE_DECIMAL_BID_FORMAT
5080 memcpy (p, "__bid_floatbitint", 17);
5081 #else
5082 memcpy (p, "__dpd_floatbitint", 17);
5083 #endif
5084 p += 17;
5086 else
5088 memcpy (p, "__floatbitint", 13);
5089 p += 13;
5091 for (q = mname; *q; q++)
5092 *p++ = TOLOWER (*q);
5093 *p = '\0';
5094 rtx fun = init_one_libfunc (libfunc_name);
5095 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
5096 rtx val = emit_library_call_value (fun, target, LCT_PURE, mode,
5097 arg0, ptr_mode, arg1, SImode);
5098 if (val != target)
5099 emit_move_insn (target, val);
5102 void
5103 expand_POPCOUNT (internal_fn fn, gcall *stmt)
5105 if (gimple_call_num_args (stmt) == 1)
5107 expand_unary_optab_fn (fn, stmt, popcount_optab);
5108 return;
5110 /* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
5111 because the result is only used in an equality comparison against 1.
5112 Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
5113 or (arg ^ (arg - 1)) > arg - 1 is cheaper. */
5114 bool speed_p = optimize_insn_for_speed_p ();
5115 tree lhs = gimple_call_lhs (stmt);
5116 tree arg = gimple_call_arg (stmt, 0);
5117 tree type = TREE_TYPE (arg);
5118 machine_mode mode = TYPE_MODE (type);
5119 do_pending_stack_adjust ();
5120 start_sequence ();
5121 expand_unary_optab_fn (fn, stmt, popcount_optab);
5122 rtx_insn *popcount_insns = get_insns ();
5123 end_sequence ();
5124 start_sequence ();
5125 rtx plhs = expand_normal (lhs);
5126 rtx pcmp = emit_store_flag (NULL_RTX, EQ, plhs, const1_rtx, mode, 0, 0);
5127 if (pcmp == NULL_RTX)
5129 fail:
5130 end_sequence ();
5131 emit_insn (popcount_insns);
5132 return;
5134 rtx_insn *popcount_cmp_insns = get_insns ();
5135 end_sequence ();
5136 start_sequence ();
5137 rtx op0 = expand_normal (arg);
5138 rtx argm1 = expand_simple_binop (mode, PLUS, op0, constm1_rtx, NULL_RTX,
5139 1, OPTAB_DIRECT);
5140 if (argm1 == NULL_RTX)
5141 goto fail;
5142 rtx argxorargm1 = expand_simple_binop (mode, XOR, op0, argm1, NULL_RTX,
5143 1, OPTAB_DIRECT);
5144 if (argxorargm1 == NULL_RTX)
5145 goto fail;
5146 rtx cmp = emit_store_flag (NULL_RTX, GTU, argxorargm1, argm1, mode, 1, 1);
5147 if (cmp == NULL_RTX)
5148 goto fail;
5149 rtx_insn *cmp_insns = get_insns ();
5150 end_sequence ();
5151 unsigned popcount_cost = (seq_cost (popcount_insns, speed_p)
5152 + seq_cost (popcount_cmp_insns, speed_p));
5153 unsigned cmp_cost = seq_cost (cmp_insns, speed_p);
5154 if (popcount_cost <= cmp_cost)
5155 emit_insn (popcount_insns);
5156 else
5158 emit_insn (cmp_insns);
5159 plhs = expand_normal (lhs);
5160 if (GET_MODE (cmp) != GET_MODE (plhs))
5161 cmp = convert_to_mode (GET_MODE (plhs), cmp, 1);
5162 emit_move_insn (plhs, cmp);