2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #define INCLUDE_ALGORITHM
25 #include "coretypes.h"
32 #include "tree-pass.h"
34 #include "optabs-tree.h"
37 #include "diagnostic-core.h"
38 #include "fold-const.h"
39 #include "stor-layout.h"
42 #include "gimple-iterator.h"
43 #include "gimplify-me.h"
44 #include "tree-ssa-loop-ivopts.h"
45 #include "tree-ssa-loop-manip.h"
46 #include "tree-ssa-loop-niter.h"
47 #include "tree-ssa-loop.h"
49 #include "tree-scalar-evolution.h"
50 #include "tree-vectorizer.h"
51 #include "gimple-fold.h"
54 #include "tree-if-conv.h"
55 #include "internal-fn.h"
56 #include "tree-vector-builder.h"
57 #include "vec-perm-indices.h"
59 #include "case-cfn-macros.h"
60 #include "langhooks.h"
62 /* Loop Vectorization Pass.
64 This pass tries to vectorize loops.
66 For example, the vectorizer transforms the following simple loop:
68 short a[N]; short b[N]; short c[N]; int i;
74 as if it was manually vectorized by rewriting the source code into:
76 typedef int __attribute__((mode(V8HI))) v8hi;
77 short a[N]; short b[N]; short c[N]; int i;
78 v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
81 for (i=0; i<N/8; i++){
88 The main entry to this pass is vectorize_loops(), in which
89 the vectorizer applies a set of analyses on a given set of loops,
90 followed by the actual vectorization transformation for the loops that
91 had successfully passed the analysis phase.
92 Throughout this pass we make a distinction between two types of
93 data: scalars (which are represented by SSA_NAMES), and memory references
94 ("data-refs"). These two types of data require different handling both
95 during analysis and transformation. The types of data-refs that the
96 vectorizer currently supports are ARRAY_REFS which base is an array DECL
97 (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
98 accesses are required to have a simple (consecutive) access pattern.
102 The driver for the analysis phase is vect_analyze_loop().
103 It applies a set of analyses, some of which rely on the scalar evolution
104 analyzer (scev) developed by Sebastian Pop.
106 During the analysis phase the vectorizer records some information
107 per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
108 loop, as well as general information about the loop as a whole, which is
109 recorded in a "loop_vec_info" struct attached to each loop.
111 Transformation phase:
112 =====================
113 The loop transformation phase scans all the stmts in the loop, and
114 creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
115 the loop that needs to be vectorized. It inserts the vector code sequence
116 just before the scalar stmt S, and records a pointer to the vector code
117 in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
118 attached to S). This pointer will be used for the vectorization of following
119 stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
120 otherwise, we rely on dead code elimination for removing it.
122 For example, say stmt S1 was vectorized into stmt VS1:
125 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
128 To vectorize stmt S2, the vectorizer first finds the stmt that defines
129 the operand 'b' (S1), and gets the relevant vector def 'vb' from the
130 vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
131 resulting sequence would be:
134 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
136 S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
138 Operands that are not SSA_NAMEs, are data-refs that appear in
139 load/store operations (like 'x[i]' in S1), and are handled differently.
143 Currently the only target specific information that is used is the
144 size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
145 Targets that can support different sizes of vectors, for now will need
146 to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
147 flexibility will be added in the future.
149 Since we only vectorize operations which vector form can be
150 expressed using existing tree codes, to verify that an operation is
151 supported, the vectorizer checks the relevant optab at the relevant
152 machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
153 the value found is CODE_FOR_nothing, then there's no target support, and
154 we can't vectorize the stmt.
156 For additional information on this project see:
157 http://gcc.gnu.org/projects/tree-ssa/vectorization.html
160 static void vect_estimate_min_profitable_iters (loop_vec_info
, int *, int *,
162 static stmt_vec_info
vect_is_simple_reduction (loop_vec_info
, stmt_vec_info
,
163 bool *, bool *, bool);
165 /* Subroutine of vect_determine_vf_for_stmt that handles only one
166 statement. VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
167 may already be set for general statements (not just data refs). */
170 vect_determine_vf_for_stmt_1 (vec_info
*vinfo
, stmt_vec_info stmt_info
,
171 bool vectype_maybe_set_p
,
174 gimple
*stmt
= stmt_info
->stmt
;
176 if ((!STMT_VINFO_RELEVANT_P (stmt_info
)
177 && !STMT_VINFO_LIVE_P (stmt_info
))
178 || gimple_clobber_p (stmt
))
180 if (dump_enabled_p ())
181 dump_printf_loc (MSG_NOTE
, vect_location
, "skip.\n");
182 return opt_result::success ();
185 tree stmt_vectype
, nunits_vectype
;
186 opt_result res
= vect_get_vector_types_for_stmt (vinfo
, stmt_info
,
194 if (STMT_VINFO_VECTYPE (stmt_info
))
195 /* The only case when a vectype had been already set is for stmts
196 that contain a data ref, or for "pattern-stmts" (stmts generated
197 by the vectorizer to represent/replace a certain idiom). */
198 gcc_assert ((STMT_VINFO_DATA_REF (stmt_info
)
199 || vectype_maybe_set_p
)
200 && STMT_VINFO_VECTYPE (stmt_info
) == stmt_vectype
);
202 STMT_VINFO_VECTYPE (stmt_info
) = stmt_vectype
;
206 vect_update_max_nunits (vf
, nunits_vectype
);
208 return opt_result::success ();
211 /* Subroutine of vect_determine_vectorization_factor. Set the vector
212 types of STMT_INFO and all attached pattern statements and update
213 the vectorization factor VF accordingly. Return true on success
214 or false if something prevented vectorization. */
217 vect_determine_vf_for_stmt (vec_info
*vinfo
,
218 stmt_vec_info stmt_info
, poly_uint64
*vf
)
220 if (dump_enabled_p ())
221 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: %G",
223 opt_result res
= vect_determine_vf_for_stmt_1 (vinfo
, stmt_info
, false, vf
);
227 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
228 && STMT_VINFO_RELATED_STMT (stmt_info
))
230 gimple
*pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
231 stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
233 /* If a pattern statement has def stmts, analyze them too. */
234 for (gimple_stmt_iterator si
= gsi_start (pattern_def_seq
);
235 !gsi_end_p (si
); gsi_next (&si
))
237 stmt_vec_info def_stmt_info
= vinfo
->lookup_stmt (gsi_stmt (si
));
238 if (dump_enabled_p ())
239 dump_printf_loc (MSG_NOTE
, vect_location
,
240 "==> examining pattern def stmt: %G",
241 def_stmt_info
->stmt
);
242 res
= vect_determine_vf_for_stmt_1 (vinfo
, def_stmt_info
, true, vf
);
247 if (dump_enabled_p ())
248 dump_printf_loc (MSG_NOTE
, vect_location
,
249 "==> examining pattern statement: %G",
251 res
= vect_determine_vf_for_stmt_1 (vinfo
, stmt_info
, true, vf
);
256 return opt_result::success ();
259 /* Function vect_determine_vectorization_factor
261 Determine the vectorization factor (VF). VF is the number of data elements
262 that are operated upon in parallel in a single iteration of the vectorized
263 loop. For example, when vectorizing a loop that operates on 4byte elements,
264 on a target with vector size (VS) 16byte, the VF is set to 4, since 4
265 elements can fit in a single vector register.
267 We currently support vectorization of loops in which all types operated upon
268 are of the same size. Therefore this function currently sets VF according to
269 the size of the types operated upon, and fails if there are multiple sizes
272 VF is also the factor by which the loop iterations are strip-mined, e.g.:
279 for (i=0; i<N; i+=VF){
280 a[i:VF] = b[i:VF] + c[i:VF];
285 vect_determine_vectorization_factor (loop_vec_info loop_vinfo
)
287 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
288 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
289 unsigned nbbs
= loop
->num_nodes
;
290 poly_uint64 vectorization_factor
= 1;
291 tree scalar_type
= NULL_TREE
;
294 stmt_vec_info stmt_info
;
297 DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
299 for (i
= 0; i
< nbbs
; i
++)
301 basic_block bb
= bbs
[i
];
303 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
307 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
308 if (dump_enabled_p ())
309 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining phi: %G",
312 gcc_assert (stmt_info
);
314 if (STMT_VINFO_RELEVANT_P (stmt_info
)
315 || STMT_VINFO_LIVE_P (stmt_info
))
317 gcc_assert (!STMT_VINFO_VECTYPE (stmt_info
));
318 scalar_type
= TREE_TYPE (PHI_RESULT (phi
));
320 if (dump_enabled_p ())
321 dump_printf_loc (MSG_NOTE
, vect_location
,
322 "get vectype for scalar type: %T\n",
325 vectype
= get_vectype_for_scalar_type (loop_vinfo
, scalar_type
);
327 return opt_result::failure_at (phi
,
328 "not vectorized: unsupported "
331 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
333 if (dump_enabled_p ())
334 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: %T\n",
337 if (dump_enabled_p ())
339 dump_printf_loc (MSG_NOTE
, vect_location
, "nunits = ");
340 dump_dec (MSG_NOTE
, TYPE_VECTOR_SUBPARTS (vectype
));
341 dump_printf (MSG_NOTE
, "\n");
344 vect_update_max_nunits (&vectorization_factor
, vectype
);
348 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
351 if (is_gimple_debug (gsi_stmt (si
)))
353 stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
355 = vect_determine_vf_for_stmt (loop_vinfo
,
356 stmt_info
, &vectorization_factor
);
362 /* TODO: Analyze cost. Decide if worth while to vectorize. */
363 if (dump_enabled_p ())
365 dump_printf_loc (MSG_NOTE
, vect_location
, "vectorization factor = ");
366 dump_dec (MSG_NOTE
, vectorization_factor
);
367 dump_printf (MSG_NOTE
, "\n");
370 if (known_le (vectorization_factor
, 1U))
371 return opt_result::failure_at (vect_location
,
372 "not vectorized: unsupported data-type\n");
373 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
374 return opt_result::success ();
378 /* Function vect_is_simple_iv_evolution.
380 FORNOW: A simple evolution of an induction variables in the loop is
381 considered a polynomial evolution. */
384 vect_is_simple_iv_evolution (unsigned loop_nb
, tree access_fn
, tree
* init
,
389 tree evolution_part
= evolution_part_in_loop_num (access_fn
, loop_nb
);
392 /* When there is no evolution in this loop, the evolution function
394 if (evolution_part
== NULL_TREE
)
397 /* When the evolution is a polynomial of degree >= 2
398 the evolution function is not "simple". */
399 if (tree_is_chrec (evolution_part
))
402 step_expr
= evolution_part
;
403 init_expr
= unshare_expr (initial_condition_in_loop_num (access_fn
, loop_nb
));
405 if (dump_enabled_p ())
406 dump_printf_loc (MSG_NOTE
, vect_location
, "step: %T, init: %T\n",
407 step_expr
, init_expr
);
412 if (TREE_CODE (step_expr
) != INTEGER_CST
413 && (TREE_CODE (step_expr
) != SSA_NAME
414 || ((bb
= gimple_bb (SSA_NAME_DEF_STMT (step_expr
)))
415 && flow_bb_inside_loop_p (get_loop (cfun
, loop_nb
), bb
))
416 || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr
))
417 && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
))
418 || !flag_associative_math
)))
419 && (TREE_CODE (step_expr
) != REAL_CST
420 || !flag_associative_math
))
422 if (dump_enabled_p ())
423 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
431 /* Function vect_is_nonlinear_iv_evolution
433 Only support nonlinear induction for integer type
436 3. lshift/rshift by constant.
438 For neg induction, return a fake step as integer -1. */
440 vect_is_nonlinear_iv_evolution (class loop
* loop
, stmt_vec_info stmt_info
,
441 gphi
* loop_phi_node
, tree
*init
, tree
*step
)
443 tree init_expr
, ev_expr
, result
, op1
, op2
;
446 if (gimple_phi_num_args (loop_phi_node
) != 2)
449 init_expr
= PHI_ARG_DEF_FROM_EDGE (loop_phi_node
, loop_preheader_edge (loop
));
450 ev_expr
= PHI_ARG_DEF_FROM_EDGE (loop_phi_node
, loop_latch_edge (loop
));
452 /* Support nonlinear induction only for integer type. */
453 if (!INTEGRAL_TYPE_P (TREE_TYPE (init_expr
)))
457 result
= PHI_RESULT (loop_phi_node
);
459 if (TREE_CODE (ev_expr
) != SSA_NAME
460 || ((def
= SSA_NAME_DEF_STMT (ev_expr
)), false)
461 || !is_gimple_assign (def
))
464 enum tree_code t_code
= gimple_assign_rhs_code (def
);
468 if (gimple_assign_rhs1 (def
) != result
)
470 *step
= build_int_cst (TREE_TYPE (init_expr
), -1);
471 STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
) = vect_step_op_neg
;
477 op1
= gimple_assign_rhs1 (def
);
478 op2
= gimple_assign_rhs2 (def
);
479 if (TREE_CODE (op2
) != INTEGER_CST
483 if (t_code
== LSHIFT_EXPR
)
484 STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
) = vect_step_op_shl
;
485 else if (t_code
== RSHIFT_EXPR
)
486 STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
) = vect_step_op_shr
;
487 /* NEGATE_EXPR and MULT_EXPR are both vect_step_op_mul. */
489 STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
) = vect_step_op_mul
;
496 STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_info
) = *init
;
497 STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info
) = *step
;
502 /* Return true if PHI, described by STMT_INFO, is the inner PHI in
503 what we are assuming is a double reduction. For example, given
504 a structure like this:
507 x_1 = PHI <x_4(outer2), ...>;
511 x_2 = PHI <x_1(outer1), ...>;
517 x_4 = PHI <x_3(inner)>;
520 outer loop analysis would treat x_1 as a double reduction phi and
521 this function would then return true for x_2. */
524 vect_inner_phi_in_double_reduction_p (loop_vec_info loop_vinfo
, gphi
*phi
)
528 FOR_EACH_PHI_ARG (use_p
, phi
, op_iter
, SSA_OP_USE
)
529 if (stmt_vec_info def_info
= loop_vinfo
->lookup_def (USE_FROM_PTR (use_p
)))
530 if (STMT_VINFO_DEF_TYPE (def_info
) == vect_double_reduction_def
)
535 /* Returns true if Phi is a first-order recurrence. A first-order
536 recurrence is a non-reduction recurrence relation in which the value of
537 the recurrence in the current loop iteration equals a value defined in
538 the previous iteration. */
541 vect_phi_first_order_recurrence_p (loop_vec_info loop_vinfo
, class loop
*loop
,
544 /* A nested cycle isn't vectorizable as first order recurrence. */
545 if (LOOP_VINFO_LOOP (loop_vinfo
) != loop
)
548 /* Ensure the loop latch definition is from within the loop. */
549 edge latch
= loop_latch_edge (loop
);
550 tree ldef
= PHI_ARG_DEF_FROM_EDGE (phi
, latch
);
551 if (TREE_CODE (ldef
) != SSA_NAME
552 || SSA_NAME_IS_DEFAULT_DEF (ldef
)
553 || is_a
<gphi
*> (SSA_NAME_DEF_STMT (ldef
))
554 || !flow_bb_inside_loop_p (loop
, gimple_bb (SSA_NAME_DEF_STMT (ldef
))))
557 tree def
= gimple_phi_result (phi
);
559 /* Ensure every use_stmt of the phi node is dominated by the latch
561 imm_use_iterator imm_iter
;
563 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, def
)
564 if (!is_gimple_debug (USE_STMT (use_p
))
565 && (SSA_NAME_DEF_STMT (ldef
) == USE_STMT (use_p
)
566 || !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (ldef
),
570 /* First-order recurrence autovectorization needs shuffle vector. */
571 tree scalar_type
= TREE_TYPE (def
);
572 tree vectype
= get_vectype_for_scalar_type (loop_vinfo
, scalar_type
);
579 /* Function vect_analyze_scalar_cycles_1.
581 Examine the cross iteration def-use cycles of scalar variables
582 in LOOP. LOOP_VINFO represents the loop that is now being
583 considered for vectorization (can be LOOP, or an outer-loop
584 enclosing LOOP). SLP indicates there will be some subsequent
585 slp analyses or not. */
588 vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo
, class loop
*loop
,
591 basic_block bb
= loop
->header
;
593 auto_vec
<stmt_vec_info
, 64> worklist
;
595 bool double_reduc
, reduc_chain
;
597 DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
599 /* First - identify all inductions. Reduction detection assumes that all the
600 inductions have been identified, therefore, this order must not be
602 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
604 gphi
*phi
= gsi
.phi ();
605 tree access_fn
= NULL
;
606 tree def
= PHI_RESULT (phi
);
607 stmt_vec_info stmt_vinfo
= loop_vinfo
->lookup_stmt (phi
);
609 if (dump_enabled_p ())
610 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: %G",
613 /* Skip virtual phi's. The data dependences that are associated with
614 virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
615 if (virtual_operand_p (def
))
618 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_unknown_def_type
;
620 /* Analyze the evolution function. */
621 access_fn
= analyze_scalar_evolution (loop
, def
);
624 STRIP_NOPS (access_fn
);
625 if (dump_enabled_p ())
626 dump_printf_loc (MSG_NOTE
, vect_location
,
627 "Access function of PHI: %T\n", access_fn
);
628 STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
)
629 = initial_condition_in_loop_num (access_fn
, loop
->num
);
630 STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
)
631 = evolution_part_in_loop_num (access_fn
, loop
->num
);
635 || vect_inner_phi_in_double_reduction_p (loop_vinfo
, phi
)
636 || !vect_is_simple_iv_evolution (loop
->num
, access_fn
,
638 || (LOOP_VINFO_LOOP (loop_vinfo
) != loop
639 && TREE_CODE (step
) != INTEGER_CST
))
640 /* Only handle nonlinear iv for same loop. */
641 && (LOOP_VINFO_LOOP (loop_vinfo
) != loop
642 || !vect_is_nonlinear_iv_evolution (loop
, stmt_vinfo
,
645 worklist
.safe_push (stmt_vinfo
);
649 gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
)
651 gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
) != NULL_TREE
);
653 if (dump_enabled_p ())
654 dump_printf_loc (MSG_NOTE
, vect_location
, "Detected induction.\n");
655 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_induction_def
;
659 /* Second - identify all reductions and nested cycles. */
660 while (worklist
.length () > 0)
662 stmt_vec_info stmt_vinfo
= worklist
.pop ();
663 gphi
*phi
= as_a
<gphi
*> (stmt_vinfo
->stmt
);
664 tree def
= PHI_RESULT (phi
);
666 if (dump_enabled_p ())
667 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: %G",
670 gcc_assert (!virtual_operand_p (def
)
671 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_unknown_def_type
);
673 stmt_vec_info reduc_stmt_info
674 = vect_is_simple_reduction (loop_vinfo
, stmt_vinfo
, &double_reduc
,
678 STMT_VINFO_REDUC_DEF (stmt_vinfo
) = reduc_stmt_info
;
679 STMT_VINFO_REDUC_DEF (reduc_stmt_info
) = stmt_vinfo
;
682 if (dump_enabled_p ())
683 dump_printf_loc (MSG_NOTE
, vect_location
,
684 "Detected double reduction.\n");
686 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_double_reduction_def
;
687 STMT_VINFO_DEF_TYPE (reduc_stmt_info
) = vect_double_reduction_def
;
691 if (loop
!= LOOP_VINFO_LOOP (loop_vinfo
))
693 if (dump_enabled_p ())
694 dump_printf_loc (MSG_NOTE
, vect_location
,
695 "Detected vectorizable nested cycle.\n");
697 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_nested_cycle
;
701 if (dump_enabled_p ())
702 dump_printf_loc (MSG_NOTE
, vect_location
,
703 "Detected reduction.\n");
705 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_reduction_def
;
706 STMT_VINFO_DEF_TYPE (reduc_stmt_info
) = vect_reduction_def
;
707 /* Store the reduction cycles for possible vectorization in
708 loop-aware SLP if it was not detected as reduction
711 LOOP_VINFO_REDUCTIONS (loop_vinfo
).safe_push
716 else if (vect_phi_first_order_recurrence_p (loop_vinfo
, loop
, phi
))
717 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_first_order_recurrence
;
719 if (dump_enabled_p ())
720 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
721 "Unknown def-use cycle pattern.\n");
726 /* Function vect_analyze_scalar_cycles.
728 Examine the cross iteration def-use cycles of scalar variables, by
729 analyzing the loop-header PHIs of scalar variables. Classify each
730 cycle as one of the following: invariant, induction, reduction, unknown.
731 We do that for the loop represented by LOOP_VINFO, and also to its
732 inner-loop, if exists.
733 Examples for scalar cycles:
748 vect_analyze_scalar_cycles (loop_vec_info loop_vinfo
, bool slp
)
750 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
752 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
, slp
);
754 /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
755 Reductions in such inner-loop therefore have different properties than
756 the reductions in the nest that gets vectorized:
757 1. When vectorized, they are executed in the same order as in the original
758 scalar loop, so we can't change the order of computation when
760 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
761 current checks are too strict. */
764 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
->inner
, slp
);
767 /* Transfer group and reduction information from STMT_INFO to its
771 vect_fixup_reduc_chain (stmt_vec_info stmt_info
)
773 stmt_vec_info firstp
= STMT_VINFO_RELATED_STMT (stmt_info
);
775 gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp
)
776 && REDUC_GROUP_FIRST_ELEMENT (stmt_info
));
777 REDUC_GROUP_SIZE (firstp
) = REDUC_GROUP_SIZE (stmt_info
);
780 stmtp
= STMT_VINFO_RELATED_STMT (stmt_info
);
781 gcc_checking_assert (STMT_VINFO_DEF_TYPE (stmtp
)
782 == STMT_VINFO_DEF_TYPE (stmt_info
));
783 REDUC_GROUP_FIRST_ELEMENT (stmtp
) = firstp
;
784 stmt_info
= REDUC_GROUP_NEXT_ELEMENT (stmt_info
);
786 REDUC_GROUP_NEXT_ELEMENT (stmtp
)
787 = STMT_VINFO_RELATED_STMT (stmt_info
);
792 /* Fixup scalar cycles that now have their stmts detected as patterns. */
795 vect_fixup_scalar_cycles_with_patterns (loop_vec_info loop_vinfo
)
800 FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
), i
, first
)
802 stmt_vec_info next
= REDUC_GROUP_NEXT_ELEMENT (first
);
805 if ((STMT_VINFO_IN_PATTERN_P (next
)
806 != STMT_VINFO_IN_PATTERN_P (first
))
807 || STMT_VINFO_REDUC_IDX (vect_stmt_to_vectorize (next
)) == -1)
809 next
= REDUC_GROUP_NEXT_ELEMENT (next
);
811 /* If all reduction chain members are well-formed patterns adjust
812 the group to group the pattern stmts instead. */
814 && STMT_VINFO_REDUC_IDX (vect_stmt_to_vectorize (first
)) != -1)
816 if (STMT_VINFO_IN_PATTERN_P (first
))
818 vect_fixup_reduc_chain (first
);
819 LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
)[i
]
820 = STMT_VINFO_RELATED_STMT (first
);
823 /* If not all stmt in the chain are patterns or if we failed
824 to update STMT_VINFO_REDUC_IDX dissolve the chain and handle
825 it as regular reduction instead. */
828 stmt_vec_info vinfo
= first
;
829 stmt_vec_info last
= NULL
;
832 next
= REDUC_GROUP_NEXT_ELEMENT (vinfo
);
833 REDUC_GROUP_FIRST_ELEMENT (vinfo
) = NULL
;
834 REDUC_GROUP_NEXT_ELEMENT (vinfo
) = NULL
;
838 STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize (first
))
840 loop_vinfo
->reductions
.safe_push (vect_stmt_to_vectorize (last
));
841 LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
).unordered_remove (i
);
847 /* Function vect_get_loop_niters.
849 Determine how many iterations the loop is executed and place it
850 in NUMBER_OF_ITERATIONS. Place the number of latch iterations
851 in NUMBER_OF_ITERATIONSM1. Place the condition under which the
852 niter information holds in ASSUMPTIONS.
854 Return the loop exit conditions. */
858 vect_get_loop_niters (class loop
*loop
, const_edge main_exit
, tree
*assumptions
,
859 tree
*number_of_iterations
, tree
*number_of_iterationsm1
)
861 auto_vec
<edge
> exits
= get_loop_exit_edges (loop
);
863 conds
.create (exits
.length ());
864 class tree_niter_desc niter_desc
;
865 tree niter_assumptions
, niter
, may_be_zero
;
867 *assumptions
= boolean_true_node
;
868 *number_of_iterationsm1
= chrec_dont_know
;
869 *number_of_iterations
= chrec_dont_know
;
871 DUMP_VECT_SCOPE ("get_loop_niters");
873 if (exits
.is_empty ())
876 if (dump_enabled_p ())
877 dump_printf_loc (MSG_NOTE
, vect_location
, "Loop has %d exits.\n",
882 FOR_EACH_VEC_ELT (exits
, i
, exit
)
884 gcond
*cond
= get_loop_exit_condition (exit
);
886 conds
.safe_push (cond
);
888 if (dump_enabled_p ())
889 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyzing exit %d...\n", i
);
891 if (exit
!= main_exit
)
894 may_be_zero
= NULL_TREE
;
895 if (!number_of_iterations_exit_assumptions (loop
, exit
, &niter_desc
, NULL
)
896 || chrec_contains_undetermined (niter_desc
.niter
))
899 niter_assumptions
= niter_desc
.assumptions
;
900 may_be_zero
= niter_desc
.may_be_zero
;
901 niter
= niter_desc
.niter
;
903 if (may_be_zero
&& integer_zerop (may_be_zero
))
904 may_be_zero
= NULL_TREE
;
908 if (COMPARISON_CLASS_P (may_be_zero
))
910 /* Try to combine may_be_zero with assumptions, this can simplify
911 computation of niter expression. */
912 if (niter_assumptions
&& !integer_nonzerop (niter_assumptions
))
913 niter_assumptions
= fold_build2 (TRUTH_AND_EXPR
, boolean_type_node
,
915 fold_build1 (TRUTH_NOT_EXPR
,
919 niter
= fold_build3 (COND_EXPR
, TREE_TYPE (niter
), may_be_zero
,
920 build_int_cst (TREE_TYPE (niter
), 0),
921 rewrite_to_non_trapping_overflow (niter
));
923 may_be_zero
= NULL_TREE
;
925 else if (integer_nonzerop (may_be_zero
))
927 *number_of_iterationsm1
= build_int_cst (TREE_TYPE (niter
), 0);
928 *number_of_iterations
= build_int_cst (TREE_TYPE (niter
), 1);
935 /* Loop assumptions are based off the normal exit. */
936 *assumptions
= niter_assumptions
;
937 *number_of_iterationsm1
= niter
;
939 /* We want the number of loop header executions which is the number
940 of latch executions plus one.
941 ??? For UINT_MAX latch executions this number overflows to zero
942 for loops like do { n++; } while (n != 0); */
943 if (niter
&& !chrec_contains_undetermined (niter
))
945 niter
= fold_build2 (PLUS_EXPR
, TREE_TYPE (niter
),
946 unshare_expr (niter
),
947 build_int_cst (TREE_TYPE (niter
), 1));
948 if (TREE_CODE (niter
) == INTEGER_CST
949 && TREE_CODE (*number_of_iterationsm1
) != INTEGER_CST
)
951 /* If we manage to fold niter + 1 into INTEGER_CST even when
952 niter is some complex expression, ensure back
953 *number_of_iterationsm1 is an INTEGER_CST as well. See
955 *number_of_iterationsm1
956 = fold_build2 (PLUS_EXPR
, TREE_TYPE (niter
), niter
,
957 build_minus_one_cst (TREE_TYPE (niter
)));
960 *number_of_iterations
= niter
;
963 if (dump_enabled_p ())
964 dump_printf_loc (MSG_NOTE
, vect_location
, "All loop exits successfully analyzed.\n");
969 /* Determine the main loop exit for the vectorizer. */
972 vec_init_loop_exit_info (class loop
*loop
)
974 /* Before we begin we must first determine which exit is the main one and
975 which are auxilary exits. */
976 auto_vec
<edge
> exits
= get_loop_exit_edges (loop
);
977 if (exits
.length () == 1)
980 /* If we have multiple exits we only support counting IV at the moment.
981 Analyze all exits and return the last one we can analyze. */
982 class tree_niter_desc niter_desc
;
983 edge candidate
= NULL
;
984 for (edge exit
: exits
)
986 if (!get_loop_exit_condition (exit
))
989 if (number_of_iterations_exit_assumptions (loop
, exit
, &niter_desc
, NULL
)
990 && !chrec_contains_undetermined (niter_desc
.niter
))
992 tree may_be_zero
= niter_desc
.may_be_zero
;
993 if ((integer_zerop (may_be_zero
)
994 /* As we are handling may_be_zero that's not false by
995 rewriting niter to may_be_zero ? 0 : niter we require
997 || (single_pred_p (loop
->latch
)
998 && exit
->src
== single_pred (loop
->latch
)
999 && (integer_nonzerop (may_be_zero
)
1000 || COMPARISON_CLASS_P (may_be_zero
))))
1002 || dominated_by_p (CDI_DOMINATORS
, exit
->src
,
1011 /* Function bb_in_loop_p
1013 Used as predicate for dfs order traversal of the loop bbs. */
1016 bb_in_loop_p (const_basic_block bb
, const void *data
)
1018 const class loop
*const loop
= (const class loop
*)data
;
1019 if (flow_bb_inside_loop_p (loop
, bb
))
1025 /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
1026 stmt_vec_info structs for all the stmts in LOOP_IN. */
1028 _loop_vec_info::_loop_vec_info (class loop
*loop_in
, vec_info_shared
*shared
)
1029 : vec_info (vec_info::loop
, shared
),
1031 bbs (XCNEWVEC (basic_block
, loop
->num_nodes
)),
1032 num_itersm1 (NULL_TREE
),
1033 num_iters (NULL_TREE
),
1034 num_iters_unchanged (NULL_TREE
),
1035 num_iters_assumptions (NULL_TREE
),
1036 vector_costs (nullptr),
1037 scalar_costs (nullptr),
1039 versioning_threshold (0),
1040 vectorization_factor (0),
1041 main_loop_edge (nullptr),
1042 skip_main_loop_edge (nullptr),
1043 skip_this_loop_edge (nullptr),
1044 reusable_accumulators (),
1045 suggested_unroll_factor (1),
1046 max_vectorization_factor (0),
1047 mask_skip_niters (NULL_TREE
),
1048 rgroup_compare_type (NULL_TREE
),
1049 simd_if_cond (NULL_TREE
),
1050 partial_vector_style (vect_partial_vectors_none
),
1051 unaligned_dr (NULL
),
1052 peeling_for_alignment (0),
1056 slp_unrolling_factor (1),
1057 inner_loop_cost_factor (param_vect_inner_loop_cost_factor
),
1058 vectorizable (false),
1059 can_use_partial_vectors_p (param_vect_partial_vector_usage
!= 0),
1060 using_partial_vectors_p (false),
1061 using_decrementing_iv_p (false),
1062 using_select_vl_p (false),
1063 epil_using_partial_vectors_p (false),
1064 partial_load_store_bias (0),
1065 peeling_for_gaps (false),
1066 peeling_for_niter (false),
1067 early_breaks (false),
1068 no_data_dependencies (false),
1069 has_mask_store (false),
1070 scalar_loop_scaling (profile_probability::uninitialized ()),
1072 orig_loop_info (NULL
),
1073 vec_loop_iv_exit (NULL
),
1074 vec_epilogue_loop_iv_exit (NULL
),
1075 scalar_loop_iv_exit (NULL
)
1077 /* CHECKME: We want to visit all BBs before their successors (except for
1078 latch blocks, for which this assertion wouldn't hold). In the simple
1079 case of the loop forms we allow, a dfs order of the BBs would the same
1080 as reversed postorder traversal, so we are safe. */
1082 unsigned int nbbs
= dfs_enumerate_from (loop
->header
, 0, bb_in_loop_p
,
1083 bbs
, loop
->num_nodes
, loop
);
1084 gcc_assert (nbbs
== loop
->num_nodes
);
1086 for (unsigned int i
= 0; i
< nbbs
; i
++)
1088 basic_block bb
= bbs
[i
];
1089 gimple_stmt_iterator si
;
1091 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
1093 gimple
*phi
= gsi_stmt (si
);
1094 gimple_set_uid (phi
, 0);
1098 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1100 gimple
*stmt
= gsi_stmt (si
);
1101 gimple_set_uid (stmt
, 0);
1102 if (is_gimple_debug (stmt
))
1105 /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
1106 third argument is the #pragma omp simd if (x) condition, when 0,
1107 loop shouldn't be vectorized, when non-zero constant, it should
1108 be vectorized normally, otherwise versioned with vectorized loop
1109 done if the condition is non-zero at runtime. */
1110 if (loop_in
->simduid
1111 && is_gimple_call (stmt
)
1112 && gimple_call_internal_p (stmt
)
1113 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
1114 && gimple_call_num_args (stmt
) >= 3
1115 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
1116 && (loop_in
->simduid
1117 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0))))
1119 tree arg
= gimple_call_arg (stmt
, 2);
1120 if (integer_zerop (arg
) || TREE_CODE (arg
) == SSA_NAME
)
1123 gcc_assert (integer_nonzerop (arg
));
1128 epilogue_vinfos
.create (6);
1131 /* Free all levels of rgroup CONTROLS. */
1134 release_vec_loop_controls (vec
<rgroup_controls
> *controls
)
1136 rgroup_controls
*rgc
;
1138 FOR_EACH_VEC_ELT (*controls
, i
, rgc
)
1139 rgc
->controls
.release ();
1140 controls
->release ();
1143 /* Free all memory used by the _loop_vec_info, as well as all the
1144 stmt_vec_info structs of all the stmts in the loop. */
1146 _loop_vec_info::~_loop_vec_info ()
1150 release_vec_loop_controls (&masks
.rgc_vec
);
1151 release_vec_loop_controls (&lens
);
1154 epilogue_vinfos
.release ();
1155 delete scalar_costs
;
1156 delete vector_costs
;
1158 /* When we release an epiloge vinfo that we do not intend to use
1159 avoid clearing AUX of the main loop which should continue to
1160 point to the main loop vinfo since otherwise we'll leak that. */
1161 if (loop
->aux
== this)
1165 /* Return an invariant or register for EXPR and emit necessary
1166 computations in the LOOP_VINFO loop preheader. */
1169 cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo
, tree expr
)
1171 if (is_gimple_reg (expr
)
1172 || is_gimple_min_invariant (expr
))
1175 if (! loop_vinfo
->ivexpr_map
)
1176 loop_vinfo
->ivexpr_map
= new hash_map
<tree_operand_hash
, tree
>;
1177 tree
&cached
= loop_vinfo
->ivexpr_map
->get_or_insert (expr
);
1180 gimple_seq stmts
= NULL
;
1181 cached
= force_gimple_operand (unshare_expr (expr
),
1182 &stmts
, true, NULL_TREE
);
1185 edge e
= loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo
));
1186 gsi_insert_seq_on_edge_immediate (e
, stmts
);
1192 /* Return true if we can use CMP_TYPE as the comparison type to produce
1193 all masks required to mask LOOP_VINFO. */
1196 can_produce_all_loop_masks_p (loop_vec_info loop_vinfo
, tree cmp_type
)
1198 rgroup_controls
*rgm
;
1200 FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
, i
, rgm
)
1201 if (rgm
->type
!= NULL_TREE
1202 && !direct_internal_fn_supported_p (IFN_WHILE_ULT
,
1203 cmp_type
, rgm
->type
,
1204 OPTIMIZE_FOR_SPEED
))
1209 /* Calculate the maximum number of scalars per iteration for every
1210 rgroup in LOOP_VINFO. */
1213 vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo
)
1215 unsigned int res
= 1;
1217 rgroup_controls
*rgm
;
1218 FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
, i
, rgm
)
1219 res
= MAX (res
, rgm
->max_nscalars_per_iter
);
1223 /* Calculate the minimum precision necessary to represent:
1227 as an unsigned integer, where MAX_NITERS is the maximum number of
1228 loop header iterations for the original scalar form of LOOP_VINFO. */
1231 vect_min_prec_for_max_niters (loop_vec_info loop_vinfo
, unsigned int factor
)
1233 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1235 /* Get the maximum number of iterations that is representable
1236 in the counter type. */
1237 tree ni_type
= TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo
));
1238 widest_int max_ni
= wi::to_widest (TYPE_MAX_VALUE (ni_type
)) + 1;
1240 /* Get a more refined estimate for the number of iterations. */
1241 widest_int max_back_edges
;
1242 if (max_loop_iterations (loop
, &max_back_edges
))
1243 max_ni
= wi::smin (max_ni
, max_back_edges
+ 1);
1245 /* Work out how many bits we need to represent the limit. */
1246 return wi::min_precision (max_ni
* factor
, UNSIGNED
);
1249 /* True if the loop needs peeling or partial vectors when vectorized. */
1252 vect_need_peeling_or_partial_vectors_p (loop_vec_info loop_vinfo
)
1254 unsigned HOST_WIDE_INT const_vf
;
1255 HOST_WIDE_INT max_niter
1256 = likely_max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo
));
1258 unsigned th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
1259 if (!th
&& LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
))
1260 th
= LOOP_VINFO_COST_MODEL_THRESHOLD (LOOP_VINFO_ORIG_LOOP_INFO
1263 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1264 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) >= 0)
1266 /* Work out the (constant) number of iterations that need to be
1267 peeled for reasons other than niters. */
1268 unsigned int peel_niter
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
1269 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
1271 if (!multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo
) - peel_niter
,
1272 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)))
1275 else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
)
1276 /* ??? When peeling for gaps but not alignment, we could
1277 try to check whether the (variable) niters is known to be
1278 VF * N + 1. That's something of a niche case though. */
1279 || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
1280 || !LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant (&const_vf
)
1281 || ((tree_ctz (LOOP_VINFO_NITERS (loop_vinfo
))
1282 < (unsigned) exact_log2 (const_vf
))
1283 /* In case of versioning, check if the maximum number of
1284 iterations is greater than th. If they are identical,
1285 the epilogue is unnecessary. */
1286 && (!LOOP_REQUIRES_VERSIONING (loop_vinfo
)
1287 || ((unsigned HOST_WIDE_INT
) max_niter
1288 /* We'd like to use LOOP_VINFO_VERSIONING_THRESHOLD
1289 but that's only computed later based on our result.
1290 The following is the most conservative approximation. */
1291 > (std::max ((unsigned HOST_WIDE_INT
) th
,
1292 const_vf
) / const_vf
) * const_vf
))))
1298 /* Each statement in LOOP_VINFO can be masked where necessary. Check
1299 whether we can actually generate the masks required. Return true if so,
1300 storing the type of the scalar IV in LOOP_VINFO_RGROUP_COMPARE_TYPE. */
1303 vect_verify_full_masking (loop_vec_info loop_vinfo
)
1305 unsigned int min_ni_width
;
1307 /* Use a normal loop if there are no statements that need masking.
1308 This only happens in rare degenerate cases: it means that the loop
1309 has no loads, no stores, and no live-out values. */
1310 if (LOOP_VINFO_MASKS (loop_vinfo
).is_empty ())
1313 /* Produce the rgroup controls. */
1314 for (auto mask
: LOOP_VINFO_MASKS (loop_vinfo
).mask_set
)
1316 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
1317 tree vectype
= mask
.first
;
1318 unsigned nvectors
= mask
.second
;
1320 if (masks
->rgc_vec
.length () < nvectors
)
1321 masks
->rgc_vec
.safe_grow_cleared (nvectors
, true);
1322 rgroup_controls
*rgm
= &(*masks
).rgc_vec
[nvectors
- 1];
1323 /* The number of scalars per iteration and the number of vectors are
1324 both compile-time constants. */
1325 unsigned int nscalars_per_iter
1326 = exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
1327 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)).to_constant ();
1329 if (rgm
->max_nscalars_per_iter
< nscalars_per_iter
)
1331 rgm
->max_nscalars_per_iter
= nscalars_per_iter
;
1332 rgm
->type
= truth_type_for (vectype
);
1337 unsigned int max_nscalars_per_iter
1338 = vect_get_max_nscalars_per_iter (loop_vinfo
);
1340 /* Work out how many bits we need to represent the limit. */
1342 = vect_min_prec_for_max_niters (loop_vinfo
, max_nscalars_per_iter
);
1344 /* Find a scalar mode for which WHILE_ULT is supported. */
1345 opt_scalar_int_mode cmp_mode_iter
;
1346 tree cmp_type
= NULL_TREE
;
1347 tree iv_type
= NULL_TREE
;
1348 widest_int iv_limit
= vect_iv_limit_for_partial_vectors (loop_vinfo
);
1349 unsigned int iv_precision
= UINT_MAX
;
1352 iv_precision
= wi::min_precision (iv_limit
* max_nscalars_per_iter
,
1355 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
1357 unsigned int cmp_bits
= GET_MODE_BITSIZE (cmp_mode_iter
.require ());
1358 if (cmp_bits
>= min_ni_width
1359 && targetm
.scalar_mode_supported_p (cmp_mode_iter
.require ()))
1361 tree this_type
= build_nonstandard_integer_type (cmp_bits
, true);
1363 && can_produce_all_loop_masks_p (loop_vinfo
, this_type
))
1365 /* Although we could stop as soon as we find a valid mode,
1366 there are at least two reasons why that's not always the
1369 - An IV that's Pmode or wider is more likely to be reusable
1370 in address calculations than an IV that's narrower than
1373 - Doing the comparison in IV_PRECISION or wider allows
1374 a natural 0-based IV, whereas using a narrower comparison
1375 type requires mitigations against wrap-around.
1377 Conversely, if the IV limit is variable, doing the comparison
1378 in a wider type than the original type can introduce
1379 unnecessary extensions, so picking the widest valid mode
1380 is not always a good choice either.
1382 Here we prefer the first IV type that's Pmode or wider,
1383 and the first comparison type that's IV_PRECISION or wider.
1384 (The comparison type must be no wider than the IV type,
1385 to avoid extensions in the vector loop.)
1387 ??? We might want to try continuing beyond Pmode for ILP32
1388 targets if CMP_BITS < IV_PRECISION. */
1389 iv_type
= this_type
;
1390 if (!cmp_type
|| iv_precision
> TYPE_PRECISION (cmp_type
))
1391 cmp_type
= this_type
;
1392 if (cmp_bits
>= GET_MODE_BITSIZE (Pmode
))
1400 LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
.release ();
1404 LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo
) = cmp_type
;
1405 LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
) = iv_type
;
1406 LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
) = vect_partial_vectors_while_ult
;
1410 /* Each statement in LOOP_VINFO can be masked where necessary. Check
1411 whether we can actually generate AVX512 style masks. Return true if so,
1412 storing the type of the scalar IV in LOOP_VINFO_RGROUP_IV_TYPE. */
1415 vect_verify_full_masking_avx512 (loop_vec_info loop_vinfo
)
1417 /* Produce differently organized rgc_vec and differently check
1418 we can produce masks. */
1420 /* Use a normal loop if there are no statements that need masking.
1421 This only happens in rare degenerate cases: it means that the loop
1422 has no loads, no stores, and no live-out values. */
1423 if (LOOP_VINFO_MASKS (loop_vinfo
).is_empty ())
1426 /* For the decrementing IV we need to represent all values in
1427 [0, niter + niter_skip] where niter_skip is the elements we
1428 skip in the first iteration for prologue peeling. */
1429 tree iv_type
= NULL_TREE
;
1430 widest_int iv_limit
= vect_iv_limit_for_partial_vectors (loop_vinfo
);
1431 unsigned int iv_precision
= UINT_MAX
;
1433 iv_precision
= wi::min_precision (iv_limit
, UNSIGNED
);
1435 /* First compute the type for the IV we use to track the remaining
1436 scalar iterations. */
1437 opt_scalar_int_mode cmp_mode_iter
;
1438 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
1440 unsigned int cmp_bits
= GET_MODE_BITSIZE (cmp_mode_iter
.require ());
1441 if (cmp_bits
>= iv_precision
1442 && targetm
.scalar_mode_supported_p (cmp_mode_iter
.require ()))
1444 iv_type
= build_nonstandard_integer_type (cmp_bits
, true);
1452 /* Produce the rgroup controls. */
1453 for (auto const &mask
: LOOP_VINFO_MASKS (loop_vinfo
).mask_set
)
1455 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
1456 tree vectype
= mask
.first
;
1457 unsigned nvectors
= mask
.second
;
1459 /* The number of scalars per iteration and the number of vectors are
1460 both compile-time constants. */
1461 unsigned int nscalars_per_iter
1462 = exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
1463 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)).to_constant ();
1465 /* We index the rgroup_controls vector with nscalars_per_iter
1466 which we keep constant and instead have a varying nvectors,
1467 remembering the vector mask with the fewest nV. */
1468 if (masks
->rgc_vec
.length () < nscalars_per_iter
)
1469 masks
->rgc_vec
.safe_grow_cleared (nscalars_per_iter
, true);
1470 rgroup_controls
*rgm
= &(*masks
).rgc_vec
[nscalars_per_iter
- 1];
1472 if (!rgm
->type
|| rgm
->factor
> nvectors
)
1474 rgm
->type
= truth_type_for (vectype
);
1475 rgm
->compare_type
= NULL_TREE
;
1476 rgm
->max_nscalars_per_iter
= nscalars_per_iter
;
1477 rgm
->factor
= nvectors
;
1478 rgm
->bias_adjusted_ctrl
= NULL_TREE
;
1482 /* There is no fixed compare type we are going to use but we have to
1483 be able to get at one for each mask group. */
1484 unsigned int min_ni_width
1485 = wi::min_precision (vect_max_vf (loop_vinfo
), UNSIGNED
);
1488 for (auto &rgc
: LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
)
1490 tree mask_type
= rgc
.type
;
1494 /* For now vect_get_loop_mask only supports integer mode masks
1495 when we need to split it. */
1496 if (GET_MODE_CLASS (TYPE_MODE (mask_type
)) != MODE_INT
1497 || TYPE_PRECISION (TREE_TYPE (mask_type
)) != 1)
1503 /* If iv_type is usable as compare type use that - we can elide the
1504 saturation in that case. */
1505 if (TYPE_PRECISION (iv_type
) >= min_ni_width
)
1508 = build_vector_type (iv_type
, TYPE_VECTOR_SUBPARTS (mask_type
));
1509 if (expand_vec_cmp_expr_p (cmp_vectype
, mask_type
, LT_EXPR
))
1510 rgc
.compare_type
= cmp_vectype
;
1512 if (!rgc
.compare_type
)
1513 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
1515 unsigned int cmp_bits
= GET_MODE_BITSIZE (cmp_mode_iter
.require ());
1516 if (cmp_bits
>= min_ni_width
1517 && targetm
.scalar_mode_supported_p (cmp_mode_iter
.require ()))
1519 tree cmp_type
= build_nonstandard_integer_type (cmp_bits
, true);
1523 /* Check whether we can produce the mask with cmp_type. */
1525 = build_vector_type (cmp_type
, TYPE_VECTOR_SUBPARTS (mask_type
));
1526 if (expand_vec_cmp_expr_p (cmp_vectype
, mask_type
, LT_EXPR
))
1528 rgc
.compare_type
= cmp_vectype
;
1533 if (!rgc
.compare_type
)
1541 release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
);
1545 LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo
) = error_mark_node
;
1546 LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
) = iv_type
;
1547 LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
) = vect_partial_vectors_avx512
;
1551 /* Check whether we can use vector access with length based on precison
1552 comparison. So far, to keep it simple, we only allow the case that the
1553 precision of the target supported length is larger than the precision
1554 required by loop niters. */
1557 vect_verify_loop_lens (loop_vec_info loop_vinfo
)
1559 if (LOOP_VINFO_LENS (loop_vinfo
).is_empty ())
1562 machine_mode len_load_mode
, len_store_mode
;
1563 if (!get_len_load_store_mode (loop_vinfo
->vector_mode
, true)
1564 .exists (&len_load_mode
))
1566 if (!get_len_load_store_mode (loop_vinfo
->vector_mode
, false)
1567 .exists (&len_store_mode
))
1570 signed char partial_load_bias
= internal_len_load_store_bias
1571 (IFN_LEN_LOAD
, len_load_mode
);
1573 signed char partial_store_bias
= internal_len_load_store_bias
1574 (IFN_LEN_STORE
, len_store_mode
);
1576 gcc_assert (partial_load_bias
== partial_store_bias
);
1578 if (partial_load_bias
== VECT_PARTIAL_BIAS_UNSUPPORTED
)
1581 /* If the backend requires a bias of -1 for LEN_LOAD, we must not emit
1582 len_loads with a length of zero. In order to avoid that we prohibit
1583 more than one loop length here. */
1584 if (partial_load_bias
== -1
1585 && LOOP_VINFO_LENS (loop_vinfo
).length () > 1)
1588 LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
) = partial_load_bias
;
1590 unsigned int max_nitems_per_iter
= 1;
1592 rgroup_controls
*rgl
;
1593 /* Find the maximum number of items per iteration for every rgroup. */
1594 FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo
), i
, rgl
)
1596 unsigned nitems_per_iter
= rgl
->max_nscalars_per_iter
* rgl
->factor
;
1597 max_nitems_per_iter
= MAX (max_nitems_per_iter
, nitems_per_iter
);
1600 /* Work out how many bits we need to represent the length limit. */
1601 unsigned int min_ni_prec
1602 = vect_min_prec_for_max_niters (loop_vinfo
, max_nitems_per_iter
);
1604 /* Now use the maximum of below precisions for one suitable IV type:
1605 - the IV's natural precision
1606 - the precision needed to hold: the maximum number of scalar
1607 iterations multiplied by the scale factor (min_ni_prec above)
1608 - the Pmode precision
1610 If min_ni_prec is less than the precision of the current niters,
1611 we perfer to still use the niters type. Prefer to use Pmode and
1612 wider IV to avoid narrow conversions. */
1614 unsigned int ni_prec
1615 = TYPE_PRECISION (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
)));
1616 min_ni_prec
= MAX (min_ni_prec
, ni_prec
);
1617 min_ni_prec
= MAX (min_ni_prec
, GET_MODE_BITSIZE (Pmode
));
1619 tree iv_type
= NULL_TREE
;
1620 opt_scalar_int_mode tmode_iter
;
1621 FOR_EACH_MODE_IN_CLASS (tmode_iter
, MODE_INT
)
1623 scalar_mode tmode
= tmode_iter
.require ();
1624 unsigned int tbits
= GET_MODE_BITSIZE (tmode
);
1626 /* ??? Do we really want to construct one IV whose precision exceeds
1628 if (tbits
> BITS_PER_WORD
)
1631 /* Find the first available standard integral type. */
1632 if (tbits
>= min_ni_prec
&& targetm
.scalar_mode_supported_p (tmode
))
1634 iv_type
= build_nonstandard_integer_type (tbits
, true);
1641 if (dump_enabled_p ())
1642 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1643 "can't vectorize with length-based partial vectors"
1644 " because there is no suitable iv type.\n");
1648 LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo
) = iv_type
;
1649 LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
) = iv_type
;
1650 LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
) = vect_partial_vectors_len
;
1655 /* Calculate the cost of one scalar iteration of the loop. */
1657 vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo
)
1659 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1660 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1661 int nbbs
= loop
->num_nodes
, factor
;
1662 int innerloop_iters
, i
;
1664 DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
1666 /* Gather costs for statements in the scalar loop. */
1669 innerloop_iters
= 1;
1671 innerloop_iters
= LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo
);
1673 for (i
= 0; i
< nbbs
; i
++)
1675 gimple_stmt_iterator si
;
1676 basic_block bb
= bbs
[i
];
1678 if (bb
->loop_father
== loop
->inner
)
1679 factor
= innerloop_iters
;
1683 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1685 gimple
*stmt
= gsi_stmt (si
);
1686 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (stmt
);
1688 if (!is_gimple_assign (stmt
) && !is_gimple_call (stmt
))
1691 /* Skip stmts that are not vectorized inside the loop. */
1692 stmt_vec_info vstmt_info
= vect_stmt_to_vectorize (stmt_info
);
1693 if (!STMT_VINFO_RELEVANT_P (vstmt_info
)
1694 && (!STMT_VINFO_LIVE_P (vstmt_info
)
1695 || !VECTORIZABLE_CYCLE_DEF
1696 (STMT_VINFO_DEF_TYPE (vstmt_info
))))
1699 vect_cost_for_stmt kind
;
1700 if (STMT_VINFO_DATA_REF (stmt_info
))
1702 if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info
)))
1705 kind
= scalar_store
;
1707 else if (vect_nop_conversion_p (stmt_info
))
1712 /* We are using vect_prologue here to avoid scaling twice
1713 by the inner loop factor. */
1714 record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
),
1715 factor
, kind
, stmt_info
, 0, vect_prologue
);
1719 /* Now accumulate cost. */
1720 loop_vinfo
->scalar_costs
= init_cost (loop_vinfo
, true);
1721 add_stmt_costs (loop_vinfo
->scalar_costs
,
1722 &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
));
1723 loop_vinfo
->scalar_costs
->finish_cost (nullptr);
1726 /* Function vect_analyze_loop_form.
1728 Verify that certain CFG restrictions hold, including:
1729 - the loop has a pre-header
1730 - the loop has a single entry
1731 - nested loops can have only a single exit.
1732 - the loop exit condition is simple enough
1733 - the number of iterations can be analyzed, i.e, a countable loop. The
1734 niter could be analyzed under some assumptions. */
1737 vect_analyze_loop_form (class loop
*loop
, vect_loop_form_info
*info
)
1739 DUMP_VECT_SCOPE ("vect_analyze_loop_form");
1741 edge exit_e
= vec_init_loop_exit_info (loop
);
1743 return opt_result::failure_at (vect_location
,
1745 " could not determine main exit from"
1746 " loop with multiple exits.\n");
1747 info
->loop_exit
= exit_e
;
1748 if (dump_enabled_p ())
1749 dump_printf_loc (MSG_NOTE
, vect_location
,
1750 "using as main loop exit: %d -> %d [AUX: %p]\n",
1751 exit_e
->src
->index
, exit_e
->dest
->index
, exit_e
->aux
);
1753 /* Check if we have any control flow that doesn't leave the loop. */
1754 class loop
*v_loop
= loop
->inner
? loop
->inner
: loop
;
1755 basic_block
*bbs
= get_loop_body (v_loop
);
1756 for (unsigned i
= 0; i
< v_loop
->num_nodes
; i
++)
1757 if (EDGE_COUNT (bbs
[i
]->succs
) != 1
1758 && (EDGE_COUNT (bbs
[i
]->succs
) != 2
1759 || !loop_exits_from_bb_p (bbs
[i
]->loop_father
, bbs
[i
])))
1762 return opt_result::failure_at (vect_location
,
1764 " unsupported control flow in loop.\n");
1768 /* Different restrictions apply when we are considering an inner-most loop,
1769 vs. an outer (nested) loop.
1770 (FORNOW. May want to relax some of these restrictions in the future). */
1772 info
->inner_loop_cond
= NULL
;
1775 /* Inner-most loop. */
1777 if (empty_block_p (loop
->header
))
1778 return opt_result::failure_at (vect_location
,
1779 "not vectorized: empty loop.\n");
1783 class loop
*innerloop
= loop
->inner
;
1786 /* Nested loop. We currently require that the loop is doubly-nested,
1787 contains a single inner loop with a single exit to the block
1788 with the single exit condition in the outer loop.
1789 Vectorizable outer-loops look like this:
1801 The inner-loop also has the properties expected of inner-most loops
1802 as described above. */
1804 if ((loop
->inner
)->inner
|| (loop
->inner
)->next
)
1805 return opt_result::failure_at (vect_location
,
1807 " multiple nested loops.\n");
1809 entryedge
= loop_preheader_edge (innerloop
);
1810 if (entryedge
->src
!= loop
->header
1811 || !single_exit (innerloop
)
1812 || single_exit (innerloop
)->dest
!= EDGE_PRED (loop
->latch
, 0)->src
)
1813 return opt_result::failure_at (vect_location
,
1815 " unsupported outerloop form.\n");
1817 /* Analyze the inner-loop. */
1818 vect_loop_form_info inner
;
1819 opt_result res
= vect_analyze_loop_form (loop
->inner
, &inner
);
1822 if (dump_enabled_p ())
1823 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1824 "not vectorized: Bad inner loop.\n");
1828 /* Don't support analyzing niter under assumptions for inner
1830 if (!integer_onep (inner
.assumptions
))
1831 return opt_result::failure_at (vect_location
,
1832 "not vectorized: Bad inner loop.\n");
1834 if (!expr_invariant_in_loop_p (loop
, inner
.number_of_iterations
))
1835 return opt_result::failure_at (vect_location
,
1836 "not vectorized: inner-loop count not"
1839 if (dump_enabled_p ())
1840 dump_printf_loc (MSG_NOTE
, vect_location
,
1841 "Considering outer-loop vectorization.\n");
1842 info
->inner_loop_cond
= inner
.conds
[0];
1845 if (EDGE_COUNT (loop
->header
->preds
) != 2)
1846 return opt_result::failure_at (vect_location
,
1848 " too many incoming edges.\n");
1850 /* We assume that the latch is empty. */
1851 if (!empty_block_p (loop
->latch
)
1852 || !gimple_seq_empty_p (phi_nodes (loop
->latch
)))
1853 return opt_result::failure_at (vect_location
,
1854 "not vectorized: latch block not empty.\n");
1856 /* Make sure there is no abnormal exit. */
1857 auto_vec
<edge
> exits
= get_loop_exit_edges (loop
);
1858 for (edge e
: exits
)
1860 if (e
->flags
& EDGE_ABNORMAL
)
1861 return opt_result::failure_at (vect_location
,
1863 " abnormal loop exit edge.\n");
1867 = vect_get_loop_niters (loop
, exit_e
, &info
->assumptions
,
1868 &info
->number_of_iterations
,
1869 &info
->number_of_iterationsm1
);
1870 if (info
->conds
.is_empty ())
1871 return opt_result::failure_at
1873 "not vectorized: complicated exit condition.\n");
1875 /* Determine what the primary and alternate exit conds are. */
1876 for (unsigned i
= 0; i
< info
->conds
.length (); i
++)
1878 gcond
*cond
= info
->conds
[i
];
1879 if (exit_e
->src
== gimple_bb (cond
))
1880 std::swap (info
->conds
[0], info
->conds
[i
]);
1883 if (integer_zerop (info
->assumptions
)
1884 || !info
->number_of_iterations
1885 || chrec_contains_undetermined (info
->number_of_iterations
))
1886 return opt_result::failure_at
1888 "not vectorized: number of iterations cannot be computed.\n");
1890 if (integer_zerop (info
->number_of_iterations
))
1891 return opt_result::failure_at
1893 "not vectorized: number of iterations = 0.\n");
1895 if (!(tree_fits_shwi_p (info
->number_of_iterations
)
1896 && tree_to_shwi (info
->number_of_iterations
) > 0))
1898 if (dump_enabled_p ())
1900 dump_printf_loc (MSG_NOTE
, vect_location
,
1901 "Symbolic number of iterations is ");
1902 dump_generic_expr (MSG_NOTE
, TDF_DETAILS
, info
->number_of_iterations
);
1903 dump_printf (MSG_NOTE
, "\n");
1907 return opt_result::success ();
1910 /* Create a loop_vec_info for LOOP with SHARED and the
1911 vect_analyze_loop_form result. */
1914 vect_create_loop_vinfo (class loop
*loop
, vec_info_shared
*shared
,
1915 const vect_loop_form_info
*info
,
1916 loop_vec_info main_loop_info
)
1918 loop_vec_info loop_vinfo
= new _loop_vec_info (loop
, shared
);
1919 LOOP_VINFO_NITERSM1 (loop_vinfo
) = info
->number_of_iterationsm1
;
1920 LOOP_VINFO_NITERS (loop_vinfo
) = info
->number_of_iterations
;
1921 LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
) = info
->number_of_iterations
;
1922 LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
) = main_loop_info
;
1923 /* Also record the assumptions for versioning. */
1924 if (!integer_onep (info
->assumptions
) && !main_loop_info
)
1925 LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo
) = info
->assumptions
;
1927 for (gcond
*cond
: info
->conds
)
1929 stmt_vec_info loop_cond_info
= loop_vinfo
->lookup_stmt (cond
);
1930 STMT_VINFO_TYPE (loop_cond_info
) = loop_exit_ctrl_vec_info_type
;
1931 /* Mark the statement as a condition. */
1932 STMT_VINFO_DEF_TYPE (loop_cond_info
) = vect_condition_def
;
1935 for (unsigned i
= 1; i
< info
->conds
.length (); i
++)
1936 LOOP_VINFO_LOOP_CONDS (loop_vinfo
).safe_push (info
->conds
[i
]);
1937 LOOP_VINFO_LOOP_IV_COND (loop_vinfo
) = info
->conds
[0];
1939 LOOP_VINFO_IV_EXIT (loop_vinfo
) = info
->loop_exit
;
1941 /* Check to see if we're vectorizing multiple exits. */
1942 LOOP_VINFO_EARLY_BREAKS (loop_vinfo
)
1943 = !LOOP_VINFO_LOOP_CONDS (loop_vinfo
).is_empty ();
1945 if (info
->inner_loop_cond
)
1947 stmt_vec_info inner_loop_cond_info
1948 = loop_vinfo
->lookup_stmt (info
->inner_loop_cond
);
1949 STMT_VINFO_TYPE (inner_loop_cond_info
) = loop_exit_ctrl_vec_info_type
;
1950 /* If we have an estimate on the number of iterations of the inner
1951 loop use that to limit the scale for costing, otherwise use
1952 --param vect-inner-loop-cost-factor literally. */
1954 if (estimated_stmt_executions (loop
->inner
, &nit
))
1955 LOOP_VINFO_INNER_LOOP_COST_FACTOR (loop_vinfo
)
1956 = wi::smin (nit
, param_vect_inner_loop_cost_factor
).to_uhwi ();
1964 /* Scan the loop stmts and dependent on whether there are any (non-)SLP
1965 statements update the vectorization factor. */
1968 vect_update_vf_for_slp (loop_vec_info loop_vinfo
)
1970 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1971 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1972 int nbbs
= loop
->num_nodes
;
1973 poly_uint64 vectorization_factor
;
1976 DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
1978 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1979 gcc_assert (known_ne (vectorization_factor
, 0U));
1981 /* If all the stmts in the loop can be SLPed, we perform only SLP, and
1982 vectorization factor of the loop is the unrolling factor required by
1983 the SLP instances. If that unrolling factor is 1, we say, that we
1984 perform pure SLP on loop - cross iteration parallelism is not
1986 bool only_slp_in_loop
= true;
1987 for (i
= 0; i
< nbbs
; i
++)
1989 basic_block bb
= bbs
[i
];
1990 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
1993 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (si
.phi ());
1996 if ((STMT_VINFO_RELEVANT_P (stmt_info
)
1997 || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
1998 && !PURE_SLP_STMT (stmt_info
))
1999 /* STMT needs both SLP and loop-based vectorization. */
2000 only_slp_in_loop
= false;
2002 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
2005 if (is_gimple_debug (gsi_stmt (si
)))
2007 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
2008 stmt_info
= vect_stmt_to_vectorize (stmt_info
);
2009 if ((STMT_VINFO_RELEVANT_P (stmt_info
)
2010 || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
2011 && !PURE_SLP_STMT (stmt_info
))
2012 /* STMT needs both SLP and loop-based vectorization. */
2013 only_slp_in_loop
= false;
2017 if (only_slp_in_loop
)
2019 if (dump_enabled_p ())
2020 dump_printf_loc (MSG_NOTE
, vect_location
,
2021 "Loop contains only SLP stmts\n");
2022 vectorization_factor
= LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
);
2026 if (dump_enabled_p ())
2027 dump_printf_loc (MSG_NOTE
, vect_location
,
2028 "Loop contains SLP and non-SLP stmts\n");
2029 /* Both the vectorization factor and unroll factor have the form
2030 GET_MODE_SIZE (loop_vinfo->vector_mode) * X for some rational X,
2031 so they must have a common multiple. */
2032 vectorization_factor
2033 = force_common_multiple (vectorization_factor
,
2034 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
));
2037 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
2038 if (dump_enabled_p ())
2040 dump_printf_loc (MSG_NOTE
, vect_location
,
2041 "Updating vectorization factor to ");
2042 dump_dec (MSG_NOTE
, vectorization_factor
);
2043 dump_printf (MSG_NOTE
, ".\n");
2047 /* Return true if STMT_INFO describes a double reduction phi and if
2048 the other phi in the reduction is also relevant for vectorization.
2049 This rejects cases such as:
2052 x_1 = PHI <x_3(outer2), ...>;
2060 x_3 = PHI <x_2(inner)>;
2062 if nothing in x_2 or elsewhere makes x_1 relevant. */
2065 vect_active_double_reduction_p (stmt_vec_info stmt_info
)
2067 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
)
2070 return STMT_VINFO_RELEVANT_P (STMT_VINFO_REDUC_DEF (stmt_info
));
2073 /* Function vect_analyze_loop_operations.
2075 Scan the loop stmts and make sure they are all vectorizable. */
2078 vect_analyze_loop_operations (loop_vec_info loop_vinfo
)
2080 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2081 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
2082 int nbbs
= loop
->num_nodes
;
2084 stmt_vec_info stmt_info
;
2085 bool need_to_vectorize
= false;
2088 DUMP_VECT_SCOPE ("vect_analyze_loop_operations");
2090 auto_vec
<stmt_info_for_cost
> cost_vec
;
2092 for (i
= 0; i
< nbbs
; i
++)
2094 basic_block bb
= bbs
[i
];
2096 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
2099 gphi
*phi
= si
.phi ();
2102 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
2103 if (dump_enabled_p ())
2104 dump_printf_loc (MSG_NOTE
, vect_location
, "examining phi: %G",
2106 if (virtual_operand_p (gimple_phi_result (phi
)))
2109 /* Inner-loop loop-closed exit phi in outer-loop vectorization
2110 (i.e., a phi in the tail of the outer-loop). */
2111 if (! is_loop_header_bb_p (bb
))
2113 /* FORNOW: we currently don't support the case that these phis
2114 are not used in the outerloop (unless it is double reduction,
2115 i.e., this phi is vect_reduction_def), cause this case
2116 requires to actually do something here. */
2117 if (STMT_VINFO_LIVE_P (stmt_info
)
2118 && !vect_active_double_reduction_p (stmt_info
))
2119 return opt_result::failure_at (phi
,
2120 "Unsupported loop-closed phi"
2121 " in outer-loop.\n");
2123 /* If PHI is used in the outer loop, we check that its operand
2124 is defined in the inner loop. */
2125 if (STMT_VINFO_RELEVANT_P (stmt_info
))
2129 if (gimple_phi_num_args (phi
) != 1)
2130 return opt_result::failure_at (phi
, "unsupported phi");
2132 phi_op
= PHI_ARG_DEF (phi
, 0);
2133 stmt_vec_info op_def_info
= loop_vinfo
->lookup_def (phi_op
);
2135 return opt_result::failure_at (phi
, "unsupported phi\n");
2137 if (STMT_VINFO_RELEVANT (op_def_info
) != vect_used_in_outer
2138 && (STMT_VINFO_RELEVANT (op_def_info
)
2139 != vect_used_in_outer_by_reduction
))
2140 return opt_result::failure_at (phi
, "unsupported phi\n");
2142 if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_internal_def
2143 || (STMT_VINFO_DEF_TYPE (stmt_info
)
2144 == vect_double_reduction_def
))
2145 && !vectorizable_lc_phi (loop_vinfo
,
2146 stmt_info
, NULL
, NULL
))
2147 return opt_result::failure_at (phi
, "unsupported phi\n");
2153 gcc_assert (stmt_info
);
2155 if ((STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_scope
2156 || STMT_VINFO_LIVE_P (stmt_info
))
2157 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_induction_def
2158 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_first_order_recurrence
)
2159 /* A scalar-dependence cycle that we don't support. */
2160 return opt_result::failure_at (phi
,
2162 " scalar dependence cycle.\n");
2164 if (STMT_VINFO_RELEVANT_P (stmt_info
))
2166 need_to_vectorize
= true;
2167 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
2168 && ! PURE_SLP_STMT (stmt_info
))
2169 ok
= vectorizable_induction (loop_vinfo
,
2170 stmt_info
, NULL
, NULL
,
2172 else if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
2173 || (STMT_VINFO_DEF_TYPE (stmt_info
)
2174 == vect_double_reduction_def
)
2175 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
2176 && ! PURE_SLP_STMT (stmt_info
))
2177 ok
= vectorizable_reduction (loop_vinfo
,
2178 stmt_info
, NULL
, NULL
, &cost_vec
);
2179 else if ((STMT_VINFO_DEF_TYPE (stmt_info
)
2180 == vect_first_order_recurrence
)
2181 && ! PURE_SLP_STMT (stmt_info
))
2182 ok
= vectorizable_recurr (loop_vinfo
, stmt_info
, NULL
, NULL
,
2186 /* SLP PHIs are tested by vect_slp_analyze_node_operations. */
2188 && STMT_VINFO_LIVE_P (stmt_info
)
2189 && !PURE_SLP_STMT (stmt_info
))
2190 ok
= vectorizable_live_operation (loop_vinfo
, stmt_info
, NULL
, NULL
,
2191 -1, false, &cost_vec
);
2194 return opt_result::failure_at (phi
,
2195 "not vectorized: relevant phi not "
2197 static_cast <gimple
*> (phi
));
2200 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
2203 gimple
*stmt
= gsi_stmt (si
);
2204 if (!gimple_clobber_p (stmt
)
2205 && !is_gimple_debug (stmt
))
2208 = vect_analyze_stmt (loop_vinfo
,
2209 loop_vinfo
->lookup_stmt (stmt
),
2211 NULL
, NULL
, &cost_vec
);
2218 add_stmt_costs (loop_vinfo
->vector_costs
, &cost_vec
);
2220 /* All operations in the loop are either irrelevant (deal with loop
2221 control, or dead), or only used outside the loop and can be moved
2222 out of the loop (e.g. invariants, inductions). The loop can be
2223 optimized away by scalar optimizations. We're better off not
2224 touching this loop. */
2225 if (!need_to_vectorize
)
2227 if (dump_enabled_p ())
2228 dump_printf_loc (MSG_NOTE
, vect_location
,
2229 "All the computation can be taken out of the loop.\n");
2230 return opt_result::failure_at
2232 "not vectorized: redundant loop. no profit to vectorize.\n");
2235 return opt_result::success ();
2238 /* Return true if we know that the iteration count is smaller than the
2239 vectorization factor. Return false if it isn't, or if we can't be sure
2243 vect_known_niters_smaller_than_vf (loop_vec_info loop_vinfo
)
2245 unsigned int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
2247 HOST_WIDE_INT max_niter
;
2248 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
2249 max_niter
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
2251 max_niter
= max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo
));
2253 if (max_niter
!= -1 && (unsigned HOST_WIDE_INT
) max_niter
< assumed_vf
)
2259 /* Analyze the cost of the loop described by LOOP_VINFO. Decide if it
2260 is worthwhile to vectorize. Return 1 if definitely yes, 0 if
2261 definitely no, or -1 if it's worth retrying. */
2264 vect_analyze_loop_costing (loop_vec_info loop_vinfo
,
2265 unsigned *suggested_unroll_factor
)
2267 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2268 unsigned int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
2270 /* Only loops that can handle partially-populated vectors can have iteration
2271 counts less than the vectorization factor. */
2272 if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
2273 && vect_known_niters_smaller_than_vf (loop_vinfo
))
2275 if (dump_enabled_p ())
2276 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2277 "not vectorized: iteration count smaller than "
2278 "vectorization factor.\n");
2282 /* If we know the number of iterations we can do better, for the
2283 epilogue we can also decide whether the main loop leaves us
2284 with enough iterations, prefering a smaller vector epilog then
2285 also possibly used for the case we skip the vector loop. */
2286 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
2288 widest_int scalar_niters
2289 = wi::to_widest (LOOP_VINFO_NITERSM1 (loop_vinfo
)) + 1;
2290 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
2292 loop_vec_info orig_loop_vinfo
2293 = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
);
2295 = constant_lower_bound (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo
));
2296 int prolog_peeling
= 0;
2297 if (!vect_use_loop_mask_for_alignment_p (loop_vinfo
))
2298 prolog_peeling
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo
);
2299 if (prolog_peeling
>= 0
2300 && known_eq (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo
),
2304 = LOOP_VINFO_PEELING_FOR_GAPS (orig_loop_vinfo
) ? 1 : 0;
2305 scalar_niters
= ((scalar_niters
- gap
- prolog_peeling
)
2309 /* Reject vectorizing for a single scalar iteration, even if
2310 we could in principle implement that using partial vectors. */
2311 unsigned peeling_gap
= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
);
2312 if (scalar_niters
<= peeling_gap
+ 1)
2314 if (dump_enabled_p ())
2315 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2316 "not vectorized: loop only has a single "
2317 "scalar iteration.\n");
2321 if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
2323 /* Check that the loop processes at least one full vector. */
2324 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2325 if (known_lt (scalar_niters
, vf
))
2327 if (dump_enabled_p ())
2328 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2329 "loop does not have enough iterations "
2330 "to support vectorization.\n");
2334 /* If we need to peel an extra epilogue iteration to handle data
2335 accesses with gaps, check that there are enough scalar iterations
2338 The check above is redundant with this one when peeling for gaps,
2339 but the distinction is useful for diagnostics. */
2340 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
2341 && known_le (scalar_niters
, vf
))
2343 if (dump_enabled_p ())
2344 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2345 "loop does not have enough iterations "
2346 "to support peeling for gaps.\n");
2352 /* If using the "very cheap" model. reject cases in which we'd keep
2353 a copy of the scalar code (even if we might be able to vectorize it). */
2354 if (loop_cost_model (loop
) == VECT_COST_MODEL_VERY_CHEAP
2355 && (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
)
2356 || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
2357 || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
)))
2359 if (dump_enabled_p ())
2360 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2361 "some scalar iterations would need to be peeled\n");
2365 int min_profitable_iters
, min_profitable_estimate
;
2366 vect_estimate_min_profitable_iters (loop_vinfo
, &min_profitable_iters
,
2367 &min_profitable_estimate
,
2368 suggested_unroll_factor
);
2370 if (min_profitable_iters
< 0)
2372 if (dump_enabled_p ())
2373 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2374 "not vectorized: vectorization not profitable.\n");
2375 if (dump_enabled_p ())
2376 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2377 "not vectorized: vector version will never be "
2382 int min_scalar_loop_bound
= (param_min_vect_loop_bound
2385 /* Use the cost model only if it is more conservative than user specified
2387 unsigned int th
= (unsigned) MAX (min_scalar_loop_bound
,
2388 min_profitable_iters
);
2390 LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
) = th
;
2392 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
2393 && LOOP_VINFO_INT_NITERS (loop_vinfo
) < th
)
2395 if (dump_enabled_p ())
2396 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2397 "not vectorized: vectorization not profitable.\n");
2398 if (dump_enabled_p ())
2399 dump_printf_loc (MSG_NOTE
, vect_location
,
2400 "not vectorized: iteration count smaller than user "
2401 "specified loop bound parameter or minimum profitable "
2402 "iterations (whichever is more conservative).\n");
2406 /* The static profitablity threshold min_profitable_estimate includes
2407 the cost of having to check at runtime whether the scalar loop
2408 should be used instead. If it turns out that we don't need or want
2409 such a check, the threshold we should use for the static estimate
2410 is simply the point at which the vector loop becomes more profitable
2411 than the scalar loop. */
2412 if (min_profitable_estimate
> min_profitable_iters
2413 && !LOOP_REQUIRES_VERSIONING (loop_vinfo
)
2414 && !LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
)
2415 && !LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
)
2416 && !vect_apply_runtime_profitability_check_p (loop_vinfo
))
2418 if (dump_enabled_p ())
2419 dump_printf_loc (MSG_NOTE
, vect_location
, "no need for a runtime"
2420 " choice between the scalar and vector loops\n");
2421 min_profitable_estimate
= min_profitable_iters
;
2424 /* If the vector loop needs multiple iterations to be beneficial then
2425 things are probably too close to call, and the conservative thing
2426 would be to stick with the scalar code. */
2427 if (loop_cost_model (loop
) == VECT_COST_MODEL_VERY_CHEAP
2428 && min_profitable_estimate
> (int) vect_vf_for_cost (loop_vinfo
))
2430 if (dump_enabled_p ())
2431 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2432 "one iteration of the vector loop would be"
2433 " more expensive than the equivalent number of"
2434 " iterations of the scalar loop\n");
2438 HOST_WIDE_INT estimated_niter
;
2440 /* If we are vectorizing an epilogue then we know the maximum number of
2441 scalar iterations it will cover is at least one lower than the
2442 vectorization factor of the main loop. */
2443 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
2445 = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
)) - 1;
2448 estimated_niter
= estimated_stmt_executions_int (loop
);
2449 if (estimated_niter
== -1)
2450 estimated_niter
= likely_max_stmt_executions_int (loop
);
2452 if (estimated_niter
!= -1
2453 && ((unsigned HOST_WIDE_INT
) estimated_niter
2454 < MAX (th
, (unsigned) min_profitable_estimate
)))
2456 if (dump_enabled_p ())
2457 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2458 "not vectorized: estimated iteration count too "
2460 if (dump_enabled_p ())
2461 dump_printf_loc (MSG_NOTE
, vect_location
,
2462 "not vectorized: estimated iteration count smaller "
2463 "than specified loop bound parameter or minimum "
2464 "profitable iterations (whichever is more "
2465 "conservative).\n");
2473 vect_get_datarefs_in_loop (loop_p loop
, basic_block
*bbs
,
2474 vec
<data_reference_p
> *datarefs
,
2475 unsigned int *n_stmts
)
2478 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2479 for (gimple_stmt_iterator gsi
= gsi_start_bb (bbs
[i
]);
2480 !gsi_end_p (gsi
); gsi_next (&gsi
))
2482 gimple
*stmt
= gsi_stmt (gsi
);
2483 if (is_gimple_debug (stmt
))
2486 opt_result res
= vect_find_stmt_data_reference (loop
, stmt
, datarefs
,
2490 if (is_gimple_call (stmt
) && loop
->safelen
)
2492 tree fndecl
= gimple_call_fndecl (stmt
), op
;
2493 if (fndecl
== NULL_TREE
2494 && gimple_call_internal_p (stmt
, IFN_MASK_CALL
))
2496 fndecl
= gimple_call_arg (stmt
, 0);
2497 gcc_checking_assert (TREE_CODE (fndecl
) == ADDR_EXPR
);
2498 fndecl
= TREE_OPERAND (fndecl
, 0);
2499 gcc_checking_assert (TREE_CODE (fndecl
) == FUNCTION_DECL
);
2501 if (fndecl
!= NULL_TREE
)
2503 cgraph_node
*node
= cgraph_node::get (fndecl
);
2504 if (node
!= NULL
&& node
->simd_clones
!= NULL
)
2506 unsigned int j
, n
= gimple_call_num_args (stmt
);
2507 for (j
= 0; j
< n
; j
++)
2509 op
= gimple_call_arg (stmt
, j
);
2511 || (REFERENCE_CLASS_P (op
)
2512 && get_base_address (op
)))
2515 op
= gimple_call_lhs (stmt
);
2516 /* Ignore #pragma omp declare simd functions
2517 if they don't have data references in the
2518 call stmt itself. */
2522 || (REFERENCE_CLASS_P (op
)
2523 && get_base_address (op
)))))
2530 /* If dependence analysis will give up due to the limit on the
2531 number of datarefs stop here and fail fatally. */
2532 if (datarefs
->length ()
2533 > (unsigned)param_loop_max_datarefs_for_datadeps
)
2534 return opt_result::failure_at (stmt
, "exceeded param "
2535 "loop-max-datarefs-for-datadeps\n");
2537 return opt_result::success ();
2540 /* Look for SLP-only access groups and turn each individual access into its own
2543 vect_dissolve_slp_only_groups (loop_vec_info loop_vinfo
)
2546 struct data_reference
*dr
;
2548 DUMP_VECT_SCOPE ("vect_dissolve_slp_only_groups");
2550 vec
<data_reference_p
> datarefs
= LOOP_VINFO_DATAREFS (loop_vinfo
);
2551 FOR_EACH_VEC_ELT (datarefs
, i
, dr
)
2553 gcc_assert (DR_REF (dr
));
2554 stmt_vec_info stmt_info
2555 = vect_stmt_to_vectorize (loop_vinfo
->lookup_stmt (DR_STMT (dr
)));
2557 /* Check if the load is a part of an interleaving chain. */
2558 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
2560 stmt_vec_info first_element
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
2561 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (first_element
);
2562 unsigned int group_size
= DR_GROUP_SIZE (first_element
);
2564 /* Check if SLP-only groups. */
2565 if (!STMT_SLP_TYPE (stmt_info
)
2566 && STMT_VINFO_SLP_VECT_ONLY (first_element
))
2568 /* Dissolve the group. */
2569 STMT_VINFO_SLP_VECT_ONLY (first_element
) = false;
2571 stmt_vec_info vinfo
= first_element
;
2574 stmt_vec_info next
= DR_GROUP_NEXT_ELEMENT (vinfo
);
2575 DR_GROUP_FIRST_ELEMENT (vinfo
) = vinfo
;
2576 DR_GROUP_NEXT_ELEMENT (vinfo
) = NULL
;
2577 DR_GROUP_SIZE (vinfo
) = 1;
2578 if (STMT_VINFO_STRIDED_P (first_element
)
2579 /* We cannot handle stores with gaps. */
2580 || DR_IS_WRITE (dr_info
->dr
))
2582 STMT_VINFO_STRIDED_P (vinfo
) = true;
2583 DR_GROUP_GAP (vinfo
) = 0;
2586 DR_GROUP_GAP (vinfo
) = group_size
- 1;
2587 /* Duplicate and adjust alignment info, it needs to
2588 be present on each group leader, see dr_misalignment. */
2589 if (vinfo
!= first_element
)
2591 dr_vec_info
*dr_info2
= STMT_VINFO_DR_INFO (vinfo
);
2592 dr_info2
->target_alignment
= dr_info
->target_alignment
;
2593 int misalignment
= dr_info
->misalignment
;
2594 if (misalignment
!= DR_MISALIGNMENT_UNKNOWN
)
2597 = (TREE_INT_CST_LOW (DR_INIT (dr_info2
->dr
))
2598 - TREE_INT_CST_LOW (DR_INIT (dr_info
->dr
)));
2599 unsigned HOST_WIDE_INT align_c
2600 = dr_info
->target_alignment
.to_constant ();
2601 misalignment
= (misalignment
+ diff
) % align_c
;
2603 dr_info2
->misalignment
= misalignment
;
2612 /* Determine if operating on full vectors for LOOP_VINFO might leave
2613 some scalar iterations still to do. If so, decide how we should
2614 handle those scalar iterations. The possibilities are:
2616 (1) Make LOOP_VINFO operate on partial vectors instead of full vectors.
2619 LOOP_VINFO_USING_PARTIAL_VECTORS_P == true
2620 LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P == false
2621 LOOP_VINFO_PEELING_FOR_NITER == false
2623 (2) Make LOOP_VINFO operate on full vectors and use an epilogue loop
2624 to handle the remaining scalar iterations. In this case:
2626 LOOP_VINFO_USING_PARTIAL_VECTORS_P == false
2627 LOOP_VINFO_PEELING_FOR_NITER == true
2629 There are two choices:
2631 (2a) Consider vectorizing the epilogue loop at the same VF as the
2632 main loop, but using partial vectors instead of full vectors.
2635 LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P == true
2637 (2b) Consider vectorizing the epilogue loop at lower VFs only.
2640 LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P == false
2644 vect_determine_partial_vectors_and_peeling (loop_vec_info loop_vinfo
)
2646 /* Determine whether there would be any scalar iterations left over. */
2647 bool need_peeling_or_partial_vectors_p
2648 = vect_need_peeling_or_partial_vectors_p (loop_vinfo
);
2650 /* Decide whether to vectorize the loop with partial vectors. */
2651 LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
) = false;
2652 LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P (loop_vinfo
) = false;
2653 if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
)
2654 && need_peeling_or_partial_vectors_p
)
2656 /* For partial-vector-usage=1, try to push the handling of partial
2657 vectors to the epilogue, with the main loop continuing to operate
2660 If we are unrolling we also do not want to use partial vectors. This
2661 is to avoid the overhead of generating multiple masks and also to
2662 avoid having to execute entire iterations of FALSE masked instructions
2663 when dealing with one or less full iterations.
2665 ??? We could then end up failing to use partial vectors if we
2666 decide to peel iterations into a prologue, and if the main loop
2667 then ends up processing fewer than VF iterations. */
2668 if ((param_vect_partial_vector_usage
== 1
2669 || loop_vinfo
->suggested_unroll_factor
> 1)
2670 && !LOOP_VINFO_EPILOGUE_P (loop_vinfo
)
2671 && !vect_known_niters_smaller_than_vf (loop_vinfo
))
2672 LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P (loop_vinfo
) = true;
2674 LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
) = true;
2677 if (dump_enabled_p ())
2678 dump_printf_loc (MSG_NOTE
, vect_location
,
2679 "operating on %s vectors%s.\n",
2680 LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
2681 ? "partial" : "full",
2682 LOOP_VINFO_EPILOGUE_P (loop_vinfo
)
2683 ? " for epilogue loop" : "");
2685 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
)
2686 = (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
2687 && need_peeling_or_partial_vectors_p
);
2689 /* We set LOOP_VINFO_USING_SELECT_VL_P as true before loop vectorization
2690 analysis that we don't know whether the loop is vectorized by partial
2691 vectors (More details see tree-vect-loop-manip.cc).
2693 However, SELECT_VL vectorizaton style should only applied on partial
2694 vectorization since SELECT_VL is the GIMPLE IR that calculates the
2695 number of elements to be process for each iteration.
2697 After loop vectorization analysis, Clear LOOP_VINFO_USING_SELECT_VL_P
2698 if it is not partial vectorized loop. */
2699 if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
2700 LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo
) = false;
2702 return opt_result::success ();
2705 /* Function vect_analyze_loop_2.
2707 Apply a set of analyses on LOOP specified by LOOP_VINFO, the different
2708 analyses will record information in some members of LOOP_VINFO. FATAL
2709 indicates if some analysis meets fatal error. If one non-NULL pointer
2710 SUGGESTED_UNROLL_FACTOR is provided, it's intent to be filled with one
2711 worked out suggested unroll factor, while one NULL pointer shows it's
2712 going to apply the suggested unroll factor. SLP_DONE_FOR_SUGGESTED_UF
2713 is to hold the slp decision when the suggested unroll factor is worked
2716 vect_analyze_loop_2 (loop_vec_info loop_vinfo
, bool &fatal
,
2717 unsigned *suggested_unroll_factor
,
2718 bool& slp_done_for_suggested_uf
)
2720 opt_result ok
= opt_result::success ();
2722 unsigned int max_vf
= MAX_VECTORIZATION_FACTOR
;
2723 poly_uint64 min_vf
= 2;
2724 loop_vec_info orig_loop_vinfo
= NULL
;
2726 /* If we are dealing with an epilogue then orig_loop_vinfo points to the
2727 loop_vec_info of the first vectorized loop. */
2728 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
2729 orig_loop_vinfo
= LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
);
2731 orig_loop_vinfo
= loop_vinfo
;
2732 gcc_assert (orig_loop_vinfo
);
2734 /* The first group of checks is independent of the vector size. */
2737 if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo
)
2738 && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo
)))
2739 return opt_result::failure_at (vect_location
,
2740 "not vectorized: simd if(0)\n");
2742 /* Find all data references in the loop (which correspond to vdefs/vuses)
2743 and analyze their evolution in the loop. */
2745 loop_p loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2747 /* Gather the data references and count stmts in the loop. */
2748 if (!LOOP_VINFO_DATAREFS (loop_vinfo
).exists ())
2751 = vect_get_datarefs_in_loop (loop
, LOOP_VINFO_BBS (loop_vinfo
),
2752 &LOOP_VINFO_DATAREFS (loop_vinfo
),
2753 &LOOP_VINFO_N_STMTS (loop_vinfo
));
2756 if (dump_enabled_p ())
2757 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2758 "not vectorized: loop contains function "
2759 "calls or data references that cannot "
2763 loop_vinfo
->shared
->save_datarefs ();
2766 loop_vinfo
->shared
->check_datarefs ();
2768 /* Analyze the data references and also adjust the minimal
2769 vectorization factor according to the loads and stores. */
2771 ok
= vect_analyze_data_refs (loop_vinfo
, &min_vf
, &fatal
);
2774 if (dump_enabled_p ())
2775 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2776 "bad data references.\n");
2780 /* Check if we are applying unroll factor now. */
2781 bool applying_suggested_uf
= loop_vinfo
->suggested_unroll_factor
> 1;
2782 gcc_assert (!applying_suggested_uf
|| !suggested_unroll_factor
);
2784 /* If the slp decision is false when suggested unroll factor is worked
2785 out, and we are applying suggested unroll factor, we can simply skip
2786 all slp related analyses this time. */
2787 bool slp
= !applying_suggested_uf
|| slp_done_for_suggested_uf
;
2789 /* Classify all cross-iteration scalar data-flow cycles.
2790 Cross-iteration cycles caused by virtual phis are analyzed separately. */
2791 vect_analyze_scalar_cycles (loop_vinfo
, slp
);
2793 vect_pattern_recog (loop_vinfo
);
2795 vect_fixup_scalar_cycles_with_patterns (loop_vinfo
);
2797 /* Analyze the access patterns of the data-refs in the loop (consecutive,
2798 complex, etc.). FORNOW: Only handle consecutive access pattern. */
2800 ok
= vect_analyze_data_ref_accesses (loop_vinfo
, NULL
);
2803 if (dump_enabled_p ())
2804 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2805 "bad data access.\n");
2809 /* Data-flow analysis to detect stmts that do not need to be vectorized. */
2811 ok
= vect_mark_stmts_to_be_vectorized (loop_vinfo
, &fatal
);
2814 if (dump_enabled_p ())
2815 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2816 "unexpected pattern.\n");
2820 /* While the rest of the analysis below depends on it in some way. */
2823 /* Analyze data dependences between the data-refs in the loop
2824 and adjust the maximum vectorization factor according to
2826 FORNOW: fail at the first data dependence that we encounter. */
2828 ok
= vect_analyze_data_ref_dependences (loop_vinfo
, &max_vf
);
2831 if (dump_enabled_p ())
2832 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2833 "bad data dependence.\n");
2836 if (max_vf
!= MAX_VECTORIZATION_FACTOR
2837 && maybe_lt (max_vf
, min_vf
))
2838 return opt_result::failure_at (vect_location
, "bad data dependence.\n");
2839 LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo
) = max_vf
;
2841 ok
= vect_determine_vectorization_factor (loop_vinfo
);
2844 if (dump_enabled_p ())
2845 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2846 "can't determine vectorization factor.\n");
2850 /* Compute the scalar iteration cost. */
2851 vect_compute_single_scalar_iteration_cost (loop_vinfo
);
2853 poly_uint64 saved_vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2857 /* Check the SLP opportunities in the loop, analyze and build
2859 ok
= vect_analyze_slp (loop_vinfo
, LOOP_VINFO_N_STMTS (loop_vinfo
));
2863 /* If there are any SLP instances mark them as pure_slp. */
2864 slp
= vect_make_slp_decision (loop_vinfo
);
2867 /* Find stmts that need to be both vectorized and SLPed. */
2868 vect_detect_hybrid_slp (loop_vinfo
);
2870 /* Update the vectorization factor based on the SLP decision. */
2871 vect_update_vf_for_slp (loop_vinfo
);
2873 /* Optimize the SLP graph with the vectorization factor fixed. */
2874 vect_optimize_slp (loop_vinfo
);
2876 /* Gather the loads reachable from the SLP graph entries. */
2877 vect_gather_slp_loads (loop_vinfo
);
2881 bool saved_can_use_partial_vectors_p
2882 = LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
);
2884 /* We don't expect to have to roll back to anything other than an empty
2886 gcc_assert (LOOP_VINFO_MASKS (loop_vinfo
).is_empty ());
2888 /* This is the point where we can re-start analysis with SLP forced off. */
2891 /* Apply the suggested unrolling factor, this was determined by the backend
2892 during finish_cost the first time we ran the analyzis for this
2894 if (applying_suggested_uf
)
2895 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) *= loop_vinfo
->suggested_unroll_factor
;
2897 /* Now the vectorization factor is final. */
2898 poly_uint64 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2899 gcc_assert (known_ne (vectorization_factor
, 0U));
2901 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) && dump_enabled_p ())
2903 dump_printf_loc (MSG_NOTE
, vect_location
,
2904 "vectorization_factor = ");
2905 dump_dec (MSG_NOTE
, vectorization_factor
);
2906 dump_printf (MSG_NOTE
, ", niters = %wd\n",
2907 LOOP_VINFO_INT_NITERS (loop_vinfo
));
2910 if (max_vf
!= MAX_VECTORIZATION_FACTOR
2911 && maybe_lt (max_vf
, LOOP_VINFO_VECT_FACTOR (loop_vinfo
)))
2912 return opt_result::failure_at (vect_location
, "bad data dependence.\n");
2914 loop_vinfo
->vector_costs
= init_cost (loop_vinfo
, false);
2916 /* Analyze the alignment of the data-refs in the loop.
2917 Fail if a data reference is found that cannot be vectorized. */
2919 ok
= vect_analyze_data_refs_alignment (loop_vinfo
);
2922 if (dump_enabled_p ())
2923 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2924 "bad data alignment.\n");
2928 /* Prune the list of ddrs to be tested at run-time by versioning for alias.
2929 It is important to call pruning after vect_analyze_data_ref_accesses,
2930 since we use grouping information gathered by interleaving analysis. */
2931 ok
= vect_prune_runtime_alias_test_list (loop_vinfo
);
2935 /* Do not invoke vect_enhance_data_refs_alignment for epilogue
2936 vectorization, since we do not want to add extra peeling or
2937 add versioning for alignment. */
2938 if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
2939 /* This pass will decide on using loop versioning and/or loop peeling in
2940 order to enhance the alignment of data references in the loop. */
2941 ok
= vect_enhance_data_refs_alignment (loop_vinfo
);
2947 /* Analyze operations in the SLP instances. Note this may
2948 remove unsupported SLP instances which makes the above
2949 SLP kind detection invalid. */
2950 unsigned old_size
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).length ();
2951 vect_slp_analyze_operations (loop_vinfo
);
2952 if (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).length () != old_size
)
2954 ok
= opt_result::failure_at (vect_location
,
2955 "unsupported SLP instances\n");
2959 /* Check whether any load in ALL SLP instances is possibly permuted. */
2960 slp_tree load_node
, slp_root
;
2962 slp_instance instance
;
2963 bool can_use_lanes
= true;
2964 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), x
, instance
)
2966 slp_root
= SLP_INSTANCE_TREE (instance
);
2967 int group_size
= SLP_TREE_LANES (slp_root
);
2968 tree vectype
= SLP_TREE_VECTYPE (slp_root
);
2969 bool loads_permuted
= false;
2970 FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance
), i
, load_node
)
2972 if (!SLP_TREE_LOAD_PERMUTATION (load_node
).exists ())
2975 stmt_vec_info load_info
;
2976 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (load_node
), j
, load_info
)
2977 if (SLP_TREE_LOAD_PERMUTATION (load_node
)[j
] != j
)
2979 loads_permuted
= true;
2984 /* If the loads and stores can be handled with load/store-lane
2985 instructions record it and move on to the next instance. */
2987 && SLP_INSTANCE_KIND (instance
) == slp_inst_kind_store
2988 && vect_store_lanes_supported (vectype
, group_size
, false)
2991 FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance
), i
, load_node
)
2992 if (STMT_VINFO_GROUPED_ACCESS
2993 (SLP_TREE_REPRESENTATIVE (load_node
)))
2995 stmt_vec_info stmt_vinfo
= DR_GROUP_FIRST_ELEMENT
2996 (SLP_TREE_REPRESENTATIVE (load_node
));
2997 /* Use SLP for strided accesses (or if we can't
2999 if (STMT_VINFO_STRIDED_P (stmt_vinfo
)
3000 || vect_load_lanes_supported
3001 (STMT_VINFO_VECTYPE (stmt_vinfo
),
3002 DR_GROUP_SIZE (stmt_vinfo
), false) == IFN_LAST
)
3007 = can_use_lanes
&& i
== SLP_INSTANCE_LOADS (instance
).length ();
3009 if (can_use_lanes
&& dump_enabled_p ())
3010 dump_printf_loc (MSG_NOTE
, vect_location
,
3011 "SLP instance %p can use load/store-lanes\n",
3016 can_use_lanes
= false;
3021 /* If all SLP instances can use load/store-lanes abort SLP and try again
3022 with SLP disabled. */
3025 ok
= opt_result::failure_at (vect_location
,
3026 "Built SLP cancelled: can use "
3027 "load/store-lanes\n");
3028 if (dump_enabled_p ())
3029 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3030 "Built SLP cancelled: all SLP instances support "
3031 "load/store-lanes\n");
3036 /* Dissolve SLP-only groups. */
3037 vect_dissolve_slp_only_groups (loop_vinfo
);
3039 /* Scan all the remaining operations in the loop that are not subject
3040 to SLP and make sure they are vectorizable. */
3041 ok
= vect_analyze_loop_operations (loop_vinfo
);
3044 if (dump_enabled_p ())
3045 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3046 "bad operation or unsupported loop bound.\n");
3050 /* For now, we don't expect to mix both masking and length approaches for one
3051 loop, disable it if both are recorded. */
3052 if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
)
3053 && !LOOP_VINFO_MASKS (loop_vinfo
).is_empty ()
3054 && !LOOP_VINFO_LENS (loop_vinfo
).is_empty ())
3056 if (dump_enabled_p ())
3057 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3058 "can't vectorize a loop with partial vectors"
3059 " because we don't expect to mix different"
3060 " approaches with partial vectors for the"
3062 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
3065 /* If we still have the option of using partial vectors,
3066 check whether we can generate the necessary loop controls. */
3067 if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
))
3069 if (!LOOP_VINFO_MASKS (loop_vinfo
).is_empty ())
3071 if (!vect_verify_full_masking (loop_vinfo
)
3072 && !vect_verify_full_masking_avx512 (loop_vinfo
))
3073 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
3075 else /* !LOOP_VINFO_LENS (loop_vinfo).is_empty () */
3076 if (!vect_verify_loop_lens (loop_vinfo
))
3077 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
3080 /* If we're vectorizing a loop that uses length "controls" and
3081 can iterate more than once, we apply decrementing IV approach
3083 if (LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
)
3084 && LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
) == vect_partial_vectors_len
3085 && LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
) == 0
3086 && !(LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
3087 && known_le (LOOP_VINFO_INT_NITERS (loop_vinfo
),
3088 LOOP_VINFO_VECT_FACTOR (loop_vinfo
))))
3089 LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo
) = true;
3091 /* If a loop uses length controls and has a decrementing loop control IV,
3092 we will normally pass that IV through a MIN_EXPR to calcaluate the
3093 basis for the length controls. E.g. in a loop that processes one
3094 element per scalar iteration, the number of elements would be
3095 MIN_EXPR <N, VF>, where N is the number of scalar iterations left.
3097 This MIN_EXPR approach allows us to use pointer IVs with an invariant
3098 step, since only the final iteration of the vector loop can have
3101 However, some targets have a dedicated instruction for calculating the
3102 preferred length, given the total number of elements that still need to
3103 be processed. This is encapsulated in the SELECT_VL internal function.
3105 If the target supports SELECT_VL, we can use it instead of MIN_EXPR
3106 to determine the basis for the length controls. However, unlike the
3107 MIN_EXPR calculation, the SELECT_VL calculation can decide to make
3108 lanes inactive in any iteration of the vector loop, not just the last
3109 iteration. This SELECT_VL approach therefore requires us to use pointer
3110 IVs with variable steps.
3112 Once we've decided how many elements should be processed by one
3113 iteration of the vector loop, we need to populate the rgroup controls.
3114 If a loop has multiple rgroups, we need to make sure that those rgroups
3115 "line up" (that is, they must be consistent about which elements are
3116 active and which aren't). This is done by vect_adjust_loop_lens_control.
3118 In principle, it would be possible to use vect_adjust_loop_lens_control
3119 on either the result of a MIN_EXPR or the result of a SELECT_VL.
3122 (1) In practice, it only makes sense to use SELECT_VL when a vector
3123 operation will be controlled directly by the result. It is not
3124 worth using SELECT_VL if it would only be the input to other
3127 (2) If we use SELECT_VL for an rgroup that has N controls, each associated
3128 pointer IV will need N updates by a variable amount (N-1 updates
3129 within the iteration and 1 update to move to the next iteration).
3131 Because of this, we prefer to use the MIN_EXPR approach whenever there
3132 is more than one length control.
3134 In addition, SELECT_VL always operates to a granularity of 1 unit.
3135 If we wanted to use it to control an SLP operation on N consecutive
3136 elements, we would need to make the SELECT_VL inputs measure scalar
3137 iterations (rather than elements) and then multiply the SELECT_VL
3138 result by N. But using SELECT_VL this way is inefficient because
3141 2. We don't apply SELECT_VL on single-rgroup when both (1) and (2) are
3144 (1). LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) is true.
3145 (2). LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant () is true.
3147 Since SELECT_VL (variable step) will make SCEV analysis failed and then
3148 we will fail to gain benefits of following unroll optimizations. We prefer
3149 using the MIN_EXPR approach in this situation. */
3150 if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo
))
3152 tree iv_type
= LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
);
3153 if (direct_internal_fn_supported_p (IFN_SELECT_VL
, iv_type
,
3155 && LOOP_VINFO_LENS (loop_vinfo
).length () == 1
3156 && LOOP_VINFO_LENS (loop_vinfo
)[0].factor
== 1 && !slp
3157 && (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
3158 || !LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant ()))
3159 LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo
) = true;
3162 /* Decide whether this loop_vinfo should use partial vectors or peeling,
3163 assuming that the loop will be used as a main loop. We will redo
3164 this analysis later if we instead decide to use the loop as an
3166 ok
= vect_determine_partial_vectors_and_peeling (loop_vinfo
);
3170 /* If we're vectorizing an epilogue loop, the vectorized loop either needs
3171 to be able to handle fewer than VF scalars, or needs to have a lower VF
3172 than the main loop. */
3173 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
)
3174 && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
3176 poly_uint64 unscaled_vf
3177 = exact_div (LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo
),
3178 orig_loop_vinfo
->suggested_unroll_factor
);
3179 if (maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo
), unscaled_vf
))
3180 return opt_result::failure_at (vect_location
,
3181 "Vectorization factor too high for"
3182 " epilogue loop.\n");
3185 /* Check the costings of the loop make vectorizing worthwhile. */
3186 res
= vect_analyze_loop_costing (loop_vinfo
, suggested_unroll_factor
);
3189 ok
= opt_result::failure_at (vect_location
,
3190 "Loop costings may not be worthwhile.\n");
3194 return opt_result::failure_at (vect_location
,
3195 "Loop costings not worthwhile.\n");
3197 /* If an epilogue loop is required make sure we can create one. */
3198 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
3199 || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
)
3200 || LOOP_VINFO_EARLY_BREAKS (loop_vinfo
))
3202 if (dump_enabled_p ())
3203 dump_printf_loc (MSG_NOTE
, vect_location
, "epilog loop required\n");
3204 if (!vect_can_advance_ivs_p (loop_vinfo
)
3205 || !slpeel_can_duplicate_loop_p (loop
,
3206 LOOP_VINFO_IV_EXIT (loop_vinfo
),
3207 LOOP_VINFO_IV_EXIT (loop_vinfo
)))
3209 ok
= opt_result::failure_at (vect_location
,
3210 "not vectorized: can't create required "
3216 /* During peeling, we need to check if number of loop iterations is
3217 enough for both peeled prolog loop and vector loop. This check
3218 can be merged along with threshold check of loop versioning, so
3219 increase threshold for this case if necessary.
3221 If we are analyzing an epilogue we still want to check what its
3222 versioning threshold would be. If we decide to vectorize the epilogues we
3223 will want to use the lowest versioning threshold of all epilogues and main
3224 loop. This will enable us to enter a vectorized epilogue even when
3225 versioning the loop. We can't simply check whether the epilogue requires
3226 versioning though since we may have skipped some versioning checks when
3227 analyzing the epilogue. For instance, checks for alias versioning will be
3228 skipped when dealing with epilogues as we assume we already checked them
3229 for the main loop. So instead we always check the 'orig_loop_vinfo'. */
3230 if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo
))
3232 poly_uint64 niters_th
= 0;
3233 unsigned int th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
3235 if (!vect_use_loop_mask_for_alignment_p (loop_vinfo
))
3237 /* Niters for peeled prolog loop. */
3238 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) < 0)
3240 dr_vec_info
*dr_info
= LOOP_VINFO_UNALIGNED_DR (loop_vinfo
);
3241 tree vectype
= STMT_VINFO_VECTYPE (dr_info
->stmt
);
3242 niters_th
+= TYPE_VECTOR_SUBPARTS (vectype
) - 1;
3245 niters_th
+= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
3248 /* Niters for at least one iteration of vectorized loop. */
3249 if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
3250 niters_th
+= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
3251 /* One additional iteration because of peeling for gap. */
3252 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
3255 /* Use the same condition as vect_transform_loop to decide when to use
3256 the cost to determine a versioning threshold. */
3257 if (vect_apply_runtime_profitability_check_p (loop_vinfo
)
3258 && ordered_p (th
, niters_th
))
3259 niters_th
= ordered_max (poly_uint64 (th
), niters_th
);
3261 LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
) = niters_th
;
3264 gcc_assert (known_eq (vectorization_factor
,
3265 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)));
3267 slp_done_for_suggested_uf
= slp
;
3269 /* Ok to vectorize! */
3270 LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
) = 1;
3271 return opt_result::success ();
3274 /* Ensure that "ok" is false (with an opt_problem if dumping is enabled). */
3277 /* Try again with SLP forced off but if we didn't do any SLP there is
3278 no point in re-trying. */
3282 /* If the slp decision is true when suggested unroll factor is worked
3283 out, and we are applying suggested unroll factor, we don't need to
3285 if (applying_suggested_uf
&& slp_done_for_suggested_uf
)
3288 /* If there are reduction chains re-trying will fail anyway. */
3289 if (! LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
).is_empty ())
3292 /* Likewise if the grouped loads or stores in the SLP cannot be handled
3293 via interleaving or lane instructions. */
3294 slp_instance instance
;
3297 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), i
, instance
)
3299 stmt_vec_info vinfo
;
3300 vinfo
= SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance
))[0];
3301 if (! STMT_VINFO_GROUPED_ACCESS (vinfo
))
3303 vinfo
= DR_GROUP_FIRST_ELEMENT (vinfo
);
3304 unsigned int size
= DR_GROUP_SIZE (vinfo
);
3305 tree vectype
= STMT_VINFO_VECTYPE (vinfo
);
3306 if (vect_store_lanes_supported (vectype
, size
, false) == IFN_LAST
3307 && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype
), 1U)
3308 && ! vect_grouped_store_supported (vectype
, size
))
3309 return opt_result::failure_at (vinfo
->stmt
,
3310 "unsupported grouped store\n");
3311 FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance
), j
, node
)
3313 vinfo
= SLP_TREE_REPRESENTATIVE (node
);
3314 if (STMT_VINFO_GROUPED_ACCESS (vinfo
))
3316 vinfo
= DR_GROUP_FIRST_ELEMENT (vinfo
);
3317 bool single_element_p
= !DR_GROUP_NEXT_ELEMENT (vinfo
);
3318 size
= DR_GROUP_SIZE (vinfo
);
3319 vectype
= STMT_VINFO_VECTYPE (vinfo
);
3320 if (vect_load_lanes_supported (vectype
, size
, false) == IFN_LAST
3321 && ! vect_grouped_load_supported (vectype
, single_element_p
,
3323 return opt_result::failure_at (vinfo
->stmt
,
3324 "unsupported grouped load\n");
3329 if (dump_enabled_p ())
3330 dump_printf_loc (MSG_NOTE
, vect_location
,
3331 "re-trying with SLP disabled\n");
3333 /* Roll back state appropriately. No SLP this time. */
3335 /* Restore vectorization factor as it were without SLP. */
3336 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = saved_vectorization_factor
;
3337 /* Free the SLP instances. */
3338 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), j
, instance
)
3339 vect_free_slp_instance (instance
);
3340 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).release ();
3341 /* Reset SLP type to loop_vect on all stmts. */
3342 for (i
= 0; i
< LOOP_VINFO_LOOP (loop_vinfo
)->num_nodes
; ++i
)
3344 basic_block bb
= LOOP_VINFO_BBS (loop_vinfo
)[i
];
3345 for (gimple_stmt_iterator si
= gsi_start_phis (bb
);
3346 !gsi_end_p (si
); gsi_next (&si
))
3348 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
3349 STMT_SLP_TYPE (stmt_info
) = loop_vect
;
3350 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
3351 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
3353 /* vectorizable_reduction adjusts reduction stmt def-types,
3354 restore them to that of the PHI. */
3355 STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info
))
3356 = STMT_VINFO_DEF_TYPE (stmt_info
);
3357 STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
3358 (STMT_VINFO_REDUC_DEF (stmt_info
)))
3359 = STMT_VINFO_DEF_TYPE (stmt_info
);
3362 for (gimple_stmt_iterator si
= gsi_start_bb (bb
);
3363 !gsi_end_p (si
); gsi_next (&si
))
3365 if (is_gimple_debug (gsi_stmt (si
)))
3367 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
3368 STMT_SLP_TYPE (stmt_info
) = loop_vect
;
3369 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
3371 stmt_vec_info pattern_stmt_info
3372 = STMT_VINFO_RELATED_STMT (stmt_info
);
3373 if (STMT_VINFO_SLP_VECT_ONLY_PATTERN (pattern_stmt_info
))
3374 STMT_VINFO_IN_PATTERN_P (stmt_info
) = false;
3376 gimple
*pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
3377 STMT_SLP_TYPE (pattern_stmt_info
) = loop_vect
;
3378 for (gimple_stmt_iterator pi
= gsi_start (pattern_def_seq
);
3379 !gsi_end_p (pi
); gsi_next (&pi
))
3380 STMT_SLP_TYPE (loop_vinfo
->lookup_stmt (gsi_stmt (pi
)))
3385 /* Free optimized alias test DDRS. */
3386 LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
).truncate (0);
3387 LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo
).release ();
3388 LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo
).release ();
3389 /* Reset target cost data. */
3390 delete loop_vinfo
->vector_costs
;
3391 loop_vinfo
->vector_costs
= nullptr;
3392 /* Reset accumulated rgroup information. */
3393 LOOP_VINFO_MASKS (loop_vinfo
).mask_set
.empty ();
3394 release_vec_loop_controls (&LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
);
3395 release_vec_loop_controls (&LOOP_VINFO_LENS (loop_vinfo
));
3396 /* Reset assorted flags. */
3397 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = false;
3398 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) = false;
3399 LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
) = 0;
3400 LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
) = 0;
3401 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
)
3402 = saved_can_use_partial_vectors_p
;
3403 LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
) = false;
3408 /* Return true if vectorizing a loop using NEW_LOOP_VINFO appears
3409 to be better than vectorizing it using OLD_LOOP_VINFO. Assume that
3410 OLD_LOOP_VINFO is better unless something specifically indicates
3413 Note that this deliberately isn't a partial order. */
3416 vect_better_loop_vinfo_p (loop_vec_info new_loop_vinfo
,
3417 loop_vec_info old_loop_vinfo
)
3419 struct loop
*loop
= LOOP_VINFO_LOOP (new_loop_vinfo
);
3420 gcc_assert (LOOP_VINFO_LOOP (old_loop_vinfo
) == loop
);
3422 poly_int64 new_vf
= LOOP_VINFO_VECT_FACTOR (new_loop_vinfo
);
3423 poly_int64 old_vf
= LOOP_VINFO_VECT_FACTOR (old_loop_vinfo
);
3425 /* Always prefer a VF of loop->simdlen over any other VF. */
3428 bool new_simdlen_p
= known_eq (new_vf
, loop
->simdlen
);
3429 bool old_simdlen_p
= known_eq (old_vf
, loop
->simdlen
);
3430 if (new_simdlen_p
!= old_simdlen_p
)
3431 return new_simdlen_p
;
3434 const auto *old_costs
= old_loop_vinfo
->vector_costs
;
3435 const auto *new_costs
= new_loop_vinfo
->vector_costs
;
3436 if (loop_vec_info main_loop
= LOOP_VINFO_ORIG_LOOP_INFO (old_loop_vinfo
))
3437 return new_costs
->better_epilogue_loop_than_p (old_costs
, main_loop
);
3439 return new_costs
->better_main_loop_than_p (old_costs
);
3442 /* Decide whether to replace OLD_LOOP_VINFO with NEW_LOOP_VINFO. Return
3443 true if we should. */
3446 vect_joust_loop_vinfos (loop_vec_info new_loop_vinfo
,
3447 loop_vec_info old_loop_vinfo
)
3449 if (!vect_better_loop_vinfo_p (new_loop_vinfo
, old_loop_vinfo
))
3452 if (dump_enabled_p ())
3453 dump_printf_loc (MSG_NOTE
, vect_location
,
3454 "***** Preferring vector mode %s to vector mode %s\n",
3455 GET_MODE_NAME (new_loop_vinfo
->vector_mode
),
3456 GET_MODE_NAME (old_loop_vinfo
->vector_mode
));
3460 /* Analyze LOOP with VECTOR_MODES[MODE_I] and as epilogue if MAIN_LOOP_VINFO is
3461 not NULL. Set AUTODETECTED_VECTOR_MODE if VOIDmode and advance
3462 MODE_I to the next mode useful to analyze.
3463 Return the loop_vinfo on success and wrapped null on failure. */
3465 static opt_loop_vec_info
3466 vect_analyze_loop_1 (class loop
*loop
, vec_info_shared
*shared
,
3467 const vect_loop_form_info
*loop_form_info
,
3468 loop_vec_info main_loop_vinfo
,
3469 const vector_modes
&vector_modes
, unsigned &mode_i
,
3470 machine_mode
&autodetected_vector_mode
,
3473 loop_vec_info loop_vinfo
3474 = vect_create_loop_vinfo (loop
, shared
, loop_form_info
, main_loop_vinfo
);
3476 machine_mode vector_mode
= vector_modes
[mode_i
];
3477 loop_vinfo
->vector_mode
= vector_mode
;
3478 unsigned int suggested_unroll_factor
= 1;
3479 bool slp_done_for_suggested_uf
= false;
3481 /* Run the main analysis. */
3482 opt_result res
= vect_analyze_loop_2 (loop_vinfo
, fatal
,
3483 &suggested_unroll_factor
,
3484 slp_done_for_suggested_uf
);
3485 if (dump_enabled_p ())
3486 dump_printf_loc (MSG_NOTE
, vect_location
,
3487 "***** Analysis %s with vector mode %s\n",
3488 res
? "succeeded" : " failed",
3489 GET_MODE_NAME (loop_vinfo
->vector_mode
));
3491 if (res
&& !main_loop_vinfo
&& suggested_unroll_factor
> 1)
3493 if (dump_enabled_p ())
3494 dump_printf_loc (MSG_NOTE
, vect_location
,
3495 "***** Re-trying analysis for unrolling"
3496 " with unroll factor %d and slp %s.\n",
3497 suggested_unroll_factor
,
3498 slp_done_for_suggested_uf
? "on" : "off");
3499 loop_vec_info unroll_vinfo
3500 = vect_create_loop_vinfo (loop
, shared
, loop_form_info
, main_loop_vinfo
);
3501 unroll_vinfo
->vector_mode
= vector_mode
;
3502 unroll_vinfo
->suggested_unroll_factor
= suggested_unroll_factor
;
3503 opt_result new_res
= vect_analyze_loop_2 (unroll_vinfo
, fatal
, NULL
,
3504 slp_done_for_suggested_uf
);
3508 loop_vinfo
= unroll_vinfo
;
3511 delete unroll_vinfo
;
3514 /* Remember the autodetected vector mode. */
3515 if (vector_mode
== VOIDmode
)
3516 autodetected_vector_mode
= loop_vinfo
->vector_mode
;
3518 /* Advance mode_i, first skipping modes that would result in the
3519 same analysis result. */
3520 while (mode_i
+ 1 < vector_modes
.length ()
3521 && vect_chooses_same_modes_p (loop_vinfo
,
3522 vector_modes
[mode_i
+ 1]))
3524 if (dump_enabled_p ())
3525 dump_printf_loc (MSG_NOTE
, vect_location
,
3526 "***** The result for vector mode %s would"
3528 GET_MODE_NAME (vector_modes
[mode_i
+ 1]));
3531 if (mode_i
+ 1 < vector_modes
.length ()
3532 && VECTOR_MODE_P (autodetected_vector_mode
)
3533 && (related_vector_mode (vector_modes
[mode_i
+ 1],
3534 GET_MODE_INNER (autodetected_vector_mode
))
3535 == autodetected_vector_mode
)
3536 && (related_vector_mode (autodetected_vector_mode
,
3537 GET_MODE_INNER (vector_modes
[mode_i
+ 1]))
3538 == vector_modes
[mode_i
+ 1]))
3540 if (dump_enabled_p ())
3541 dump_printf_loc (MSG_NOTE
, vect_location
,
3542 "***** Skipping vector mode %s, which would"
3543 " repeat the analysis for %s\n",
3544 GET_MODE_NAME (vector_modes
[mode_i
+ 1]),
3545 GET_MODE_NAME (autodetected_vector_mode
));
3554 gcc_checking_assert (main_loop_vinfo
== NULL
);
3555 return opt_loop_vec_info::propagate_failure (res
);
3558 return opt_loop_vec_info::success (loop_vinfo
);
3561 /* Function vect_analyze_loop.
3563 Apply a set of analyses on LOOP, and create a loop_vec_info struct
3564 for it. The different analyses will record information in the
3565 loop_vec_info struct. */
3567 vect_analyze_loop (class loop
*loop
, vec_info_shared
*shared
)
3569 DUMP_VECT_SCOPE ("analyze_loop_nest");
3571 if (loop_outer (loop
)
3572 && loop_vec_info_for_loop (loop_outer (loop
))
3573 && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop
))))
3574 return opt_loop_vec_info::failure_at (vect_location
,
3575 "outer-loop already vectorized.\n");
3577 if (!find_loop_nest (loop
, &shared
->loop_nest
))
3578 return opt_loop_vec_info::failure_at
3580 "not vectorized: loop nest containing two or more consecutive inner"
3581 " loops cannot be vectorized\n");
3583 /* Analyze the loop form. */
3584 vect_loop_form_info loop_form_info
;
3585 opt_result res
= vect_analyze_loop_form (loop
, &loop_form_info
);
3588 if (dump_enabled_p ())
3589 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3590 "bad loop form.\n");
3591 return opt_loop_vec_info::propagate_failure (res
);
3593 if (!integer_onep (loop_form_info
.assumptions
))
3595 /* We consider to vectorize this loop by versioning it under
3596 some assumptions. In order to do this, we need to clear
3597 existing information computed by scev and niter analyzer. */
3599 free_numbers_of_iterations_estimates (loop
);
3600 /* Also set flag for this loop so that following scev and niter
3601 analysis are done under the assumptions. */
3602 loop_constraint_set (loop
, LOOP_C_FINITE
);
3605 /* Clear the existing niter information to make sure the nonwrapping flag
3606 will be calculated and set propriately. */
3607 free_numbers_of_iterations_estimates (loop
);
3609 auto_vector_modes vector_modes
;
3610 /* Autodetect first vector size we try. */
3611 vector_modes
.safe_push (VOIDmode
);
3612 unsigned int autovec_flags
3613 = targetm
.vectorize
.autovectorize_vector_modes (&vector_modes
,
3614 loop
->simdlen
!= 0);
3615 bool pick_lowest_cost_p
= ((autovec_flags
& VECT_COMPARE_COSTS
)
3616 && !unlimited_cost_model (loop
));
3617 machine_mode autodetected_vector_mode
= VOIDmode
;
3618 opt_loop_vec_info first_loop_vinfo
= opt_loop_vec_info::success (NULL
);
3619 unsigned int mode_i
= 0;
3620 unsigned HOST_WIDE_INT simdlen
= loop
->simdlen
;
3622 /* Keep track of the VF for each mode. Initialize all to 0 which indicates
3623 a mode has not been analyzed. */
3624 auto_vec
<poly_uint64
, 8> cached_vf_per_mode
;
3625 for (unsigned i
= 0; i
< vector_modes
.length (); ++i
)
3626 cached_vf_per_mode
.safe_push (0);
3628 /* First determine the main loop vectorization mode, either the first
3629 one that works, starting with auto-detecting the vector mode and then
3630 following the targets order of preference, or the one with the
3631 lowest cost if pick_lowest_cost_p. */
3635 unsigned int last_mode_i
= mode_i
;
3636 /* Set cached VF to -1 prior to analysis, which indicates a mode has
3638 cached_vf_per_mode
[last_mode_i
] = -1;
3639 opt_loop_vec_info loop_vinfo
3640 = vect_analyze_loop_1 (loop
, shared
, &loop_form_info
,
3641 NULL
, vector_modes
, mode_i
,
3642 autodetected_vector_mode
, fatal
);
3648 /* Analyzis has been successful so update the VF value. The
3649 VF should always be a multiple of unroll_factor and we want to
3650 capture the original VF here. */
3651 cached_vf_per_mode
[last_mode_i
]
3652 = exact_div (LOOP_VINFO_VECT_FACTOR (loop_vinfo
),
3653 loop_vinfo
->suggested_unroll_factor
);
3654 /* Once we hit the desired simdlen for the first time,
3655 discard any previous attempts. */
3657 && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo
), simdlen
))
3659 delete first_loop_vinfo
;
3660 first_loop_vinfo
= opt_loop_vec_info::success (NULL
);
3663 else if (pick_lowest_cost_p
3665 && vect_joust_loop_vinfos (loop_vinfo
, first_loop_vinfo
))
3667 /* Pick loop_vinfo over first_loop_vinfo. */
3668 delete first_loop_vinfo
;
3669 first_loop_vinfo
= opt_loop_vec_info::success (NULL
);
3671 if (first_loop_vinfo
== NULL
)
3672 first_loop_vinfo
= loop_vinfo
;
3676 loop_vinfo
= opt_loop_vec_info::success (NULL
);
3679 /* Commit to first_loop_vinfo if we have no reason to try
3681 if (!simdlen
&& !pick_lowest_cost_p
)
3684 if (mode_i
== vector_modes
.length ()
3685 || autodetected_vector_mode
== VOIDmode
)
3688 /* Try the next biggest vector size. */
3689 if (dump_enabled_p ())
3690 dump_printf_loc (MSG_NOTE
, vect_location
,
3691 "***** Re-trying analysis with vector mode %s\n",
3692 GET_MODE_NAME (vector_modes
[mode_i
]));
3694 if (!first_loop_vinfo
)
3695 return opt_loop_vec_info::propagate_failure (res
);
3697 if (dump_enabled_p ())
3698 dump_printf_loc (MSG_NOTE
, vect_location
,
3699 "***** Choosing vector mode %s\n",
3700 GET_MODE_NAME (first_loop_vinfo
->vector_mode
));
3702 /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
3703 enabled, SIMDUID is not set, it is the innermost loop and we have
3704 either already found the loop's SIMDLEN or there was no SIMDLEN to
3706 TODO: Enable epilogue vectorization for loops with SIMDUID set. */
3707 bool vect_epilogues
= (!simdlen
3708 && loop
->inner
== NULL
3709 && param_vect_epilogues_nomask
3710 && LOOP_VINFO_PEELING_FOR_NITER (first_loop_vinfo
)
3711 /* No code motion support for multiple epilogues so for now
3712 not supported when multiple exits. */
3713 && !LOOP_VINFO_EARLY_BREAKS (first_loop_vinfo
)
3715 if (!vect_epilogues
)
3716 return first_loop_vinfo
;
3718 /* Now analyze first_loop_vinfo for epilogue vectorization. */
3719 poly_uint64 lowest_th
= LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo
);
3721 /* For epilogues start the analysis from the first mode. The motivation
3722 behind starting from the beginning comes from cases where the VECTOR_MODES
3723 array may contain length-agnostic and length-specific modes. Their
3724 ordering is not guaranteed, so we could end up picking a mode for the main
3725 loop that is after the epilogue's optimal mode. */
3726 vector_modes
[0] = autodetected_vector_mode
;
3729 bool supports_partial_vectors
=
3730 partial_vectors_supported_p () && param_vect_partial_vector_usage
!= 0;
3731 poly_uint64 first_vinfo_vf
= LOOP_VINFO_VECT_FACTOR (first_loop_vinfo
);
3735 /* If the target does not support partial vectors we can shorten the
3736 number of modes to analyze for the epilogue as we know we can't pick a
3737 mode that would lead to a VF at least as big as the
3739 if (!supports_partial_vectors
3740 && maybe_ge (cached_vf_per_mode
[mode_i
], first_vinfo_vf
))
3743 if (mode_i
== vector_modes
.length ())
3748 if (dump_enabled_p ())
3749 dump_printf_loc (MSG_NOTE
, vect_location
,
3750 "***** Re-trying epilogue analysis with vector "
3751 "mode %s\n", GET_MODE_NAME (vector_modes
[mode_i
]));
3754 opt_loop_vec_info loop_vinfo
3755 = vect_analyze_loop_1 (loop
, shared
, &loop_form_info
,
3757 vector_modes
, mode_i
,
3758 autodetected_vector_mode
, fatal
);
3764 if (pick_lowest_cost_p
)
3766 /* Keep trying to roll back vectorization attempts while the
3767 loop_vec_infos they produced were worse than this one. */
3768 vec
<loop_vec_info
> &vinfos
= first_loop_vinfo
->epilogue_vinfos
;
3769 while (!vinfos
.is_empty ()
3770 && vect_joust_loop_vinfos (loop_vinfo
, vinfos
.last ()))
3772 gcc_assert (vect_epilogues
);
3773 delete vinfos
.pop ();
3776 /* For now only allow one epilogue loop. */
3777 if (first_loop_vinfo
->epilogue_vinfos
.is_empty ())
3779 first_loop_vinfo
->epilogue_vinfos
.safe_push (loop_vinfo
);
3780 poly_uint64 th
= LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
);
3781 gcc_assert (!LOOP_REQUIRES_VERSIONING (loop_vinfo
)
3782 || maybe_ne (lowest_th
, 0U));
3783 /* Keep track of the known smallest versioning
3785 if (ordered_p (lowest_th
, th
))
3786 lowest_th
= ordered_min (lowest_th
, th
);
3791 loop_vinfo
= opt_loop_vec_info::success (NULL
);
3794 /* For now only allow one epilogue loop, but allow
3795 pick_lowest_cost_p to replace it, so commit to the
3796 first epilogue if we have no reason to try alternatives. */
3797 if (!pick_lowest_cost_p
)
3801 if (mode_i
== vector_modes
.length ())
3806 if (!first_loop_vinfo
->epilogue_vinfos
.is_empty ())
3808 LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo
) = lowest_th
;
3809 if (dump_enabled_p ())
3810 dump_printf_loc (MSG_NOTE
, vect_location
,
3811 "***** Choosing epilogue vector mode %s\n",
3813 (first_loop_vinfo
->epilogue_vinfos
[0]->vector_mode
));
3816 return first_loop_vinfo
;
3819 /* Return true if there is an in-order reduction function for CODE, storing
3820 it in *REDUC_FN if so. */
3823 fold_left_reduction_fn (code_helper code
, internal_fn
*reduc_fn
)
3825 /* We support MINUS_EXPR by negating the operand. This also preserves an
3826 initial -0.0 since -0.0 - 0.0 (neutral op for MINUS_EXPR) == -0.0 +
3828 if (code
== PLUS_EXPR
|| code
== MINUS_EXPR
)
3830 *reduc_fn
= IFN_FOLD_LEFT_PLUS
;
3836 /* Function reduction_fn_for_scalar_code
3839 CODE - tree_code of a reduction operations.
3842 REDUC_FN - the corresponding internal function to be used to reduce the
3843 vector of partial results into a single scalar result, or IFN_LAST
3844 if the operation is a supported reduction operation, but does not have
3845 such an internal function.
3847 Return FALSE if CODE currently cannot be vectorized as reduction. */
3850 reduction_fn_for_scalar_code (code_helper code
, internal_fn
*reduc_fn
)
3852 if (code
.is_tree_code ())
3853 switch (tree_code (code
))
3856 *reduc_fn
= IFN_REDUC_MAX
;
3860 *reduc_fn
= IFN_REDUC_MIN
;
3864 *reduc_fn
= IFN_REDUC_PLUS
;
3868 *reduc_fn
= IFN_REDUC_AND
;
3872 *reduc_fn
= IFN_REDUC_IOR
;
3876 *reduc_fn
= IFN_REDUC_XOR
;
3881 *reduc_fn
= IFN_LAST
;
3888 switch (combined_fn (code
))
3891 *reduc_fn
= IFN_REDUC_FMAX
;
3895 *reduc_fn
= IFN_REDUC_FMIN
;
3903 /* If there is a neutral value X such that a reduction would not be affected
3904 by the introduction of additional X elements, return that X, otherwise
3905 return null. CODE is the code of the reduction and SCALAR_TYPE is type
3906 of the scalar elements. If the reduction has just a single initial value
3907 then INITIAL_VALUE is that value, otherwise it is null.
3908 If AS_INITIAL is TRUE the value is supposed to be used as initial value.
3909 In that case no signed zero is returned. */
3912 neutral_op_for_reduction (tree scalar_type
, code_helper code
,
3913 tree initial_value
, bool as_initial
)
3915 if (code
.is_tree_code ())
3916 switch (tree_code (code
))
3923 return build_zero_cst (scalar_type
);
3924 case WIDEN_SUM_EXPR
:
3926 if (!as_initial
&& HONOR_SIGNED_ZEROS (scalar_type
))
3927 return build_real (scalar_type
, dconstm0
);
3929 return build_zero_cst (scalar_type
);
3932 return build_one_cst (scalar_type
);
3935 return build_all_ones_cst (scalar_type
);
3939 return initial_value
;
3945 switch (combined_fn (code
))
3949 return initial_value
;
3956 /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
3957 STMT is printed with a message MSG. */
3960 report_vect_op (dump_flags_t msg_type
, gimple
*stmt
, const char *msg
)
3962 dump_printf_loc (msg_type
, vect_location
, "%s%G", msg
, stmt
);
3965 /* Return true if we need an in-order reduction for operation CODE
3966 on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
3967 overflow must wrap. */
3970 needs_fold_left_reduction_p (tree type
, code_helper code
)
3972 /* CHECKME: check for !flag_finite_math_only too? */
3973 if (SCALAR_FLOAT_TYPE_P (type
))
3975 if (code
.is_tree_code ())
3976 switch (tree_code (code
))
3983 return !flag_associative_math
;
3986 switch (combined_fn (code
))
3993 return !flag_associative_math
;
3997 if (INTEGRAL_TYPE_P (type
))
3998 return (!code
.is_tree_code ()
3999 || !operation_no_trapping_overflow (type
, tree_code (code
)));
4001 if (SAT_FIXED_POINT_TYPE_P (type
))
4007 /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
4008 has a handled computation expression. Store the main reduction
4009 operation in *CODE. */
4012 check_reduction_path (dump_user_location_t loc
, loop_p loop
, gphi
*phi
,
4013 tree loop_arg
, code_helper
*code
,
4014 vec
<std::pair
<ssa_op_iter
, use_operand_p
> > &path
)
4016 auto_bitmap visited
;
4017 tree lookfor
= PHI_RESULT (phi
);
4019 use_operand_p curr
= op_iter_init_phiuse (&curri
, phi
, SSA_OP_USE
);
4020 while (USE_FROM_PTR (curr
) != loop_arg
)
4021 curr
= op_iter_next_use (&curri
);
4022 curri
.i
= curri
.numops
;
4025 path
.safe_push (std::make_pair (curri
, curr
));
4026 tree use
= USE_FROM_PTR (curr
);
4029 gimple
*def
= SSA_NAME_DEF_STMT (use
);
4030 if (gimple_nop_p (def
)
4031 || ! flow_bb_inside_loop_p (loop
, gimple_bb (def
)))
4036 std::pair
<ssa_op_iter
, use_operand_p
> x
= path
.pop ();
4040 curr
= op_iter_next_use (&curri
);
4041 /* Skip already visited or non-SSA operands (from iterating
4043 while (curr
!= NULL_USE_OPERAND_P
4044 && (TREE_CODE (USE_FROM_PTR (curr
)) != SSA_NAME
4045 || ! bitmap_set_bit (visited
,
4047 (USE_FROM_PTR (curr
)))));
4049 while (curr
== NULL_USE_OPERAND_P
&& ! path
.is_empty ());
4050 if (curr
== NULL_USE_OPERAND_P
)
4055 if (gimple_code (def
) == GIMPLE_PHI
)
4056 curr
= op_iter_init_phiuse (&curri
, as_a
<gphi
*>(def
), SSA_OP_USE
);
4058 curr
= op_iter_init_use (&curri
, def
, SSA_OP_USE
);
4059 while (curr
!= NULL_USE_OPERAND_P
4060 && (TREE_CODE (USE_FROM_PTR (curr
)) != SSA_NAME
4061 || ! bitmap_set_bit (visited
,
4063 (USE_FROM_PTR (curr
)))))
4064 curr
= op_iter_next_use (&curri
);
4065 if (curr
== NULL_USE_OPERAND_P
)
4070 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
4072 dump_printf_loc (MSG_NOTE
, loc
, "reduction path: ");
4074 std::pair
<ssa_op_iter
, use_operand_p
> *x
;
4075 FOR_EACH_VEC_ELT (path
, i
, x
)
4076 dump_printf (MSG_NOTE
, "%T ", USE_FROM_PTR (x
->second
));
4077 dump_printf (MSG_NOTE
, "\n");
4080 /* Check whether the reduction path detected is valid. */
4081 bool fail
= path
.length () == 0;
4085 for (unsigned i
= 1; i
< path
.length (); ++i
)
4087 gimple
*use_stmt
= USE_STMT (path
[i
].second
);
4089 if (!gimple_extract_op (use_stmt
, &op
))
4094 unsigned int opi
= op
.num_ops
;
4095 if (gassign
*assign
= dyn_cast
<gassign
*> (use_stmt
))
4097 /* The following make sure we can compute the operand index
4098 easily plus it mostly disallows chaining via COND_EXPR condition
4100 for (opi
= 0; opi
< op
.num_ops
; ++opi
)
4101 if (gimple_assign_rhs1_ptr (assign
) + opi
== path
[i
].second
->use
)
4104 else if (gcall
*call
= dyn_cast
<gcall
*> (use_stmt
))
4106 for (opi
= 0; opi
< op
.num_ops
; ++opi
)
4107 if (gimple_call_arg_ptr (call
, opi
) == path
[i
].second
->use
)
4110 if (opi
== op
.num_ops
)
4115 op
.code
= canonicalize_code (op
.code
, op
.type
);
4116 if (op
.code
== MINUS_EXPR
)
4118 op
.code
= PLUS_EXPR
;
4119 /* Track whether we negate the reduction value each iteration. */
4120 if (op
.ops
[1] == op
.ops
[opi
])
4123 else if (op
.code
== IFN_COND_SUB
)
4125 op
.code
= IFN_COND_ADD
;
4126 /* Track whether we negate the reduction value each iteration. */
4127 if (op
.ops
[2] == op
.ops
[opi
])
4130 if (CONVERT_EXPR_CODE_P (op
.code
)
4131 && tree_nop_conversion_p (op
.type
, TREE_TYPE (op
.ops
[0])))
4133 else if (*code
== ERROR_MARK
)
4136 sign
= TYPE_SIGN (op
.type
);
4138 else if (op
.code
!= *code
)
4143 else if ((op
.code
== MIN_EXPR
4144 || op
.code
== MAX_EXPR
)
4145 && sign
!= TYPE_SIGN (op
.type
))
4150 /* Check there's only a single stmt the op is used on. For the
4151 not value-changing tail and the last stmt allow out-of-loop uses.
4152 ??? We could relax this and handle arbitrary live stmts by
4153 forcing a scalar epilogue for example. */
4154 imm_use_iterator imm_iter
;
4155 use_operand_p use_p
;
4156 gimple
*op_use_stmt
;
4158 bool cond_fn_p
= op
.code
.is_internal_fn ()
4159 && (conditional_internal_fn_code (internal_fn (op
.code
))
4162 FOR_EACH_IMM_USE_STMT (op_use_stmt
, imm_iter
, op
.ops
[opi
])
4164 /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
4165 op1 twice (once as definition, once as else) in the same operation.
4167 if (cond_fn_p
&& op_use_stmt
== use_stmt
)
4169 gcall
*call
= as_a
<gcall
*> (use_stmt
);
4171 = internal_fn_else_index (internal_fn (op
.code
));
4173 for (unsigned int j
= 0; j
< gimple_call_num_args (call
); ++j
)
4177 if (gimple_call_arg (call
, j
) == op
.ops
[opi
])
4181 else if (!is_gimple_debug (op_use_stmt
)
4182 && (*code
!= ERROR_MARK
4183 || flow_bb_inside_loop_p (loop
,
4184 gimple_bb (op_use_stmt
))))
4185 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
4195 return ! fail
&& ! neg
&& *code
!= ERROR_MARK
;
4199 check_reduction_path (dump_user_location_t loc
, loop_p loop
, gphi
*phi
,
4200 tree loop_arg
, enum tree_code code
)
4202 auto_vec
<std::pair
<ssa_op_iter
, use_operand_p
> > path
;
4204 return (check_reduction_path (loc
, loop
, phi
, loop_arg
, &code_
, path
)
4210 /* Function vect_is_simple_reduction
4212 (1) Detect a cross-iteration def-use cycle that represents a simple
4213 reduction computation. We look for the following pattern:
4218 a2 = operation (a3, a1)
4225 a2 = operation (a3, a1)
4228 1. operation is commutative and associative and it is safe to
4229 change the order of the computation
4230 2. no uses for a2 in the loop (a2 is used out of the loop)
4231 3. no uses of a1 in the loop besides the reduction operation
4232 4. no uses of a1 outside the loop.
4234 Conditions 1,4 are tested here.
4235 Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
4237 (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
4240 (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
4244 inner loop (def of a3)
4247 (4) Detect condition expressions, ie:
4248 for (int i = 0; i < N; i++)
4254 static stmt_vec_info
4255 vect_is_simple_reduction (loop_vec_info loop_info
, stmt_vec_info phi_info
,
4256 bool *double_reduc
, bool *reduc_chain_p
, bool slp
)
4258 gphi
*phi
= as_a
<gphi
*> (phi_info
->stmt
);
4259 gimple
*phi_use_stmt
= NULL
;
4260 imm_use_iterator imm_iter
;
4261 use_operand_p use_p
;
4263 *double_reduc
= false;
4264 *reduc_chain_p
= false;
4265 STMT_VINFO_REDUC_TYPE (phi_info
) = TREE_CODE_REDUCTION
;
4267 tree phi_name
= PHI_RESULT (phi
);
4268 /* ??? If there are no uses of the PHI result the inner loop reduction
4269 won't be detected as possibly double-reduction by vectorizable_reduction
4270 because that tries to walk the PHI arg from the preheader edge which
4271 can be constant. See PR60382. */
4272 if (has_zero_uses (phi_name
))
4274 class loop
*loop
= (gimple_bb (phi
))->loop_father
;
4275 unsigned nphi_def_loop_uses
= 0;
4276 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, phi_name
)
4278 gimple
*use_stmt
= USE_STMT (use_p
);
4279 if (is_gimple_debug (use_stmt
))
4282 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
4284 if (dump_enabled_p ())
4285 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4286 "intermediate value used outside loop.\n");
4291 /* In case of a COND_OP (mask, op1, op2, op1) reduction we might have
4292 op1 twice (once as definition, once as else) in the same operation.
4293 Only count it as one. */
4294 if (use_stmt
!= phi_use_stmt
)
4296 nphi_def_loop_uses
++;
4297 phi_use_stmt
= use_stmt
;
4301 tree latch_def
= PHI_ARG_DEF_FROM_EDGE (phi
, loop_latch_edge (loop
));
4302 if (TREE_CODE (latch_def
) != SSA_NAME
)
4304 if (dump_enabled_p ())
4305 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4306 "reduction: not ssa_name: %T\n", latch_def
);
4310 stmt_vec_info def_stmt_info
= loop_info
->lookup_def (latch_def
);
4312 || !flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt_info
->stmt
)))
4315 bool nested_in_vect_loop
4316 = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info
), loop
);
4317 unsigned nlatch_def_loop_uses
= 0;
4318 auto_vec
<gphi
*, 3> lcphis
;
4319 bool inner_loop_of_double_reduc
= false;
4320 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, latch_def
)
4322 gimple
*use_stmt
= USE_STMT (use_p
);
4323 if (is_gimple_debug (use_stmt
))
4325 if (flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
4326 nlatch_def_loop_uses
++;
4329 /* We can have more than one loop-closed PHI. */
4330 lcphis
.safe_push (as_a
<gphi
*> (use_stmt
));
4331 if (nested_in_vect_loop
4332 && (STMT_VINFO_DEF_TYPE (loop_info
->lookup_stmt (use_stmt
))
4333 == vect_double_reduction_def
))
4334 inner_loop_of_double_reduc
= true;
4338 /* If we are vectorizing an inner reduction we are executing that
4339 in the original order only in case we are not dealing with a
4340 double reduction. */
4341 if (nested_in_vect_loop
&& !inner_loop_of_double_reduc
)
4343 if (dump_enabled_p ())
4344 report_vect_op (MSG_NOTE
, def_stmt_info
->stmt
,
4345 "detected nested cycle: ");
4346 return def_stmt_info
;
4349 /* When the inner loop of a double reduction ends up with more than
4350 one loop-closed PHI we have failed to classify alternate such
4351 PHIs as double reduction, leading to wrong code. See PR103237. */
4352 if (inner_loop_of_double_reduc
&& lcphis
.length () != 1)
4354 if (dump_enabled_p ())
4355 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4356 "unhandle double reduction\n");
4360 /* If this isn't a nested cycle or if the nested cycle reduction value
4361 is used ouside of the inner loop we cannot handle uses of the reduction
4363 if (nlatch_def_loop_uses
> 1 || nphi_def_loop_uses
> 1)
4365 if (dump_enabled_p ())
4366 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4367 "reduction used in loop.\n");
4371 /* If DEF_STMT is a phi node itself, we expect it to have a single argument
4372 defined in the inner loop. */
4373 if (gphi
*def_stmt
= dyn_cast
<gphi
*> (def_stmt_info
->stmt
))
4375 tree op1
= PHI_ARG_DEF (def_stmt
, 0);
4376 if (gimple_phi_num_args (def_stmt
) != 1
4377 || TREE_CODE (op1
) != SSA_NAME
)
4379 if (dump_enabled_p ())
4380 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4381 "unsupported phi node definition.\n");
4386 /* Verify there is an inner cycle composed of the PHI phi_use_stmt
4387 and the latch definition op1. */
4388 gimple
*def1
= SSA_NAME_DEF_STMT (op1
);
4389 if (gimple_bb (def1
)
4390 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
4392 && flow_bb_inside_loop_p (loop
->inner
, gimple_bb (def1
))
4393 && (is_gimple_assign (def1
) || is_gimple_call (def1
))
4394 && is_a
<gphi
*> (phi_use_stmt
)
4395 && flow_bb_inside_loop_p (loop
->inner
, gimple_bb (phi_use_stmt
))
4396 && (op1
== PHI_ARG_DEF_FROM_EDGE (phi_use_stmt
,
4397 loop_latch_edge (loop
->inner
))))
4399 if (dump_enabled_p ())
4400 report_vect_op (MSG_NOTE
, def_stmt
,
4401 "detected double reduction: ");
4403 *double_reduc
= true;
4404 return def_stmt_info
;
4410 /* Look for the expression computing latch_def from then loop PHI result. */
4411 auto_vec
<std::pair
<ssa_op_iter
, use_operand_p
> > path
;
4413 if (check_reduction_path (vect_location
, loop
, phi
, latch_def
, &code
,
4416 STMT_VINFO_REDUC_CODE (phi_info
) = code
;
4417 if (code
== COND_EXPR
&& !nested_in_vect_loop
)
4418 STMT_VINFO_REDUC_TYPE (phi_info
) = COND_REDUCTION
;
4420 /* Fill in STMT_VINFO_REDUC_IDX and gather stmts for an SLP
4421 reduction chain for which the additional restriction is that
4422 all operations in the chain are the same. */
4423 auto_vec
<stmt_vec_info
, 8> reduc_chain
;
4425 bool is_slp_reduc
= !nested_in_vect_loop
&& code
!= COND_EXPR
;
4426 for (i
= path
.length () - 1; i
>= 1; --i
)
4428 gimple
*stmt
= USE_STMT (path
[i
].second
);
4429 stmt_vec_info stmt_info
= loop_info
->lookup_stmt (stmt
);
4431 if (!gimple_extract_op (stmt
, &op
))
4433 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
4434 STMT_VINFO_REDUC_IDX (stmt_info
)
4435 = path
[i
].second
->use
- gimple_assign_rhs1_ptr (assign
);
4438 gcall
*call
= as_a
<gcall
*> (stmt
);
4439 STMT_VINFO_REDUC_IDX (stmt_info
)
4440 = path
[i
].second
->use
- gimple_call_arg_ptr (call
, 0);
4442 bool leading_conversion
= (CONVERT_EXPR_CODE_P (op
.code
)
4443 && (i
== 1 || i
== path
.length () - 1));
4444 if ((op
.code
!= code
&& !leading_conversion
)
4445 /* We can only handle the final value in epilogue
4446 generation for reduction chains. */
4447 || (i
!= 1 && !has_single_use (gimple_get_lhs (stmt
))))
4448 is_slp_reduc
= false;
4449 /* For reduction chains we support a trailing/leading
4450 conversions. We do not store those in the actual chain. */
4451 if (leading_conversion
)
4453 reduc_chain
.safe_push (stmt_info
);
4455 if (slp
&& is_slp_reduc
&& reduc_chain
.length () > 1)
4457 for (unsigned i
= 0; i
< reduc_chain
.length () - 1; ++i
)
4459 REDUC_GROUP_FIRST_ELEMENT (reduc_chain
[i
]) = reduc_chain
[0];
4460 REDUC_GROUP_NEXT_ELEMENT (reduc_chain
[i
]) = reduc_chain
[i
+1];
4462 REDUC_GROUP_FIRST_ELEMENT (reduc_chain
.last ()) = reduc_chain
[0];
4463 REDUC_GROUP_NEXT_ELEMENT (reduc_chain
.last ()) = NULL
;
4465 /* Save the chain for further analysis in SLP detection. */
4466 LOOP_VINFO_REDUCTION_CHAINS (loop_info
).safe_push (reduc_chain
[0]);
4467 REDUC_GROUP_SIZE (reduc_chain
[0]) = reduc_chain
.length ();
4469 *reduc_chain_p
= true;
4470 if (dump_enabled_p ())
4471 dump_printf_loc (MSG_NOTE
, vect_location
,
4472 "reduction: detected reduction chain\n");
4474 else if (dump_enabled_p ())
4475 dump_printf_loc (MSG_NOTE
, vect_location
,
4476 "reduction: detected reduction\n");
4478 return def_stmt_info
;
4481 if (dump_enabled_p ())
4482 dump_printf_loc (MSG_NOTE
, vect_location
,
4483 "reduction: unknown pattern\n");
4488 /* Estimate the number of peeled epilogue iterations for LOOP_VINFO.
4489 PEEL_ITERS_PROLOGUE is the number of peeled prologue iterations,
4490 or -1 if not known. */
4493 vect_get_peel_iters_epilogue (loop_vec_info loop_vinfo
, int peel_iters_prologue
)
4495 int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
4496 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) || peel_iters_prologue
== -1)
4498 if (dump_enabled_p ())
4499 dump_printf_loc (MSG_NOTE
, vect_location
,
4500 "cost model: epilogue peel iters set to vf/2 "
4501 "because loop iterations are unknown .\n");
4502 return assumed_vf
/ 2;
4506 int niters
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
4507 peel_iters_prologue
= MIN (niters
, peel_iters_prologue
);
4508 int peel_iters_epilogue
= (niters
- peel_iters_prologue
) % assumed_vf
;
4509 /* If we need to peel for gaps, but no peeling is required, we have to
4510 peel VF iterations. */
4511 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) && !peel_iters_epilogue
)
4512 peel_iters_epilogue
= assumed_vf
;
4513 return peel_iters_epilogue
;
4517 /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */
4519 vect_get_known_peeling_cost (loop_vec_info loop_vinfo
, int peel_iters_prologue
,
4520 int *peel_iters_epilogue
,
4521 stmt_vector_for_cost
*scalar_cost_vec
,
4522 stmt_vector_for_cost
*prologue_cost_vec
,
4523 stmt_vector_for_cost
*epilogue_cost_vec
)
4527 *peel_iters_epilogue
4528 = vect_get_peel_iters_epilogue (loop_vinfo
, peel_iters_prologue
);
4530 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
4532 /* If peeled iterations are known but number of scalar loop
4533 iterations are unknown, count a taken branch per peeled loop. */
4534 if (peel_iters_prologue
> 0)
4535 retval
= record_stmt_cost (prologue_cost_vec
, 1, cond_branch_taken
,
4537 if (*peel_iters_epilogue
> 0)
4538 retval
+= record_stmt_cost (epilogue_cost_vec
, 1, cond_branch_taken
,
4542 stmt_info_for_cost
*si
;
4544 if (peel_iters_prologue
)
4545 FOR_EACH_VEC_ELT (*scalar_cost_vec
, j
, si
)
4546 retval
+= record_stmt_cost (prologue_cost_vec
,
4547 si
->count
* peel_iters_prologue
,
4548 si
->kind
, si
->stmt_info
, si
->misalign
,
4550 if (*peel_iters_epilogue
)
4551 FOR_EACH_VEC_ELT (*scalar_cost_vec
, j
, si
)
4552 retval
+= record_stmt_cost (epilogue_cost_vec
,
4553 si
->count
* *peel_iters_epilogue
,
4554 si
->kind
, si
->stmt_info
, si
->misalign
,
4560 /* Function vect_estimate_min_profitable_iters
4562 Return the number of iterations required for the vector version of the
4563 loop to be profitable relative to the cost of the scalar version of the
4566 *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
4567 of iterations for vectorization. -1 value means loop vectorization
4568 is not profitable. This returned value may be used for dynamic
4569 profitability check.
4571 *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
4572 for static check against estimated number of iterations. */
4575 vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo
,
4576 int *ret_min_profitable_niters
,
4577 int *ret_min_profitable_estimate
,
4578 unsigned *suggested_unroll_factor
)
4580 int min_profitable_iters
;
4581 int min_profitable_estimate
;
4582 int peel_iters_prologue
;
4583 int peel_iters_epilogue
;
4584 unsigned vec_inside_cost
= 0;
4585 int vec_outside_cost
= 0;
4586 unsigned vec_prologue_cost
= 0;
4587 unsigned vec_epilogue_cost
= 0;
4588 int scalar_single_iter_cost
= 0;
4589 int scalar_outside_cost
= 0;
4590 int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
4591 int npeel
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
4592 vector_costs
*target_cost_data
= loop_vinfo
->vector_costs
;
4594 /* Cost model disabled. */
4595 if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo
)))
4597 if (dump_enabled_p ())
4598 dump_printf_loc (MSG_NOTE
, vect_location
, "cost model disabled.\n");
4599 *ret_min_profitable_niters
= 0;
4600 *ret_min_profitable_estimate
= 0;
4604 /* Requires loop versioning tests to handle misalignment. */
4605 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
))
4607 /* FIXME: Make cost depend on complexity of individual check. */
4608 unsigned len
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
).length ();
4609 (void) add_stmt_cost (target_cost_data
, len
, scalar_stmt
, vect_prologue
);
4610 if (dump_enabled_p ())
4611 dump_printf (MSG_NOTE
,
4612 "cost model: Adding cost of checks for loop "
4613 "versioning to treat misalignment.\n");
4616 /* Requires loop versioning with alias checks. */
4617 if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
4619 /* FIXME: Make cost depend on complexity of individual check. */
4620 unsigned len
= LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo
).length ();
4621 (void) add_stmt_cost (target_cost_data
, len
, scalar_stmt
, vect_prologue
);
4622 len
= LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo
).length ();
4624 /* Count LEN - 1 ANDs and LEN comparisons. */
4625 (void) add_stmt_cost (target_cost_data
, len
* 2 - 1,
4626 scalar_stmt
, vect_prologue
);
4627 len
= LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
).length ();
4630 /* Count LEN - 1 ANDs and LEN comparisons. */
4631 unsigned int nstmts
= len
* 2 - 1;
4632 /* +1 for each bias that needs adding. */
4633 for (unsigned int i
= 0; i
< len
; ++i
)
4634 if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
)[i
].unsigned_p
)
4636 (void) add_stmt_cost (target_cost_data
, nstmts
,
4637 scalar_stmt
, vect_prologue
);
4639 if (dump_enabled_p ())
4640 dump_printf (MSG_NOTE
,
4641 "cost model: Adding cost of checks for loop "
4642 "versioning aliasing.\n");
4645 /* Requires loop versioning with niter checks. */
4646 if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo
))
4648 /* FIXME: Make cost depend on complexity of individual check. */
4649 (void) add_stmt_cost (target_cost_data
, 1, vector_stmt
,
4650 NULL
, NULL
, NULL_TREE
, 0, vect_prologue
);
4651 if (dump_enabled_p ())
4652 dump_printf (MSG_NOTE
,
4653 "cost model: Adding cost of checks for loop "
4654 "versioning niters.\n");
4657 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
4658 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
,
4661 /* Count statements in scalar loop. Using this as scalar cost for a single
4664 TODO: Add outer loop support.
4666 TODO: Consider assigning different costs to different scalar
4669 scalar_single_iter_cost
= loop_vinfo
->scalar_costs
->total_cost ();
4671 /* Add additional cost for the peeled instructions in prologue and epilogue
4672 loop. (For fully-masked loops there will be no peeling.)
4674 FORNOW: If we don't know the value of peel_iters for prologue or epilogue
4675 at compile-time - we assume it's vf/2 (the worst would be vf-1).
4677 TODO: Build an expression that represents peel_iters for prologue and
4678 epilogue to be used in a run-time test. */
4680 bool prologue_need_br_taken_cost
= false;
4681 bool prologue_need_br_not_taken_cost
= false;
4683 /* Calculate peel_iters_prologue. */
4684 if (vect_use_loop_mask_for_alignment_p (loop_vinfo
))
4685 peel_iters_prologue
= 0;
4688 peel_iters_prologue
= assumed_vf
/ 2;
4689 if (dump_enabled_p ())
4690 dump_printf (MSG_NOTE
, "cost model: "
4691 "prologue peel iters set to vf/2.\n");
4693 /* If peeled iterations are unknown, count a taken branch and a not taken
4694 branch per peeled loop. Even if scalar loop iterations are known,
4695 vector iterations are not known since peeled prologue iterations are
4696 not known. Hence guards remain the same. */
4697 prologue_need_br_taken_cost
= true;
4698 prologue_need_br_not_taken_cost
= true;
4702 peel_iters_prologue
= npeel
;
4703 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) && peel_iters_prologue
> 0)
4704 /* If peeled iterations are known but number of scalar loop
4705 iterations are unknown, count a taken branch per peeled loop. */
4706 prologue_need_br_taken_cost
= true;
4709 bool epilogue_need_br_taken_cost
= false;
4710 bool epilogue_need_br_not_taken_cost
= false;
4712 /* Calculate peel_iters_epilogue. */
4713 if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
4714 /* We need to peel exactly one iteration for gaps. */
4715 peel_iters_epilogue
= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) ? 1 : 0;
4718 /* If peeling for alignment is unknown, loop bound of main loop
4720 peel_iters_epilogue
= assumed_vf
/ 2;
4721 if (dump_enabled_p ())
4722 dump_printf (MSG_NOTE
, "cost model: "
4723 "epilogue peel iters set to vf/2 because "
4724 "peeling for alignment is unknown.\n");
4726 /* See the same reason above in peel_iters_prologue calculation. */
4727 epilogue_need_br_taken_cost
= true;
4728 epilogue_need_br_not_taken_cost
= true;
4732 peel_iters_epilogue
= vect_get_peel_iters_epilogue (loop_vinfo
, npeel
);
4733 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) && peel_iters_epilogue
> 0)
4734 /* If peeled iterations are known but number of scalar loop
4735 iterations are unknown, count a taken branch per peeled loop. */
4736 epilogue_need_br_taken_cost
= true;
4739 stmt_info_for_cost
*si
;
4741 /* Add costs associated with peel_iters_prologue. */
4742 if (peel_iters_prologue
)
4743 FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
), j
, si
)
4745 (void) add_stmt_cost (target_cost_data
,
4746 si
->count
* peel_iters_prologue
, si
->kind
,
4747 si
->stmt_info
, si
->node
, si
->vectype
,
4748 si
->misalign
, vect_prologue
);
4751 /* Add costs associated with peel_iters_epilogue. */
4752 if (peel_iters_epilogue
)
4753 FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
), j
, si
)
4755 (void) add_stmt_cost (target_cost_data
,
4756 si
->count
* peel_iters_epilogue
, si
->kind
,
4757 si
->stmt_info
, si
->node
, si
->vectype
,
4758 si
->misalign
, vect_epilogue
);
4761 /* Add possible cond_branch_taken/cond_branch_not_taken cost. */
4763 if (prologue_need_br_taken_cost
)
4764 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
,
4767 if (prologue_need_br_not_taken_cost
)
4768 (void) add_stmt_cost (target_cost_data
, 1,
4769 cond_branch_not_taken
, vect_prologue
);
4771 if (epilogue_need_br_taken_cost
)
4772 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
,
4775 if (epilogue_need_br_not_taken_cost
)
4776 (void) add_stmt_cost (target_cost_data
, 1,
4777 cond_branch_not_taken
, vect_epilogue
);
4779 /* Take care of special costs for rgroup controls of partial vectors. */
4780 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
4781 && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
)
4782 == vect_partial_vectors_avx512
))
4784 /* Calculate how many masks we need to generate. */
4785 unsigned int num_masks
= 0;
4786 bool need_saturation
= false;
4787 for (auto rgm
: LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
)
4790 unsigned nvectors
= rgm
.factor
;
4791 num_masks
+= nvectors
;
4792 if (TYPE_PRECISION (TREE_TYPE (rgm
.compare_type
))
4793 < TYPE_PRECISION (LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
)))
4794 need_saturation
= true;
4797 /* ??? The target isn't able to identify the costs below as
4798 producing masks so it cannot penaltize cases where we'd run
4799 out of mask registers for example. */
4801 /* ??? We are also failing to account for smaller vector masks
4802 we generate by splitting larger masks in vect_get_loop_mask. */
4804 /* In the worst case, we need to generate each mask in the prologue
4805 and in the loop body. We need one splat per group and one
4808 Sometimes the prologue mask will fold to a constant,
4809 so the actual prologue cost might be smaller. However, it's
4810 simpler and safer to use the worst-case cost; if this ends up
4811 being the tie-breaker between vectorizing or not, then it's
4812 probably better not to vectorize. */
4813 (void) add_stmt_cost (target_cost_data
,
4815 + LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
.length (),
4816 vector_stmt
, NULL
, NULL
, NULL_TREE
, 0,
4818 (void) add_stmt_cost (target_cost_data
,
4820 + LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
.length (),
4821 vector_stmt
, NULL
, NULL
, NULL_TREE
, 0, vect_body
);
4823 /* When we need saturation we need it both in the prologue and
4825 if (need_saturation
)
4827 (void) add_stmt_cost (target_cost_data
, 1, scalar_stmt
,
4828 NULL
, NULL
, NULL_TREE
, 0, vect_prologue
);
4829 (void) add_stmt_cost (target_cost_data
, 1, scalar_stmt
,
4830 NULL
, NULL
, NULL_TREE
, 0, vect_body
);
4833 else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
4834 && (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
)
4835 == vect_partial_vectors_while_ult
))
4837 /* Calculate how many masks we need to generate. */
4838 unsigned int num_masks
= 0;
4839 rgroup_controls
*rgm
;
4840 unsigned int num_vectors_m1
;
4841 FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo
).rgc_vec
,
4842 num_vectors_m1
, rgm
)
4844 num_masks
+= num_vectors_m1
+ 1;
4845 gcc_assert (num_masks
> 0);
4847 /* In the worst case, we need to generate each mask in the prologue
4848 and in the loop body. One of the loop body mask instructions
4849 replaces the comparison in the scalar loop, and since we don't
4850 count the scalar comparison against the scalar body, we shouldn't
4851 count that vector instruction against the vector body either.
4853 Sometimes we can use unpacks instead of generating prologue
4854 masks and sometimes the prologue mask will fold to a constant,
4855 so the actual prologue cost might be smaller. However, it's
4856 simpler and safer to use the worst-case cost; if this ends up
4857 being the tie-breaker between vectorizing or not, then it's
4858 probably better not to vectorize. */
4859 (void) add_stmt_cost (target_cost_data
, num_masks
,
4860 vector_stmt
, NULL
, NULL
, NULL_TREE
, 0,
4862 (void) add_stmt_cost (target_cost_data
, num_masks
- 1,
4863 vector_stmt
, NULL
, NULL
, NULL_TREE
, 0,
4866 else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo
))
4868 /* Referring to the functions vect_set_loop_condition_partial_vectors
4869 and vect_set_loop_controls_directly, we need to generate each
4870 length in the prologue and in the loop body if required. Although
4871 there are some possible optimizations, we consider the worst case
4874 bool niters_known_p
= LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
);
4875 signed char partial_load_store_bias
4876 = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
);
4878 = (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
)
4879 && !vect_known_niters_smaller_than_vf (loop_vinfo
));
4881 /* Calculate how many statements to be added. */
4882 unsigned int prologue_stmts
= 0;
4883 unsigned int body_stmts
= 0;
4885 rgroup_controls
*rgc
;
4886 unsigned int num_vectors_m1
;
4887 FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo
), num_vectors_m1
, rgc
)
4890 /* May need one SHIFT for nitems_total computation. */
4891 unsigned nitems
= rgc
->max_nscalars_per_iter
* rgc
->factor
;
4892 if (nitems
!= 1 && !niters_known_p
)
4893 prologue_stmts
+= 1;
4895 /* May need one MAX and one MINUS for wrap around. */
4896 if (vect_rgroup_iv_might_wrap_p (loop_vinfo
, rgc
))
4897 prologue_stmts
+= 2;
4899 /* Need one MAX and one MINUS for each batch limit excepting for
4901 prologue_stmts
+= num_vectors_m1
* 2;
4903 unsigned int num_vectors
= num_vectors_m1
+ 1;
4905 /* Need to set up lengths in prologue, only one MIN required
4906 for each since start index is zero. */
4907 prologue_stmts
+= num_vectors
;
4909 /* If we have a non-zero partial load bias, we need one PLUS
4910 to adjust the load length. */
4911 if (partial_load_store_bias
!= 0)
4914 unsigned int length_update_cost
= 0;
4915 if (LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo
))
4916 /* For decrement IV style, Each only need a single SELECT_VL
4917 or MIN since beginning to calculate the number of elements
4918 need to be processed in current iteration. */
4919 length_update_cost
= 1;
4921 /* For increment IV stype, Each may need two MINs and one MINUS to
4922 update lengths in body for next iteration. */
4923 length_update_cost
= 3;
4926 body_stmts
+= length_update_cost
* num_vectors
;
4929 (void) add_stmt_cost (target_cost_data
, prologue_stmts
,
4930 scalar_stmt
, vect_prologue
);
4931 (void) add_stmt_cost (target_cost_data
, body_stmts
,
4932 scalar_stmt
, vect_body
);
4935 /* FORNOW: The scalar outside cost is incremented in one of the
4938 1. The vectorizer checks for alignment and aliasing and generates
4939 a condition that allows dynamic vectorization. A cost model
4940 check is ANDED with the versioning condition. Hence scalar code
4941 path now has the added cost of the versioning check.
4943 if (cost > th & versioning_check)
4946 Hence run-time scalar is incremented by not-taken branch cost.
4948 2. The vectorizer then checks if a prologue is required. If the
4949 cost model check was not done before during versioning, it has to
4950 be done before the prologue check.
4953 prologue = scalar_iters
4958 if (prologue == num_iters)
4961 Hence the run-time scalar cost is incremented by a taken branch,
4962 plus a not-taken branch, plus a taken branch cost.
4964 3. The vectorizer then checks if an epilogue is required. If the
4965 cost model check was not done before during prologue check, it
4966 has to be done with the epilogue check.
4972 if (prologue == num_iters)
4975 if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
4978 Hence the run-time scalar cost should be incremented by 2 taken
4981 TODO: The back end may reorder the BBS's differently and reverse
4982 conditions/branch directions. Change the estimates below to
4983 something more reasonable. */
4985 /* If the number of iterations is known and we do not do versioning, we can
4986 decide whether to vectorize at compile time. Hence the scalar version
4987 do not carry cost model guard costs. */
4988 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
4989 || LOOP_REQUIRES_VERSIONING (loop_vinfo
))
4991 /* Cost model check occurs at versioning. */
4992 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
4993 scalar_outside_cost
+= vect_get_stmt_cost (cond_branch_not_taken
);
4996 /* Cost model check occurs at prologue generation. */
4997 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) < 0)
4998 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
)
4999 + vect_get_stmt_cost (cond_branch_not_taken
);
5000 /* Cost model check occurs at epilogue generation. */
5002 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
);
5006 /* Complete the target-specific cost calculations. */
5007 finish_cost (loop_vinfo
->vector_costs
, loop_vinfo
->scalar_costs
,
5008 &vec_prologue_cost
, &vec_inside_cost
, &vec_epilogue_cost
,
5009 suggested_unroll_factor
);
5011 if (suggested_unroll_factor
&& *suggested_unroll_factor
> 1
5012 && LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo
) != MAX_VECTORIZATION_FACTOR
5013 && !known_le (LOOP_VINFO_VECT_FACTOR (loop_vinfo
) *
5014 *suggested_unroll_factor
,
5015 LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo
)))
5017 if (dump_enabled_p ())
5018 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5019 "can't unroll as unrolled vectorization factor larger"
5020 " than maximum vectorization factor: "
5021 HOST_WIDE_INT_PRINT_UNSIGNED
"\n",
5022 LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo
));
5023 *suggested_unroll_factor
= 1;
5026 vec_outside_cost
= (int)(vec_prologue_cost
+ vec_epilogue_cost
);
5028 if (dump_enabled_p ())
5030 dump_printf_loc (MSG_NOTE
, vect_location
, "Cost model analysis: \n");
5031 dump_printf (MSG_NOTE
, " Vector inside of loop cost: %d\n",
5033 dump_printf (MSG_NOTE
, " Vector prologue cost: %d\n",
5035 dump_printf (MSG_NOTE
, " Vector epilogue cost: %d\n",
5037 dump_printf (MSG_NOTE
, " Scalar iteration cost: %d\n",
5038 scalar_single_iter_cost
);
5039 dump_printf (MSG_NOTE
, " Scalar outside cost: %d\n",
5040 scalar_outside_cost
);
5041 dump_printf (MSG_NOTE
, " Vector outside cost: %d\n",
5043 dump_printf (MSG_NOTE
, " prologue iterations: %d\n",
5044 peel_iters_prologue
);
5045 dump_printf (MSG_NOTE
, " epilogue iterations: %d\n",
5046 peel_iters_epilogue
);
5049 /* Calculate number of iterations required to make the vector version
5050 profitable, relative to the loop bodies only. The following condition
5052 SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
5054 SIC = scalar iteration cost, VIC = vector iteration cost,
5055 VOC = vector outside cost, VF = vectorization factor,
5056 NPEEL = prologue iterations + epilogue iterations,
5057 SOC = scalar outside cost for run time cost model check. */
5059 int saving_per_viter
= (scalar_single_iter_cost
* assumed_vf
5061 if (saving_per_viter
<= 0)
5063 if (LOOP_VINFO_LOOP (loop_vinfo
)->force_vectorize
)
5064 warning_at (vect_location
.get_location_t (), OPT_Wopenmp_simd
,
5065 "vectorization did not happen for a simd loop");
5067 if (dump_enabled_p ())
5068 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5069 "cost model: the vector iteration cost = %d "
5070 "divided by the scalar iteration cost = %d "
5071 "is greater or equal to the vectorization factor = %d"
5073 vec_inside_cost
, scalar_single_iter_cost
, assumed_vf
);
5074 *ret_min_profitable_niters
= -1;
5075 *ret_min_profitable_estimate
= -1;
5079 /* ??? The "if" arm is written to handle all cases; see below for what
5080 we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
5081 if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
5083 /* Rewriting the condition above in terms of the number of
5084 vector iterations (vniters) rather than the number of
5085 scalar iterations (niters) gives:
5087 SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
5089 <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
5091 For integer N, X and Y when X > 0:
5093 N * X > Y <==> N >= (Y /[floor] X) + 1. */
5094 int outside_overhead
= (vec_outside_cost
5095 - scalar_single_iter_cost
* peel_iters_prologue
5096 - scalar_single_iter_cost
* peel_iters_epilogue
5097 - scalar_outside_cost
);
5098 /* We're only interested in cases that require at least one
5099 vector iteration. */
5100 int min_vec_niters
= 1;
5101 if (outside_overhead
> 0)
5102 min_vec_niters
= outside_overhead
/ saving_per_viter
+ 1;
5104 if (dump_enabled_p ())
5105 dump_printf (MSG_NOTE
, " Minimum number of vector iterations: %d\n",
5108 if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
5110 /* Now that we know the minimum number of vector iterations,
5111 find the minimum niters for which the scalar cost is larger:
5113 SIC * niters > VIC * vniters + VOC - SOC
5115 We know that the minimum niters is no more than
5116 vniters * VF + NPEEL, but it might be (and often is) less
5117 than that if a partial vector iteration is cheaper than the
5118 equivalent scalar code. */
5119 int threshold
= (vec_inside_cost
* min_vec_niters
5121 - scalar_outside_cost
);
5123 min_profitable_iters
= 1;
5125 min_profitable_iters
= threshold
/ scalar_single_iter_cost
+ 1;
5128 /* Convert the number of vector iterations into a number of
5129 scalar iterations. */
5130 min_profitable_iters
= (min_vec_niters
* assumed_vf
5131 + peel_iters_prologue
5132 + peel_iters_epilogue
);
5136 min_profitable_iters
= ((vec_outside_cost
- scalar_outside_cost
)
5138 - vec_inside_cost
* peel_iters_prologue
5139 - vec_inside_cost
* peel_iters_epilogue
);
5140 if (min_profitable_iters
<= 0)
5141 min_profitable_iters
= 0;
5144 min_profitable_iters
/= saving_per_viter
;
5146 if ((scalar_single_iter_cost
* assumed_vf
* min_profitable_iters
)
5147 <= (((int) vec_inside_cost
* min_profitable_iters
)
5148 + (((int) vec_outside_cost
- scalar_outside_cost
)
5150 min_profitable_iters
++;
5154 if (dump_enabled_p ())
5155 dump_printf (MSG_NOTE
,
5156 " Calculated minimum iters for profitability: %d\n",
5157 min_profitable_iters
);
5159 if (!LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
5160 && min_profitable_iters
< (assumed_vf
+ peel_iters_prologue
))
5161 /* We want the vectorized loop to execute at least once. */
5162 min_profitable_iters
= assumed_vf
+ peel_iters_prologue
;
5163 else if (min_profitable_iters
< peel_iters_prologue
)
5164 /* For LOOP_VINFO_USING_PARTIAL_VECTORS_P, we need to ensure the
5165 vectorized loop executes at least once. */
5166 min_profitable_iters
= peel_iters_prologue
;
5168 if (dump_enabled_p ())
5169 dump_printf_loc (MSG_NOTE
, vect_location
,
5170 " Runtime profitability threshold = %d\n",
5171 min_profitable_iters
);
5173 *ret_min_profitable_niters
= min_profitable_iters
;
5175 /* Calculate number of iterations required to make the vector version
5176 profitable, relative to the loop bodies only.
5178 Non-vectorized variant is SIC * niters and it must win over vector
5179 variant on the expected loop trip count. The following condition must hold true:
5180 SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC */
5182 if (vec_outside_cost
<= 0)
5183 min_profitable_estimate
= 0;
5184 /* ??? This "else if" arm is written to handle all cases; see below for
5185 what we would do for !LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
5186 else if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
5188 /* This is a repeat of the code above, but with + SOC rather
5190 int outside_overhead
= (vec_outside_cost
5191 - scalar_single_iter_cost
* peel_iters_prologue
5192 - scalar_single_iter_cost
* peel_iters_epilogue
5193 + scalar_outside_cost
);
5194 int min_vec_niters
= 1;
5195 if (outside_overhead
> 0)
5196 min_vec_niters
= outside_overhead
/ saving_per_viter
+ 1;
5198 if (LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
5200 int threshold
= (vec_inside_cost
* min_vec_niters
5202 + scalar_outside_cost
);
5203 min_profitable_estimate
= threshold
/ scalar_single_iter_cost
+ 1;
5206 min_profitable_estimate
= (min_vec_niters
* assumed_vf
5207 + peel_iters_prologue
5208 + peel_iters_epilogue
);
5212 min_profitable_estimate
= ((vec_outside_cost
+ scalar_outside_cost
)
5214 - vec_inside_cost
* peel_iters_prologue
5215 - vec_inside_cost
* peel_iters_epilogue
)
5216 / ((scalar_single_iter_cost
* assumed_vf
)
5219 min_profitable_estimate
= MAX (min_profitable_estimate
, min_profitable_iters
);
5220 if (dump_enabled_p ())
5221 dump_printf_loc (MSG_NOTE
, vect_location
,
5222 " Static estimate profitability threshold = %d\n",
5223 min_profitable_estimate
);
5225 *ret_min_profitable_estimate
= min_profitable_estimate
;
5228 /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
5229 vector elements (not bits) for a vector with NELT elements. */
5231 calc_vec_perm_mask_for_shift (unsigned int offset
, unsigned int nelt
,
5232 vec_perm_builder
*sel
)
5234 /* The encoding is a single stepped pattern. Any wrap-around is handled
5235 by vec_perm_indices. */
5236 sel
->new_vector (nelt
, 1, 3);
5237 for (unsigned int i
= 0; i
< 3; i
++)
5238 sel
->quick_push (i
+ offset
);
5241 /* Checks whether the target supports whole-vector shifts for vectors of mode
5242 MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
5243 it supports vec_perm_const with masks for all necessary shift amounts. */
5245 have_whole_vector_shift (machine_mode mode
)
5247 if (optab_handler (vec_shr_optab
, mode
) != CODE_FOR_nothing
)
5250 /* Variable-length vectors should be handled via the optab. */
5252 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
5255 vec_perm_builder sel
;
5256 vec_perm_indices indices
;
5257 for (unsigned int i
= nelt
/ 2; i
>= 1; i
/= 2)
5259 calc_vec_perm_mask_for_shift (i
, nelt
, &sel
);
5260 indices
.new_vector (sel
, 2, nelt
);
5261 if (!can_vec_perm_const_p (mode
, mode
, indices
, false))
5267 /* Return true if (a) STMT_INFO is a DOT_PROD_EXPR reduction whose
5268 multiplication operands have differing signs and (b) we intend
5269 to emulate the operation using a series of signed DOT_PROD_EXPRs.
5270 See vect_emulate_mixed_dot_prod for the actual sequence used. */
5273 vect_is_emulated_mixed_dot_prod (loop_vec_info loop_vinfo
,
5274 stmt_vec_info stmt_info
)
5276 gassign
*assign
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
5277 if (!assign
|| gimple_assign_rhs_code (assign
) != DOT_PROD_EXPR
)
5280 tree rhs1
= gimple_assign_rhs1 (assign
);
5281 tree rhs2
= gimple_assign_rhs2 (assign
);
5282 if (TYPE_SIGN (TREE_TYPE (rhs1
)) == TYPE_SIGN (TREE_TYPE (rhs2
)))
5285 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
5286 gcc_assert (reduc_info
->is_reduc_info
);
5287 return !directly_supported_p (DOT_PROD_EXPR
,
5288 STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
),
5289 optab_vector_mixed_sign
);
5292 /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
5293 functions. Design better to avoid maintenance issues. */
5295 /* Function vect_model_reduction_cost.
5297 Models cost for a reduction operation, including the vector ops
5298 generated within the strip-mine loop in some cases, the initial
5299 definition before the loop, and the epilogue code that must be generated. */
5302 vect_model_reduction_cost (loop_vec_info loop_vinfo
,
5303 stmt_vec_info stmt_info
, internal_fn reduc_fn
,
5304 vect_reduction_type reduction_type
,
5305 int ncopies
, stmt_vector_for_cost
*cost_vec
)
5307 int prologue_cost
= 0, epilogue_cost
= 0, inside_cost
= 0;
5310 class loop
*loop
= NULL
;
5313 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5315 /* Condition reductions generate two reductions in the loop. */
5316 if (reduction_type
== COND_REDUCTION
)
5319 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5320 mode
= TYPE_MODE (vectype
);
5321 stmt_vec_info orig_stmt_info
= vect_orig_stmt (stmt_info
);
5324 if (!gimple_extract_op (orig_stmt_info
->stmt
, &op
))
5327 bool emulated_mixed_dot_prod
5328 = vect_is_emulated_mixed_dot_prod (loop_vinfo
, stmt_info
);
5329 if (reduction_type
== EXTRACT_LAST_REDUCTION
)
5330 /* No extra instructions are needed in the prologue. The loop body
5331 operations are costed in vectorizable_condition. */
5333 else if (reduction_type
== FOLD_LEFT_REDUCTION
)
5335 /* No extra instructions needed in the prologue. */
5338 if (reduc_fn
!= IFN_LAST
)
5339 /* Count one reduction-like operation per vector. */
5340 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vec_to_scalar
,
5341 stmt_info
, 0, vect_body
);
5344 /* Use NELEMENTS extracts and NELEMENTS scalar ops. */
5345 unsigned int nelements
= ncopies
* vect_nunits_for_cost (vectype
);
5346 inside_cost
= record_stmt_cost (cost_vec
, nelements
,
5347 vec_to_scalar
, stmt_info
, 0,
5349 inside_cost
+= record_stmt_cost (cost_vec
, nelements
,
5350 scalar_stmt
, stmt_info
, 0,
5356 /* Add in the cost of the initial definitions. */
5358 if (reduction_type
== COND_REDUCTION
)
5359 /* For cond reductions we have four vectors: initial index, step,
5360 initial result of the data reduction, initial value of the index
5363 else if (emulated_mixed_dot_prod
)
5364 /* We need the initial reduction value and two invariants:
5365 one that contains the minimum signed value and one that
5366 contains half of its negative. */
5370 prologue_cost
+= record_stmt_cost (cost_vec
, prologue_stmts
,
5371 scalar_to_vec
, stmt_info
, 0,
5375 /* Determine cost of epilogue code.
5377 We have a reduction operator that will reduce the vector in one statement.
5378 Also requires scalar extract. */
5380 if (!loop
|| !nested_in_vect_loop_p (loop
, orig_stmt_info
))
5382 if (reduc_fn
!= IFN_LAST
)
5384 if (reduction_type
== COND_REDUCTION
)
5386 /* An EQ stmt and an COND_EXPR stmt. */
5387 epilogue_cost
+= record_stmt_cost (cost_vec
, 2,
5388 vector_stmt
, stmt_info
, 0,
5390 /* Reduction of the max index and a reduction of the found
5392 epilogue_cost
+= record_stmt_cost (cost_vec
, 2,
5393 vec_to_scalar
, stmt_info
, 0,
5395 /* A broadcast of the max value. */
5396 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
5397 scalar_to_vec
, stmt_info
, 0,
5402 epilogue_cost
+= record_stmt_cost (cost_vec
, 1, vector_stmt
,
5403 stmt_info
, 0, vect_epilogue
);
5404 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
5405 vec_to_scalar
, stmt_info
, 0,
5409 else if (reduction_type
== COND_REDUCTION
)
5411 unsigned estimated_nunits
= vect_nunits_for_cost (vectype
);
5412 /* Extraction of scalar elements. */
5413 epilogue_cost
+= record_stmt_cost (cost_vec
,
5414 2 * estimated_nunits
,
5415 vec_to_scalar
, stmt_info
, 0,
5417 /* Scalar max reductions via COND_EXPR / MAX_EXPR. */
5418 epilogue_cost
+= record_stmt_cost (cost_vec
,
5419 2 * estimated_nunits
- 3,
5420 scalar_stmt
, stmt_info
, 0,
5423 else if (reduction_type
== EXTRACT_LAST_REDUCTION
5424 || reduction_type
== FOLD_LEFT_REDUCTION
)
5425 /* No extra instructions need in the epilogue. */
5429 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype
));
5430 tree bitsize
= TYPE_SIZE (op
.type
);
5431 int element_bitsize
= tree_to_uhwi (bitsize
);
5432 int nelements
= vec_size_in_bits
/ element_bitsize
;
5434 if (op
.code
== COND_EXPR
)
5437 /* We have a whole vector shift available. */
5438 if (VECTOR_MODE_P (mode
)
5439 && directly_supported_p (op
.code
, vectype
)
5440 && have_whole_vector_shift (mode
))
5442 /* Final reduction via vector shifts and the reduction operator.
5443 Also requires scalar extract. */
5444 epilogue_cost
+= record_stmt_cost (cost_vec
,
5445 exact_log2 (nelements
) * 2,
5446 vector_stmt
, stmt_info
, 0,
5448 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
5449 vec_to_scalar
, stmt_info
, 0,
5453 /* Use extracts and reduction op for final reduction. For N
5454 elements, we have N extracts and N-1 reduction ops. */
5455 epilogue_cost
+= record_stmt_cost (cost_vec
,
5456 nelements
+ nelements
- 1,
5457 vector_stmt
, stmt_info
, 0,
5462 if (dump_enabled_p ())
5463 dump_printf (MSG_NOTE
,
5464 "vect_model_reduction_cost: inside_cost = %d, "
5465 "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost
,
5466 prologue_cost
, epilogue_cost
);
5469 /* SEQ is a sequence of instructions that initialize the reduction
5470 described by REDUC_INFO. Emit them in the appropriate place. */
5473 vect_emit_reduction_init_stmts (loop_vec_info loop_vinfo
,
5474 stmt_vec_info reduc_info
, gimple
*seq
)
5476 if (reduc_info
->reused_accumulator
)
5478 /* When reusing an accumulator from the main loop, we only need
5479 initialization instructions if the main loop can be skipped.
5480 In that case, emit the initialization instructions at the end
5481 of the guard block that does the skip. */
5482 edge skip_edge
= loop_vinfo
->skip_main_loop_edge
;
5483 gcc_assert (skip_edge
);
5484 gimple_stmt_iterator gsi
= gsi_last_bb (skip_edge
->src
);
5485 gsi_insert_seq_before (&gsi
, seq
, GSI_SAME_STMT
);
5489 /* The normal case: emit the initialization instructions on the
5491 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5492 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), seq
);
5496 /* Function get_initial_def_for_reduction
5499 REDUC_INFO - the info_for_reduction
5500 INIT_VAL - the initial value of the reduction variable
5501 NEUTRAL_OP - a value that has no effect on the reduction, as per
5502 neutral_op_for_reduction
5505 Return a vector variable, initialized according to the operation that
5506 STMT_VINFO performs. This vector will be used as the initial value
5507 of the vector of partial results.
5509 The value we need is a vector in which element 0 has value INIT_VAL
5510 and every other element has value NEUTRAL_OP. */
5513 get_initial_def_for_reduction (loop_vec_info loop_vinfo
,
5514 stmt_vec_info reduc_info
,
5515 tree init_val
, tree neutral_op
)
5517 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5518 tree scalar_type
= TREE_TYPE (init_val
);
5519 tree vectype
= get_vectype_for_scalar_type (loop_vinfo
, scalar_type
);
5521 gimple_seq stmts
= NULL
;
5523 gcc_assert (vectype
);
5525 gcc_assert (POINTER_TYPE_P (scalar_type
) || INTEGRAL_TYPE_P (scalar_type
)
5526 || SCALAR_FLOAT_TYPE_P (scalar_type
));
5528 gcc_assert (nested_in_vect_loop_p (loop
, reduc_info
)
5529 || loop
== (gimple_bb (reduc_info
->stmt
))->loop_father
);
5531 if (operand_equal_p (init_val
, neutral_op
))
5533 /* If both elements are equal then the vector described above is
5535 neutral_op
= gimple_convert (&stmts
, TREE_TYPE (vectype
), neutral_op
);
5536 init_def
= gimple_build_vector_from_val (&stmts
, vectype
, neutral_op
);
5540 neutral_op
= gimple_convert (&stmts
, TREE_TYPE (vectype
), neutral_op
);
5541 init_val
= gimple_convert (&stmts
, TREE_TYPE (vectype
), init_val
);
5542 if (!TYPE_VECTOR_SUBPARTS (vectype
).is_constant ())
5544 /* Construct a splat of NEUTRAL_OP and insert INIT_VAL into
5546 init_def
= gimple_build_vector_from_val (&stmts
, vectype
,
5548 init_def
= gimple_build (&stmts
, CFN_VEC_SHL_INSERT
,
5549 vectype
, init_def
, init_val
);
5553 /* Build {INIT_VAL, NEUTRAL_OP, NEUTRAL_OP, ...}. */
5554 tree_vector_builder
elts (vectype
, 1, 2);
5555 elts
.quick_push (init_val
);
5556 elts
.quick_push (neutral_op
);
5557 init_def
= gimple_build_vector (&stmts
, &elts
);
5562 vect_emit_reduction_init_stmts (loop_vinfo
, reduc_info
, stmts
);
5566 /* Get at the initial defs for the reduction PHIs for REDUC_INFO,
5567 which performs a reduction involving GROUP_SIZE scalar statements.
5568 NUMBER_OF_VECTORS is the number of vector defs to create. If NEUTRAL_OP
5569 is nonnull, introducing extra elements of that value will not change the
5573 get_initial_defs_for_reduction (loop_vec_info loop_vinfo
,
5574 stmt_vec_info reduc_info
,
5575 vec
<tree
> *vec_oprnds
,
5576 unsigned int number_of_vectors
,
5577 unsigned int group_size
, tree neutral_op
)
5579 vec
<tree
> &initial_values
= reduc_info
->reduc_initial_values
;
5580 unsigned HOST_WIDE_INT nunits
;
5581 unsigned j
, number_of_places_left_in_vector
;
5582 tree vector_type
= STMT_VINFO_VECTYPE (reduc_info
);
5585 gcc_assert (group_size
== initial_values
.length () || neutral_op
);
5587 /* NUMBER_OF_COPIES is the number of times we need to use the same values in
5588 created vectors. It is greater than 1 if unrolling is performed.
5590 For example, we have two scalar operands, s1 and s2 (e.g., group of
5591 strided accesses of size two), while NUNITS is four (i.e., four scalars
5592 of this type can be packed in a vector). The output vector will contain
5593 two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
5596 If REDUC_GROUP_SIZE > NUNITS, the scalars will be split into several
5597 vectors containing the operands.
5599 For example, NUNITS is four as before, and the group size is 8
5600 (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
5601 {s5, s6, s7, s8}. */
5603 if (!TYPE_VECTOR_SUBPARTS (vector_type
).is_constant (&nunits
))
5604 nunits
= group_size
;
5606 number_of_places_left_in_vector
= nunits
;
5607 bool constant_p
= true;
5608 tree_vector_builder
elts (vector_type
, nunits
, 1);
5609 elts
.quick_grow (nunits
);
5610 gimple_seq ctor_seq
= NULL
;
5611 for (j
= 0; j
< nunits
* number_of_vectors
; ++j
)
5616 /* Get the def before the loop. In reduction chain we have only
5617 one initial value. Else we have as many as PHIs in the group. */
5618 if (i
>= initial_values
.length () || (j
> i
&& neutral_op
))
5621 op
= initial_values
[i
];
5623 /* Create 'vect_ = {op0,op1,...,opn}'. */
5624 number_of_places_left_in_vector
--;
5625 elts
[nunits
- number_of_places_left_in_vector
- 1] = op
;
5626 if (!CONSTANT_CLASS_P (op
))
5629 if (number_of_places_left_in_vector
== 0)
5632 if (constant_p
&& !neutral_op
5633 ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type
), nunits
)
5634 : known_eq (TYPE_VECTOR_SUBPARTS (vector_type
), nunits
))
5635 /* Build the vector directly from ELTS. */
5636 init
= gimple_build_vector (&ctor_seq
, &elts
);
5637 else if (neutral_op
)
5639 /* Build a vector of the neutral value and shift the
5640 other elements into place. */
5641 init
= gimple_build_vector_from_val (&ctor_seq
, vector_type
,
5644 while (k
> 0 && elts
[k
- 1] == neutral_op
)
5649 init
= gimple_build (&ctor_seq
, CFN_VEC_SHL_INSERT
,
5650 vector_type
, init
, elts
[k
]);
5655 /* First time round, duplicate ELTS to fill the
5656 required number of vectors. */
5657 duplicate_and_interleave (loop_vinfo
, &ctor_seq
, vector_type
,
5658 elts
, number_of_vectors
, *vec_oprnds
);
5661 vec_oprnds
->quick_push (init
);
5663 number_of_places_left_in_vector
= nunits
;
5664 elts
.new_vector (vector_type
, nunits
, 1);
5665 elts
.quick_grow (nunits
);
5669 if (ctor_seq
!= NULL
)
5670 vect_emit_reduction_init_stmts (loop_vinfo
, reduc_info
, ctor_seq
);
5673 /* For a statement STMT_INFO taking part in a reduction operation return
5674 the stmt_vec_info the meta information is stored on. */
5677 info_for_reduction (vec_info
*vinfo
, stmt_vec_info stmt_info
)
5679 stmt_info
= vect_orig_stmt (stmt_info
);
5680 gcc_assert (STMT_VINFO_REDUC_DEF (stmt_info
));
5681 if (!is_a
<gphi
*> (stmt_info
->stmt
)
5682 || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
5683 stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
5684 gphi
*phi
= as_a
<gphi
*> (stmt_info
->stmt
);
5685 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
5687 if (gimple_phi_num_args (phi
) == 1)
5688 stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
5690 else if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
5692 stmt_vec_info info
= vinfo
->lookup_def (vect_phi_initial_value (phi
));
5693 if (info
&& STMT_VINFO_DEF_TYPE (info
) == vect_double_reduction_def
)
5699 /* See if LOOP_VINFO is an epilogue loop whose main loop had a reduction that
5700 REDUC_INFO can build on. Adjust REDUC_INFO and return true if so, otherwise
5704 vect_find_reusable_accumulator (loop_vec_info loop_vinfo
,
5705 stmt_vec_info reduc_info
)
5707 loop_vec_info main_loop_vinfo
= LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
);
5708 if (!main_loop_vinfo
)
5711 if (STMT_VINFO_REDUC_TYPE (reduc_info
) != TREE_CODE_REDUCTION
)
5714 unsigned int num_phis
= reduc_info
->reduc_initial_values
.length ();
5715 auto_vec
<tree
, 16> main_loop_results (num_phis
);
5716 auto_vec
<tree
, 16> initial_values (num_phis
);
5717 if (edge main_loop_edge
= loop_vinfo
->main_loop_edge
)
5719 /* The epilogue loop can be entered either from the main loop or
5720 from an earlier guard block. */
5721 edge skip_edge
= loop_vinfo
->skip_main_loop_edge
;
5722 for (tree incoming_value
: reduc_info
->reduc_initial_values
)
5726 INCOMING_VALUE = phi<MAIN_LOOP_RESULT(main loop),
5727 INITIAL_VALUE(guard block)>. */
5728 gcc_assert (TREE_CODE (incoming_value
) == SSA_NAME
);
5730 gphi
*phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (incoming_value
));
5731 gcc_assert (gimple_bb (phi
) == main_loop_edge
->dest
);
5733 tree from_main_loop
= PHI_ARG_DEF_FROM_EDGE (phi
, main_loop_edge
);
5734 tree from_skip
= PHI_ARG_DEF_FROM_EDGE (phi
, skip_edge
);
5736 main_loop_results
.quick_push (from_main_loop
);
5737 initial_values
.quick_push (from_skip
);
5741 /* The main loop dominates the epilogue loop. */
5742 main_loop_results
.splice (reduc_info
->reduc_initial_values
);
5744 /* See if the main loop has the kind of accumulator we need. */
5745 vect_reusable_accumulator
*accumulator
5746 = main_loop_vinfo
->reusable_accumulators
.get (main_loop_results
[0]);
5748 || num_phis
!= accumulator
->reduc_info
->reduc_scalar_results
.length ()
5749 || !std::equal (main_loop_results
.begin (), main_loop_results
.end (),
5750 accumulator
->reduc_info
->reduc_scalar_results
.begin ()))
5753 /* Handle the case where we can reduce wider vectors to narrower ones. */
5754 tree vectype
= STMT_VINFO_VECTYPE (reduc_info
);
5755 tree old_vectype
= TREE_TYPE (accumulator
->reduc_input
);
5756 unsigned HOST_WIDE_INT m
;
5757 if (!constant_multiple_p (TYPE_VECTOR_SUBPARTS (old_vectype
),
5758 TYPE_VECTOR_SUBPARTS (vectype
), &m
))
5760 /* Check the intermediate vector types and operations are available. */
5761 tree prev_vectype
= old_vectype
;
5762 poly_uint64 intermediate_nunits
= TYPE_VECTOR_SUBPARTS (old_vectype
);
5763 while (known_gt (intermediate_nunits
, TYPE_VECTOR_SUBPARTS (vectype
)))
5765 intermediate_nunits
= exact_div (intermediate_nunits
, 2);
5766 tree intermediate_vectype
= get_related_vectype_for_scalar_type
5767 (TYPE_MODE (vectype
), TREE_TYPE (vectype
), intermediate_nunits
);
5768 if (!intermediate_vectype
5769 || !directly_supported_p (STMT_VINFO_REDUC_CODE (reduc_info
),
5770 intermediate_vectype
)
5771 || !can_vec_extract (TYPE_MODE (prev_vectype
),
5772 TYPE_MODE (intermediate_vectype
)))
5774 prev_vectype
= intermediate_vectype
;
5777 /* Non-SLP reductions might apply an adjustment after the reduction
5778 operation, in order to simplify the initialization of the accumulator.
5779 If the epilogue loop carries on from where the main loop left off,
5780 it should apply the same adjustment to the final reduction result.
5782 If the epilogue loop can also be entered directly (rather than via
5783 the main loop), we need to be able to handle that case in the same way,
5784 with the same adjustment. (In principle we could add a PHI node
5785 to select the correct adjustment, but in practice that shouldn't be
5787 tree main_adjustment
5788 = STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (accumulator
->reduc_info
);
5789 if (loop_vinfo
->main_loop_edge
&& main_adjustment
)
5791 gcc_assert (num_phis
== 1);
5792 tree initial_value
= initial_values
[0];
5793 /* Check that we can use INITIAL_VALUE as the adjustment and
5794 initialize the accumulator with a neutral value instead. */
5795 if (!operand_equal_p (initial_value
, main_adjustment
))
5797 code_helper code
= STMT_VINFO_REDUC_CODE (reduc_info
);
5798 initial_values
[0] = neutral_op_for_reduction (TREE_TYPE (initial_value
),
5799 code
, initial_value
);
5801 STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info
) = main_adjustment
;
5802 reduc_info
->reduc_initial_values
.truncate (0);
5803 reduc_info
->reduc_initial_values
.splice (initial_values
);
5804 reduc_info
->reused_accumulator
= accumulator
;
5808 /* Reduce the vector VEC_DEF down to VECTYPE with reduction operation
5809 CODE emitting stmts before GSI. Returns a vector def of VECTYPE. */
5812 vect_create_partial_epilog (tree vec_def
, tree vectype
, code_helper code
,
5815 unsigned nunits
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec_def
)).to_constant ();
5816 unsigned nunits1
= TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
5817 tree stype
= TREE_TYPE (vectype
);
5818 tree new_temp
= vec_def
;
5819 while (nunits
> nunits1
)
5822 tree vectype1
= get_related_vectype_for_scalar_type (TYPE_MODE (vectype
),
5824 unsigned int bitsize
= tree_to_uhwi (TYPE_SIZE (vectype1
));
5826 /* The target has to make sure we support lowpart/highpart
5827 extraction, either via direct vector extract or through
5828 an integer mode punning. */
5830 gimple
*epilog_stmt
;
5831 if (convert_optab_handler (vec_extract_optab
,
5832 TYPE_MODE (TREE_TYPE (new_temp
)),
5833 TYPE_MODE (vectype1
))
5834 != CODE_FOR_nothing
)
5836 /* Extract sub-vectors directly once vec_extract becomes
5837 a conversion optab. */
5838 dst1
= make_ssa_name (vectype1
);
5840 = gimple_build_assign (dst1
, BIT_FIELD_REF
,
5841 build3 (BIT_FIELD_REF
, vectype1
,
5842 new_temp
, TYPE_SIZE (vectype1
),
5844 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5845 dst2
= make_ssa_name (vectype1
);
5847 = gimple_build_assign (dst2
, BIT_FIELD_REF
,
5848 build3 (BIT_FIELD_REF
, vectype1
,
5849 new_temp
, TYPE_SIZE (vectype1
),
5850 bitsize_int (bitsize
)));
5851 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5855 /* Extract via punning to appropriately sized integer mode
5857 tree eltype
= build_nonstandard_integer_type (bitsize
, 1);
5858 tree etype
= build_vector_type (eltype
, 2);
5859 gcc_assert (convert_optab_handler (vec_extract_optab
,
5862 != CODE_FOR_nothing
);
5863 tree tem
= make_ssa_name (etype
);
5864 epilog_stmt
= gimple_build_assign (tem
, VIEW_CONVERT_EXPR
,
5865 build1 (VIEW_CONVERT_EXPR
,
5867 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5869 tem
= make_ssa_name (eltype
);
5871 = gimple_build_assign (tem
, BIT_FIELD_REF
,
5872 build3 (BIT_FIELD_REF
, eltype
,
5873 new_temp
, TYPE_SIZE (eltype
),
5875 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5876 dst1
= make_ssa_name (vectype1
);
5877 epilog_stmt
= gimple_build_assign (dst1
, VIEW_CONVERT_EXPR
,
5878 build1 (VIEW_CONVERT_EXPR
,
5880 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5881 tem
= make_ssa_name (eltype
);
5883 = gimple_build_assign (tem
, BIT_FIELD_REF
,
5884 build3 (BIT_FIELD_REF
, eltype
,
5885 new_temp
, TYPE_SIZE (eltype
),
5886 bitsize_int (bitsize
)));
5887 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5888 dst2
= make_ssa_name (vectype1
);
5889 epilog_stmt
= gimple_build_assign (dst2
, VIEW_CONVERT_EXPR
,
5890 build1 (VIEW_CONVERT_EXPR
,
5892 gimple_seq_add_stmt_without_update (seq
, epilog_stmt
);
5895 new_temp
= gimple_build (seq
, code
, vectype1
, dst1
, dst2
);
5901 /* Function vect_create_epilog_for_reduction
5903 Create code at the loop-epilog to finalize the result of a reduction
5906 STMT_INFO is the scalar reduction stmt that is being vectorized.
5907 SLP_NODE is an SLP node containing a group of reduction statements. The
5908 first one in this group is STMT_INFO.
5909 SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
5910 REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
5912 LOOP_EXIT is the edge to update in the merge block. In the case of a single
5913 exit this edge is always the main loop exit.
5916 1. Completes the reduction def-use cycles.
5917 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
5918 by calling the function specified by REDUC_FN if available, or by
5919 other means (whole-vector shifts or a scalar loop).
5920 The function also creates a new phi node at the loop exit to preserve
5921 loop-closed form, as illustrated below.
5923 The flow at the entry to this function:
5926 vec_def = phi <vec_init, null> # REDUCTION_PHI
5927 VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5928 s_loop = scalar_stmt # (scalar) STMT_INFO
5930 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5934 The above is transformed by this function into:
5937 vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
5938 VECT_DEF = vector_stmt # vectorized form of STMT_INFO
5939 s_loop = scalar_stmt # (scalar) STMT_INFO
5941 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5942 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5943 v_out2 = reduce <v_out1>
5944 s_out3 = extract_field <v_out2, 0>
5945 s_out4 = adjust_result <s_out3>
5951 vect_create_epilog_for_reduction (loop_vec_info loop_vinfo
,
5952 stmt_vec_info stmt_info
,
5954 slp_instance slp_node_instance
,
5957 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
5958 gcc_assert (reduc_info
->is_reduc_info
);
5959 /* For double reductions we need to get at the inner loop reduction
5960 stmt which has the meta info attached. Our stmt_info is that of the
5961 loop-closed PHI of the inner loop which we remember as
5962 def for the reduction PHI generation. */
5963 bool double_reduc
= false;
5964 stmt_vec_info rdef_info
= stmt_info
;
5965 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
5967 gcc_assert (!slp_node
);
5968 double_reduc
= true;
5969 stmt_info
= loop_vinfo
->lookup_def (gimple_phi_arg_def
5970 (stmt_info
->stmt
, 0));
5971 stmt_info
= vect_stmt_to_vectorize (stmt_info
);
5973 code_helper code
= STMT_VINFO_REDUC_CODE (reduc_info
);
5974 internal_fn reduc_fn
= STMT_VINFO_REDUC_FN (reduc_info
);
5977 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
), *outer_loop
= NULL
;
5978 basic_block exit_bb
;
5981 gimple
*new_phi
= NULL
, *phi
= NULL
;
5982 gimple_stmt_iterator exit_gsi
;
5983 tree new_temp
= NULL_TREE
, new_name
, new_scalar_dest
;
5984 gimple
*epilog_stmt
= NULL
;
5988 tree orig_name
, scalar_result
;
5989 imm_use_iterator imm_iter
, phi_imm_iter
;
5990 use_operand_p use_p
, phi_use_p
;
5992 auto_vec
<tree
> reduc_inputs
;
5994 vec
<tree
> &scalar_results
= reduc_info
->reduc_scalar_results
;
5995 unsigned int group_size
= 1, k
;
5996 /* SLP reduction without reduction chain, e.g.,
6000 b2 = operation (b1) */
6001 bool slp_reduc
= (slp_node
&& !REDUC_GROUP_FIRST_ELEMENT (stmt_info
));
6002 bool direct_slp_reduc
;
6003 tree induction_index
= NULL_TREE
;
6006 group_size
= SLP_TREE_LANES (slp_node
);
6008 if (nested_in_vect_loop_p (loop
, stmt_info
))
6012 gcc_assert (!slp_node
&& double_reduc
);
6015 vectype
= STMT_VINFO_REDUC_VECTYPE (reduc_info
);
6016 gcc_assert (vectype
);
6017 mode
= TYPE_MODE (vectype
);
6019 tree induc_val
= NULL_TREE
;
6020 tree adjustment_def
= NULL
;
6025 /* Optimize: for induction condition reduction, if we can't use zero
6026 for induc_val, use initial_def. */
6027 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
6028 induc_val
= STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
);
6029 else if (double_reduc
)
6032 adjustment_def
= STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info
);
6035 stmt_vec_info single_live_out_stmt
[] = { stmt_info
};
6036 array_slice
<const stmt_vec_info
> live_out_stmts
= single_live_out_stmt
;
6038 /* All statements produce live-out values. */
6039 live_out_stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
6045 vec_num
= SLP_TREE_VEC_DEFS (slp_node_instance
->reduc_phis
).length ();
6051 ncopies
= STMT_VINFO_VEC_STMTS (reduc_info
).length ();
6054 /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
6055 which is updated with the current index of the loop for every match of
6056 the original loop's cond_expr (VEC_STMT). This results in a vector
6057 containing the last time the condition passed for that vector lane.
6058 The first match will be a 1 to allow 0 to be used for non-matching
6059 indexes. If there are no matches at all then the vector will be all
6062 PR92772: This algorithm is broken for architectures that support
6063 masked vectors, but do not provide fold_extract_last. */
6064 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
)
6066 auto_vec
<std::pair
<tree
, bool>, 2> ccompares
;
6067 stmt_vec_info cond_info
= STMT_VINFO_REDUC_DEF (reduc_info
);
6068 cond_info
= vect_stmt_to_vectorize (cond_info
);
6069 while (cond_info
!= reduc_info
)
6071 if (gimple_assign_rhs_code (cond_info
->stmt
) == COND_EXPR
)
6073 gimple
*vec_stmt
= STMT_VINFO_VEC_STMTS (cond_info
)[0];
6074 gcc_assert (gimple_assign_rhs_code (vec_stmt
) == VEC_COND_EXPR
);
6076 (std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt
)),
6077 STMT_VINFO_REDUC_IDX (cond_info
) == 2));
6080 = loop_vinfo
->lookup_def (gimple_op (cond_info
->stmt
,
6081 1 + STMT_VINFO_REDUC_IDX
6083 cond_info
= vect_stmt_to_vectorize (cond_info
);
6085 gcc_assert (ccompares
.length () != 0);
6087 tree indx_before_incr
, indx_after_incr
;
6088 poly_uint64 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype
);
6089 int scalar_precision
6090 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype
)));
6091 tree cr_index_scalar_type
= make_unsigned_type (scalar_precision
);
6092 tree cr_index_vector_type
= get_related_vectype_for_scalar_type
6093 (TYPE_MODE (vectype
), cr_index_scalar_type
,
6094 TYPE_VECTOR_SUBPARTS (vectype
));
6096 /* First we create a simple vector induction variable which starts
6097 with the values {1,2,3,...} (SERIES_VECT) and increments by the
6098 vector size (STEP). */
6100 /* Create a {1,2,3,...} vector. */
6101 tree series_vect
= build_index_vector (cr_index_vector_type
, 1, 1);
6103 /* Create a vector of the step value. */
6104 tree step
= build_int_cst (cr_index_scalar_type
, nunits_out
);
6105 tree vec_step
= build_vector_from_val (cr_index_vector_type
, step
);
6107 /* Create an induction variable. */
6108 gimple_stmt_iterator incr_gsi
;
6110 vect_iv_increment_position (loop_exit
, &incr_gsi
, &insert_after
);
6111 create_iv (series_vect
, PLUS_EXPR
, vec_step
, NULL_TREE
, loop
, &incr_gsi
,
6112 insert_after
, &indx_before_incr
, &indx_after_incr
);
6114 /* Next create a new phi node vector (NEW_PHI_TREE) which starts
6115 filled with zeros (VEC_ZERO). */
6117 /* Create a vector of 0s. */
6118 tree zero
= build_zero_cst (cr_index_scalar_type
);
6119 tree vec_zero
= build_vector_from_val (cr_index_vector_type
, zero
);
6121 /* Create a vector phi node. */
6122 tree new_phi_tree
= make_ssa_name (cr_index_vector_type
);
6123 new_phi
= create_phi_node (new_phi_tree
, loop
->header
);
6124 add_phi_arg (as_a
<gphi
*> (new_phi
), vec_zero
,
6125 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
6127 /* Now take the condition from the loops original cond_exprs
6128 and produce a new cond_exprs (INDEX_COND_EXPR) which for
6129 every match uses values from the induction variable
6130 (INDEX_BEFORE_INCR) otherwise uses values from the phi node
6132 Finally, we update the phi (NEW_PHI_TREE) to take the value of
6133 the new cond_expr (INDEX_COND_EXPR). */
6134 gimple_seq stmts
= NULL
;
6135 for (int i
= ccompares
.length () - 1; i
!= -1; --i
)
6137 tree ccompare
= ccompares
[i
].first
;
6138 if (ccompares
[i
].second
)
6139 new_phi_tree
= gimple_build (&stmts
, VEC_COND_EXPR
,
6140 cr_index_vector_type
,
6142 indx_before_incr
, new_phi_tree
);
6144 new_phi_tree
= gimple_build (&stmts
, VEC_COND_EXPR
,
6145 cr_index_vector_type
,
6147 new_phi_tree
, indx_before_incr
);
6149 gsi_insert_seq_before (&incr_gsi
, stmts
, GSI_SAME_STMT
);
6151 /* Update the phi with the vec cond. */
6152 induction_index
= new_phi_tree
;
6153 add_phi_arg (as_a
<gphi
*> (new_phi
), induction_index
,
6154 loop_latch_edge (loop
), UNKNOWN_LOCATION
);
6157 /* 2. Create epilog code.
6158 The reduction epilog code operates across the elements of the vector
6159 of partial results computed by the vectorized loop.
6160 The reduction epilog code consists of:
6162 step 1: compute the scalar result in a vector (v_out2)
6163 step 2: extract the scalar result (s_out3) from the vector (v_out2)
6164 step 3: adjust the scalar result (s_out3) if needed.
6166 Step 1 can be accomplished using one the following three schemes:
6167 (scheme 1) using reduc_fn, if available.
6168 (scheme 2) using whole-vector shifts, if available.
6169 (scheme 3) using a scalar loop. In this case steps 1+2 above are
6172 The overall epilog code looks like this:
6174 s_out0 = phi <s_loop> # original EXIT_PHI
6175 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6176 v_out2 = reduce <v_out1> # step 1
6177 s_out3 = extract_field <v_out2, 0> # step 2
6178 s_out4 = adjust_result <s_out3> # step 3
6180 (step 3 is optional, and steps 1 and 2 may be combined).
6181 Lastly, the uses of s_out0 are replaced by s_out4. */
6184 /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
6185 v_out1 = phi <VECT_DEF>
6186 Store them in NEW_PHIS. */
6189 /* We need to reduce values in all exits. */
6190 exit_bb
= loop_exit
->dest
;
6191 exit_gsi
= gsi_after_labels (exit_bb
);
6192 reduc_inputs
.create (slp_node
? vec_num
: ncopies
);
6193 for (unsigned i
= 0; i
< vec_num
; i
++)
6195 gimple_seq stmts
= NULL
;
6197 def
= vect_get_slp_vect_def (slp_node
, i
);
6199 def
= gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info
)[0]);
6200 for (j
= 0; j
< ncopies
; j
++)
6202 tree new_def
= copy_ssa_name (def
);
6203 phi
= create_phi_node (new_def
, exit_bb
);
6205 def
= gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info
)[j
]);
6206 if (LOOP_VINFO_IV_EXIT (loop_vinfo
) == loop_exit
)
6207 SET_PHI_ARG_DEF (phi
, loop_exit
->dest_idx
, def
);
6210 for (unsigned k
= 0; k
< gimple_phi_num_args (phi
); k
++)
6211 SET_PHI_ARG_DEF (phi
, k
, def
);
6213 new_def
= gimple_convert (&stmts
, vectype
, new_def
);
6214 reduc_inputs
.quick_push (new_def
);
6216 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6219 /* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
6220 (i.e. when reduc_fn is not available) and in the final adjustment
6221 code (if needed). Also get the original scalar reduction variable as
6222 defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
6223 represents a reduction pattern), the tree-code and scalar-def are
6224 taken from the original stmt that the pattern-stmt (STMT) replaces.
6225 Otherwise (it is a regular reduction) - the tree-code and scalar-def
6226 are taken from STMT. */
6228 stmt_vec_info orig_stmt_info
= vect_orig_stmt (stmt_info
);
6229 if (orig_stmt_info
!= stmt_info
)
6231 /* Reduction pattern */
6232 gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info
));
6233 gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info
) == stmt_info
);
6236 scalar_dest
= gimple_get_lhs (orig_stmt_info
->stmt
);
6237 scalar_type
= TREE_TYPE (scalar_dest
);
6238 scalar_results
.truncate (0);
6239 scalar_results
.reserve_exact (group_size
);
6240 new_scalar_dest
= vect_create_destination_var (scalar_dest
, NULL
);
6241 bitsize
= TYPE_SIZE (scalar_type
);
6243 /* True if we should implement SLP_REDUC using native reduction operations
6244 instead of scalar operations. */
6245 direct_slp_reduc
= (reduc_fn
!= IFN_LAST
6247 && !TYPE_VECTOR_SUBPARTS (vectype
).is_constant ());
6249 /* In case of reduction chain, e.g.,
6252 a3 = operation (a2),
6254 we may end up with more than one vector result. Here we reduce them
6257 The same is true for a SLP reduction, e.g.,
6261 b2 = operation (a2),
6263 where we can end up with more than one vector as well. We can
6264 easily accumulate vectors when the number of vector elements is
6265 a multiple of the SLP group size.
6267 The same is true if we couldn't use a single defuse cycle. */
6268 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
)
6271 && constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype
), group_size
))
6274 gimple_seq stmts
= NULL
;
6275 tree single_input
= reduc_inputs
[0];
6276 for (k
= 1; k
< reduc_inputs
.length (); k
++)
6277 single_input
= gimple_build (&stmts
, code
, vectype
,
6278 single_input
, reduc_inputs
[k
]);
6279 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6281 reduc_inputs
.truncate (0);
6282 reduc_inputs
.safe_push (single_input
);
6285 tree orig_reduc_input
= reduc_inputs
[0];
6287 /* If this loop is an epilogue loop that can be skipped after the
6288 main loop, we can only share a reduction operation between the
6289 main loop and the epilogue if we put it at the target of the
6292 We can still reuse accumulators if this check fails. Doing so has
6293 the minor(?) benefit of making the epilogue loop's scalar result
6294 independent of the main loop's scalar result. */
6295 bool unify_with_main_loop_p
= false;
6296 if (reduc_info
->reused_accumulator
6297 && loop_vinfo
->skip_this_loop_edge
6298 && single_succ_p (exit_bb
)
6299 && single_succ (exit_bb
) == loop_vinfo
->skip_this_loop_edge
->dest
)
6301 unify_with_main_loop_p
= true;
6303 basic_block reduc_block
= loop_vinfo
->skip_this_loop_edge
->dest
;
6304 reduc_inputs
[0] = make_ssa_name (vectype
);
6305 gphi
*new_phi
= create_phi_node (reduc_inputs
[0], reduc_block
);
6306 add_phi_arg (new_phi
, orig_reduc_input
, single_succ_edge (exit_bb
),
6308 add_phi_arg (new_phi
, reduc_info
->reused_accumulator
->reduc_input
,
6309 loop_vinfo
->skip_this_loop_edge
, UNKNOWN_LOCATION
);
6310 exit_gsi
= gsi_after_labels (reduc_block
);
6313 /* Shouldn't be used beyond this point. */
6316 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
6317 && reduc_fn
!= IFN_LAST
)
6319 /* For condition reductions, we have a vector (REDUC_INPUTS 0) containing
6320 various data values where the condition matched and another vector
6321 (INDUCTION_INDEX) containing all the indexes of those matches. We
6322 need to extract the last matching index (which will be the index with
6323 highest value) and use this to index into the data vector.
6324 For the case where there were no matches, the data vector will contain
6325 all default values and the index vector will be all zeros. */
6327 /* Get various versions of the type of the vector of indexes. */
6328 tree index_vec_type
= TREE_TYPE (induction_index
);
6329 gcc_checking_assert (TYPE_UNSIGNED (index_vec_type
));
6330 tree index_scalar_type
= TREE_TYPE (index_vec_type
);
6331 tree index_vec_cmp_type
= truth_type_for (index_vec_type
);
6333 /* Get an unsigned integer version of the type of the data vector. */
6334 int scalar_precision
6335 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type
));
6336 tree scalar_type_unsigned
= make_unsigned_type (scalar_precision
);
6337 tree vectype_unsigned
= get_same_sized_vectype (scalar_type_unsigned
,
6340 /* First we need to create a vector (ZERO_VEC) of zeros and another
6341 vector (MAX_INDEX_VEC) filled with the last matching index, which we
6342 can create using a MAX reduction and then expanding.
6343 In the case where the loop never made any matches, the max index will
6346 /* Vector of {0, 0, 0,...}. */
6347 tree zero_vec
= build_zero_cst (vectype
);
6349 /* Find maximum value from the vector of found indexes. */
6350 tree max_index
= make_ssa_name (index_scalar_type
);
6351 gcall
*max_index_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
6352 1, induction_index
);
6353 gimple_call_set_lhs (max_index_stmt
, max_index
);
6354 gsi_insert_before (&exit_gsi
, max_index_stmt
, GSI_SAME_STMT
);
6356 /* Vector of {max_index, max_index, max_index,...}. */
6357 tree max_index_vec
= make_ssa_name (index_vec_type
);
6358 tree max_index_vec_rhs
= build_vector_from_val (index_vec_type
,
6360 gimple
*max_index_vec_stmt
= gimple_build_assign (max_index_vec
,
6362 gsi_insert_before (&exit_gsi
, max_index_vec_stmt
, GSI_SAME_STMT
);
6364 /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
6365 with the vector (INDUCTION_INDEX) of found indexes, choosing values
6366 from the data vector (REDUC_INPUTS 0) for matches, 0 (ZERO_VEC)
6367 otherwise. Only one value should match, resulting in a vector
6368 (VEC_COND) with one data value and the rest zeros.
6369 In the case where the loop never made any matches, every index will
6370 match, resulting in a vector with all data values (which will all be
6371 the default value). */
6373 /* Compare the max index vector to the vector of found indexes to find
6374 the position of the max value. */
6375 tree vec_compare
= make_ssa_name (index_vec_cmp_type
);
6376 gimple
*vec_compare_stmt
= gimple_build_assign (vec_compare
, EQ_EXPR
,
6379 gsi_insert_before (&exit_gsi
, vec_compare_stmt
, GSI_SAME_STMT
);
6381 /* Use the compare to choose either values from the data vector or
6383 tree vec_cond
= make_ssa_name (vectype
);
6384 gimple
*vec_cond_stmt
= gimple_build_assign (vec_cond
, VEC_COND_EXPR
,
6388 gsi_insert_before (&exit_gsi
, vec_cond_stmt
, GSI_SAME_STMT
);
6390 /* Finally we need to extract the data value from the vector (VEC_COND)
6391 into a scalar (MATCHED_DATA_REDUC). Logically we want to do a OR
6392 reduction, but because this doesn't exist, we can use a MAX reduction
6393 instead. The data value might be signed or a float so we need to cast
6395 In the case where the loop never made any matches, the data values are
6396 all identical, and so will reduce down correctly. */
6398 /* Make the matched data values unsigned. */
6399 tree vec_cond_cast
= make_ssa_name (vectype_unsigned
);
6400 tree vec_cond_cast_rhs
= build1 (VIEW_CONVERT_EXPR
, vectype_unsigned
,
6402 gimple
*vec_cond_cast_stmt
= gimple_build_assign (vec_cond_cast
,
6405 gsi_insert_before (&exit_gsi
, vec_cond_cast_stmt
, GSI_SAME_STMT
);
6407 /* Reduce down to a scalar value. */
6408 tree data_reduc
= make_ssa_name (scalar_type_unsigned
);
6409 gcall
*data_reduc_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
6411 gimple_call_set_lhs (data_reduc_stmt
, data_reduc
);
6412 gsi_insert_before (&exit_gsi
, data_reduc_stmt
, GSI_SAME_STMT
);
6414 /* Convert the reduced value back to the result type and set as the
6416 gimple_seq stmts
= NULL
;
6417 new_temp
= gimple_build (&stmts
, VIEW_CONVERT_EXPR
, scalar_type
,
6419 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6420 scalar_results
.safe_push (new_temp
);
6422 else if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
6423 && reduc_fn
== IFN_LAST
)
6425 /* Condition reduction without supported IFN_REDUC_MAX. Generate
6427 idx_val = induction_index[0];
6428 val = data_reduc[0];
6429 for (idx = 0, val = init, i = 0; i < nelts; ++i)
6430 if (induction_index[i] > idx_val)
6431 val = data_reduc[i], idx_val = induction_index[i];
6434 tree data_eltype
= TREE_TYPE (vectype
);
6435 tree idx_eltype
= TREE_TYPE (TREE_TYPE (induction_index
));
6436 unsigned HOST_WIDE_INT el_size
= tree_to_uhwi (TYPE_SIZE (idx_eltype
));
6437 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index
));
6438 /* Enforced by vectorizable_reduction, which ensures we have target
6439 support before allowing a conditional reduction on variable-length
6441 unsigned HOST_WIDE_INT v_size
= el_size
* nunits
.to_constant ();
6442 tree idx_val
= NULL_TREE
, val
= NULL_TREE
;
6443 for (unsigned HOST_WIDE_INT off
= 0; off
< v_size
; off
+= el_size
)
6445 tree old_idx_val
= idx_val
;
6447 idx_val
= make_ssa_name (idx_eltype
);
6448 epilog_stmt
= gimple_build_assign (idx_val
, BIT_FIELD_REF
,
6449 build3 (BIT_FIELD_REF
, idx_eltype
,
6451 bitsize_int (el_size
),
6452 bitsize_int (off
)));
6453 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6454 val
= make_ssa_name (data_eltype
);
6455 epilog_stmt
= gimple_build_assign (val
, BIT_FIELD_REF
,
6456 build3 (BIT_FIELD_REF
,
6459 bitsize_int (el_size
),
6460 bitsize_int (off
)));
6461 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6464 tree new_idx_val
= idx_val
;
6465 if (off
!= v_size
- el_size
)
6467 new_idx_val
= make_ssa_name (idx_eltype
);
6468 epilog_stmt
= gimple_build_assign (new_idx_val
,
6471 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6473 tree cond
= make_ssa_name (boolean_type_node
);
6474 epilog_stmt
= gimple_build_assign (cond
, GT_EXPR
,
6475 idx_val
, old_idx_val
);
6476 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6477 tree new_val
= make_ssa_name (data_eltype
);
6478 epilog_stmt
= gimple_build_assign (new_val
, COND_EXPR
,
6479 cond
, val
, old_val
);
6480 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6481 idx_val
= new_idx_val
;
6485 /* Convert the reduced value back to the result type and set as the
6487 gimple_seq stmts
= NULL
;
6488 val
= gimple_convert (&stmts
, scalar_type
, val
);
6489 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6490 scalar_results
.safe_push (val
);
6493 /* 2.3 Create the reduction code, using one of the three schemes described
6494 above. In SLP we simply need to extract all the elements from the
6495 vector (without reducing them), so we use scalar shifts. */
6496 else if (reduc_fn
!= IFN_LAST
&& !slp_reduc
)
6502 v_out2 = reduc_expr <v_out1> */
6504 if (dump_enabled_p ())
6505 dump_printf_loc (MSG_NOTE
, vect_location
,
6506 "Reduce using direct vector reduction.\n");
6508 gimple_seq stmts
= NULL
;
6509 vec_elem_type
= TREE_TYPE (vectype
);
6510 new_temp
= gimple_build (&stmts
, as_combined_fn (reduc_fn
),
6511 vec_elem_type
, reduc_inputs
[0]);
6512 new_temp
= gimple_convert (&stmts
, scalar_type
, new_temp
);
6513 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6515 if ((STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
6518 /* Earlier we set the initial value to be a vector if induc_val
6519 values. Check the result and if it is induc_val then replace
6520 with the original initial value, unless induc_val is
6521 the same as initial_def already. */
6522 tree zcompare
= make_ssa_name (boolean_type_node
);
6523 epilog_stmt
= gimple_build_assign (zcompare
, EQ_EXPR
,
6524 new_temp
, induc_val
);
6525 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6526 tree initial_def
= reduc_info
->reduc_initial_values
[0];
6527 tmp
= make_ssa_name (new_scalar_dest
);
6528 epilog_stmt
= gimple_build_assign (tmp
, COND_EXPR
, zcompare
,
6529 initial_def
, new_temp
);
6530 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6534 scalar_results
.safe_push (new_temp
);
6536 else if (direct_slp_reduc
)
6538 /* Here we create one vector for each of the REDUC_GROUP_SIZE results,
6539 with the elements for other SLP statements replaced with the
6540 neutral value. We can then do a normal reduction on each vector. */
6542 /* Enforced by vectorizable_reduction. */
6543 gcc_assert (reduc_inputs
.length () == 1);
6544 gcc_assert (pow2p_hwi (group_size
));
6546 gimple_seq seq
= NULL
;
6548 /* Build a vector {0, 1, 2, ...}, with the same number of elements
6549 and the same element size as VECTYPE. */
6550 tree index
= build_index_vector (vectype
, 0, 1);
6551 tree index_type
= TREE_TYPE (index
);
6552 tree index_elt_type
= TREE_TYPE (index_type
);
6553 tree mask_type
= truth_type_for (index_type
);
6555 /* Create a vector that, for each element, identifies which of
6556 the REDUC_GROUP_SIZE results should use it. */
6557 tree index_mask
= build_int_cst (index_elt_type
, group_size
- 1);
6558 index
= gimple_build (&seq
, BIT_AND_EXPR
, index_type
, index
,
6559 build_vector_from_val (index_type
, index_mask
));
6561 /* Get a neutral vector value. This is simply a splat of the neutral
6562 scalar value if we have one, otherwise the initial scalar value
6563 is itself a neutral value. */
6564 tree vector_identity
= NULL_TREE
;
6565 tree neutral_op
= NULL_TREE
;
6568 tree initial_value
= NULL_TREE
;
6569 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
6570 initial_value
= reduc_info
->reduc_initial_values
[0];
6571 neutral_op
= neutral_op_for_reduction (TREE_TYPE (vectype
), code
,
6572 initial_value
, false);
6575 vector_identity
= gimple_build_vector_from_val (&seq
, vectype
,
6577 for (unsigned int i
= 0; i
< group_size
; ++i
)
6579 /* If there's no univeral neutral value, we can use the
6580 initial scalar value from the original PHI. This is used
6581 for MIN and MAX reduction, for example. */
6584 tree scalar_value
= reduc_info
->reduc_initial_values
[i
];
6585 scalar_value
= gimple_convert (&seq
, TREE_TYPE (vectype
),
6587 vector_identity
= gimple_build_vector_from_val (&seq
, vectype
,
6591 /* Calculate the equivalent of:
6593 sel[j] = (index[j] == i);
6595 which selects the elements of REDUC_INPUTS[0] that should
6596 be included in the result. */
6597 tree compare_val
= build_int_cst (index_elt_type
, i
);
6598 compare_val
= build_vector_from_val (index_type
, compare_val
);
6599 tree sel
= gimple_build (&seq
, EQ_EXPR
, mask_type
,
6600 index
, compare_val
);
6602 /* Calculate the equivalent of:
6604 vec = seq ? reduc_inputs[0] : vector_identity;
6606 VEC is now suitable for a full vector reduction. */
6607 tree vec
= gimple_build (&seq
, VEC_COND_EXPR
, vectype
,
6608 sel
, reduc_inputs
[0], vector_identity
);
6610 /* Do the reduction and convert it to the appropriate type. */
6611 tree scalar
= gimple_build (&seq
, as_combined_fn (reduc_fn
),
6612 TREE_TYPE (vectype
), vec
);
6613 scalar
= gimple_convert (&seq
, scalar_type
, scalar
);
6614 scalar_results
.safe_push (scalar
);
6616 gsi_insert_seq_before (&exit_gsi
, seq
, GSI_SAME_STMT
);
6620 bool reduce_with_shift
;
6623 gcc_assert (slp_reduc
|| reduc_inputs
.length () == 1);
6625 /* See if the target wants to do the final (shift) reduction
6626 in a vector mode of smaller size and first reduce upper/lower
6627 halves against each other. */
6628 enum machine_mode mode1
= mode
;
6629 tree stype
= TREE_TYPE (vectype
);
6630 unsigned nunits
= TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
6631 unsigned nunits1
= nunits
;
6632 if ((mode1
= targetm
.vectorize
.split_reduction (mode
)) != mode
6633 && reduc_inputs
.length () == 1)
6635 nunits1
= GET_MODE_NUNITS (mode1
).to_constant ();
6636 /* For SLP reductions we have to make sure lanes match up, but
6637 since we're doing individual element final reduction reducing
6638 vector width here is even more important.
6639 ??? We can also separate lanes with permutes, for the common
6640 case of power-of-two group-size odd/even extracts would work. */
6641 if (slp_reduc
&& nunits
!= nunits1
)
6643 nunits1
= least_common_multiple (nunits1
, group_size
);
6644 gcc_assert (exact_log2 (nunits1
) != -1 && nunits1
<= nunits
);
6648 && (mode1
= targetm
.vectorize
.split_reduction (mode
)) != mode
)
6649 nunits1
= GET_MODE_NUNITS (mode1
).to_constant ();
6651 tree vectype1
= get_related_vectype_for_scalar_type (TYPE_MODE (vectype
),
6653 reduce_with_shift
= have_whole_vector_shift (mode1
);
6654 if (!VECTOR_MODE_P (mode1
)
6655 || !directly_supported_p (code
, vectype1
))
6656 reduce_with_shift
= false;
6658 /* First reduce the vector to the desired vector size we should
6659 do shift reduction on by combining upper and lower halves. */
6660 gimple_seq stmts
= NULL
;
6661 new_temp
= vect_create_partial_epilog (reduc_inputs
[0], vectype1
,
6663 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6664 reduc_inputs
[0] = new_temp
;
6666 if (reduce_with_shift
&& !slp_reduc
)
6668 int element_bitsize
= tree_to_uhwi (bitsize
);
6669 /* Enforced by vectorizable_reduction, which disallows SLP reductions
6670 for variable-length vectors and also requires direct target support
6671 for loop reductions. */
6672 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype1
));
6673 int nelements
= vec_size_in_bits
/ element_bitsize
;
6674 vec_perm_builder sel
;
6675 vec_perm_indices indices
;
6679 tree zero_vec
= build_zero_cst (vectype1
);
6681 for (offset = nelements/2; offset >= 1; offset/=2)
6683 Create: va' = vec_shift <va, offset>
6684 Create: va = vop <va, va'>
6689 if (dump_enabled_p ())
6690 dump_printf_loc (MSG_NOTE
, vect_location
,
6691 "Reduce using vector shifts\n");
6693 gimple_seq stmts
= NULL
;
6694 new_temp
= gimple_convert (&stmts
, vectype1
, new_temp
);
6695 for (elt_offset
= nelements
/ 2;
6699 calc_vec_perm_mask_for_shift (elt_offset
, nelements
, &sel
);
6700 indices
.new_vector (sel
, 2, nelements
);
6701 tree mask
= vect_gen_perm_mask_any (vectype1
, indices
);
6702 new_name
= gimple_build (&stmts
, VEC_PERM_EXPR
, vectype1
,
6703 new_temp
, zero_vec
, mask
);
6704 new_temp
= gimple_build (&stmts
, code
,
6705 vectype1
, new_name
, new_temp
);
6707 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6709 /* 2.4 Extract the final scalar result. Create:
6710 s_out3 = extract_field <v_out2, bitpos> */
6712 if (dump_enabled_p ())
6713 dump_printf_loc (MSG_NOTE
, vect_location
,
6714 "extract scalar result\n");
6716 rhs
= build3 (BIT_FIELD_REF
, scalar_type
, new_temp
,
6717 bitsize
, bitsize_zero_node
);
6718 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
6719 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
6720 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
6721 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6722 scalar_results
.safe_push (new_temp
);
6727 s = extract_field <v_out2, 0>
6728 for (offset = element_size;
6729 offset < vector_size;
6730 offset += element_size;)
6732 Create: s' = extract_field <v_out2, offset>
6733 Create: s = op <s, s'> // For non SLP cases
6736 if (dump_enabled_p ())
6737 dump_printf_loc (MSG_NOTE
, vect_location
,
6738 "Reduce using scalar code.\n");
6740 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype1
));
6741 int element_bitsize
= tree_to_uhwi (bitsize
);
6742 tree compute_type
= TREE_TYPE (vectype
);
6743 gimple_seq stmts
= NULL
;
6744 FOR_EACH_VEC_ELT (reduc_inputs
, i
, vec_temp
)
6747 new_temp
= gimple_build (&stmts
, BIT_FIELD_REF
, compute_type
,
6748 vec_temp
, bitsize
, bitsize_zero_node
);
6750 /* In SLP we don't need to apply reduction operation, so we just
6751 collect s' values in SCALAR_RESULTS. */
6753 scalar_results
.safe_push (new_temp
);
6755 for (bit_offset
= element_bitsize
;
6756 bit_offset
< vec_size_in_bits
;
6757 bit_offset
+= element_bitsize
)
6759 tree bitpos
= bitsize_int (bit_offset
);
6760 new_name
= gimple_build (&stmts
, BIT_FIELD_REF
,
6761 compute_type
, vec_temp
,
6765 /* In SLP we don't need to apply reduction operation, so
6766 we just collect s' values in SCALAR_RESULTS. */
6767 new_temp
= new_name
;
6768 scalar_results
.safe_push (new_name
);
6771 new_temp
= gimple_build (&stmts
, code
, compute_type
,
6772 new_name
, new_temp
);
6776 /* The only case where we need to reduce scalar results in SLP, is
6777 unrolling. If the size of SCALAR_RESULTS is greater than
6778 REDUC_GROUP_SIZE, we reduce them combining elements modulo
6779 REDUC_GROUP_SIZE. */
6782 tree res
, first_res
, new_res
;
6784 /* Reduce multiple scalar results in case of SLP unrolling. */
6785 for (j
= group_size
; scalar_results
.iterate (j
, &res
);
6788 first_res
= scalar_results
[j
% group_size
];
6789 new_res
= gimple_build (&stmts
, code
, compute_type
,
6791 scalar_results
[j
% group_size
] = new_res
;
6793 scalar_results
.truncate (group_size
);
6794 for (k
= 0; k
< group_size
; k
++)
6795 scalar_results
[k
] = gimple_convert (&stmts
, scalar_type
,
6800 /* Not SLP - we have one scalar to keep in SCALAR_RESULTS. */
6801 new_temp
= gimple_convert (&stmts
, scalar_type
, new_temp
);
6802 scalar_results
.safe_push (new_temp
);
6805 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6808 if ((STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
6811 /* Earlier we set the initial value to be a vector if induc_val
6812 values. Check the result and if it is induc_val then replace
6813 with the original initial value, unless induc_val is
6814 the same as initial_def already. */
6815 tree zcompare
= make_ssa_name (boolean_type_node
);
6816 epilog_stmt
= gimple_build_assign (zcompare
, EQ_EXPR
, new_temp
,
6818 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6819 tree initial_def
= reduc_info
->reduc_initial_values
[0];
6820 tree tmp
= make_ssa_name (new_scalar_dest
);
6821 epilog_stmt
= gimple_build_assign (tmp
, COND_EXPR
, zcompare
,
6822 initial_def
, new_temp
);
6823 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
6824 scalar_results
[0] = tmp
;
6828 /* 2.5 Adjust the final result by the initial value of the reduction
6829 variable. (When such adjustment is not needed, then
6830 'adjustment_def' is zero). For example, if code is PLUS we create:
6831 new_temp = loop_exit_def + adjustment_def */
6835 gcc_assert (!slp_reduc
);
6836 gimple_seq stmts
= NULL
;
6839 gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def
)));
6840 adjustment_def
= gimple_convert (&stmts
, vectype
, adjustment_def
);
6841 new_temp
= gimple_build (&stmts
, code
, vectype
,
6842 reduc_inputs
[0], adjustment_def
);
6846 new_temp
= scalar_results
[0];
6847 gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def
)) != VECTOR_TYPE
);
6848 adjustment_def
= gimple_convert (&stmts
, TREE_TYPE (vectype
),
6850 new_temp
= gimple_convert (&stmts
, TREE_TYPE (vectype
), new_temp
);
6851 new_temp
= gimple_build (&stmts
, code
, TREE_TYPE (vectype
),
6852 new_temp
, adjustment_def
);
6853 new_temp
= gimple_convert (&stmts
, scalar_type
, new_temp
);
6856 epilog_stmt
= gimple_seq_last_stmt (stmts
);
6857 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
6858 scalar_results
[0] = new_temp
;
6861 /* Record this operation if it could be reused by the epilogue loop. */
6862 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == TREE_CODE_REDUCTION
6863 && reduc_inputs
.length () == 1)
6864 loop_vinfo
->reusable_accumulators
.put (scalar_results
[0],
6865 { orig_reduc_input
, reduc_info
});
6870 /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
6871 phis with new adjusted scalar results, i.e., replace use <s_out0>
6876 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6877 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6878 v_out2 = reduce <v_out1>
6879 s_out3 = extract_field <v_out2, 0>
6880 s_out4 = adjust_result <s_out3>
6887 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
6888 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
6889 v_out2 = reduce <v_out1>
6890 s_out3 = extract_field <v_out2, 0>
6891 s_out4 = adjust_result <s_out3>
6895 gcc_assert (live_out_stmts
.size () == scalar_results
.length ());
6896 auto_vec
<gimple
*> phis
;
6897 for (k
= 0; k
< live_out_stmts
.size (); k
++)
6899 stmt_vec_info scalar_stmt_info
= vect_orig_stmt (live_out_stmts
[k
]);
6900 scalar_dest
= gimple_get_lhs (scalar_stmt_info
->stmt
);
6902 /* Find the loop-closed-use at the loop exit of the original scalar
6903 result. (The reduction result is expected to have two immediate uses,
6904 one at the latch block, and one at the loop exit). For double
6905 reductions we are looking for exit phis of the outer loop. */
6906 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
6908 if (!flow_bb_inside_loop_p (loop
, gimple_bb (USE_STMT (use_p
))))
6910 if (!is_gimple_debug (USE_STMT (use_p
))
6911 && gimple_bb (USE_STMT (use_p
)) == loop_exit
->dest
)
6912 phis
.safe_push (USE_STMT (use_p
));
6916 if (double_reduc
&& gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
)
6918 tree phi_res
= PHI_RESULT (USE_STMT (use_p
));
6920 FOR_EACH_IMM_USE_FAST (phi_use_p
, phi_imm_iter
, phi_res
)
6922 if (!flow_bb_inside_loop_p (loop
,
6923 gimple_bb (USE_STMT (phi_use_p
)))
6924 && !is_gimple_debug (USE_STMT (phi_use_p
)))
6925 phis
.safe_push (USE_STMT (phi_use_p
));
6931 FOR_EACH_VEC_ELT (phis
, i
, exit_phi
)
6933 /* Replace the uses: */
6934 orig_name
= PHI_RESULT (exit_phi
);
6936 /* Look for a single use at the target of the skip edge. */
6937 if (unify_with_main_loop_p
)
6939 use_operand_p use_p
;
6941 if (!single_imm_use (orig_name
, &use_p
, &user
))
6943 orig_name
= gimple_get_lhs (user
);
6946 scalar_result
= scalar_results
[k
];
6947 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, orig_name
)
6949 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
6950 SET_USE (use_p
, scalar_result
);
6951 update_stmt (use_stmt
);
6959 /* Return a vector of type VECTYPE that is equal to the vector select
6960 operation "MASK ? VEC : IDENTITY". Insert the select statements
6964 merge_with_identity (gimple_stmt_iterator
*gsi
, tree mask
, tree vectype
,
6965 tree vec
, tree identity
)
6967 tree cond
= make_temp_ssa_name (vectype
, NULL
, "cond");
6968 gimple
*new_stmt
= gimple_build_assign (cond
, VEC_COND_EXPR
,
6969 mask
, vec
, identity
);
6970 gsi_insert_before (gsi
, new_stmt
, GSI_SAME_STMT
);
6974 /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
6975 order, starting with LHS. Insert the extraction statements before GSI and
6976 associate the new scalar SSA names with variable SCALAR_DEST.
6977 If MASK is nonzero mask the input and then operate on it unconditionally.
6978 Return the SSA name for the result. */
6981 vect_expand_fold_left (gimple_stmt_iterator
*gsi
, tree scalar_dest
,
6982 tree_code code
, tree lhs
, tree vector_rhs
,
6985 tree vectype
= TREE_TYPE (vector_rhs
);
6986 tree scalar_type
= TREE_TYPE (vectype
);
6987 tree bitsize
= TYPE_SIZE (scalar_type
);
6988 unsigned HOST_WIDE_INT vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype
));
6989 unsigned HOST_WIDE_INT element_bitsize
= tree_to_uhwi (bitsize
);
6991 /* Re-create a VEC_COND_EXPR to mask the input here in order to be able
6992 to perform an unconditional element-wise reduction of it. */
6995 tree masked_vector_rhs
= make_temp_ssa_name (vectype
, NULL
,
6996 "masked_vector_rhs");
6997 tree neutral_op
= neutral_op_for_reduction (scalar_type
, code
, NULL_TREE
,
6999 tree vector_identity
= build_vector_from_val (vectype
, neutral_op
);
7000 gassign
*select
= gimple_build_assign (masked_vector_rhs
, VEC_COND_EXPR
,
7001 mask
, vector_rhs
, vector_identity
);
7002 gsi_insert_before (gsi
, select
, GSI_SAME_STMT
);
7003 vector_rhs
= masked_vector_rhs
;
7006 for (unsigned HOST_WIDE_INT bit_offset
= 0;
7007 bit_offset
< vec_size_in_bits
;
7008 bit_offset
+= element_bitsize
)
7010 tree bitpos
= bitsize_int (bit_offset
);
7011 tree rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vector_rhs
,
7014 gassign
*stmt
= gimple_build_assign (scalar_dest
, rhs
);
7015 rhs
= make_ssa_name (scalar_dest
, stmt
);
7016 gimple_assign_set_lhs (stmt
, rhs
);
7017 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
7019 stmt
= gimple_build_assign (scalar_dest
, code
, lhs
, rhs
);
7020 tree new_name
= make_ssa_name (scalar_dest
, stmt
);
7021 gimple_assign_set_lhs (stmt
, new_name
);
7022 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
7028 /* Get a masked internal function equivalent to REDUC_FN. VECTYPE_IN is the
7029 type of the vector input. */
7032 get_masked_reduction_fn (internal_fn reduc_fn
, tree vectype_in
)
7034 internal_fn mask_reduc_fn
;
7035 internal_fn mask_len_reduc_fn
;
7039 case IFN_FOLD_LEFT_PLUS
:
7040 mask_reduc_fn
= IFN_MASK_FOLD_LEFT_PLUS
;
7041 mask_len_reduc_fn
= IFN_MASK_LEN_FOLD_LEFT_PLUS
;
7048 if (direct_internal_fn_supported_p (mask_reduc_fn
, vectype_in
,
7049 OPTIMIZE_FOR_SPEED
))
7050 return mask_reduc_fn
;
7051 if (direct_internal_fn_supported_p (mask_len_reduc_fn
, vectype_in
,
7052 OPTIMIZE_FOR_SPEED
))
7053 return mask_len_reduc_fn
;
7057 /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
7058 statement that sets the live-out value. REDUC_DEF_STMT is the phi
7059 statement. CODE is the operation performed by STMT_INFO and OPS are
7060 its scalar operands. REDUC_INDEX is the index of the operand in
7061 OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
7062 implements in-order reduction, or IFN_LAST if we should open-code it.
7063 VECTYPE_IN is the type of the vector input. MASKS specifies the masks
7064 that should be used to control the operation in a fully-masked loop. */
7067 vectorize_fold_left_reduction (loop_vec_info loop_vinfo
,
7068 stmt_vec_info stmt_info
,
7069 gimple_stmt_iterator
*gsi
,
7070 gimple
**vec_stmt
, slp_tree slp_node
,
7071 gimple
*reduc_def_stmt
,
7072 code_helper code
, internal_fn reduc_fn
,
7073 tree
*ops
, int num_ops
, tree vectype_in
,
7074 int reduc_index
, vec_loop_masks
*masks
,
7075 vec_loop_lens
*lens
)
7077 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7078 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
7079 internal_fn mask_reduc_fn
= get_masked_reduction_fn (reduc_fn
, vectype_in
);
7085 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
7087 gcc_assert (!nested_in_vect_loop_p (loop
, stmt_info
));
7088 gcc_assert (ncopies
== 1);
7090 bool is_cond_op
= false;
7091 if (!code
.is_tree_code ())
7093 code
= conditional_internal_fn_code (internal_fn (code
));
7094 gcc_assert (code
!= ERROR_MARK
);
7098 gcc_assert (TREE_CODE_LENGTH (tree_code (code
)) == binary_op
);
7104 if (dump_enabled_p ())
7105 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7106 "fold-left reduction on SLP not supported.\n");
7110 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out
),
7111 TYPE_VECTOR_SUBPARTS (vectype_in
)));
7114 /* The operands either come from a binary operation or an IFN_COND operation.
7115 The former is a gimple assign with binary rhs and the latter is a
7116 gimple call with four arguments. */
7117 gcc_assert (num_ops
== 2 || num_ops
== 4);
7120 op0
= ops
[1 - reduc_index
];
7123 op0
= ops
[2 + (1 - reduc_index
)];
7125 gcc_assert (!slp_node
);
7129 stmt_vec_info scalar_dest_def_info
;
7130 auto_vec
<tree
> vec_oprnds0
, vec_opmask
;
7133 auto_vec
<vec
<tree
> > vec_defs (2);
7134 vect_get_slp_defs (loop_vinfo
, slp_node
, &vec_defs
);
7135 vec_oprnds0
.safe_splice (vec_defs
[1 - reduc_index
]);
7136 vec_defs
[0].release ();
7137 vec_defs
[1].release ();
7138 group_size
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
7139 scalar_dest_def_info
= SLP_TREE_SCALAR_STMTS (slp_node
)[group_size
- 1];
7143 vect_get_vec_defs_for_operand (loop_vinfo
, stmt_info
, 1,
7145 scalar_dest_def_info
= stmt_info
;
7147 /* For an IFN_COND_OP we also need the vector mask operand. */
7149 vect_get_vec_defs_for_operand (loop_vinfo
, stmt_info
, 1,
7150 opmask
, &vec_opmask
);
7153 gimple
*sdef
= vect_orig_stmt (scalar_dest_def_info
)->stmt
;
7154 tree scalar_dest
= gimple_get_lhs (sdef
);
7155 tree scalar_type
= TREE_TYPE (scalar_dest
);
7156 tree reduc_var
= gimple_phi_result (reduc_def_stmt
);
7158 int vec_num
= vec_oprnds0
.length ();
7159 gcc_assert (vec_num
== 1 || slp_node
);
7160 tree vec_elem_type
= TREE_TYPE (vectype_out
);
7161 gcc_checking_assert (useless_type_conversion_p (scalar_type
, vec_elem_type
));
7163 tree vector_identity
= NULL_TREE
;
7164 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
7166 vector_identity
= build_zero_cst (vectype_out
);
7167 if (!HONOR_SIGNED_ZEROS (vectype_out
))
7171 gcc_assert (!HONOR_SIGN_DEPENDENT_ROUNDING (vectype_out
));
7172 vector_identity
= const_unop (NEGATE_EXPR
, vectype_out
,
7177 tree scalar_dest_var
= vect_create_destination_var (scalar_dest
, NULL
);
7180 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, def0
)
7183 tree mask
= NULL_TREE
;
7184 tree len
= NULL_TREE
;
7185 tree bias
= NULL_TREE
;
7186 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
7187 mask
= vect_get_loop_mask (loop_vinfo
, gsi
, masks
, vec_num
, vectype_in
, i
);
7188 else if (is_cond_op
)
7189 mask
= vec_opmask
[0];
7190 if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo
))
7192 len
= vect_get_loop_len (loop_vinfo
, gsi
, lens
, vec_num
, vectype_in
,
7194 signed char biasval
= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
);
7195 bias
= build_int_cst (intQI_type_node
, biasval
);
7197 mask
= build_minus_one_cst (truth_type_for (vectype_in
));
7200 /* Handle MINUS by adding the negative. */
7201 if (reduc_fn
!= IFN_LAST
&& code
== MINUS_EXPR
)
7203 tree negated
= make_ssa_name (vectype_out
);
7204 new_stmt
= gimple_build_assign (negated
, NEGATE_EXPR
, def0
);
7205 gsi_insert_before (gsi
, new_stmt
, GSI_SAME_STMT
);
7209 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
7210 && mask
&& mask_reduc_fn
== IFN_LAST
)
7211 def0
= merge_with_identity (gsi
, mask
, vectype_out
, def0
,
7214 /* On the first iteration the input is simply the scalar phi
7215 result, and for subsequent iterations it is the output of
7216 the preceding operation. */
7217 if (reduc_fn
!= IFN_LAST
|| (mask
&& mask_reduc_fn
!= IFN_LAST
))
7219 if (mask
&& len
&& mask_reduc_fn
== IFN_MASK_LEN_FOLD_LEFT_PLUS
)
7220 new_stmt
= gimple_build_call_internal (mask_reduc_fn
, 5, reduc_var
,
7221 def0
, mask
, len
, bias
);
7222 else if (mask
&& mask_reduc_fn
== IFN_MASK_FOLD_LEFT_PLUS
)
7223 new_stmt
= gimple_build_call_internal (mask_reduc_fn
, 3, reduc_var
,
7226 new_stmt
= gimple_build_call_internal (reduc_fn
, 2, reduc_var
,
7228 /* For chained SLP reductions the output of the previous reduction
7229 operation serves as the input of the next. For the final statement
7230 the output cannot be a temporary - we reuse the original
7231 scalar destination of the last statement. */
7232 if (i
!= vec_num
- 1)
7234 gimple_set_lhs (new_stmt
, scalar_dest_var
);
7235 reduc_var
= make_ssa_name (scalar_dest_var
, new_stmt
);
7236 gimple_set_lhs (new_stmt
, reduc_var
);
7241 reduc_var
= vect_expand_fold_left (gsi
, scalar_dest_var
,
7242 tree_code (code
), reduc_var
, def0
,
7244 new_stmt
= SSA_NAME_DEF_STMT (reduc_var
);
7245 /* Remove the statement, so that we can use the same code paths
7246 as for statements that we've just created. */
7247 gimple_stmt_iterator tmp_gsi
= gsi_for_stmt (new_stmt
);
7248 gsi_remove (&tmp_gsi
, true);
7251 if (i
== vec_num
- 1)
7253 gimple_set_lhs (new_stmt
, scalar_dest
);
7254 vect_finish_replace_stmt (loop_vinfo
,
7255 scalar_dest_def_info
,
7259 vect_finish_stmt_generation (loop_vinfo
,
7260 scalar_dest_def_info
,
7264 slp_node
->push_vec_def (new_stmt
);
7267 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_stmt
);
7268 *vec_stmt
= new_stmt
;
7275 /* Function is_nonwrapping_integer_induction.
7277 Check if STMT_VINO (which is part of loop LOOP) both increments and
7278 does not cause overflow. */
7281 is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo
, class loop
*loop
)
7283 gphi
*phi
= as_a
<gphi
*> (stmt_vinfo
->stmt
);
7284 tree base
= STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
);
7285 tree step
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
);
7286 tree lhs_type
= TREE_TYPE (gimple_phi_result (phi
));
7287 widest_int ni
, max_loop_value
, lhs_max
;
7288 wi::overflow_type overflow
= wi::OVF_NONE
;
7290 /* Make sure the loop is integer based. */
7291 if (TREE_CODE (base
) != INTEGER_CST
7292 || TREE_CODE (step
) != INTEGER_CST
)
7295 /* Check that the max size of the loop will not wrap. */
7297 if (TYPE_OVERFLOW_UNDEFINED (lhs_type
))
7300 if (! max_stmt_executions (loop
, &ni
))
7303 max_loop_value
= wi::mul (wi::to_widest (step
), ni
, TYPE_SIGN (lhs_type
),
7308 max_loop_value
= wi::add (wi::to_widest (base
), max_loop_value
,
7309 TYPE_SIGN (lhs_type
), &overflow
);
7313 return (wi::min_precision (max_loop_value
, TYPE_SIGN (lhs_type
))
7314 <= TYPE_PRECISION (lhs_type
));
7317 /* Check if masking can be supported by inserting a conditional expression.
7318 CODE is the code for the operation. COND_FN is the conditional internal
7319 function, if it exists. VECTYPE_IN is the type of the vector input. */
7321 use_mask_by_cond_expr_p (code_helper code
, internal_fn cond_fn
,
7324 if (cond_fn
!= IFN_LAST
7325 && direct_internal_fn_supported_p (cond_fn
, vectype_in
,
7326 OPTIMIZE_FOR_SPEED
))
7329 if (code
.is_tree_code ())
7330 switch (tree_code (code
))
7342 /* Insert a conditional expression to enable masked vectorization. CODE is the
7343 code for the operation. VOP is the array of operands. MASK is the loop
7344 mask. GSI is a statement iterator used to place the new conditional
7347 build_vect_cond_expr (code_helper code
, tree vop
[3], tree mask
,
7348 gimple_stmt_iterator
*gsi
)
7350 switch (tree_code (code
))
7354 tree vectype
= TREE_TYPE (vop
[1]);
7355 tree zero
= build_zero_cst (vectype
);
7356 tree masked_op1
= make_temp_ssa_name (vectype
, NULL
, "masked_op1");
7357 gassign
*select
= gimple_build_assign (masked_op1
, VEC_COND_EXPR
,
7358 mask
, vop
[1], zero
);
7359 gsi_insert_before (gsi
, select
, GSI_SAME_STMT
);
7360 vop
[1] = masked_op1
;
7366 tree vectype
= TREE_TYPE (vop
[1]);
7367 tree masked_op1
= make_temp_ssa_name (vectype
, NULL
, "masked_op1");
7368 gassign
*select
= gimple_build_assign (masked_op1
, VEC_COND_EXPR
,
7369 mask
, vop
[1], vop
[0]);
7370 gsi_insert_before (gsi
, select
, GSI_SAME_STMT
);
7371 vop
[1] = masked_op1
;
7380 /* Function vectorizable_reduction.
7382 Check if STMT_INFO performs a reduction operation that can be vectorized.
7383 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
7384 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
7385 Return true if STMT_INFO is vectorizable in this way.
7387 This function also handles reduction idioms (patterns) that have been
7388 recognized in advance during vect_pattern_recog. In this case, STMT_INFO
7389 may be of this form:
7390 X = pattern_expr (arg0, arg1, ..., X)
7391 and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
7392 sequence that had been detected and replaced by the pattern-stmt
7395 This function also handles reduction of condition expressions, for example:
7396 for (int i = 0; i < N; i++)
7399 This is handled by vectorising the loop and creating an additional vector
7400 containing the loop indexes for which "a[i] < value" was true. In the
7401 function epilogue this is reduced to a single max value and then used to
7402 index into the vector of results.
7404 In some cases of reduction patterns, the type of the reduction variable X is
7405 different than the type of the other arguments of STMT_INFO.
7406 In such cases, the vectype that is used when transforming STMT_INFO into
7407 a vector stmt is different than the vectype that is used to determine the
7408 vectorization factor, because it consists of a different number of elements
7409 than the actual number of elements that are being operated upon in parallel.
7411 For example, consider an accumulation of shorts into an int accumulator.
7412 On some targets it's possible to vectorize this pattern operating on 8
7413 shorts at a time (hence, the vectype for purposes of determining the
7414 vectorization factor should be V8HI); on the other hand, the vectype that
7415 is used to create the vector form is actually V4SI (the type of the result).
7417 Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
7418 indicates what is the actual level of parallelism (V8HI in the example), so
7419 that the right vectorization factor would be derived. This vectype
7420 corresponds to the type of arguments to the reduction stmt, and should *NOT*
7421 be used to create the vectorized stmt. The right vectype for the vectorized
7422 stmt is obtained from the type of the result X:
7423 get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
7425 This means that, contrary to "regular" reductions (or "regular" stmts in
7426 general), the following equation:
7427 STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
7428 does *NOT* necessarily hold for reduction patterns. */
7431 vectorizable_reduction (loop_vec_info loop_vinfo
,
7432 stmt_vec_info stmt_info
, slp_tree slp_node
,
7433 slp_instance slp_node_instance
,
7434 stmt_vector_for_cost
*cost_vec
)
7436 tree vectype_in
= NULL_TREE
;
7437 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7438 enum vect_def_type cond_reduc_dt
= vect_unknown_def_type
;
7439 stmt_vec_info cond_stmt_vinfo
= NULL
;
7442 bool single_defuse_cycle
= false;
7443 bool nested_cycle
= false;
7444 bool double_reduc
= false;
7446 tree cr_index_scalar_type
= NULL_TREE
, cr_index_vector_type
= NULL_TREE
;
7447 tree cond_reduc_val
= NULL_TREE
;
7449 /* Make sure it was already recognized as a reduction computation. */
7450 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_reduction_def
7451 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
7452 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_nested_cycle
)
7455 /* The stmt we store reduction analysis meta on. */
7456 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
7457 reduc_info
->is_reduc_info
= true;
7459 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
7461 if (is_a
<gphi
*> (stmt_info
->stmt
))
7465 /* We eventually need to set a vector type on invariant
7469 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node
), j
, child
)
7470 if (!vect_maybe_update_slp_op_vectype
7471 (child
, SLP_TREE_VECTYPE (slp_node
)))
7473 if (dump_enabled_p ())
7474 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7475 "incompatible vector types for "
7480 /* Analysis for double-reduction is done on the outer
7481 loop PHI, nested cycles have no further restrictions. */
7482 STMT_VINFO_TYPE (stmt_info
) = cycle_phi_info_type
;
7485 STMT_VINFO_TYPE (stmt_info
) = reduc_vec_info_type
;
7489 stmt_vec_info orig_stmt_of_analysis
= stmt_info
;
7490 stmt_vec_info phi_info
= stmt_info
;
7491 if (!is_a
<gphi
*> (stmt_info
->stmt
))
7493 STMT_VINFO_TYPE (stmt_info
) = reduc_vec_info_type
;
7498 slp_node_instance
->reduc_phis
= slp_node
;
7499 /* ??? We're leaving slp_node to point to the PHIs, we only
7500 need it to get at the number of vector stmts which wasn't
7501 yet initialized for the instance root. */
7503 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
7505 use_operand_p use_p
;
7507 bool res
= single_imm_use (gimple_phi_result (stmt_info
->stmt
),
7510 phi_info
= loop_vinfo
->lookup_stmt (use_stmt
);
7513 /* PHIs should not participate in patterns. */
7514 gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info
));
7515 gphi
*reduc_def_phi
= as_a
<gphi
*> (phi_info
->stmt
);
7517 /* Verify following REDUC_IDX from the latch def leads us back to the PHI
7518 and compute the reduction chain length. Discover the real
7519 reduction operation stmt on the way (stmt_info and slp_for_stmt_info). */
7521 = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi
,
7523 (gimple_bb (reduc_def_phi
)->loop_father
));
7524 unsigned reduc_chain_length
= 0;
7525 bool only_slp_reduc_chain
= true;
7527 slp_tree slp_for_stmt_info
= slp_node
? slp_node_instance
->root
: NULL
;
7528 while (reduc_def
!= PHI_RESULT (reduc_def_phi
))
7530 stmt_vec_info def
= loop_vinfo
->lookup_def (reduc_def
);
7531 stmt_vec_info vdef
= vect_stmt_to_vectorize (def
);
7532 if (STMT_VINFO_REDUC_IDX (vdef
) == -1)
7534 if (dump_enabled_p ())
7535 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7536 "reduction chain broken by patterns.\n");
7539 if (!REDUC_GROUP_FIRST_ELEMENT (vdef
))
7540 only_slp_reduc_chain
= false;
7541 /* For epilogue generation live members of the chain need
7542 to point back to the PHI via their original stmt for
7543 info_for_reduction to work. For SLP we need to look at
7544 all lanes here - even though we only will vectorize from
7545 the SLP node with live lane zero the other live lanes also
7546 need to be identified as part of a reduction to be able
7547 to skip code generation for them. */
7548 if (slp_for_stmt_info
)
7550 for (auto s
: SLP_TREE_SCALAR_STMTS (slp_for_stmt_info
))
7551 if (STMT_VINFO_LIVE_P (s
))
7552 STMT_VINFO_REDUC_DEF (vect_orig_stmt (s
)) = phi_info
;
7554 else if (STMT_VINFO_LIVE_P (vdef
))
7555 STMT_VINFO_REDUC_DEF (def
) = phi_info
;
7557 if (!gimple_extract_op (vdef
->stmt
, &op
))
7559 if (dump_enabled_p ())
7560 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7561 "reduction chain includes unsupported"
7562 " statement type.\n");
7565 if (CONVERT_EXPR_CODE_P (op
.code
))
7567 if (!tree_nop_conversion_p (op
.type
, TREE_TYPE (op
.ops
[0])))
7569 if (dump_enabled_p ())
7570 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7571 "conversion in the reduction chain.\n");
7575 else if (!stmt_info
)
7576 /* First non-conversion stmt. */
7578 reduc_def
= op
.ops
[STMT_VINFO_REDUC_IDX (vdef
)];
7579 reduc_chain_length
++;
7580 if (!stmt_info
&& slp_node
)
7581 slp_for_stmt_info
= SLP_TREE_CHILDREN (slp_for_stmt_info
)[0];
7583 /* PHIs should not participate in patterns. */
7584 gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info
));
7586 if (nested_in_vect_loop_p (loop
, stmt_info
))
7589 nested_cycle
= true;
7592 /* STMT_VINFO_REDUC_DEF doesn't point to the first but the last
7594 if (slp_node
&& REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
7596 gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (stmt_info
));
7597 stmt_info
= REDUC_GROUP_FIRST_ELEMENT (stmt_info
);
7599 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
7600 gcc_assert (slp_node
7601 && REDUC_GROUP_FIRST_ELEMENT (stmt_info
) == stmt_info
);
7603 /* 1. Is vectorizable reduction? */
7604 /* Not supportable if the reduction variable is used in the loop, unless
7605 it's a reduction chain. */
7606 if (STMT_VINFO_RELEVANT (stmt_info
) > vect_used_in_outer
7607 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
7610 /* Reductions that are not used even in an enclosing outer-loop,
7611 are expected to be "live" (used out of the loop). */
7612 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_unused_in_scope
7613 && !STMT_VINFO_LIVE_P (stmt_info
))
7616 /* 2. Has this been recognized as a reduction pattern?
7618 Check if STMT represents a pattern that has been recognized
7619 in earlier analysis stages. For stmts that represent a pattern,
7620 the STMT_VINFO_RELATED_STMT field records the last stmt in
7621 the original sequence that constitutes the pattern. */
7623 stmt_vec_info orig_stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
7626 gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info
));
7627 gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info
));
7630 /* 3. Check the operands of the operation. The first operands are defined
7631 inside the loop body. The last operand is the reduction variable,
7632 which is defined by the loop-header-phi. */
7634 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
7635 STMT_VINFO_REDUC_VECTYPE (reduc_info
) = vectype_out
;
7637 if (!gimple_extract_op (stmt_info
->stmt
, &op
))
7639 bool lane_reduc_code_p
= (op
.code
== DOT_PROD_EXPR
7640 || op
.code
== WIDEN_SUM_EXPR
7641 || op
.code
== SAD_EXPR
);
7643 if (!POINTER_TYPE_P (op
.type
) && !INTEGRAL_TYPE_P (op
.type
)
7644 && !SCALAR_FLOAT_TYPE_P (op
.type
))
7647 /* Do not try to vectorize bit-precision reductions. */
7648 if (!type_has_mode_precision_p (op
.type
))
7651 /* For lane-reducing ops we're reducing the number of reduction PHIs
7652 which means the only use of that may be in the lane-reducing operation. */
7653 if (lane_reduc_code_p
7654 && reduc_chain_length
!= 1
7655 && !only_slp_reduc_chain
)
7657 if (dump_enabled_p ())
7658 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7659 "lane-reducing reduction with extra stmts.\n");
7663 /* All uses but the last are expected to be defined in the loop.
7664 The last use is the reduction variable. In case of nested cycle this
7665 assumption is not true: we use reduc_index to record the index of the
7666 reduction variable. */
7667 slp_tree
*slp_op
= XALLOCAVEC (slp_tree
, op
.num_ops
);
7668 tree
*vectype_op
= XALLOCAVEC (tree
, op
.num_ops
);
7669 /* We need to skip an extra operand for COND_EXPRs with embedded
7671 unsigned opno_adjust
= 0;
7672 if (op
.code
== COND_EXPR
&& COMPARISON_CLASS_P (op
.ops
[0]))
7674 for (i
= 0; i
< (int) op
.num_ops
; i
++)
7676 /* The condition of COND_EXPR is checked in vectorizable_condition(). */
7677 if (i
== 0 && op
.code
== COND_EXPR
)
7680 stmt_vec_info def_stmt_info
;
7681 enum vect_def_type dt
;
7682 if (!vect_is_simple_use (loop_vinfo
, stmt_info
, slp_for_stmt_info
,
7683 i
+ opno_adjust
, &op
.ops
[i
], &slp_op
[i
], &dt
,
7684 &vectype_op
[i
], &def_stmt_info
))
7686 if (dump_enabled_p ())
7687 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7688 "use not simple.\n");
7691 if (i
== STMT_VINFO_REDUC_IDX (stmt_info
))
7694 /* For an IFN_COND_OP we might hit the reduction definition operand
7695 twice (once as definition, once as else). */
7696 if (op
.ops
[i
] == op
.ops
[STMT_VINFO_REDUC_IDX (stmt_info
)])
7699 /* There should be only one cycle def in the stmt, the one
7700 leading to reduc_def. */
7701 if (VECTORIZABLE_CYCLE_DEF (dt
))
7706 = get_vectype_for_scalar_type (loop_vinfo
,
7707 TREE_TYPE (op
.ops
[i
]), slp_op
[i
]);
7709 /* To properly compute ncopies we are interested in the widest
7710 non-reduction input type in case we're looking at a widening
7711 accumulation that we later handle in vect_transform_reduction. */
7712 if (lane_reduc_code_p
7715 || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in
)))
7716 < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_op
[i
]))))))
7717 vectype_in
= vectype_op
[i
];
7719 /* Record how the non-reduction-def value of COND_EXPR is defined.
7720 ??? For a chain of multiple CONDs we'd have to match them up all. */
7721 if (op
.code
== COND_EXPR
&& reduc_chain_length
== 1)
7723 if (dt
== vect_constant_def
)
7726 cond_reduc_val
= op
.ops
[i
];
7728 else if (dt
== vect_induction_def
7730 && is_nonwrapping_integer_induction (def_stmt_info
, loop
))
7733 cond_stmt_vinfo
= def_stmt_info
;
7738 vectype_in
= STMT_VINFO_VECTYPE (phi_info
);
7739 STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
) = vectype_in
;
7741 enum vect_reduction_type v_reduc_type
= STMT_VINFO_REDUC_TYPE (phi_info
);
7742 STMT_VINFO_REDUC_TYPE (reduc_info
) = v_reduc_type
;
7743 /* If we have a condition reduction, see if we can simplify it further. */
7744 if (v_reduc_type
== COND_REDUCTION
)
7749 /* When the condition uses the reduction value in the condition, fail. */
7750 if (STMT_VINFO_REDUC_IDX (stmt_info
) == 0)
7752 if (dump_enabled_p ())
7753 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7754 "condition depends on previous iteration\n");
7758 if (reduc_chain_length
== 1
7759 && (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST
, vectype_in
,
7761 || direct_internal_fn_supported_p (IFN_LEN_FOLD_EXTRACT_LAST
,
7763 OPTIMIZE_FOR_SPEED
)))
7765 if (dump_enabled_p ())
7766 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7767 "optimizing condition reduction with"
7768 " FOLD_EXTRACT_LAST.\n");
7769 STMT_VINFO_REDUC_TYPE (reduc_info
) = EXTRACT_LAST_REDUCTION
;
7771 else if (cond_reduc_dt
== vect_induction_def
)
7774 = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo
);
7775 tree step
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo
);
7777 gcc_assert (TREE_CODE (base
) == INTEGER_CST
7778 && TREE_CODE (step
) == INTEGER_CST
);
7779 cond_reduc_val
= NULL_TREE
;
7780 enum tree_code cond_reduc_op_code
= ERROR_MARK
;
7781 tree res
= PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo
));
7782 if (!types_compatible_p (TREE_TYPE (res
), TREE_TYPE (base
)))
7784 /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
7785 above base; punt if base is the minimum value of the type for
7786 MAX_EXPR or maximum value of the type for MIN_EXPR for now. */
7787 else if (tree_int_cst_sgn (step
) == -1)
7789 cond_reduc_op_code
= MIN_EXPR
;
7790 if (tree_int_cst_sgn (base
) == -1)
7791 cond_reduc_val
= build_int_cst (TREE_TYPE (base
), 0);
7792 else if (tree_int_cst_lt (base
,
7793 TYPE_MAX_VALUE (TREE_TYPE (base
))))
7795 = int_const_binop (PLUS_EXPR
, base
, integer_one_node
);
7799 cond_reduc_op_code
= MAX_EXPR
;
7800 if (tree_int_cst_sgn (base
) == 1)
7801 cond_reduc_val
= build_int_cst (TREE_TYPE (base
), 0);
7802 else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base
)),
7805 = int_const_binop (MINUS_EXPR
, base
, integer_one_node
);
7809 if (dump_enabled_p ())
7810 dump_printf_loc (MSG_NOTE
, vect_location
,
7811 "condition expression based on "
7812 "integer induction.\n");
7813 STMT_VINFO_REDUC_CODE (reduc_info
) = cond_reduc_op_code
;
7814 STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
)
7816 STMT_VINFO_REDUC_TYPE (reduc_info
) = INTEGER_INDUC_COND_REDUCTION
;
7819 else if (cond_reduc_dt
== vect_constant_def
)
7821 enum vect_def_type cond_initial_dt
;
7822 tree cond_initial_val
= vect_phi_initial_value (reduc_def_phi
);
7823 vect_is_simple_use (cond_initial_val
, loop_vinfo
, &cond_initial_dt
);
7824 if (cond_initial_dt
== vect_constant_def
7825 && types_compatible_p (TREE_TYPE (cond_initial_val
),
7826 TREE_TYPE (cond_reduc_val
)))
7828 tree e
= fold_binary (LE_EXPR
, boolean_type_node
,
7829 cond_initial_val
, cond_reduc_val
);
7830 if (e
&& (integer_onep (e
) || integer_zerop (e
)))
7832 if (dump_enabled_p ())
7833 dump_printf_loc (MSG_NOTE
, vect_location
,
7834 "condition expression based on "
7835 "compile time constant.\n");
7836 /* Record reduction code at analysis stage. */
7837 STMT_VINFO_REDUC_CODE (reduc_info
)
7838 = integer_onep (e
) ? MAX_EXPR
: MIN_EXPR
;
7839 STMT_VINFO_REDUC_TYPE (reduc_info
) = CONST_COND_REDUCTION
;
7845 if (STMT_VINFO_LIVE_P (phi_info
))
7851 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
7853 gcc_assert (ncopies
>= 1);
7855 poly_uint64 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
7859 gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info
)
7860 == vect_double_reduction_def
);
7861 double_reduc
= true;
7864 /* 4.2. Check support for the epilog operation.
7866 If STMT represents a reduction pattern, then the type of the
7867 reduction variable may be different than the type of the rest
7868 of the arguments. For example, consider the case of accumulation
7869 of shorts into an int accumulator; The original code:
7870 S1: int_a = (int) short_a;
7871 orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
7874 STMT: int_acc = widen_sum <short_a, int_acc>
7877 1. The tree-code that is used to create the vector operation in the
7878 epilog code (that reduces the partial results) is not the
7879 tree-code of STMT, but is rather the tree-code of the original
7880 stmt from the pattern that STMT is replacing. I.e, in the example
7881 above we want to use 'widen_sum' in the loop, but 'plus' in the
7883 2. The type (mode) we use to check available target support
7884 for the vector operation to be created in the *epilog*, is
7885 determined by the type of the reduction variable (in the example
7886 above we'd check this: optab_handler (plus_optab, vect_int_mode])).
7887 However the type (mode) we use to check available target support
7888 for the vector operation to be created *inside the loop*, is
7889 determined by the type of the other arguments to STMT (in the
7890 example we'd check this: optab_handler (widen_sum_optab,
7893 This is contrary to "regular" reductions, in which the types of all
7894 the arguments are the same as the type of the reduction variable.
7895 For "regular" reductions we can therefore use the same vector type
7896 (and also the same tree-code) when generating the epilog code and
7897 when generating the code inside the loop. */
7899 code_helper orig_code
= STMT_VINFO_REDUC_CODE (phi_info
);
7901 /* If conversion might have created a conditional operation like
7902 IFN_COND_ADD already. Use the internal code for the following checks. */
7903 if (orig_code
.is_internal_fn ())
7905 tree_code new_code
= conditional_internal_fn_code (internal_fn (orig_code
));
7906 orig_code
= new_code
!= ERROR_MARK
? new_code
: orig_code
;
7909 STMT_VINFO_REDUC_CODE (reduc_info
) = orig_code
;
7911 vect_reduction_type reduction_type
= STMT_VINFO_REDUC_TYPE (reduc_info
);
7912 if (reduction_type
== TREE_CODE_REDUCTION
)
7914 /* Check whether it's ok to change the order of the computation.
7915 Generally, when vectorizing a reduction we change the order of the
7916 computation. This may change the behavior of the program in some
7917 cases, so we need to check that this is ok. One exception is when
7918 vectorizing an outer-loop: the inner-loop is executed sequentially,
7919 and therefore vectorizing reductions in the inner-loop during
7920 outer-loop vectorization is safe. Likewise when we are vectorizing
7921 a series of reductions using SLP and the VF is one the reductions
7922 are performed in scalar order. */
7924 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
)
7925 && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo
), 1u))
7927 else if (needs_fold_left_reduction_p (op
.type
, orig_code
))
7929 /* When vectorizing a reduction chain w/o SLP the reduction PHI
7930 is not directy used in stmt. */
7931 if (!only_slp_reduc_chain
7932 && reduc_chain_length
!= 1)
7934 if (dump_enabled_p ())
7935 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7936 "in-order reduction chain without SLP.\n");
7939 STMT_VINFO_REDUC_TYPE (reduc_info
)
7940 = reduction_type
= FOLD_LEFT_REDUCTION
;
7942 else if (!commutative_binary_op_p (orig_code
, op
.type
)
7943 || !associative_binary_op_p (orig_code
, op
.type
))
7945 if (dump_enabled_p ())
7946 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7947 "reduction: not commutative/associative\n");
7952 if ((double_reduc
|| reduction_type
!= TREE_CODE_REDUCTION
)
7955 if (dump_enabled_p ())
7956 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7957 "multiple types in double reduction or condition "
7958 "reduction or fold-left reduction.\n");
7962 internal_fn reduc_fn
= IFN_LAST
;
7963 if (reduction_type
== TREE_CODE_REDUCTION
7964 || reduction_type
== FOLD_LEFT_REDUCTION
7965 || reduction_type
== INTEGER_INDUC_COND_REDUCTION
7966 || reduction_type
== CONST_COND_REDUCTION
)
7968 if (reduction_type
== FOLD_LEFT_REDUCTION
7969 ? fold_left_reduction_fn (orig_code
, &reduc_fn
)
7970 : reduction_fn_for_scalar_code (orig_code
, &reduc_fn
))
7972 if (reduc_fn
!= IFN_LAST
7973 && !direct_internal_fn_supported_p (reduc_fn
, vectype_out
,
7974 OPTIMIZE_FOR_SPEED
))
7976 if (dump_enabled_p ())
7977 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7978 "reduc op not supported by target.\n");
7980 reduc_fn
= IFN_LAST
;
7985 if (!nested_cycle
|| double_reduc
)
7987 if (dump_enabled_p ())
7988 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7989 "no reduc code for scalar code.\n");
7995 else if (reduction_type
== COND_REDUCTION
)
7997 int scalar_precision
7998 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (op
.type
));
7999 cr_index_scalar_type
= make_unsigned_type (scalar_precision
);
8000 cr_index_vector_type
= get_same_sized_vectype (cr_index_scalar_type
,
8003 if (direct_internal_fn_supported_p (IFN_REDUC_MAX
, cr_index_vector_type
,
8004 OPTIMIZE_FOR_SPEED
))
8005 reduc_fn
= IFN_REDUC_MAX
;
8007 STMT_VINFO_REDUC_FN (reduc_info
) = reduc_fn
;
8009 if (reduction_type
!= EXTRACT_LAST_REDUCTION
8010 && (!nested_cycle
|| double_reduc
)
8011 && reduc_fn
== IFN_LAST
8012 && !nunits_out
.is_constant ())
8014 if (dump_enabled_p ())
8015 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8016 "missing target support for reduction on"
8017 " variable-length vectors.\n");
8021 /* For SLP reductions, see if there is a neutral value we can use. */
8022 tree neutral_op
= NULL_TREE
;
8025 tree initial_value
= NULL_TREE
;
8026 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
) != NULL
)
8027 initial_value
= vect_phi_initial_value (reduc_def_phi
);
8028 neutral_op
= neutral_op_for_reduction (TREE_TYPE (vectype_out
),
8029 orig_code
, initial_value
);
8032 if (double_reduc
&& reduction_type
== FOLD_LEFT_REDUCTION
)
8034 /* We can't support in-order reductions of code such as this:
8036 for (int i = 0; i < n1; ++i)
8037 for (int j = 0; j < n2; ++j)
8040 since GCC effectively transforms the loop when vectorizing:
8042 for (int i = 0; i < n1 / VF; ++i)
8043 for (int j = 0; j < n2; ++j)
8044 for (int k = 0; k < VF; ++k)
8047 which is a reassociation of the original operation. */
8048 if (dump_enabled_p ())
8049 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8050 "in-order double reduction not supported.\n");
8055 if (reduction_type
== FOLD_LEFT_REDUCTION
8057 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
8059 /* We cannot use in-order reductions in this case because there is
8060 an implicit reassociation of the operations involved. */
8061 if (dump_enabled_p ())
8062 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8063 "in-order unchained SLP reductions not supported.\n");
8067 /* For double reductions, and for SLP reductions with a neutral value,
8068 we construct a variable-length initial vector by loading a vector
8069 full of the neutral value and then shift-and-inserting the start
8070 values into the low-numbered elements. */
8071 if ((double_reduc
|| neutral_op
)
8072 && !nunits_out
.is_constant ()
8073 && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT
,
8074 vectype_out
, OPTIMIZE_FOR_SPEED
))
8076 if (dump_enabled_p ())
8077 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8078 "reduction on variable-length vectors requires"
8079 " target support for a vector-shift-and-insert"
8084 /* Check extra constraints for variable-length unchained SLP reductions. */
8086 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
)
8087 && !nunits_out
.is_constant ())
8089 /* We checked above that we could build the initial vector when
8090 there's a neutral element value. Check here for the case in
8091 which each SLP statement has its own initial value and in which
8092 that value needs to be repeated for every instance of the
8093 statement within the initial vector. */
8094 unsigned int group_size
= SLP_TREE_LANES (slp_node
);
8096 && !can_duplicate_and_interleave_p (loop_vinfo
, group_size
,
8097 TREE_TYPE (vectype_out
)))
8099 if (dump_enabled_p ())
8100 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8101 "unsupported form of SLP reduction for"
8102 " variable-length vectors: cannot build"
8103 " initial vector.\n");
8106 /* The epilogue code relies on the number of elements being a multiple
8107 of the group size. The duplicate-and-interleave approach to setting
8108 up the initial vector does too. */
8109 if (!multiple_p (nunits_out
, group_size
))
8111 if (dump_enabled_p ())
8112 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8113 "unsupported form of SLP reduction for"
8114 " variable-length vectors: the vector size"
8115 " is not a multiple of the number of results.\n");
8120 if (reduction_type
== COND_REDUCTION
)
8124 if (! max_loop_iterations (loop
, &ni
))
8126 if (dump_enabled_p ())
8127 dump_printf_loc (MSG_NOTE
, vect_location
,
8128 "loop count not known, cannot create cond "
8132 /* Convert backedges to iterations. */
8135 /* The additional index will be the same type as the condition. Check
8136 that the loop can fit into this less one (because we'll use up the
8137 zero slot for when there are no matches). */
8138 tree max_index
= TYPE_MAX_VALUE (cr_index_scalar_type
);
8139 if (wi::geu_p (ni
, wi::to_widest (max_index
)))
8141 if (dump_enabled_p ())
8142 dump_printf_loc (MSG_NOTE
, vect_location
,
8143 "loop size is greater than data size.\n");
8148 /* In case the vectorization factor (VF) is bigger than the number
8149 of elements that we can fit in a vectype (nunits), we have to generate
8150 more than one vector stmt - i.e - we need to "unroll" the
8151 vector stmt by a factor VF/nunits. For more details see documentation
8152 in vectorizable_operation. */
8154 /* If the reduction is used in an outer loop we need to generate
8155 VF intermediate results, like so (e.g. for ncopies=2):
8160 (i.e. we generate VF results in 2 registers).
8161 In this case we have a separate def-use cycle for each copy, and therefore
8162 for each copy we get the vector def for the reduction variable from the
8163 respective phi node created for this copy.
8165 Otherwise (the reduction is unused in the loop nest), we can combine
8166 together intermediate results, like so (e.g. for ncopies=2):
8170 (i.e. we generate VF/2 results in a single register).
8171 In this case for each copy we get the vector def for the reduction variable
8172 from the vectorized reduction operation generated in the previous iteration.
8174 This only works when we see both the reduction PHI and its only consumer
8175 in vectorizable_reduction and there are no intermediate stmts
8176 participating. When unrolling we want each unrolled iteration to have its
8177 own reduction accumulator since one of the main goals of unrolling a
8178 reduction is to reduce the aggregate loop-carried latency. */
8180 && (STMT_VINFO_RELEVANT (stmt_info
) <= vect_used_only_live
)
8181 && reduc_chain_length
== 1
8182 && loop_vinfo
->suggested_unroll_factor
== 1)
8183 single_defuse_cycle
= true;
8185 if (single_defuse_cycle
|| lane_reduc_code_p
)
8187 gcc_assert (op
.code
!= COND_EXPR
);
8189 /* 4. Supportable by target? */
8192 /* 4.1. check support for the operation in the loop
8194 This isn't necessary for the lane reduction codes, since they
8195 can only be produced by pattern matching, and it's up to the
8196 pattern matcher to test for support. The main reason for
8197 specifically skipping this step is to avoid rechecking whether
8198 mixed-sign dot-products can be implemented using signed
8200 machine_mode vec_mode
= TYPE_MODE (vectype_in
);
8201 if (!lane_reduc_code_p
8202 && !directly_supported_p (op
.code
, vectype_in
, optab_vector
))
8204 if (dump_enabled_p ())
8205 dump_printf (MSG_NOTE
, "op not supported by target.\n");
8206 if (maybe_ne (GET_MODE_SIZE (vec_mode
), UNITS_PER_WORD
)
8207 || !vect_can_vectorize_without_simd_p (op
.code
))
8210 if (dump_enabled_p ())
8211 dump_printf (MSG_NOTE
, "proceeding using word mode.\n");
8214 if (vect_emulated_vector_p (vectype_in
)
8215 && !vect_can_vectorize_without_simd_p (op
.code
))
8217 if (dump_enabled_p ())
8218 dump_printf (MSG_NOTE
, "using word mode not possible.\n");
8222 /* lane-reducing operations have to go through vect_transform_reduction.
8223 For the other cases try without the single cycle optimization. */
8226 if (lane_reduc_code_p
)
8229 single_defuse_cycle
= false;
8232 STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
) = single_defuse_cycle
;
8234 /* If the reduction stmt is one of the patterns that have lane
8235 reduction embedded we cannot handle the case of ! single_defuse_cycle. */
8236 if ((ncopies
> 1 && ! single_defuse_cycle
)
8237 && lane_reduc_code_p
)
8239 if (dump_enabled_p ())
8240 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8241 "multi def-use cycle not possible for lane-reducing "
8242 "reduction operation\n");
8247 && !(!single_defuse_cycle
8248 && !lane_reduc_code_p
8249 && reduction_type
!= FOLD_LEFT_REDUCTION
))
8250 for (i
= 0; i
< (int) op
.num_ops
; i
++)
8251 if (!vect_maybe_update_slp_op_vectype (slp_op
[i
], vectype_op
[i
]))
8253 if (dump_enabled_p ())
8254 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8255 "incompatible vector types for invariants\n");
8260 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
8264 vect_model_reduction_cost (loop_vinfo
, stmt_info
, reduc_fn
,
8265 reduction_type
, ncopies
, cost_vec
);
8266 /* Cost the reduction op inside the loop if transformed via
8267 vect_transform_reduction. Otherwise this is costed by the
8268 separate vectorizable_* routines. */
8269 if (single_defuse_cycle
|| lane_reduc_code_p
)
8272 if (vect_is_emulated_mixed_dot_prod (loop_vinfo
, stmt_info
))
8273 /* Three dot-products and a subtraction. */
8275 record_stmt_cost (cost_vec
, ncopies
* factor
, vector_stmt
,
8276 stmt_info
, 0, vect_body
);
8279 if (dump_enabled_p ()
8280 && reduction_type
== FOLD_LEFT_REDUCTION
)
8281 dump_printf_loc (MSG_NOTE
, vect_location
,
8282 "using an in-order (fold-left) reduction.\n");
8283 STMT_VINFO_TYPE (orig_stmt_of_analysis
) = cycle_phi_info_type
;
8284 /* All but single defuse-cycle optimized, lane-reducing and fold-left
8285 reductions go through their own vectorizable_* routines. */
8286 if (!single_defuse_cycle
8287 && !lane_reduc_code_p
8288 && reduction_type
!= FOLD_LEFT_REDUCTION
)
8291 = vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (phi_info
));
8292 if (slp_node
&& REDUC_GROUP_FIRST_ELEMENT (tem
))
8294 gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (tem
));
8295 tem
= REDUC_GROUP_FIRST_ELEMENT (tem
);
8297 STMT_VINFO_DEF_TYPE (vect_orig_stmt (tem
)) = vect_internal_def
;
8298 STMT_VINFO_DEF_TYPE (tem
) = vect_internal_def
;
8300 else if (loop_vinfo
&& LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
))
8302 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
8303 vec_loop_lens
*lens
= &LOOP_VINFO_LENS (loop_vinfo
);
8304 internal_fn cond_fn
= get_conditional_internal_fn (op
.code
, op
.type
);
8306 if (reduction_type
!= FOLD_LEFT_REDUCTION
8307 && !use_mask_by_cond_expr_p (op
.code
, cond_fn
, vectype_in
)
8308 && (cond_fn
== IFN_LAST
8309 || !direct_internal_fn_supported_p (cond_fn
, vectype_in
,
8310 OPTIMIZE_FOR_SPEED
)))
8312 if (dump_enabled_p ())
8313 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8314 "can't operate on partial vectors because"
8315 " no conditional operation is available.\n");
8316 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
8318 else if (reduction_type
== FOLD_LEFT_REDUCTION
8319 && reduc_fn
== IFN_LAST
8320 && !expand_vec_cond_expr_p (vectype_in
,
8321 truth_type_for (vectype_in
),
8324 if (dump_enabled_p ())
8325 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8326 "can't operate on partial vectors because"
8327 " no conditional operation is available.\n");
8328 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
8330 else if (reduction_type
== FOLD_LEFT_REDUCTION
8331 && internal_fn_mask_index (reduc_fn
) == -1
8332 && FLOAT_TYPE_P (vectype_in
)
8333 && HONOR_SIGN_DEPENDENT_ROUNDING (vectype_in
))
8335 if (dump_enabled_p ())
8336 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8337 "can't operate on partial vectors because"
8338 " signed zeros cannot be preserved.\n");
8339 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
8343 internal_fn mask_reduc_fn
8344 = get_masked_reduction_fn (reduc_fn
, vectype_in
);
8346 if (mask_reduc_fn
== IFN_MASK_LEN_FOLD_LEFT_PLUS
)
8347 vect_record_loop_len (loop_vinfo
, lens
, ncopies
* vec_num
,
8350 vect_record_loop_mask (loop_vinfo
, masks
, ncopies
* vec_num
,
8357 /* STMT_INFO is a dot-product reduction whose multiplication operands
8358 have different signs. Emit a sequence to emulate the operation
8359 using a series of signed DOT_PROD_EXPRs and return the last
8360 statement generated. VEC_DEST is the result of the vector operation
8361 and VOP lists its inputs. */
8364 vect_emulate_mixed_dot_prod (loop_vec_info loop_vinfo
, stmt_vec_info stmt_info
,
8365 gimple_stmt_iterator
*gsi
, tree vec_dest
,
8368 tree wide_vectype
= signed_type_for (TREE_TYPE (vec_dest
));
8369 tree narrow_vectype
= signed_type_for (TREE_TYPE (vop
[0]));
8370 tree narrow_elttype
= TREE_TYPE (narrow_vectype
);
8373 /* Make VOP[0] the unsigned operand VOP[1] the signed operand. */
8374 if (!TYPE_UNSIGNED (TREE_TYPE (vop
[0])))
8375 std::swap (vop
[0], vop
[1]);
8377 /* Convert all inputs to signed types. */
8378 for (int i
= 0; i
< 3; ++i
)
8379 if (TYPE_UNSIGNED (TREE_TYPE (vop
[i
])))
8381 tree tmp
= make_ssa_name (signed_type_for (TREE_TYPE (vop
[i
])));
8382 new_stmt
= gimple_build_assign (tmp
, NOP_EXPR
, vop
[i
]);
8383 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8387 /* In the comments below we assume 8-bit inputs for simplicity,
8388 but the approach works for any full integer type. */
8390 /* Create a vector of -128. */
8391 tree min_narrow_elttype
= TYPE_MIN_VALUE (narrow_elttype
);
8392 tree min_narrow
= build_vector_from_val (narrow_vectype
,
8393 min_narrow_elttype
);
8395 /* Create a vector of 64. */
8396 auto half_wi
= wi::lrshift (wi::to_wide (min_narrow_elttype
), 1);
8397 tree half_narrow
= wide_int_to_tree (narrow_elttype
, half_wi
);
8398 half_narrow
= build_vector_from_val (narrow_vectype
, half_narrow
);
8400 /* Emit: SUB_RES = VOP[0] - 128. */
8401 tree sub_res
= make_ssa_name (narrow_vectype
);
8402 new_stmt
= gimple_build_assign (sub_res
, PLUS_EXPR
, vop
[0], min_narrow
);
8403 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8407 STAGE1 = DOT_PROD_EXPR <VOP[1], 64, VOP[2]>;
8408 STAGE2 = DOT_PROD_EXPR <VOP[1], 64, STAGE1>;
8409 STAGE3 = DOT_PROD_EXPR <SUB_RES, -128, STAGE2>;
8411 on the basis that x * y == (x - 128) * y + 64 * y + 64 * y
8412 Doing the two 64 * y steps first allows more time to compute x. */
8413 tree stage1
= make_ssa_name (wide_vectype
);
8414 new_stmt
= gimple_build_assign (stage1
, DOT_PROD_EXPR
,
8415 vop
[1], half_narrow
, vop
[2]);
8416 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8418 tree stage2
= make_ssa_name (wide_vectype
);
8419 new_stmt
= gimple_build_assign (stage2
, DOT_PROD_EXPR
,
8420 vop
[1], half_narrow
, stage1
);
8421 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8423 tree stage3
= make_ssa_name (wide_vectype
);
8424 new_stmt
= gimple_build_assign (stage3
, DOT_PROD_EXPR
,
8425 sub_res
, vop
[1], stage2
);
8426 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8428 /* Convert STAGE3 to the reduction type. */
8429 return gimple_build_assign (vec_dest
, CONVERT_EXPR
, stage3
);
8432 /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
8436 vect_transform_reduction (loop_vec_info loop_vinfo
,
8437 stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
8438 gimple
**vec_stmt
, slp_tree slp_node
)
8440 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
8441 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
8446 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
8447 gcc_assert (reduc_info
->is_reduc_info
);
8449 if (nested_in_vect_loop_p (loop
, stmt_info
))
8452 gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info
) == vect_double_reduction_def
);
8456 if (!gimple_extract_op (stmt_info
->stmt
, &op
))
8459 /* All uses but the last are expected to be defined in the loop.
8460 The last use is the reduction variable. In case of nested cycle this
8461 assumption is not true: we use reduc_index to record the index of the
8462 reduction variable. */
8463 stmt_vec_info phi_info
= STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info
));
8464 gphi
*reduc_def_phi
= as_a
<gphi
*> (phi_info
->stmt
);
8465 int reduc_index
= STMT_VINFO_REDUC_IDX (stmt_info
);
8466 tree vectype_in
= STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
);
8471 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
8475 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
8479 code_helper code
= canonicalize_code (op
.code
, op
.type
);
8480 internal_fn cond_fn
= get_conditional_internal_fn (code
, op
.type
);
8482 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
8483 vec_loop_lens
*lens
= &LOOP_VINFO_LENS (loop_vinfo
);
8484 bool mask_by_cond_expr
= use_mask_by_cond_expr_p (code
, cond_fn
, vectype_in
);
8487 tree new_temp
= NULL_TREE
;
8488 auto_vec
<tree
> vec_oprnds0
;
8489 auto_vec
<tree
> vec_oprnds1
;
8490 auto_vec
<tree
> vec_oprnds2
;
8493 if (dump_enabled_p ())
8494 dump_printf_loc (MSG_NOTE
, vect_location
, "transform reduction.\n");
8496 /* FORNOW: Multiple types are not supported for condition. */
8497 if (code
== COND_EXPR
)
8498 gcc_assert (ncopies
== 1);
8500 /* A binary COND_OP reduction must have the same definition and else
8502 bool cond_fn_p
= code
.is_internal_fn ()
8503 && conditional_internal_fn_code (internal_fn (code
)) != ERROR_MARK
;
8506 gcc_assert (code
== IFN_COND_ADD
|| code
== IFN_COND_SUB
8507 || code
== IFN_COND_MUL
|| code
== IFN_COND_AND
8508 || code
== IFN_COND_IOR
|| code
== IFN_COND_XOR
8509 || code
== IFN_COND_MIN
|| code
== IFN_COND_MAX
);
8510 gcc_assert (op
.num_ops
== 4
8511 && (op
.ops
[reduc_index
]
8512 == op
.ops
[internal_fn_else_index ((internal_fn
) code
)]));
8515 bool masked_loop_p
= LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
);
8517 vect_reduction_type reduction_type
= STMT_VINFO_REDUC_TYPE (reduc_info
);
8518 if (reduction_type
== FOLD_LEFT_REDUCTION
)
8520 internal_fn reduc_fn
= STMT_VINFO_REDUC_FN (reduc_info
);
8521 gcc_assert (code
.is_tree_code () || cond_fn_p
);
8522 return vectorize_fold_left_reduction
8523 (loop_vinfo
, stmt_info
, gsi
, vec_stmt
, slp_node
, reduc_def_phi
,
8524 code
, reduc_fn
, op
.ops
, op
.num_ops
, vectype_in
,
8525 reduc_index
, masks
, lens
);
8528 bool single_defuse_cycle
= STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
);
8529 gcc_assert (single_defuse_cycle
8530 || code
== DOT_PROD_EXPR
8531 || code
== WIDEN_SUM_EXPR
8532 || code
== SAD_EXPR
);
8534 /* Create the destination vector */
8535 tree scalar_dest
= gimple_get_lhs (stmt_info
->stmt
);
8536 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
8538 /* Get NCOPIES vector definitions for all operands except the reduction
8542 vect_get_vec_defs (loop_vinfo
, stmt_info
, slp_node
, ncopies
,
8543 single_defuse_cycle
&& reduc_index
== 0
8544 ? NULL_TREE
: op
.ops
[0], &vec_oprnds0
,
8545 single_defuse_cycle
&& reduc_index
== 1
8546 ? NULL_TREE
: op
.ops
[1], &vec_oprnds1
,
8548 && !(single_defuse_cycle
&& reduc_index
== 2)
8549 ? op
.ops
[2] : NULL_TREE
, &vec_oprnds2
);
8553 /* For a conditional operation pass the truth type as mask
8555 gcc_assert (single_defuse_cycle
8556 && (reduc_index
== 1 || reduc_index
== 2));
8557 vect_get_vec_defs (loop_vinfo
, stmt_info
, slp_node
, ncopies
,
8558 op
.ops
[0], truth_type_for (vectype_in
), &vec_oprnds0
,
8559 reduc_index
== 1 ? NULL_TREE
: op
.ops
[1],
8560 NULL_TREE
, &vec_oprnds1
,
8561 reduc_index
== 2 ? NULL_TREE
: op
.ops
[2],
8562 NULL_TREE
, &vec_oprnds2
);
8565 /* For single def-use cycles get one copy of the vectorized reduction
8567 if (single_defuse_cycle
)
8569 gcc_assert (!slp_node
);
8570 vect_get_vec_defs_for_operand (loop_vinfo
, stmt_info
, 1,
8571 op
.ops
[reduc_index
],
8572 reduc_index
== 0 ? &vec_oprnds0
8573 : (reduc_index
== 1 ? &vec_oprnds1
8577 bool emulated_mixed_dot_prod
8578 = vect_is_emulated_mixed_dot_prod (loop_vinfo
, stmt_info
);
8579 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, def0
)
8582 tree vop
[3] = { def0
, vec_oprnds1
[i
], NULL_TREE
};
8583 if (masked_loop_p
&& !mask_by_cond_expr
)
8585 /* No conditional ifns have been defined for dot-product yet. */
8586 gcc_assert (code
!= DOT_PROD_EXPR
);
8588 /* Make sure that the reduction accumulator is vop[0]. */
8589 if (reduc_index
== 1)
8591 gcc_assert (commutative_binary_op_p (code
, op
.type
));
8592 std::swap (vop
[0], vop
[1]);
8594 tree mask
= vect_get_loop_mask (loop_vinfo
, gsi
, masks
,
8595 vec_num
* ncopies
, vectype_in
, i
);
8596 gcall
*call
= gimple_build_call_internal (cond_fn
, 4, mask
,
8597 vop
[0], vop
[1], vop
[0]);
8598 new_temp
= make_ssa_name (vec_dest
, call
);
8599 gimple_call_set_lhs (call
, new_temp
);
8600 gimple_call_set_nothrow (call
, true);
8601 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, call
, gsi
);
8606 if (op
.num_ops
>= 3)
8607 vop
[2] = vec_oprnds2
[i
];
8609 if (masked_loop_p
&& mask_by_cond_expr
)
8611 tree mask
= vect_get_loop_mask (loop_vinfo
, gsi
, masks
,
8612 vec_num
* ncopies
, vectype_in
, i
);
8613 build_vect_cond_expr (code
, vop
, mask
, gsi
);
8616 if (emulated_mixed_dot_prod
)
8617 new_stmt
= vect_emulate_mixed_dot_prod (loop_vinfo
, stmt_info
, gsi
,
8620 else if (code
.is_internal_fn () && !cond_fn_p
)
8621 new_stmt
= gimple_build_call_internal (internal_fn (code
),
8623 vop
[0], vop
[1], vop
[2]);
8624 else if (code
.is_internal_fn () && cond_fn_p
)
8625 new_stmt
= gimple_build_call_internal (internal_fn (code
),
8627 vop
[0], vop
[1], vop
[2],
8630 new_stmt
= gimple_build_assign (vec_dest
, tree_code (op
.code
),
8631 vop
[0], vop
[1], vop
[2]);
8632 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
8633 gimple_set_lhs (new_stmt
, new_temp
);
8634 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, new_stmt
, gsi
);
8638 slp_node
->push_vec_def (new_stmt
);
8639 else if (single_defuse_cycle
8642 if (reduc_index
== 0)
8643 vec_oprnds0
.safe_push (gimple_get_lhs (new_stmt
));
8644 else if (reduc_index
== 1)
8645 vec_oprnds1
.safe_push (gimple_get_lhs (new_stmt
));
8646 else if (reduc_index
== 2)
8647 vec_oprnds2
.safe_push (gimple_get_lhs (new_stmt
));
8650 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_stmt
);
8654 *vec_stmt
= STMT_VINFO_VEC_STMTS (stmt_info
)[0];
8659 /* Transform phase of a cycle PHI. */
8662 vect_transform_cycle_phi (loop_vec_info loop_vinfo
,
8663 stmt_vec_info stmt_info
, gimple
**vec_stmt
,
8664 slp_tree slp_node
, slp_instance slp_node_instance
)
8666 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
8667 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
8671 bool nested_cycle
= false;
8674 if (nested_in_vect_loop_p (loop
, stmt_info
))
8677 nested_cycle
= true;
8680 stmt_vec_info reduc_stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
8681 reduc_stmt_info
= vect_stmt_to_vectorize (reduc_stmt_info
);
8682 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
8683 gcc_assert (reduc_info
->is_reduc_info
);
8685 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == EXTRACT_LAST_REDUCTION
8686 || STMT_VINFO_REDUC_TYPE (reduc_info
) == FOLD_LEFT_REDUCTION
)
8687 /* Leave the scalar phi in place. */
8690 tree vectype_in
= STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
);
8691 /* For a nested cycle we do not fill the above. */
8693 vectype_in
= STMT_VINFO_VECTYPE (stmt_info
);
8694 gcc_assert (vectype_in
);
8698 /* The size vect_schedule_slp_instance computes is off for us. */
8699 vec_num
= vect_get_num_vectors (LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
8700 * SLP_TREE_LANES (slp_node
), vectype_in
);
8706 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
8709 /* Check whether we should use a single PHI node and accumulate
8710 vectors to one before the backedge. */
8711 if (STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
))
8714 /* Create the destination vector */
8715 gphi
*phi
= as_a
<gphi
*> (stmt_info
->stmt
);
8716 tree vec_dest
= vect_create_destination_var (gimple_phi_result (phi
),
8719 /* Get the loop-entry arguments. */
8720 tree vec_initial_def
= NULL_TREE
;
8721 auto_vec
<tree
> vec_initial_defs
;
8724 vec_initial_defs
.reserve (vec_num
);
8727 unsigned phi_idx
= loop_preheader_edge (loop
)->dest_idx
;
8728 vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node
)[phi_idx
],
8733 gcc_assert (slp_node
== slp_node_instance
->reduc_phis
);
8734 vec
<tree
> &initial_values
= reduc_info
->reduc_initial_values
;
8735 vec
<stmt_vec_info
> &stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
8737 unsigned int num_phis
= stmts
.length ();
8738 if (REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info
))
8740 initial_values
.reserve (num_phis
);
8741 for (unsigned int i
= 0; i
< num_phis
; ++i
)
8743 gphi
*this_phi
= as_a
<gphi
*> (stmts
[i
]->stmt
);
8744 initial_values
.quick_push (vect_phi_initial_value (this_phi
));
8747 vect_find_reusable_accumulator (loop_vinfo
, reduc_info
);
8748 if (!initial_values
.is_empty ())
8751 = (num_phis
== 1 ? initial_values
[0] : NULL_TREE
);
8752 code_helper code
= STMT_VINFO_REDUC_CODE (reduc_info
);
8754 = neutral_op_for_reduction (TREE_TYPE (vectype_out
),
8755 code
, initial_value
);
8756 get_initial_defs_for_reduction (loop_vinfo
, reduc_info
,
8757 &vec_initial_defs
, vec_num
,
8758 stmts
.length (), neutral_op
);
8764 /* Get at the scalar def before the loop, that defines the initial
8765 value of the reduction variable. */
8766 tree initial_def
= vect_phi_initial_value (phi
);
8767 reduc_info
->reduc_initial_values
.safe_push (initial_def
);
8768 /* Optimize: if initial_def is for REDUC_MAX smaller than the base
8769 and we can't use zero for induc_val, use initial_def. Similarly
8770 for REDUC_MIN and initial_def larger than the base. */
8771 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
8773 tree induc_val
= STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
);
8774 if (TREE_CODE (initial_def
) == INTEGER_CST
8775 && !integer_zerop (induc_val
)
8776 && ((STMT_VINFO_REDUC_CODE (reduc_info
) == MAX_EXPR
8777 && tree_int_cst_lt (initial_def
, induc_val
))
8778 || (STMT_VINFO_REDUC_CODE (reduc_info
) == MIN_EXPR
8779 && tree_int_cst_lt (induc_val
, initial_def
))))
8781 induc_val
= initial_def
;
8782 /* Communicate we used the initial_def to epilouge
8784 STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
) = NULL_TREE
;
8786 vec_initial_def
= build_vector_from_val (vectype_out
, induc_val
);
8788 else if (nested_cycle
)
8790 /* Do not use an adjustment def as that case is not supported
8791 correctly if ncopies is not one. */
8792 vect_get_vec_defs_for_operand (loop_vinfo
, reduc_stmt_info
,
8793 ncopies
, initial_def
,
8796 else if (STMT_VINFO_REDUC_TYPE (reduc_info
) == CONST_COND_REDUCTION
8797 || STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
)
8798 /* Fill the initial vector with the initial scalar value. */
8800 = get_initial_def_for_reduction (loop_vinfo
, reduc_stmt_info
,
8801 initial_def
, initial_def
);
8805 vect_find_reusable_accumulator (loop_vinfo
, reduc_info
);
8806 if (!reduc_info
->reduc_initial_values
.is_empty ())
8808 initial_def
= reduc_info
->reduc_initial_values
[0];
8809 code_helper code
= STMT_VINFO_REDUC_CODE (reduc_info
);
8811 = neutral_op_for_reduction (TREE_TYPE (initial_def
),
8813 gcc_assert (neutral_op
);
8814 /* Try to simplify the vector initialization by applying an
8815 adjustment after the reduction has been performed. */
8816 if (!reduc_info
->reused_accumulator
8817 && STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
8818 && !operand_equal_p (neutral_op
, initial_def
))
8820 STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info
)
8822 initial_def
= neutral_op
;
8825 = get_initial_def_for_reduction (loop_vinfo
, reduc_info
,
8826 initial_def
, neutral_op
);
8831 if (vec_initial_def
)
8833 vec_initial_defs
.create (ncopies
);
8834 for (i
= 0; i
< ncopies
; ++i
)
8835 vec_initial_defs
.quick_push (vec_initial_def
);
8838 if (auto *accumulator
= reduc_info
->reused_accumulator
)
8840 tree def
= accumulator
->reduc_input
;
8841 if (!useless_type_conversion_p (vectype_out
, TREE_TYPE (def
)))
8843 unsigned int nreduc
;
8844 bool res
= constant_multiple_p (TYPE_VECTOR_SUBPARTS
8846 TYPE_VECTOR_SUBPARTS (vectype_out
),
8849 gimple_seq stmts
= NULL
;
8850 /* Reduce the single vector to a smaller one. */
8853 /* Perform the reduction in the appropriate type. */
8854 tree rvectype
= vectype_out
;
8855 if (!useless_type_conversion_p (TREE_TYPE (vectype_out
),
8856 TREE_TYPE (TREE_TYPE (def
))))
8857 rvectype
= build_vector_type (TREE_TYPE (TREE_TYPE (def
)),
8858 TYPE_VECTOR_SUBPARTS
8860 def
= vect_create_partial_epilog (def
, rvectype
,
8861 STMT_VINFO_REDUC_CODE
8865 /* The epilogue loop might use a different vector mode, like
8867 if (TYPE_MODE (vectype_out
) != TYPE_MODE (TREE_TYPE (def
)))
8869 tree reduc_type
= build_vector_type_for_mode
8870 (TREE_TYPE (TREE_TYPE (def
)), TYPE_MODE (vectype_out
));
8871 def
= gimple_convert (&stmts
, reduc_type
, def
);
8873 /* Adjust the input so we pick up the partially reduced value
8874 for the skip edge in vect_create_epilog_for_reduction. */
8875 accumulator
->reduc_input
= def
;
8876 /* And the reduction could be carried out using a different sign. */
8877 if (!useless_type_conversion_p (vectype_out
, TREE_TYPE (def
)))
8878 def
= gimple_convert (&stmts
, vectype_out
, def
);
8879 if (loop_vinfo
->main_loop_edge
)
8881 /* While we'd like to insert on the edge this will split
8882 blocks and disturb bookkeeping, we also will eventually
8883 need this on the skip edge. Rely on sinking to
8884 fixup optimal placement and insert in the pred. */
8885 gimple_stmt_iterator gsi
8886 = gsi_last_bb (loop_vinfo
->main_loop_edge
->src
);
8887 /* Insert before a cond that eventually skips the
8889 if (!gsi_end_p (gsi
) && stmt_ends_bb_p (gsi_stmt (gsi
)))
8891 gsi_insert_seq_after (&gsi
, stmts
, GSI_CONTINUE_LINKING
);
8894 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
),
8897 if (loop_vinfo
->main_loop_edge
)
8899 = vect_get_main_loop_result (loop_vinfo
, def
,
8900 vec_initial_defs
[0]);
8902 vec_initial_defs
.safe_push (def
);
8905 /* Generate the reduction PHIs upfront. */
8906 for (i
= 0; i
< vec_num
; i
++)
8908 tree vec_init_def
= vec_initial_defs
[i
];
8909 for (j
= 0; j
< ncopies
; j
++)
8911 /* Create the reduction-phi that defines the reduction
8913 gphi
*new_phi
= create_phi_node (vec_dest
, loop
->header
);
8915 /* Set the loop-entry arg of the reduction-phi. */
8916 if (j
!= 0 && nested_cycle
)
8917 vec_init_def
= vec_initial_defs
[j
];
8918 add_phi_arg (new_phi
, vec_init_def
, loop_preheader_edge (loop
),
8921 /* The loop-latch arg is set in epilogue processing. */
8924 slp_node
->push_vec_def (new_phi
);
8928 *vec_stmt
= new_phi
;
8929 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_phi
);
8937 /* Vectorizes LC PHIs. */
8940 vectorizable_lc_phi (loop_vec_info loop_vinfo
,
8941 stmt_vec_info stmt_info
, gimple
**vec_stmt
,
8945 || !is_a
<gphi
*> (stmt_info
->stmt
)
8946 || gimple_phi_num_args (stmt_info
->stmt
) != 1)
8949 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
8950 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
)
8953 if (!vec_stmt
) /* transformation not required. */
8955 /* Deal with copies from externs or constants that disguise as
8956 loop-closed PHI nodes (PR97886). */
8958 && !vect_maybe_update_slp_op_vectype (SLP_TREE_CHILDREN (slp_node
)[0],
8959 SLP_TREE_VECTYPE (slp_node
)))
8961 if (dump_enabled_p ())
8962 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8963 "incompatible vector types for invariants\n");
8966 STMT_VINFO_TYPE (stmt_info
) = lc_phi_info_type
;
8970 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8971 tree scalar_dest
= gimple_phi_result (stmt_info
->stmt
);
8972 basic_block bb
= gimple_bb (stmt_info
->stmt
);
8973 edge e
= single_pred_edge (bb
);
8974 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
8975 auto_vec
<tree
> vec_oprnds
;
8976 vect_get_vec_defs (loop_vinfo
, stmt_info
, slp_node
,
8977 !slp_node
? vect_get_num_copies (loop_vinfo
, vectype
) : 1,
8978 gimple_phi_arg_def (stmt_info
->stmt
, 0), &vec_oprnds
);
8979 for (unsigned i
= 0; i
< vec_oprnds
.length (); i
++)
8981 /* Create the vectorized LC PHI node. */
8982 gphi
*new_phi
= create_phi_node (vec_dest
, bb
);
8983 add_phi_arg (new_phi
, vec_oprnds
[i
], e
, UNKNOWN_LOCATION
);
8985 slp_node
->push_vec_def (new_phi
);
8987 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_phi
);
8990 *vec_stmt
= STMT_VINFO_VEC_STMTS (stmt_info
)[0];
8995 /* Vectorizes PHIs. */
8998 vectorizable_phi (vec_info
*,
8999 stmt_vec_info stmt_info
, gimple
**vec_stmt
,
9000 slp_tree slp_node
, stmt_vector_for_cost
*cost_vec
)
9002 if (!is_a
<gphi
*> (stmt_info
->stmt
) || !slp_node
)
9005 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
9008 tree vectype
= SLP_TREE_VECTYPE (slp_node
);
9010 if (!vec_stmt
) /* transformation not required. */
9014 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node
), i
, child
)
9017 if (dump_enabled_p ())
9018 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9019 "PHI node with unvectorized backedge def\n");
9022 else if (!vect_maybe_update_slp_op_vectype (child
, vectype
))
9024 if (dump_enabled_p ())
9025 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9026 "incompatible vector types for invariants\n");
9029 else if (SLP_TREE_DEF_TYPE (child
) == vect_internal_def
9030 && !useless_type_conversion_p (vectype
,
9031 SLP_TREE_VECTYPE (child
)))
9033 /* With bools we can have mask and non-mask precision vectors
9034 or different non-mask precisions. while pattern recog is
9035 supposed to guarantee consistency here bugs in it can cause
9036 mismatches (PR103489 and PR103800 for example).
9037 Deal with them here instead of ICEing later. */
9038 if (dump_enabled_p ())
9039 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9040 "incompatible vector type setup from "
9041 "bool pattern detection\n");
9045 /* For single-argument PHIs assume coalescing which means zero cost
9046 for the scalar and the vector PHIs. This avoids artificially
9047 favoring the vector path (but may pessimize it in some cases). */
9048 if (gimple_phi_num_args (as_a
<gphi
*> (stmt_info
->stmt
)) > 1)
9049 record_stmt_cost (cost_vec
, SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
),
9050 vector_stmt
, stmt_info
, vectype
, 0, vect_body
);
9051 STMT_VINFO_TYPE (stmt_info
) = phi_info_type
;
9055 tree scalar_dest
= gimple_phi_result (stmt_info
->stmt
);
9056 basic_block bb
= gimple_bb (stmt_info
->stmt
);
9057 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
9058 auto_vec
<gphi
*> new_phis
;
9059 for (unsigned i
= 0; i
< gimple_phi_num_args (stmt_info
->stmt
); ++i
)
9061 slp_tree child
= SLP_TREE_CHILDREN (slp_node
)[i
];
9063 /* Skip not yet vectorized defs. */
9064 if (SLP_TREE_DEF_TYPE (child
) == vect_internal_def
9065 && SLP_TREE_VEC_DEFS (child
).is_empty ())
9068 auto_vec
<tree
> vec_oprnds
;
9069 vect_get_slp_defs (SLP_TREE_CHILDREN (slp_node
)[i
], &vec_oprnds
);
9070 if (!new_phis
.exists ())
9072 new_phis
.create (vec_oprnds
.length ());
9073 for (unsigned j
= 0; j
< vec_oprnds
.length (); j
++)
9075 /* Create the vectorized LC PHI node. */
9076 new_phis
.quick_push (create_phi_node (vec_dest
, bb
));
9077 slp_node
->push_vec_def (new_phis
[j
]);
9080 edge e
= gimple_phi_arg_edge (as_a
<gphi
*> (stmt_info
->stmt
), i
);
9081 for (unsigned j
= 0; j
< vec_oprnds
.length (); j
++)
9082 add_phi_arg (new_phis
[j
], vec_oprnds
[j
], e
, UNKNOWN_LOCATION
);
9084 /* We should have at least one already vectorized child. */
9085 gcc_assert (new_phis
.exists ());
9090 /* Vectorizes first order recurrences. An overview of the transformation
9091 is described below. Suppose we have the following loop.
9094 for (int i = 0; i < n; ++i)
9100 There is a first-order recurrence on 'a'. For this loop, the scalar IR
9101 looks (simplified) like:
9107 i = PHI <0(scalar.preheader), i+1(scalar.body)>
9108 _2 = PHI <(init(scalar.preheader), <_1(scalar.body)>
9111 if (i < n) goto scalar.body
9113 In this example, _2 is a recurrence because it's value depends on the
9114 previous iteration. We vectorize this as (VF = 4)
9117 vect_init = vect_cst(..., ..., ..., 0)
9120 i = PHI <0(vector.preheader), i+4(vector.body)>
9121 vect_1 = PHI <vect_init(vector.preheader), v2(vector.body)>
9122 vect_2 = a[i, i+1, i+2, i+3];
9123 vect_3 = vec_perm (vect_1, vect_2, { 3, 4, 5, 6 })
9124 b[i, i+1, i+2, i+3] = vect_2 - vect_3
9125 if (..) goto vector.body
9127 In this function, vectorizable_recurr, we code generate both the
9128 vector PHI node and the permute since those together compute the
9129 vectorized value of the scalar PHI. We do not yet have the
9130 backedge value to fill in there nor into the vec_perm. Those
9131 are filled in maybe_set_vectorized_backedge_value and
9134 TODO: Since the scalar loop does not have a use of the recurrence
9135 outside of the loop the natural way to implement peeling via
9136 vectorizing the live value doesn't work. For now peeling of loops
9137 with a recurrence is not implemented. For SLP the supported cases
9138 are restricted to those requiring a single vector recurrence PHI. */
9141 vectorizable_recurr (loop_vec_info loop_vinfo
, stmt_vec_info stmt_info
,
9142 gimple
**vec_stmt
, slp_tree slp_node
,
9143 stmt_vector_for_cost
*cost_vec
)
9145 if (!loop_vinfo
|| !is_a
<gphi
*> (stmt_info
->stmt
))
9148 gphi
*phi
= as_a
<gphi
*> (stmt_info
->stmt
);
9150 /* So far we only support first-order recurrence auto-vectorization. */
9151 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_first_order_recurrence
)
9154 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
9157 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
9159 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
9160 poly_int64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
9161 unsigned dist
= slp_node
? SLP_TREE_LANES (slp_node
) : 1;
9162 /* We need to be able to make progress with a single vector. */
9163 if (maybe_gt (dist
* 2, nunits
))
9165 if (dump_enabled_p ())
9166 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9167 "first order recurrence exceeds half of "
9172 /* First-order recurrence autovectorization needs to handle permutation
9173 with indices = [nunits-1, nunits, nunits+1, ...]. */
9174 vec_perm_builder
sel (nunits
, 1, 3);
9175 for (int i
= 0; i
< 3; ++i
)
9176 sel
.quick_push (nunits
- dist
+ i
);
9177 vec_perm_indices
indices (sel
, 2, nunits
);
9179 if (!vec_stmt
) /* transformation not required. */
9181 if (!can_vec_perm_const_p (TYPE_MODE (vectype
), TYPE_MODE (vectype
),
9187 /* We eventually need to set a vector type on invariant
9191 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node
), j
, child
)
9192 if (!vect_maybe_update_slp_op_vectype
9193 (child
, SLP_TREE_VECTYPE (slp_node
)))
9195 if (dump_enabled_p ())
9196 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9197 "incompatible vector types for "
9203 /* Verify we have set up compatible types. */
9204 edge le
= loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo
));
9205 tree latch_vectype
= NULL_TREE
;
9208 slp_tree latch_def
= SLP_TREE_CHILDREN (slp_node
)[le
->dest_idx
];
9209 latch_vectype
= SLP_TREE_VECTYPE (latch_def
);
9213 tree latch_def
= PHI_ARG_DEF_FROM_EDGE (phi
, le
);
9214 if (TREE_CODE (latch_def
) == SSA_NAME
)
9216 stmt_vec_info latch_def_info
= loop_vinfo
->lookup_def (latch_def
);
9217 latch_def_info
= vect_stmt_to_vectorize (latch_def_info
);
9218 latch_vectype
= STMT_VINFO_VECTYPE (latch_def_info
);
9221 if (!types_compatible_p (latch_vectype
, vectype
))
9224 /* The recurrence costs the initialization vector and one permute
9226 unsigned prologue_cost
= record_stmt_cost (cost_vec
, 1, scalar_to_vec
,
9227 stmt_info
, 0, vect_prologue
);
9228 unsigned inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
9229 stmt_info
, 0, vect_body
);
9230 if (dump_enabled_p ())
9231 dump_printf_loc (MSG_NOTE
, vect_location
,
9232 "vectorizable_recurr: inside_cost = %d, "
9233 "prologue_cost = %d .\n", inside_cost
,
9236 STMT_VINFO_TYPE (stmt_info
) = recurr_info_type
;
9240 edge pe
= loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo
));
9241 basic_block bb
= gimple_bb (phi
);
9242 tree preheader
= PHI_ARG_DEF_FROM_EDGE (phi
, pe
);
9243 if (!useless_type_conversion_p (TREE_TYPE (vectype
), TREE_TYPE (preheader
)))
9245 gimple_seq stmts
= NULL
;
9246 preheader
= gimple_convert (&stmts
, TREE_TYPE (vectype
), preheader
);
9247 gsi_insert_seq_on_edge_immediate (pe
, stmts
);
9249 tree vec_init
= build_vector_from_val (vectype
, preheader
);
9250 vec_init
= vect_init_vector (loop_vinfo
, stmt_info
, vec_init
, vectype
, NULL
);
9252 /* Create the vectorized first-order PHI node. */
9253 tree vec_dest
= vect_get_new_vect_var (vectype
,
9254 vect_simple_var
, "vec_recur_");
9255 gphi
*new_phi
= create_phi_node (vec_dest
, bb
);
9256 add_phi_arg (new_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
9258 /* Insert shuffles the first-order recurrence autovectorization.
9259 result = VEC_PERM <vec_recur, vect_1, index[nunits-1, nunits, ...]>. */
9260 tree perm
= vect_gen_perm_mask_checked (vectype
, indices
);
9262 /* Insert the required permute after the latch definition. The
9263 second and later operands are tentative and will be updated when we have
9264 vectorized the latch definition. */
9265 edge le
= loop_latch_edge (LOOP_VINFO_LOOP (loop_vinfo
));
9266 gimple
*latch_def
= SSA_NAME_DEF_STMT (PHI_ARG_DEF_FROM_EDGE (phi
, le
));
9267 gimple_stmt_iterator gsi2
= gsi_for_stmt (latch_def
);
9270 for (unsigned i
= 0; i
< ncopies
; ++i
)
9272 vec_dest
= make_ssa_name (vectype
);
9274 = gimple_build_assign (vec_dest
, VEC_PERM_EXPR
,
9275 i
== 0 ? gimple_phi_result (new_phi
) : NULL
,
9277 vect_finish_stmt_generation (loop_vinfo
, stmt_info
, vperm
, &gsi2
);
9280 slp_node
->push_vec_def (vperm
);
9282 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (vperm
);
9286 *vec_stmt
= STMT_VINFO_VEC_STMTS (stmt_info
)[0];
9290 /* Return true if VECTYPE represents a vector that requires lowering
9291 by the vector lowering pass. */
9294 vect_emulated_vector_p (tree vectype
)
9296 return (!VECTOR_MODE_P (TYPE_MODE (vectype
))
9297 && (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9298 || TYPE_PRECISION (TREE_TYPE (vectype
)) != 1));
9301 /* Return true if we can emulate CODE on an integer mode representation
9305 vect_can_vectorize_without_simd_p (tree_code code
)
9323 /* Likewise, but taking a code_helper. */
9326 vect_can_vectorize_without_simd_p (code_helper code
)
9328 return (code
.is_tree_code ()
9329 && vect_can_vectorize_without_simd_p (tree_code (code
)));
9332 /* Create vector init for vectorized iv. */
9334 vect_create_nonlinear_iv_init (gimple_seq
* stmts
, tree init_expr
,
9335 tree step_expr
, poly_uint64 nunits
,
9337 enum vect_induction_op_type induction_type
)
9339 unsigned HOST_WIDE_INT const_nunits
;
9340 tree vec_shift
, vec_init
, new_name
;
9342 tree itype
= TREE_TYPE (vectype
);
9344 /* iv_loop is the loop to be vectorized. Create:
9345 vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr). */
9346 new_name
= gimple_convert (stmts
, itype
, init_expr
);
9347 switch (induction_type
)
9349 case vect_step_op_shr
:
9350 case vect_step_op_shl
:
9351 /* Build the Initial value from shift_expr. */
9352 vec_init
= gimple_build_vector_from_val (stmts
,
9355 vec_shift
= gimple_build (stmts
, VEC_SERIES_EXPR
, vectype
,
9356 build_zero_cst (itype
), step_expr
);
9357 vec_init
= gimple_build (stmts
,
9358 (induction_type
== vect_step_op_shr
9359 ? RSHIFT_EXPR
: LSHIFT_EXPR
),
9360 vectype
, vec_init
, vec_shift
);
9363 case vect_step_op_neg
:
9365 vec_init
= gimple_build_vector_from_val (stmts
,
9368 tree vec_neg
= gimple_build (stmts
, NEGATE_EXPR
,
9370 /* The encoding has 2 interleaved stepped patterns. */
9371 vec_perm_builder
sel (nunits
, 2, 3);
9373 for (i
= 0; i
< 3; i
++)
9376 sel
[2 * i
+ 1] = i
+ nunits
;
9378 vec_perm_indices
indices (sel
, 2, nunits
);
9379 /* Don't use vect_gen_perm_mask_checked since can_vec_perm_const_p may
9380 fail when vec_init is const vector. In that situation vec_perm is not
9383 = vect_gen_perm_mask_any (vectype
, indices
);
9384 vec_init
= gimple_build (stmts
, VEC_PERM_EXPR
,
9391 case vect_step_op_mul
:
9393 /* Use unsigned mult to avoid UD integer overflow. */
9394 gcc_assert (nunits
.is_constant (&const_nunits
));
9395 tree utype
= unsigned_type_for (itype
);
9396 tree uvectype
= build_vector_type (utype
,
9397 TYPE_VECTOR_SUBPARTS (vectype
));
9398 new_name
= gimple_convert (stmts
, utype
, new_name
);
9399 vec_init
= gimple_build_vector_from_val (stmts
,
9402 tree_vector_builder
elts (uvectype
, const_nunits
, 1);
9403 tree elt_step
= build_one_cst (utype
);
9405 elts
.quick_push (elt_step
);
9406 for (i
= 1; i
< const_nunits
; i
++)
9408 /* Create: new_name_i = new_name + step_expr. */
9409 elt_step
= gimple_build (stmts
, MULT_EXPR
,
9410 utype
, elt_step
, step_expr
);
9411 elts
.quick_push (elt_step
);
9413 /* Create a vector from [new_name_0, new_name_1, ...,
9414 new_name_nunits-1]. */
9415 tree vec_mul
= gimple_build_vector (stmts
, &elts
);
9416 vec_init
= gimple_build (stmts
, MULT_EXPR
, uvectype
,
9418 vec_init
= gimple_convert (stmts
, vectype
, vec_init
);
9429 /* Peel init_expr by skip_niter for induction_type. */
9431 vect_peel_nonlinear_iv_init (gimple_seq
* stmts
, tree init_expr
,
9432 tree skip_niters
, tree step_expr
,
9433 enum vect_induction_op_type induction_type
)
9435 gcc_assert (TREE_CODE (skip_niters
) == INTEGER_CST
);
9436 tree type
= TREE_TYPE (init_expr
);
9437 unsigned prec
= TYPE_PRECISION (type
);
9438 switch (induction_type
)
9440 case vect_step_op_neg
:
9441 if (TREE_INT_CST_LOW (skip_niters
) % 2)
9442 init_expr
= gimple_build (stmts
, NEGATE_EXPR
, type
, init_expr
);
9443 /* else no change. */
9446 case vect_step_op_shr
:
9447 case vect_step_op_shl
:
9448 skip_niters
= gimple_convert (stmts
, type
, skip_niters
);
9449 step_expr
= gimple_build (stmts
, MULT_EXPR
, type
, step_expr
, skip_niters
);
9450 /* When shift mount >= precision, need to avoid UD.
9451 In the original loop, there's no UD, and according to semantic,
9452 init_expr should be 0 for lshr, ashl, and >>= (prec - 1) for ashr. */
9453 if (!tree_fits_uhwi_p (step_expr
)
9454 || tree_to_uhwi (step_expr
) >= prec
)
9456 if (induction_type
== vect_step_op_shl
9457 || TYPE_UNSIGNED (type
))
9458 init_expr
= build_zero_cst (type
);
9460 init_expr
= gimple_build (stmts
, RSHIFT_EXPR
, type
,
9462 wide_int_to_tree (type
, prec
- 1));
9465 init_expr
= gimple_build (stmts
, (induction_type
== vect_step_op_shr
9466 ? RSHIFT_EXPR
: LSHIFT_EXPR
),
9467 type
, init_expr
, step_expr
);
9470 case vect_step_op_mul
:
9472 tree utype
= unsigned_type_for (type
);
9473 init_expr
= gimple_convert (stmts
, utype
, init_expr
);
9474 wide_int skipn
= wi::to_wide (skip_niters
);
9475 wide_int begin
= wi::to_wide (step_expr
);
9476 auto_mpz base
, exp
, mod
, res
;
9477 wi::to_mpz (begin
, base
, TYPE_SIGN (type
));
9478 wi::to_mpz (skipn
, exp
, UNSIGNED
);
9479 mpz_ui_pow_ui (mod
, 2, TYPE_PRECISION (type
));
9480 mpz_powm (res
, base
, exp
, mod
);
9481 begin
= wi::from_mpz (utype
, res
, true);
9482 tree mult_expr
= wide_int_to_tree (utype
, begin
);
9483 init_expr
= gimple_build (stmts
, MULT_EXPR
, utype
,
9484 init_expr
, mult_expr
);
9485 init_expr
= gimple_convert (stmts
, type
, init_expr
);
9496 /* Create vector step for vectorized iv. */
9498 vect_create_nonlinear_iv_step (gimple_seq
* stmts
, tree step_expr
,
9500 enum vect_induction_op_type induction_type
)
9502 tree expr
= build_int_cst (TREE_TYPE (step_expr
), vf
);
9503 tree new_name
= NULL
;
9504 /* Step should be pow (step, vf) for mult induction. */
9505 if (induction_type
== vect_step_op_mul
)
9507 gcc_assert (vf
.is_constant ());
9508 wide_int begin
= wi::to_wide (step_expr
);
9510 for (unsigned i
= 0; i
!= vf
.to_constant () - 1; i
++)
9511 begin
= wi::mul (begin
, wi::to_wide (step_expr
));
9513 new_name
= wide_int_to_tree (TREE_TYPE (step_expr
), begin
);
9515 else if (induction_type
== vect_step_op_neg
)
9519 new_name
= gimple_build (stmts
, MULT_EXPR
, TREE_TYPE (step_expr
),
9525 vect_create_nonlinear_iv_vec_step (loop_vec_info loop_vinfo
,
9526 stmt_vec_info stmt_info
,
9527 tree new_name
, tree vectype
,
9528 enum vect_induction_op_type induction_type
)
9530 /* No step is needed for neg induction. */
9531 if (induction_type
== vect_step_op_neg
)
9534 tree t
= unshare_expr (new_name
);
9535 gcc_assert (CONSTANT_CLASS_P (new_name
)
9536 || TREE_CODE (new_name
) == SSA_NAME
);
9537 tree new_vec
= build_vector_from_val (vectype
, t
);
9538 tree vec_step
= vect_init_vector (loop_vinfo
, stmt_info
,
9539 new_vec
, vectype
, NULL
);
9543 /* Update vectorized iv with vect_step, induc_def is init. */
9545 vect_update_nonlinear_iv (gimple_seq
* stmts
, tree vectype
,
9546 tree induc_def
, tree vec_step
,
9547 enum vect_induction_op_type induction_type
)
9549 tree vec_def
= induc_def
;
9550 switch (induction_type
)
9552 case vect_step_op_mul
:
9554 /* Use unsigned mult to avoid UD integer overflow. */
9556 = build_vector_type (unsigned_type_for (TREE_TYPE (vectype
)),
9557 TYPE_VECTOR_SUBPARTS (vectype
));
9558 vec_def
= gimple_convert (stmts
, uvectype
, vec_def
);
9559 vec_step
= gimple_convert (stmts
, uvectype
, vec_step
);
9560 vec_def
= gimple_build (stmts
, MULT_EXPR
, uvectype
,
9562 vec_def
= gimple_convert (stmts
, vectype
, vec_def
);
9566 case vect_step_op_shr
:
9567 vec_def
= gimple_build (stmts
, RSHIFT_EXPR
, vectype
,
9571 case vect_step_op_shl
:
9572 vec_def
= gimple_build (stmts
, LSHIFT_EXPR
, vectype
,
9575 case vect_step_op_neg
:
9576 vec_def
= induc_def
;
9587 /* Function vectorizable_induction
9589 Check if STMT_INFO performs an nonlinear induction computation that can be
9590 vectorized. If VEC_STMT is also passed, vectorize the induction PHI: create
9591 a vectorized phi to replace it, put it in VEC_STMT, and add it to the same
9593 Return true if STMT_INFO is vectorizable in this way. */
9596 vectorizable_nonlinear_induction (loop_vec_info loop_vinfo
,
9597 stmt_vec_info stmt_info
,
9598 gimple
**vec_stmt
, slp_tree slp_node
,
9599 stmt_vector_for_cost
*cost_vec
)
9601 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
9603 bool nested_in_vect_loop
= false;
9604 class loop
*iv_loop
;
9606 edge pe
= loop_preheader_edge (loop
);
9608 tree vec_init
, vec_step
;
9611 gphi
*induction_phi
;
9612 tree induc_def
, vec_dest
;
9613 tree init_expr
, step_expr
;
9615 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
9617 gimple_stmt_iterator si
;
9619 gphi
*phi
= dyn_cast
<gphi
*> (stmt_info
->stmt
);
9621 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
9622 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
9623 enum vect_induction_op_type induction_type
9624 = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
);
9626 gcc_assert (induction_type
> vect_step_op_add
);
9631 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
9632 gcc_assert (ncopies
>= 1);
9634 /* FORNOW. Only handle nonlinear induction in the same loop. */
9635 if (nested_in_vect_loop_p (loop
, stmt_info
))
9637 if (dump_enabled_p ())
9638 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9639 "nonlinear induction in nested loop.\n");
9644 gcc_assert (iv_loop
== (gimple_bb (phi
))->loop_father
);
9646 /* TODO: Support slp for nonlinear iv. There should be separate vector iv
9647 update for each iv and a permutation to generate wanted vector iv. */
9650 if (dump_enabled_p ())
9651 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9652 "SLP induction not supported for nonlinear"
9657 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype
)))
9659 if (dump_enabled_p ())
9660 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9661 "floating point nonlinear induction vectorization"
9662 " not supported.\n");
9666 step_expr
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info
);
9667 init_expr
= vect_phi_initial_value (phi
);
9668 gcc_assert (step_expr
!= NULL_TREE
&& init_expr
!= NULL
9669 && TREE_CODE (step_expr
) == INTEGER_CST
);
9670 /* step_expr should be aligned with init_expr,
9671 .i.e. uint64 a >> 1, step is int, but vector<uint64> shift is used. */
9672 step_expr
= fold_convert (TREE_TYPE (vectype
), step_expr
);
9674 if (TREE_CODE (init_expr
) == INTEGER_CST
)
9675 init_expr
= fold_convert (TREE_TYPE (vectype
), init_expr
);
9676 else if (!tree_nop_conversion_p (TREE_TYPE (vectype
), TREE_TYPE (init_expr
)))
9678 /* INIT_EXPR could be a bit_field, bail out for such case. */
9679 if (dump_enabled_p ())
9680 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9681 "nonlinear induction vectorization failed:"
9682 " component type of vectype is not a nop conversion"
9683 " from type of init_expr.\n");
9687 switch (induction_type
)
9689 case vect_step_op_neg
:
9690 if (maybe_eq (TYPE_VECTOR_SUBPARTS (vectype
), 1u))
9692 if (TREE_CODE (init_expr
) != INTEGER_CST
9693 && TREE_CODE (init_expr
) != REAL_CST
)
9695 /* Check for backend support of NEGATE_EXPR and vec_perm. */
9696 if (!directly_supported_p (NEGATE_EXPR
, vectype
))
9699 /* The encoding has 2 interleaved stepped patterns. */
9700 vec_perm_builder
sel (nunits
, 2, 3);
9701 machine_mode mode
= TYPE_MODE (vectype
);
9703 for (i
= 0; i
< 3; i
++)
9706 sel
[i
* 2 + 1] = i
+ nunits
;
9708 vec_perm_indices
indices (sel
, 2, nunits
);
9709 if (!can_vec_perm_const_p (mode
, mode
, indices
))
9714 case vect_step_op_mul
:
9716 /* Check for backend support of MULT_EXPR. */
9717 if (!directly_supported_p (MULT_EXPR
, vectype
))
9720 /* ?? How to construct vector step for variable number vector.
9721 [ 1, step, pow (step, 2), pow (step, 4), .. ]. */
9722 if (!vf
.is_constant ())
9727 case vect_step_op_shr
:
9728 /* Check for backend support of RSHIFT_EXPR. */
9729 if (!directly_supported_p (RSHIFT_EXPR
, vectype
, optab_vector
))
9732 /* Don't shift more than type precision to avoid UD. */
9733 if (!tree_fits_uhwi_p (step_expr
)
9734 || maybe_ge (nunits
* tree_to_uhwi (step_expr
),
9735 TYPE_PRECISION (TREE_TYPE (init_expr
))))
9739 case vect_step_op_shl
:
9740 /* Check for backend support of RSHIFT_EXPR. */
9741 if (!directly_supported_p (LSHIFT_EXPR
, vectype
, optab_vector
))
9744 /* Don't shift more than type precision to avoid UD. */
9745 if (!tree_fits_uhwi_p (step_expr
)
9746 || maybe_ge (nunits
* tree_to_uhwi (step_expr
),
9747 TYPE_PRECISION (TREE_TYPE (init_expr
))))
9756 if (!vec_stmt
) /* transformation not required. */
9758 unsigned inside_cost
= 0, prologue_cost
= 0;
9759 /* loop cost for vec_loop. Neg induction doesn't have any
9761 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
9762 stmt_info
, 0, vect_body
);
9764 /* loop cost for vec_loop. Neg induction doesn't have any
9766 if (induction_type
== vect_step_op_neg
)
9769 /* prologue cost for vec_init and vec_step. */
9770 prologue_cost
= record_stmt_cost (cost_vec
, 2, scalar_to_vec
,
9771 stmt_info
, 0, vect_prologue
);
9773 if (dump_enabled_p ())
9774 dump_printf_loc (MSG_NOTE
, vect_location
,
9775 "vect_model_induction_cost: inside_cost = %d, "
9776 "prologue_cost = %d. \n", inside_cost
,
9779 STMT_VINFO_TYPE (stmt_info
) = induc_vec_info_type
;
9780 DUMP_VECT_SCOPE ("vectorizable_nonlinear_induction");
9786 /* Compute a vector variable, initialized with the first VF values of
9787 the induction variable. E.g., for an iv with IV_PHI='X' and
9788 evolution S, for a vector of 4 units, we want to compute:
9789 [X, X + S, X + 2*S, X + 3*S]. */
9791 if (dump_enabled_p ())
9792 dump_printf_loc (MSG_NOTE
, vect_location
, "transform induction phi.\n");
9794 pe
= loop_preheader_edge (iv_loop
);
9795 /* Find the first insertion point in the BB. */
9796 basic_block bb
= gimple_bb (phi
);
9797 si
= gsi_after_labels (bb
);
9799 gimple_seq stmts
= NULL
;
9801 niters_skip
= LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
);
9802 /* If we are using the loop mask to "peel" for alignment then we need
9803 to adjust the start value here. */
9804 if (niters_skip
!= NULL_TREE
)
9805 init_expr
= vect_peel_nonlinear_iv_init (&stmts
, init_expr
, niters_skip
,
9806 step_expr
, induction_type
);
9808 vec_init
= vect_create_nonlinear_iv_init (&stmts
, init_expr
,
9809 step_expr
, nunits
, vectype
,
9813 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
9814 gcc_assert (!new_bb
);
9818 new_name
= vect_create_nonlinear_iv_step (&stmts
, step_expr
,
9819 vf
, induction_type
);
9822 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
9823 gcc_assert (!new_bb
);
9826 vec_step
= vect_create_nonlinear_iv_vec_step (loop_vinfo
, stmt_info
,
9829 /* Create the following def-use cycle:
9834 vec_iv = PHI <vec_init, vec_loop>
9838 vec_loop = vec_iv + vec_step; */
9840 /* Create the induction-phi that defines the induction-operand. */
9841 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_");
9842 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
9843 induc_def
= PHI_RESULT (induction_phi
);
9845 /* Create the iv update inside the loop. */
9847 vec_def
= vect_update_nonlinear_iv (&stmts
, vectype
,
9848 induc_def
, vec_step
,
9851 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
9852 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
9854 /* Set the arguments of the phi node: */
9855 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
9856 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
9859 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (induction_phi
);
9860 *vec_stmt
= induction_phi
;
9862 /* In case that vectorization factor (VF) is bigger than the number
9863 of elements that we can fit in a vectype (nunits), we have to generate
9864 more than one vector stmt - i.e - we need to "unroll" the
9865 vector stmt by a factor VF/nunits. For more details see documentation
9866 in vectorizable_operation. */
9871 /* FORNOW. This restriction should be relaxed. */
9872 gcc_assert (!nested_in_vect_loop
);
9874 new_name
= vect_create_nonlinear_iv_step (&stmts
, step_expr
,
9875 nunits
, induction_type
);
9877 vec_step
= vect_create_nonlinear_iv_vec_step (loop_vinfo
, stmt_info
,
9880 vec_def
= induc_def
;
9881 for (i
= 1; i
< ncopies
; i
++)
9883 /* vec_i = vec_prev + vec_step. */
9885 vec_def
= vect_update_nonlinear_iv (&stmts
, vectype
,
9888 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
9889 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
9890 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_stmt
);
9894 if (dump_enabled_p ())
9895 dump_printf_loc (MSG_NOTE
, vect_location
,
9896 "transform induction: created def-use cycle: %G%G",
9897 (gimple
*) induction_phi
, SSA_NAME_DEF_STMT (vec_def
));
9902 /* Function vectorizable_induction
9904 Check if STMT_INFO performs an induction computation that can be vectorized.
9905 If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
9906 phi to replace it, put it in VEC_STMT, and add it to the same basic block.
9907 Return true if STMT_INFO is vectorizable in this way. */
9910 vectorizable_induction (loop_vec_info loop_vinfo
,
9911 stmt_vec_info stmt_info
,
9912 gimple
**vec_stmt
, slp_tree slp_node
,
9913 stmt_vector_for_cost
*cost_vec
)
9915 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
9917 bool nested_in_vect_loop
= false;
9918 class loop
*iv_loop
;
9920 edge pe
= loop_preheader_edge (loop
);
9922 tree new_vec
, vec_init
, vec_step
, t
;
9925 gphi
*induction_phi
;
9926 tree induc_def
, vec_dest
;
9927 tree init_expr
, step_expr
;
9928 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
9931 gimple_stmt_iterator si
;
9932 enum vect_induction_op_type induction_type
9933 = STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE (stmt_info
);
9935 gphi
*phi
= dyn_cast
<gphi
*> (stmt_info
->stmt
);
9939 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
9942 /* Make sure it was recognized as induction computation. */
9943 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_induction_def
)
9946 /* Handle nonlinear induction in a separate place. */
9947 if (induction_type
!= vect_step_op_add
)
9948 return vectorizable_nonlinear_induction (loop_vinfo
, stmt_info
,
9949 vec_stmt
, slp_node
, cost_vec
);
9951 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
9952 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
9957 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
9958 gcc_assert (ncopies
>= 1);
9960 /* FORNOW. These restrictions should be relaxed. */
9961 if (nested_in_vect_loop_p (loop
, stmt_info
))
9963 imm_use_iterator imm_iter
;
9964 use_operand_p use_p
;
9971 if (dump_enabled_p ())
9972 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9973 "multiple types in nested loop.\n");
9978 latch_e
= loop_latch_edge (loop
->inner
);
9979 loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
9980 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, loop_arg
)
9982 gimple
*use_stmt
= USE_STMT (use_p
);
9983 if (is_gimple_debug (use_stmt
))
9986 if (!flow_bb_inside_loop_p (loop
->inner
, gimple_bb (use_stmt
)))
9988 exit_phi
= use_stmt
;
9994 stmt_vec_info exit_phi_vinfo
= loop_vinfo
->lookup_stmt (exit_phi
);
9995 if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo
)
9996 && !STMT_VINFO_LIVE_P (exit_phi_vinfo
)))
9998 if (dump_enabled_p ())
9999 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10000 "inner-loop induction only used outside "
10001 "of the outer vectorized loop.\n");
10006 nested_in_vect_loop
= true;
10007 iv_loop
= loop
->inner
;
10011 gcc_assert (iv_loop
== (gimple_bb (phi
))->loop_father
);
10013 if (slp_node
&& !nunits
.is_constant ())
10015 /* The current SLP code creates the step value element-by-element. */
10016 if (dump_enabled_p ())
10017 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10018 "SLP induction not supported for variable-length"
10023 if (FLOAT_TYPE_P (vectype
) && !param_vect_induction_float
)
10025 if (dump_enabled_p ())
10026 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10027 "floating point induction vectorization disabled\n");
10031 step_expr
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info
);
10032 gcc_assert (step_expr
!= NULL_TREE
);
10033 if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr
))
10034 && !type_has_mode_precision_p (TREE_TYPE (step_expr
)))
10036 if (dump_enabled_p ())
10037 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10038 "bit-precision induction vectorization not "
10042 tree step_vectype
= get_same_sized_vectype (TREE_TYPE (step_expr
), vectype
);
10044 /* Check for backend support of PLUS/MINUS_EXPR. */
10045 if (!directly_supported_p (PLUS_EXPR
, step_vectype
)
10046 || !directly_supported_p (MINUS_EXPR
, step_vectype
))
10049 if (!vec_stmt
) /* transformation not required. */
10051 unsigned inside_cost
= 0, prologue_cost
= 0;
10054 /* We eventually need to set a vector type on invariant
10058 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (slp_node
), j
, child
)
10059 if (!vect_maybe_update_slp_op_vectype
10060 (child
, SLP_TREE_VECTYPE (slp_node
)))
10062 if (dump_enabled_p ())
10063 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10064 "incompatible vector types for "
10068 /* loop cost for vec_loop. */
10070 = record_stmt_cost (cost_vec
,
10071 SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
),
10072 vector_stmt
, stmt_info
, 0, vect_body
);
10073 /* prologue cost for vec_init (if not nested) and step. */
10074 prologue_cost
= record_stmt_cost (cost_vec
, 1 + !nested_in_vect_loop
,
10076 stmt_info
, 0, vect_prologue
);
10078 else /* if (!slp_node) */
10080 /* loop cost for vec_loop. */
10081 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
10082 stmt_info
, 0, vect_body
);
10083 /* prologue cost for vec_init and vec_step. */
10084 prologue_cost
= record_stmt_cost (cost_vec
, 2, scalar_to_vec
,
10085 stmt_info
, 0, vect_prologue
);
10087 if (dump_enabled_p ())
10088 dump_printf_loc (MSG_NOTE
, vect_location
,
10089 "vect_model_induction_cost: inside_cost = %d, "
10090 "prologue_cost = %d .\n", inside_cost
,
10093 STMT_VINFO_TYPE (stmt_info
) = induc_vec_info_type
;
10094 DUMP_VECT_SCOPE ("vectorizable_induction");
10100 /* Compute a vector variable, initialized with the first VF values of
10101 the induction variable. E.g., for an iv with IV_PHI='X' and
10102 evolution S, for a vector of 4 units, we want to compute:
10103 [X, X + S, X + 2*S, X + 3*S]. */
10105 if (dump_enabled_p ())
10106 dump_printf_loc (MSG_NOTE
, vect_location
, "transform induction phi.\n");
10108 pe
= loop_preheader_edge (iv_loop
);
10109 /* Find the first insertion point in the BB. */
10110 basic_block bb
= gimple_bb (phi
);
10111 si
= gsi_after_labels (bb
);
10113 /* For SLP induction we have to generate several IVs as for example
10114 with group size 3 we need
10115 [i0, i1, i2, i0 + S0] [i1 + S1, i2 + S2, i0 + 2*S0, i1 + 2*S1]
10116 [i2 + 2*S2, i0 + 3*S0, i1 + 3*S1, i2 + 3*S2]. */
10119 /* Enforced above. */
10120 unsigned int const_nunits
= nunits
.to_constant ();
10122 /* The initial values are vectorized, but any lanes > group_size
10123 need adjustment. */
10125 = SLP_TREE_CHILDREN (slp_node
)[pe
->dest_idx
];
10127 /* Gather steps. Since we do not vectorize inductions as
10128 cycles we have to reconstruct the step from SCEV data. */
10129 unsigned group_size
= SLP_TREE_LANES (slp_node
);
10130 tree
*steps
= XALLOCAVEC (tree
, group_size
);
10131 tree
*inits
= XALLOCAVEC (tree
, group_size
);
10132 stmt_vec_info phi_info
;
10133 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node
), i
, phi_info
)
10135 steps
[i
] = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info
);
10137 inits
[i
] = gimple_phi_arg_def (as_a
<gphi
*> (phi_info
->stmt
),
10141 /* Now generate the IVs. */
10142 unsigned nvects
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
10143 gcc_assert ((const_nunits
* nvects
) % group_size
== 0);
10145 if (nested_in_vect_loop
)
10149 /* Compute the number of distinct IVs we need. First reduce
10150 group_size if it is a multiple of const_nunits so we get
10151 one IV for a group_size of 4 but const_nunits 2. */
10152 unsigned group_sizep
= group_size
;
10153 if (group_sizep
% const_nunits
== 0)
10154 group_sizep
= group_sizep
/ const_nunits
;
10155 nivs
= least_common_multiple (group_sizep
,
10156 const_nunits
) / const_nunits
;
10158 tree stept
= TREE_TYPE (step_vectype
);
10159 tree lupdate_mul
= NULL_TREE
;
10160 if (!nested_in_vect_loop
)
10162 /* The number of iterations covered in one vector iteration. */
10163 unsigned lup_mul
= (nvects
* const_nunits
) / group_size
;
10165 = build_vector_from_val (step_vectype
,
10166 SCALAR_FLOAT_TYPE_P (stept
)
10167 ? build_real_from_wide (stept
, lup_mul
,
10169 : build_int_cstu (stept
, lup_mul
));
10171 tree peel_mul
= NULL_TREE
;
10172 gimple_seq init_stmts
= NULL
;
10173 if (LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
))
10175 if (SCALAR_FLOAT_TYPE_P (stept
))
10176 peel_mul
= gimple_build (&init_stmts
, FLOAT_EXPR
, stept
,
10177 LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
));
10179 peel_mul
= gimple_convert (&init_stmts
, stept
,
10180 LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
));
10181 peel_mul
= gimple_build_vector_from_val (&init_stmts
,
10182 step_vectype
, peel_mul
);
10185 auto_vec
<tree
> vec_steps
;
10186 for (ivn
= 0; ivn
< nivs
; ++ivn
)
10188 tree_vector_builder
step_elts (step_vectype
, const_nunits
, 1);
10189 tree_vector_builder
init_elts (vectype
, const_nunits
, 1);
10190 tree_vector_builder
mul_elts (step_vectype
, const_nunits
, 1);
10191 for (unsigned eltn
= 0; eltn
< const_nunits
; ++eltn
)
10193 /* The scalar steps of the IVs. */
10194 tree elt
= steps
[(ivn
*const_nunits
+ eltn
) % group_size
];
10195 elt
= gimple_convert (&init_stmts
, TREE_TYPE (step_vectype
), elt
);
10196 step_elts
.quick_push (elt
);
10199 /* The scalar inits of the IVs if not vectorized. */
10200 elt
= inits
[(ivn
*const_nunits
+ eltn
) % group_size
];
10201 if (!useless_type_conversion_p (TREE_TYPE (vectype
),
10203 elt
= gimple_build (&init_stmts
, VIEW_CONVERT_EXPR
,
10204 TREE_TYPE (vectype
), elt
);
10205 init_elts
.quick_push (elt
);
10207 /* The number of steps to add to the initial values. */
10208 unsigned mul_elt
= (ivn
*const_nunits
+ eltn
) / group_size
;
10209 mul_elts
.quick_push (SCALAR_FLOAT_TYPE_P (stept
)
10210 ? build_real_from_wide (stept
,
10212 : build_int_cstu (stept
, mul_elt
));
10214 vec_step
= gimple_build_vector (&init_stmts
, &step_elts
);
10215 vec_steps
.safe_push (vec_step
);
10216 tree step_mul
= gimple_build_vector (&init_stmts
, &mul_elts
);
10218 step_mul
= gimple_build (&init_stmts
, PLUS_EXPR
, step_vectype
,
10219 step_mul
, peel_mul
);
10221 vec_init
= gimple_build_vector (&init_stmts
, &init_elts
);
10223 /* Create the induction-phi that defines the induction-operand. */
10224 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
,
10226 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
10227 induc_def
= PHI_RESULT (induction_phi
);
10229 /* Create the iv update inside the loop */
10230 tree up
= vec_step
;
10232 up
= gimple_build (&init_stmts
, MULT_EXPR
, step_vectype
,
10233 vec_step
, lupdate_mul
);
10234 gimple_seq stmts
= NULL
;
10235 vec_def
= gimple_convert (&stmts
, step_vectype
, induc_def
);
10236 vec_def
= gimple_build (&stmts
,
10237 PLUS_EXPR
, step_vectype
, vec_def
, up
);
10238 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
10239 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
10240 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
10244 vec_init
= vect_get_slp_vect_def (init_node
, ivn
);
10245 if (!nested_in_vect_loop
10246 && !integer_zerop (step_mul
))
10248 vec_def
= gimple_convert (&init_stmts
, step_vectype
, vec_init
);
10249 up
= gimple_build (&init_stmts
, MULT_EXPR
, step_vectype
,
10250 vec_step
, step_mul
);
10251 vec_def
= gimple_build (&init_stmts
, PLUS_EXPR
, step_vectype
,
10253 vec_init
= gimple_convert (&init_stmts
, vectype
, vec_def
);
10256 /* Set the arguments of the phi node: */
10257 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
10259 slp_node
->push_vec_def (induction_phi
);
10261 if (!nested_in_vect_loop
)
10263 /* Fill up to the number of vectors we need for the whole group. */
10264 nivs
= least_common_multiple (group_size
,
10265 const_nunits
) / const_nunits
;
10266 vec_steps
.reserve (nivs
-ivn
);
10267 for (; ivn
< nivs
; ++ivn
)
10269 slp_node
->push_vec_def (SLP_TREE_VEC_DEFS (slp_node
)[0]);
10270 vec_steps
.quick_push (vec_steps
[0]);
10274 /* Re-use IVs when we can. We are generating further vector
10275 stmts by adding VF' * stride to the IVs generated above. */
10279 = least_common_multiple (group_size
, const_nunits
) / group_size
;
10281 = build_vector_from_val (step_vectype
,
10282 SCALAR_FLOAT_TYPE_P (stept
)
10283 ? build_real_from_wide (stept
,
10285 : build_int_cstu (stept
, vfp
));
10286 for (; ivn
< nvects
; ++ivn
)
10289 = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (slp_node
)[ivn
- nivs
]);
10290 tree def
= gimple_get_lhs (iv
);
10292 vec_steps
[ivn
- nivs
]
10293 = gimple_build (&init_stmts
, MULT_EXPR
, step_vectype
,
10294 vec_steps
[ivn
- nivs
], lupdate_mul
);
10295 gimple_seq stmts
= NULL
;
10296 def
= gimple_convert (&stmts
, step_vectype
, def
);
10297 def
= gimple_build (&stmts
, PLUS_EXPR
, step_vectype
,
10298 def
, vec_steps
[ivn
% nivs
]);
10299 def
= gimple_convert (&stmts
, vectype
, def
);
10300 if (gimple_code (iv
) == GIMPLE_PHI
)
10301 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
10304 gimple_stmt_iterator tgsi
= gsi_for_stmt (iv
);
10305 gsi_insert_seq_after (&tgsi
, stmts
, GSI_CONTINUE_LINKING
);
10307 slp_node
->push_vec_def (def
);
10311 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, init_stmts
);
10312 gcc_assert (!new_bb
);
10317 init_expr
= vect_phi_initial_value (phi
);
10319 gimple_seq stmts
= NULL
;
10320 if (!nested_in_vect_loop
)
10322 /* Convert the initial value to the IV update type. */
10323 tree new_type
= TREE_TYPE (step_expr
);
10324 init_expr
= gimple_convert (&stmts
, new_type
, init_expr
);
10326 /* If we are using the loop mask to "peel" for alignment then we need
10327 to adjust the start value here. */
10328 tree skip_niters
= LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
);
10329 if (skip_niters
!= NULL_TREE
)
10331 if (FLOAT_TYPE_P (vectype
))
10332 skip_niters
= gimple_build (&stmts
, FLOAT_EXPR
, new_type
,
10335 skip_niters
= gimple_convert (&stmts
, new_type
, skip_niters
);
10336 tree skip_step
= gimple_build (&stmts
, MULT_EXPR
, new_type
,
10337 skip_niters
, step_expr
);
10338 init_expr
= gimple_build (&stmts
, MINUS_EXPR
, new_type
,
10339 init_expr
, skip_step
);
10345 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
10346 gcc_assert (!new_bb
);
10349 /* Create the vector that holds the initial_value of the induction. */
10350 if (nested_in_vect_loop
)
10352 /* iv_loop is nested in the loop to be vectorized. init_expr had already
10353 been created during vectorization of previous stmts. We obtain it
10354 from the STMT_VINFO_VEC_STMT of the defining stmt. */
10355 auto_vec
<tree
> vec_inits
;
10356 vect_get_vec_defs_for_operand (loop_vinfo
, stmt_info
, 1,
10357 init_expr
, &vec_inits
);
10358 vec_init
= vec_inits
[0];
10359 /* If the initial value is not of proper type, convert it. */
10360 if (!useless_type_conversion_p (vectype
, TREE_TYPE (vec_init
)))
10363 = gimple_build_assign (vect_get_new_ssa_name (vectype
,
10367 build1 (VIEW_CONVERT_EXPR
, vectype
,
10369 vec_init
= gimple_assign_lhs (new_stmt
);
10370 new_bb
= gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop
),
10372 gcc_assert (!new_bb
);
10377 /* iv_loop is the loop to be vectorized. Create:
10378 vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr) */
10380 new_name
= gimple_convert (&stmts
, TREE_TYPE (step_expr
), init_expr
);
10382 unsigned HOST_WIDE_INT const_nunits
;
10383 if (nunits
.is_constant (&const_nunits
))
10385 tree_vector_builder
elts (step_vectype
, const_nunits
, 1);
10386 elts
.quick_push (new_name
);
10387 for (i
= 1; i
< const_nunits
; i
++)
10389 /* Create: new_name_i = new_name + step_expr */
10390 new_name
= gimple_build (&stmts
, PLUS_EXPR
, TREE_TYPE (new_name
),
10391 new_name
, step_expr
);
10392 elts
.quick_push (new_name
);
10394 /* Create a vector from [new_name_0, new_name_1, ...,
10395 new_name_nunits-1] */
10396 vec_init
= gimple_build_vector (&stmts
, &elts
);
10398 else if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr
)))
10399 /* Build the initial value directly from a VEC_SERIES_EXPR. */
10400 vec_init
= gimple_build (&stmts
, VEC_SERIES_EXPR
, step_vectype
,
10401 new_name
, step_expr
);
10405 [base, base, base, ...]
10406 + (vectype) [0, 1, 2, ...] * [step, step, step, ...]. */
10407 gcc_assert (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)));
10408 gcc_assert (flag_associative_math
);
10409 tree index
= build_index_vector (step_vectype
, 0, 1);
10410 tree base_vec
= gimple_build_vector_from_val (&stmts
, step_vectype
,
10412 tree step_vec
= gimple_build_vector_from_val (&stmts
, step_vectype
,
10414 vec_init
= gimple_build (&stmts
, FLOAT_EXPR
, step_vectype
, index
);
10415 vec_init
= gimple_build (&stmts
, MULT_EXPR
, step_vectype
,
10416 vec_init
, step_vec
);
10417 vec_init
= gimple_build (&stmts
, PLUS_EXPR
, step_vectype
,
10418 vec_init
, base_vec
);
10420 vec_init
= gimple_convert (&stmts
, vectype
, vec_init
);
10424 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
10425 gcc_assert (!new_bb
);
10430 /* Create the vector that holds the step of the induction. */
10431 gimple_stmt_iterator
*step_iv_si
= NULL
;
10432 if (nested_in_vect_loop
)
10433 /* iv_loop is nested in the loop to be vectorized. Generate:
10434 vec_step = [S, S, S, S] */
10435 new_name
= step_expr
;
10436 else if (LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo
))
10438 /* When we're using loop_len produced by SELEC_VL, the non-final
10439 iterations are not always processing VF elements. So vectorize
10440 induction variable instead of
10442 _21 = vect_vec_iv_.6_22 + { VF, ... };
10444 We should generate:
10446 _35 = .SELECT_VL (ivtmp_33, VF);
10447 vect_cst__22 = [vec_duplicate_expr] _35;
10448 _21 = vect_vec_iv_.6_22 + vect_cst__22; */
10449 gcc_assert (!slp_node
);
10450 gimple_seq seq
= NULL
;
10451 vec_loop_lens
*lens
= &LOOP_VINFO_LENS (loop_vinfo
);
10452 tree len
= vect_get_loop_len (loop_vinfo
, NULL
, lens
, 1, vectype
, 0, 0);
10453 expr
= force_gimple_operand (fold_convert (TREE_TYPE (step_expr
),
10454 unshare_expr (len
)),
10455 &seq
, true, NULL_TREE
);
10456 new_name
= gimple_build (&seq
, MULT_EXPR
, TREE_TYPE (step_expr
), expr
,
10458 gsi_insert_seq_before (&si
, seq
, GSI_SAME_STMT
);
10463 /* iv_loop is the loop to be vectorized. Generate:
10464 vec_step = [VF*S, VF*S, VF*S, VF*S] */
10465 gimple_seq seq
= NULL
;
10466 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
10468 expr
= build_int_cst (integer_type_node
, vf
);
10469 expr
= gimple_build (&seq
, FLOAT_EXPR
, TREE_TYPE (step_expr
), expr
);
10472 expr
= build_int_cst (TREE_TYPE (step_expr
), vf
);
10473 new_name
= gimple_build (&seq
, MULT_EXPR
, TREE_TYPE (step_expr
),
10477 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
10478 gcc_assert (!new_bb
);
10482 t
= unshare_expr (new_name
);
10483 gcc_assert (CONSTANT_CLASS_P (new_name
)
10484 || TREE_CODE (new_name
) == SSA_NAME
);
10485 new_vec
= build_vector_from_val (step_vectype
, t
);
10486 vec_step
= vect_init_vector (loop_vinfo
, stmt_info
,
10487 new_vec
, step_vectype
, step_iv_si
);
10490 /* Create the following def-use cycle:
10495 vec_iv = PHI <vec_init, vec_loop>
10499 vec_loop = vec_iv + vec_step; */
10501 /* Create the induction-phi that defines the induction-operand. */
10502 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_");
10503 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
10504 induc_def
= PHI_RESULT (induction_phi
);
10506 /* Create the iv update inside the loop */
10508 vec_def
= gimple_convert (&stmts
, step_vectype
, induc_def
);
10509 vec_def
= gimple_build (&stmts
, PLUS_EXPR
, step_vectype
, vec_def
, vec_step
);
10510 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
10511 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
10512 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
10514 /* Set the arguments of the phi node: */
10515 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
10516 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
10519 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (induction_phi
);
10520 *vec_stmt
= induction_phi
;
10522 /* In case that vectorization factor (VF) is bigger than the number
10523 of elements that we can fit in a vectype (nunits), we have to generate
10524 more than one vector stmt - i.e - we need to "unroll" the
10525 vector stmt by a factor VF/nunits. For more details see documentation
10526 in vectorizable_operation. */
10530 gimple_seq seq
= NULL
;
10531 /* FORNOW. This restriction should be relaxed. */
10532 gcc_assert (!nested_in_vect_loop
);
10533 /* We expect LOOP_VINFO_USING_SELECT_VL_P to be false if ncopies > 1. */
10534 gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo
));
10536 /* Create the vector that holds the step of the induction. */
10537 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
10539 expr
= build_int_cst (integer_type_node
, nunits
);
10540 expr
= gimple_build (&seq
, FLOAT_EXPR
, TREE_TYPE (step_expr
), expr
);
10543 expr
= build_int_cst (TREE_TYPE (step_expr
), nunits
);
10544 new_name
= gimple_build (&seq
, MULT_EXPR
, TREE_TYPE (step_expr
),
10548 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
10549 gcc_assert (!new_bb
);
10552 t
= unshare_expr (new_name
);
10553 gcc_assert (CONSTANT_CLASS_P (new_name
)
10554 || TREE_CODE (new_name
) == SSA_NAME
);
10555 new_vec
= build_vector_from_val (step_vectype
, t
);
10556 vec_step
= vect_init_vector (loop_vinfo
, stmt_info
,
10557 new_vec
, step_vectype
, NULL
);
10559 vec_def
= induc_def
;
10560 for (i
= 1; i
< ncopies
+ 1; i
++)
10562 /* vec_i = vec_prev + vec_step */
10563 gimple_seq stmts
= NULL
;
10564 vec_def
= gimple_convert (&stmts
, step_vectype
, vec_def
);
10565 vec_def
= gimple_build (&stmts
,
10566 PLUS_EXPR
, step_vectype
, vec_def
, vec_step
);
10567 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
10569 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
10572 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
10573 STMT_VINFO_VEC_STMTS (stmt_info
).safe_push (new_stmt
);
10577 /* vec_1 = vec_iv + (VF/n * S)
10578 vec_2 = vec_1 + (VF/n * S)
10580 vec_n = vec_prev + (VF/n * S) = vec_iv + VF * S = vec_loop
10582 vec_n is used as vec_loop to save the large step register and
10583 related operations. */
10584 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
10590 if (dump_enabled_p ())
10591 dump_printf_loc (MSG_NOTE
, vect_location
,
10592 "transform induction: created def-use cycle: %G%G",
10593 (gimple
*) induction_phi
, SSA_NAME_DEF_STMT (vec_def
));
10598 /* Function vectorizable_live_operation_1.
10600 helper function for vectorizable_live_operation. */
10603 vectorizable_live_operation_1 (loop_vec_info loop_vinfo
,
10604 stmt_vec_info stmt_info
, basic_block exit_bb
,
10605 tree vectype
, int ncopies
, slp_tree slp_node
,
10606 tree bitsize
, tree bitstart
, tree vec_lhs
,
10607 tree lhs_type
, gimple_stmt_iterator
*exit_gsi
)
10609 gcc_assert (single_pred_p (exit_bb
) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo
));
10611 tree vec_lhs_phi
= copy_ssa_name (vec_lhs
);
10612 gimple
*phi
= create_phi_node (vec_lhs_phi
, exit_bb
);
10613 for (unsigned i
= 0; i
< gimple_phi_num_args (phi
); i
++)
10614 SET_PHI_ARG_DEF (phi
, i
, vec_lhs
);
10616 gimple_seq stmts
= NULL
;
10619 /* If bitstart is 0 then we can use a BIT_FIELD_REF */
10620 if (integer_zerop (bitstart
))
10622 tree scalar_res
= gimple_build (&stmts
, BIT_FIELD_REF
, TREE_TYPE (vectype
),
10623 vec_lhs_phi
, bitsize
, bitstart
);
10625 /* Convert the extracted vector element to the scalar type. */
10626 new_tree
= gimple_convert (&stmts
, lhs_type
, scalar_res
);
10628 else if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo
))
10632 SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
10634 where VEC_LHS is the vectorized live-out result and MASK is
10635 the loop mask for the final iteration. */
10636 gcc_assert (ncopies
== 1 && !slp_node
);
10637 gimple_seq tem
= NULL
;
10638 gimple_stmt_iterator gsi
= gsi_last (tem
);
10639 tree len
= vect_get_loop_len (loop_vinfo
, &gsi
,
10640 &LOOP_VINFO_LENS (loop_vinfo
),
10644 signed char biasval
= LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
);
10645 tree bias_minus_one
10646 = int_const_binop (MINUS_EXPR
,
10647 build_int_cst (TREE_TYPE (len
), biasval
),
10648 build_one_cst (TREE_TYPE (len
)));
10650 /* LAST_INDEX = LEN + (BIAS - 1). */
10651 tree last_index
= gimple_build (&stmts
, PLUS_EXPR
, TREE_TYPE (len
),
10652 len
, bias_minus_one
);
10654 /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
10656 = gimple_build (&stmts
, CFN_VEC_EXTRACT
, TREE_TYPE (vectype
),
10657 vec_lhs_phi
, last_index
);
10659 /* Convert the extracted vector element to the scalar type. */
10660 new_tree
= gimple_convert (&stmts
, lhs_type
, scalar_res
);
10662 else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
10666 SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
10668 where VEC_LHS is the vectorized live-out result and MASK is
10669 the loop mask for the final iteration. */
10670 gcc_assert (!slp_node
);
10671 tree scalar_type
= TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info
));
10672 gimple_seq tem
= NULL
;
10673 gimple_stmt_iterator gsi
= gsi_last (tem
);
10674 tree mask
= vect_get_loop_mask (loop_vinfo
, &gsi
,
10675 &LOOP_VINFO_MASKS (loop_vinfo
),
10678 gimple_seq_add_seq (&stmts
, tem
);
10680 scalar_res
= gimple_build (&stmts
, CFN_EXTRACT_LAST
, scalar_type
,
10681 mask
, vec_lhs_phi
);
10683 /* Convert the extracted vector element to the scalar type. */
10684 new_tree
= gimple_convert (&stmts
, lhs_type
, scalar_res
);
10688 tree bftype
= TREE_TYPE (vectype
);
10689 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
10690 bftype
= build_nonstandard_integer_type (tree_to_uhwi (bitsize
), 1);
10691 new_tree
= build3 (BIT_FIELD_REF
, bftype
, vec_lhs_phi
, bitsize
, bitstart
);
10692 new_tree
= force_gimple_operand (fold_convert (lhs_type
, new_tree
),
10693 &stmts
, true, NULL_TREE
);
10696 *exit_gsi
= gsi_after_labels (exit_bb
);
10698 gsi_insert_seq_before (exit_gsi
, stmts
, GSI_SAME_STMT
);
10703 /* Function vectorizable_live_operation.
10705 STMT_INFO computes a value that is used outside the loop. Check if
10706 it can be supported. */
10709 vectorizable_live_operation (vec_info
*vinfo
, stmt_vec_info stmt_info
,
10710 slp_tree slp_node
, slp_instance slp_node_instance
,
10711 int slp_index
, bool vec_stmt_p
,
10712 stmt_vector_for_cost
*cost_vec
)
10714 loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (vinfo
);
10715 imm_use_iterator imm_iter
;
10716 tree lhs
, lhs_type
, bitsize
;
10717 tree vectype
= (slp_node
10718 ? SLP_TREE_VECTYPE (slp_node
)
10719 : STMT_VINFO_VECTYPE (stmt_info
));
10720 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
10723 use_operand_p use_p
;
10724 auto_vec
<tree
> vec_oprnds
;
10726 poly_uint64 vec_index
= 0;
10728 gcc_assert (STMT_VINFO_LIVE_P (stmt_info
)
10729 || LOOP_VINFO_EARLY_BREAKS (loop_vinfo
));
10731 /* If a stmt of a reduction is live, vectorize it via
10732 vect_create_epilog_for_reduction. vectorizable_reduction assessed
10733 validity so just trigger the transform here. */
10734 if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info
)))
10738 /* For SLP reductions we vectorize the epilogue for all involved stmts
10740 if (slp_node
&& !REDUC_GROUP_FIRST_ELEMENT (stmt_info
) && slp_index
!= 0)
10742 stmt_vec_info reduc_info
= info_for_reduction (loop_vinfo
, stmt_info
);
10743 gcc_assert (reduc_info
->is_reduc_info
);
10744 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == FOLD_LEFT_REDUCTION
10745 || STMT_VINFO_REDUC_TYPE (reduc_info
) == EXTRACT_LAST_REDUCTION
)
10748 if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo
)
10749 || !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo
))
10750 vect_create_epilog_for_reduction (loop_vinfo
, stmt_info
, slp_node
,
10752 LOOP_VINFO_IV_EXIT (loop_vinfo
));
10754 /* If early break we only have to materialize the reduction on the merge
10755 block, but we have to find an alternate exit first. */
10756 if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo
))
10758 slp_tree phis_node
= slp_node
? slp_node_instance
->reduc_phis
: NULL
;
10759 for (auto exit
: get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo
)))
10760 if (exit
!= LOOP_VINFO_IV_EXIT (loop_vinfo
))
10762 vect_create_epilog_for_reduction (loop_vinfo
, reduc_info
,
10763 phis_node
, slp_node_instance
,
10767 if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo
))
10768 vect_create_epilog_for_reduction (loop_vinfo
, reduc_info
,
10769 phis_node
, slp_node_instance
,
10770 LOOP_VINFO_IV_EXIT (loop_vinfo
));
10776 /* If STMT is not relevant and it is a simple assignment and its inputs are
10777 invariant then it can remain in place, unvectorized. The original last
10778 scalar value that it computes will be used. */
10779 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
10781 gcc_assert (is_simple_and_all_uses_invariant (stmt_info
, loop_vinfo
));
10782 if (dump_enabled_p ())
10783 dump_printf_loc (MSG_NOTE
, vect_location
,
10784 "statement is simple and uses invariant. Leaving in "
10792 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
10796 gcc_assert (slp_index
>= 0);
10798 /* Get the last occurrence of the scalar index from the concatenation of
10799 all the slp vectors. Calculate which slp vector it is and the index
10801 int num_scalar
= SLP_TREE_LANES (slp_node
);
10802 int num_vec
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
10803 poly_uint64 pos
= (num_vec
* nunits
) - num_scalar
+ slp_index
;
10805 /* Calculate which vector contains the result, and which lane of
10806 that vector we need. */
10807 if (!can_div_trunc_p (pos
, nunits
, &vec_entry
, &vec_index
))
10809 if (dump_enabled_p ())
10810 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10811 "Cannot determine which vector holds the"
10812 " final result.\n");
10819 /* No transformation required. */
10820 if (loop_vinfo
&& LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
))
10824 if (dump_enabled_p ())
10825 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10826 "can't operate on partial vectors "
10827 "because an SLP statement is live after "
10829 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
10831 else if (ncopies
> 1)
10833 if (dump_enabled_p ())
10834 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10835 "can't operate on partial vectors "
10836 "because ncopies is greater than 1.\n");
10837 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
10841 gcc_assert (ncopies
== 1 && !slp_node
);
10842 if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST
, vectype
,
10843 OPTIMIZE_FOR_SPEED
))
10844 vect_record_loop_mask (loop_vinfo
,
10845 &LOOP_VINFO_MASKS (loop_vinfo
),
10847 else if (can_vec_extract_var_idx_p (
10848 TYPE_MODE (vectype
), TYPE_MODE (TREE_TYPE (vectype
))))
10849 vect_record_loop_len (loop_vinfo
,
10850 &LOOP_VINFO_LENS (loop_vinfo
),
10854 if (dump_enabled_p ())
10856 MSG_MISSED_OPTIMIZATION
, vect_location
,
10857 "can't operate on partial vectors "
10858 "because the target doesn't support extract "
10859 "last reduction.\n");
10860 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo
) = false;
10864 /* ??? Enable for loop costing as well. */
10866 record_stmt_cost (cost_vec
, 1, vec_to_scalar
, stmt_info
, NULL_TREE
,
10871 /* Use the lhs of the original scalar statement. */
10872 gimple
*stmt
= vect_orig_stmt (stmt_info
)->stmt
;
10873 if (dump_enabled_p ())
10874 dump_printf_loc (MSG_NOTE
, vect_location
, "extracting lane for live "
10877 lhs
= gimple_get_lhs (stmt
);
10878 lhs_type
= TREE_TYPE (lhs
);
10880 bitsize
= vector_element_bits_tree (vectype
);
10882 /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
10883 tree vec_lhs
, vec_lhs0
, bitstart
;
10884 gimple
*vec_stmt
, *vec_stmt0
;
10887 gcc_assert (!loop_vinfo
10888 || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
10889 && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo
)));
10891 /* Get the correct slp vectorized stmt. */
10892 vec_lhs
= SLP_TREE_VEC_DEFS (slp_node
)[vec_entry
];
10893 vec_stmt
= SSA_NAME_DEF_STMT (vec_lhs
);
10895 /* In case we need to early break vectorize also get the first stmt. */
10896 vec_lhs0
= SLP_TREE_VEC_DEFS (slp_node
)[0];
10897 vec_stmt0
= SSA_NAME_DEF_STMT (vec_lhs0
);
10899 /* Get entry to use. */
10900 bitstart
= bitsize_int (vec_index
);
10901 bitstart
= int_const_binop (MULT_EXPR
, bitsize
, bitstart
);
10905 /* For multiple copies, get the last copy. */
10906 vec_stmt
= STMT_VINFO_VEC_STMTS (stmt_info
).last ();
10907 vec_lhs
= gimple_get_lhs (vec_stmt
);
10909 /* In case we need to early break vectorize also get the first stmt. */
10910 vec_stmt0
= STMT_VINFO_VEC_STMTS (stmt_info
)[0];
10911 vec_lhs0
= gimple_get_lhs (vec_stmt0
);
10913 /* Get the last lane in the vector. */
10914 bitstart
= int_const_binop (MULT_EXPR
, bitsize
, bitsize_int (nunits
- 1));
10919 /* Ensure the VEC_LHS for lane extraction stmts satisfy loop-closed PHI
10920 requirement, insert one phi node for it. It looks like:
10927 # vec_lhs' = PHI <vec_lhs>
10928 new_tree = lane_extract <vec_lhs', ...>;
10929 lhs' = new_tree; */
10931 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
10932 /* Check if we have a loop where the chosen exit is not the main exit,
10933 in these cases for an early break we restart the iteration the vector code
10934 did. For the live values we want the value at the start of the iteration
10935 rather than at the end. */
10936 edge main_e
= LOOP_VINFO_IV_EXIT (loop_vinfo
);
10937 bool all_exits_as_early_p
= LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo
);
10938 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, lhs
)
10939 if (!is_gimple_debug (use_stmt
)
10940 && !flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
10941 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
10943 edge e
= gimple_phi_arg_edge (as_a
<gphi
*> (use_stmt
),
10944 phi_arg_index_from_use (use_p
));
10945 gcc_assert (loop_exit_edge_p (loop
, e
));
10946 bool main_exit_edge
= e
== main_e
;
10947 tree tmp_vec_lhs
= vec_lhs
;
10948 tree tmp_bitstart
= bitstart
;
10950 /* For early exit where the exit is not in the BB that leads
10951 to the latch then we're restarting the iteration in the
10952 scalar loop. So get the first live value. */
10953 if ((all_exits_as_early_p
|| !main_exit_edge
)
10954 && STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
)
10956 tmp_vec_lhs
= vec_lhs0
;
10957 tmp_bitstart
= build_zero_cst (TREE_TYPE (bitstart
));
10960 gimple_stmt_iterator exit_gsi
;
10962 = vectorizable_live_operation_1 (loop_vinfo
, stmt_info
,
10963 e
->dest
, vectype
, ncopies
,
10965 tmp_bitstart
, tmp_vec_lhs
,
10966 lhs_type
, &exit_gsi
);
10968 auto gsi
= gsi_for_stmt (use_stmt
);
10969 tree lhs_phi
= gimple_phi_result (use_stmt
);
10970 remove_phi_node (&gsi
, false);
10971 gimple
*copy
= gimple_build_assign (lhs_phi
, new_tree
);
10972 gsi_insert_before (&exit_gsi
, copy
, GSI_SAME_STMT
);
10976 /* There a no further out-of-loop uses of lhs by LC-SSA construction. */
10977 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, lhs
)
10978 gcc_assert (flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)));
10982 /* For basic-block vectorization simply insert the lane-extraction. */
10983 tree bftype
= TREE_TYPE (vectype
);
10984 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
10985 bftype
= build_nonstandard_integer_type (tree_to_uhwi (bitsize
), 1);
10986 tree new_tree
= build3 (BIT_FIELD_REF
, bftype
,
10987 vec_lhs
, bitsize
, bitstart
);
10988 gimple_seq stmts
= NULL
;
10989 new_tree
= force_gimple_operand (fold_convert (lhs_type
, new_tree
),
10990 &stmts
, true, NULL_TREE
);
10991 if (TREE_CODE (new_tree
) == SSA_NAME
10992 && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs
))
10993 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree
) = 1;
10994 if (is_a
<gphi
*> (vec_stmt
))
10996 gimple_stmt_iterator si
= gsi_after_labels (gimple_bb (vec_stmt
));
10997 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
11001 gimple_stmt_iterator si
= gsi_for_stmt (vec_stmt
);
11002 gsi_insert_seq_after (&si
, stmts
, GSI_SAME_STMT
);
11005 /* Replace use of lhs with newly computed result. If the use stmt is a
11006 single arg PHI, just replace all uses of PHI result. It's necessary
11007 because lcssa PHI defining lhs may be before newly inserted stmt. */
11008 use_operand_p use_p
;
11009 stmt_vec_info use_stmt_info
;
11010 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, lhs
)
11011 if (!is_gimple_debug (use_stmt
)
11012 && (!(use_stmt_info
= vinfo
->lookup_stmt (use_stmt
))
11013 || !PURE_SLP_STMT (vect_stmt_to_vectorize (use_stmt_info
))))
11015 /* ??? This can happen when the live lane ends up being
11016 rooted in a vector construction code-generated by an
11017 external SLP node (and code-generation for that already
11018 happened). See gcc.dg/vect/bb-slp-47.c.
11019 Doing this is what would happen if that vector CTOR
11020 were not code-generated yet so it is not too bad.
11021 ??? In fact we'd likely want to avoid this situation
11022 in the first place. */
11023 if (TREE_CODE (new_tree
) == SSA_NAME
11024 && !SSA_NAME_IS_DEFAULT_DEF (new_tree
)
11025 && gimple_code (use_stmt
) != GIMPLE_PHI
11026 && !vect_stmt_dominates_stmt_p (SSA_NAME_DEF_STMT (new_tree
),
11029 if (dump_enabled_p ())
11030 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
11031 "Using original scalar computation for "
11032 "live lane because use preceeds vector "
11036 /* ??? It can also happen that we end up pulling a def into
11037 a loop where replacing out-of-loop uses would require
11038 a new LC SSA PHI node. Retain the original scalar in
11039 those cases as well. PR98064. */
11040 if (TREE_CODE (new_tree
) == SSA_NAME
11041 && !SSA_NAME_IS_DEFAULT_DEF (new_tree
)
11042 && (gimple_bb (use_stmt
)->loop_father
11043 != gimple_bb (vec_stmt
)->loop_father
)
11044 && !flow_loop_nested_p (gimple_bb (vec_stmt
)->loop_father
,
11045 gimple_bb (use_stmt
)->loop_father
))
11047 if (dump_enabled_p ())
11048 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
11049 "Using original scalar computation for "
11050 "live lane because there is an out-of-loop "
11051 "definition for it\n");
11054 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
11055 SET_USE (use_p
, new_tree
);
11056 update_stmt (use_stmt
);
11063 /* Kill any debug uses outside LOOP of SSA names defined in STMT_INFO. */
11066 vect_loop_kill_debug_uses (class loop
*loop
, stmt_vec_info stmt_info
)
11068 ssa_op_iter op_iter
;
11069 imm_use_iterator imm_iter
;
11070 def_operand_p def_p
;
11073 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt_info
->stmt
, op_iter
, SSA_OP_DEF
)
11075 FOR_EACH_IMM_USE_STMT (ustmt
, imm_iter
, DEF_FROM_PTR (def_p
))
11079 if (!is_gimple_debug (ustmt
))
11082 bb
= gimple_bb (ustmt
);
11084 if (!flow_bb_inside_loop_p (loop
, bb
))
11086 if (gimple_debug_bind_p (ustmt
))
11088 if (dump_enabled_p ())
11089 dump_printf_loc (MSG_NOTE
, vect_location
,
11090 "killing debug use\n");
11092 gimple_debug_bind_reset_value (ustmt
);
11093 update_stmt (ustmt
);
11096 gcc_unreachable ();
11102 /* Given loop represented by LOOP_VINFO, return true if computation of
11103 LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
11107 loop_niters_no_overflow (loop_vec_info loop_vinfo
)
11109 /* Constant case. */
11110 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
11112 tree cst_niters
= LOOP_VINFO_NITERS (loop_vinfo
);
11113 tree cst_nitersm1
= LOOP_VINFO_NITERSM1 (loop_vinfo
);
11115 gcc_assert (TREE_CODE (cst_niters
) == INTEGER_CST
);
11116 gcc_assert (TREE_CODE (cst_nitersm1
) == INTEGER_CST
);
11117 if (wi::to_widest (cst_nitersm1
) < wi::to_widest (cst_niters
))
11122 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
11123 /* Check the upper bound of loop niters. */
11124 if (get_max_loop_iterations (loop
, &max
))
11126 tree type
= TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
));
11127 signop sgn
= TYPE_SIGN (type
);
11128 widest_int type_max
= widest_int::from (wi::max_value (type
), sgn
);
11129 if (max
< type_max
)
11135 /* Return a mask type with half the number of elements as OLD_TYPE,
11136 given that it should have mode NEW_MODE. */
11139 vect_halve_mask_nunits (tree old_type
, machine_mode new_mode
)
11141 poly_uint64 nunits
= exact_div (TYPE_VECTOR_SUBPARTS (old_type
), 2);
11142 return build_truth_vector_type_for_mode (nunits
, new_mode
);
11145 /* Return a mask type with twice as many elements as OLD_TYPE,
11146 given that it should have mode NEW_MODE. */
11149 vect_double_mask_nunits (tree old_type
, machine_mode new_mode
)
11151 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (old_type
) * 2;
11152 return build_truth_vector_type_for_mode (nunits
, new_mode
);
11155 /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
11156 contain a sequence of NVECTORS masks that each control a vector of type
11157 VECTYPE. If SCALAR_MASK is nonnull, the fully-masked loop would AND
11158 these vector masks with the vector version of SCALAR_MASK. */
11161 vect_record_loop_mask (loop_vec_info loop_vinfo
, vec_loop_masks
*masks
,
11162 unsigned int nvectors
, tree vectype
, tree scalar_mask
)
11164 gcc_assert (nvectors
!= 0);
11168 scalar_cond_masked_key
cond (scalar_mask
, nvectors
);
11169 loop_vinfo
->scalar_cond_masked_set
.add (cond
);
11172 masks
->mask_set
.add (std::make_pair (vectype
, nvectors
));
11175 /* Given a complete set of masks MASKS, extract mask number INDEX
11176 for an rgroup that operates on NVECTORS vectors of type VECTYPE,
11177 where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
11179 See the comment above vec_loop_masks for more details about the mask
11183 vect_get_loop_mask (loop_vec_info loop_vinfo
,
11184 gimple_stmt_iterator
*gsi
, vec_loop_masks
*masks
,
11185 unsigned int nvectors
, tree vectype
, unsigned int index
)
11187 if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
)
11188 == vect_partial_vectors_while_ult
)
11190 rgroup_controls
*rgm
= &(masks
->rgc_vec
)[nvectors
- 1];
11191 tree mask_type
= rgm
->type
;
11193 /* Populate the rgroup's mask array, if this is the first time we've
11195 if (rgm
->controls
.is_empty ())
11197 rgm
->controls
.safe_grow_cleared (nvectors
, true);
11198 for (unsigned int i
= 0; i
< nvectors
; ++i
)
11200 tree mask
= make_temp_ssa_name (mask_type
, NULL
, "loop_mask");
11201 /* Provide a dummy definition until the real one is available. */
11202 SSA_NAME_DEF_STMT (mask
) = gimple_build_nop ();
11203 rgm
->controls
[i
] = mask
;
11207 tree mask
= rgm
->controls
[index
];
11208 if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type
),
11209 TYPE_VECTOR_SUBPARTS (vectype
)))
11211 /* A loop mask for data type X can be reused for data type Y
11212 if X has N times more elements than Y and if Y's elements
11213 are N times bigger than X's. In this case each sequence
11214 of N elements in the loop mask will be all-zero or all-one.
11215 We can then view-convert the mask so that each sequence of
11216 N elements is replaced by a single element. */
11217 gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type
),
11218 TYPE_VECTOR_SUBPARTS (vectype
)));
11219 gimple_seq seq
= NULL
;
11220 mask_type
= truth_type_for (vectype
);
11221 mask
= gimple_build (&seq
, VIEW_CONVERT_EXPR
, mask_type
, mask
);
11223 gsi_insert_seq_before (gsi
, seq
, GSI_SAME_STMT
);
11227 else if (LOOP_VINFO_PARTIAL_VECTORS_STYLE (loop_vinfo
)
11228 == vect_partial_vectors_avx512
)
11230 /* The number of scalars per iteration and the number of vectors are
11231 both compile-time constants. */
11232 unsigned int nscalars_per_iter
11233 = exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
11234 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)).to_constant ();
11236 rgroup_controls
*rgm
= &masks
->rgc_vec
[nscalars_per_iter
- 1];
11238 /* The stored nV is dependent on the mask type produced. */
11239 gcc_assert (exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
11240 TYPE_VECTOR_SUBPARTS (rgm
->type
)).to_constant ()
11242 nvectors
= rgm
->factor
;
11244 /* Populate the rgroup's mask array, if this is the first time we've
11246 if (rgm
->controls
.is_empty ())
11248 rgm
->controls
.safe_grow_cleared (nvectors
, true);
11249 for (unsigned int i
= 0; i
< nvectors
; ++i
)
11251 tree mask
= make_temp_ssa_name (rgm
->type
, NULL
, "loop_mask");
11252 /* Provide a dummy definition until the real one is available. */
11253 SSA_NAME_DEF_STMT (mask
) = gimple_build_nop ();
11254 rgm
->controls
[i
] = mask
;
11257 if (known_eq (TYPE_VECTOR_SUBPARTS (rgm
->type
),
11258 TYPE_VECTOR_SUBPARTS (vectype
)))
11259 return rgm
->controls
[index
];
11261 /* Split the vector if needed. Since we are dealing with integer mode
11262 masks with AVX512 we can operate on the integer representation
11263 performing the whole vector shifting. */
11264 unsigned HOST_WIDE_INT factor
;
11265 bool ok
= constant_multiple_p (TYPE_VECTOR_SUBPARTS (rgm
->type
),
11266 TYPE_VECTOR_SUBPARTS (vectype
), &factor
);
11268 gcc_assert (GET_MODE_CLASS (TYPE_MODE (rgm
->type
)) == MODE_INT
);
11269 tree mask_type
= truth_type_for (vectype
);
11270 gcc_assert (GET_MODE_CLASS (TYPE_MODE (mask_type
)) == MODE_INT
);
11271 unsigned vi
= index
/ factor
;
11272 unsigned vpart
= index
% factor
;
11273 tree vec
= rgm
->controls
[vi
];
11274 gimple_seq seq
= NULL
;
11275 vec
= gimple_build (&seq
, VIEW_CONVERT_EXPR
,
11276 lang_hooks
.types
.type_for_mode
11277 (TYPE_MODE (rgm
->type
), 1), vec
);
11278 /* For integer mode masks simply shift the right bits into position. */
11280 vec
= gimple_build (&seq
, RSHIFT_EXPR
, TREE_TYPE (vec
), vec
,
11281 build_int_cst (integer_type_node
,
11282 (TYPE_VECTOR_SUBPARTS (vectype
)
11284 vec
= gimple_convert (&seq
, lang_hooks
.types
.type_for_mode
11285 (TYPE_MODE (mask_type
), 1), vec
);
11286 vec
= gimple_build (&seq
, VIEW_CONVERT_EXPR
, mask_type
, vec
);
11288 gsi_insert_seq_before (gsi
, seq
, GSI_SAME_STMT
);
11292 gcc_unreachable ();
11295 /* Record that LOOP_VINFO would need LENS to contain a sequence of NVECTORS
11296 lengths for controlling an operation on VECTYPE. The operation splits
11297 each element of VECTYPE into FACTOR separate subelements, measuring the
11298 length as a number of these subelements. */
11301 vect_record_loop_len (loop_vec_info loop_vinfo
, vec_loop_lens
*lens
,
11302 unsigned int nvectors
, tree vectype
, unsigned int factor
)
11304 gcc_assert (nvectors
!= 0);
11305 if (lens
->length () < nvectors
)
11306 lens
->safe_grow_cleared (nvectors
, true);
11307 rgroup_controls
*rgl
= &(*lens
)[nvectors
- 1];
11309 /* The number of scalars per iteration, scalar occupied bytes and
11310 the number of vectors are both compile-time constants. */
11311 unsigned int nscalars_per_iter
11312 = exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
11313 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)).to_constant ();
11315 if (rgl
->max_nscalars_per_iter
< nscalars_per_iter
)
11317 /* For now, we only support cases in which all loads and stores fall back
11318 to VnQI or none do. */
11319 gcc_assert (!rgl
->max_nscalars_per_iter
11320 || (rgl
->factor
== 1 && factor
== 1)
11321 || (rgl
->max_nscalars_per_iter
* rgl
->factor
11322 == nscalars_per_iter
* factor
));
11323 rgl
->max_nscalars_per_iter
= nscalars_per_iter
;
11324 rgl
->type
= vectype
;
11325 rgl
->factor
= factor
;
11329 /* Given a complete set of lengths LENS, extract length number INDEX
11330 for an rgroup that operates on NVECTORS vectors of type VECTYPE,
11331 where 0 <= INDEX < NVECTORS. Return a value that contains FACTOR
11332 multipled by the number of elements that should be processed.
11333 Insert any set-up statements before GSI. */
11336 vect_get_loop_len (loop_vec_info loop_vinfo
, gimple_stmt_iterator
*gsi
,
11337 vec_loop_lens
*lens
, unsigned int nvectors
, tree vectype
,
11338 unsigned int index
, unsigned int factor
)
11340 rgroup_controls
*rgl
= &(*lens
)[nvectors
- 1];
11341 bool use_bias_adjusted_len
=
11342 LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo
) != 0;
11344 /* Populate the rgroup's len array, if this is the first time we've
11346 if (rgl
->controls
.is_empty ())
11348 rgl
->controls
.safe_grow_cleared (nvectors
, true);
11349 for (unsigned int i
= 0; i
< nvectors
; ++i
)
11351 tree len_type
= LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo
);
11352 gcc_assert (len_type
!= NULL_TREE
);
11354 tree len
= make_temp_ssa_name (len_type
, NULL
, "loop_len");
11356 /* Provide a dummy definition until the real one is available. */
11357 SSA_NAME_DEF_STMT (len
) = gimple_build_nop ();
11358 rgl
->controls
[i
] = len
;
11360 if (use_bias_adjusted_len
)
11362 gcc_assert (i
== 0);
11363 tree adjusted_len
=
11364 make_temp_ssa_name (len_type
, NULL
, "adjusted_loop_len");
11365 SSA_NAME_DEF_STMT (adjusted_len
) = gimple_build_nop ();
11366 rgl
->bias_adjusted_ctrl
= adjusted_len
;
11371 if (use_bias_adjusted_len
)
11372 return rgl
->bias_adjusted_ctrl
;
11374 tree loop_len
= rgl
->controls
[index
];
11375 if (rgl
->factor
== 1 && factor
== 1)
11377 poly_int64 nunits1
= TYPE_VECTOR_SUBPARTS (rgl
->type
);
11378 poly_int64 nunits2
= TYPE_VECTOR_SUBPARTS (vectype
);
11379 if (maybe_ne (nunits1
, nunits2
))
11381 /* A loop len for data type X can be reused for data type Y
11382 if X has N times more elements than Y and if Y's elements
11383 are N times bigger than X's. */
11384 gcc_assert (multiple_p (nunits1
, nunits2
));
11385 factor
= exact_div (nunits1
, nunits2
).to_constant ();
11386 tree iv_type
= LOOP_VINFO_RGROUP_IV_TYPE (loop_vinfo
);
11387 gimple_seq seq
= NULL
;
11388 loop_len
= gimple_build (&seq
, RDIV_EXPR
, iv_type
, loop_len
,
11389 build_int_cst (iv_type
, factor
));
11391 gsi_insert_seq_before (gsi
, seq
, GSI_SAME_STMT
);
11397 /* Scale profiling counters by estimation for LOOP which is vectorized
11399 If FLAT is true, the loop we started with had unrealistically flat
11403 scale_profile_for_vect_loop (class loop
*loop
, edge exit_e
, unsigned vf
, bool flat
)
11405 /* For flat profiles do not scale down proportionally by VF and only
11406 cap by known iteration count bounds. */
11409 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
11410 fprintf (dump_file
,
11411 "Vectorized loop profile seems flat; not scaling iteration "
11412 "count down by the vectorization factor %i\n", vf
);
11413 scale_loop_profile (loop
, profile_probability::always (),
11414 get_likely_max_loop_iterations_int (loop
));
11417 /* Loop body executes VF fewer times and exit increases VF times. */
11418 profile_count entry_count
= loop_preheader_edge (loop
)->count ();
11420 /* If we have unreliable loop profile avoid dropping entry
11421 count bellow header count. This can happen since loops
11422 has unrealistically low trip counts. */
11424 && loop
->header
->count
> entry_count
11425 && loop
->header
->count
< entry_count
* vf
)
11427 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
11428 fprintf (dump_file
,
11429 "Vectorization factor %i seems too large for profile "
11430 "prevoiusly believed to be consistent; reducing.\n", vf
);
11434 if (entry_count
.nonzero_p ())
11435 set_edge_probability_and_rescale_others
11437 entry_count
.probability_in (loop
->header
->count
/ vf
));
11438 /* Avoid producing very large exit probability when we do not have
11439 sensible profile. */
11440 else if (exit_e
->probability
< profile_probability::always () / (vf
* 2))
11441 set_edge_probability_and_rescale_others (exit_e
, exit_e
->probability
* vf
);
11442 loop
->latch
->count
= single_pred_edge (loop
->latch
)->count ();
11444 scale_loop_profile (loop
, profile_probability::always () / vf
,
11445 get_likely_max_loop_iterations_int (loop
));
11448 /* For a vectorized stmt DEF_STMT_INFO adjust all vectorized PHI
11449 latch edge values originally defined by it. */
11452 maybe_set_vectorized_backedge_value (loop_vec_info loop_vinfo
,
11453 stmt_vec_info def_stmt_info
)
11455 tree def
= gimple_get_lhs (vect_orig_stmt (def_stmt_info
)->stmt
);
11456 if (!def
|| TREE_CODE (def
) != SSA_NAME
)
11458 stmt_vec_info phi_info
;
11459 imm_use_iterator iter
;
11460 use_operand_p use_p
;
11461 FOR_EACH_IMM_USE_FAST (use_p
, iter
, def
)
11463 gphi
*phi
= dyn_cast
<gphi
*> (USE_STMT (use_p
));
11466 if (!(gimple_bb (phi
)->loop_father
->header
== gimple_bb (phi
)
11467 && (phi_info
= loop_vinfo
->lookup_stmt (phi
))
11468 && STMT_VINFO_RELEVANT_P (phi_info
)))
11470 loop_p loop
= gimple_bb (phi
)->loop_father
;
11471 edge e
= loop_latch_edge (loop
);
11472 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) != def
)
11475 if (VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (phi_info
))
11476 && STMT_VINFO_REDUC_TYPE (phi_info
) != FOLD_LEFT_REDUCTION
11477 && STMT_VINFO_REDUC_TYPE (phi_info
) != EXTRACT_LAST_REDUCTION
)
11479 vec
<gimple
*> &phi_defs
= STMT_VINFO_VEC_STMTS (phi_info
);
11480 vec
<gimple
*> &latch_defs
= STMT_VINFO_VEC_STMTS (def_stmt_info
);
11481 gcc_assert (phi_defs
.length () == latch_defs
.length ());
11482 for (unsigned i
= 0; i
< phi_defs
.length (); ++i
)
11483 add_phi_arg (as_a
<gphi
*> (phi_defs
[i
]),
11484 gimple_get_lhs (latch_defs
[i
]), e
,
11485 gimple_phi_arg_location (phi
, e
->dest_idx
));
11487 else if (STMT_VINFO_DEF_TYPE (phi_info
) == vect_first_order_recurrence
)
11489 /* For first order recurrences we have to update both uses of
11490 the latch definition, the one in the PHI node and the one
11491 in the generated VEC_PERM_EXPR. */
11492 vec
<gimple
*> &phi_defs
= STMT_VINFO_VEC_STMTS (phi_info
);
11493 vec
<gimple
*> &latch_defs
= STMT_VINFO_VEC_STMTS (def_stmt_info
);
11494 gcc_assert (phi_defs
.length () == latch_defs
.length ());
11495 tree phidef
= gimple_assign_rhs1 (phi_defs
[0]);
11496 gphi
*vphi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (phidef
));
11497 for (unsigned i
= 0; i
< phi_defs
.length (); ++i
)
11499 gassign
*perm
= as_a
<gassign
*> (phi_defs
[i
]);
11501 gimple_assign_set_rhs1 (perm
, gimple_get_lhs (latch_defs
[i
-1]));
11502 gimple_assign_set_rhs2 (perm
, gimple_get_lhs (latch_defs
[i
]));
11503 update_stmt (perm
);
11505 add_phi_arg (vphi
, gimple_get_lhs (latch_defs
.last ()), e
,
11506 gimple_phi_arg_location (phi
, e
->dest_idx
));
11511 /* Vectorize STMT_INFO if relevant, inserting any new instructions before GSI.
11512 When vectorizing STMT_INFO as a store, set *SEEN_STORE to its
11516 vect_transform_loop_stmt (loop_vec_info loop_vinfo
, stmt_vec_info stmt_info
,
11517 gimple_stmt_iterator
*gsi
, stmt_vec_info
*seen_store
)
11519 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
11520 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
11522 if (dump_enabled_p ())
11523 dump_printf_loc (MSG_NOTE
, vect_location
,
11524 "------>vectorizing statement: %G", stmt_info
->stmt
);
11526 if (MAY_HAVE_DEBUG_BIND_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
11527 vect_loop_kill_debug_uses (loop
, stmt_info
);
11529 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
11530 && !STMT_VINFO_LIVE_P (stmt_info
))
11532 if (is_gimple_call (stmt_info
->stmt
)
11533 && gimple_call_internal_p (stmt_info
->stmt
, IFN_MASK_CALL
))
11535 gcc_assert (!gimple_call_lhs (stmt_info
->stmt
));
11536 *seen_store
= stmt_info
;
11542 if (STMT_VINFO_VECTYPE (stmt_info
))
11545 = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info
));
11546 if (!STMT_SLP_TYPE (stmt_info
)
11547 && maybe_ne (nunits
, vf
)
11548 && dump_enabled_p ())
11549 /* For SLP VF is set according to unrolling factor, and not
11550 to vector size, hence for SLP this print is not valid. */
11551 dump_printf_loc (MSG_NOTE
, vect_location
, "multiple-types.\n");
11554 /* Pure SLP statements have already been vectorized. We still need
11555 to apply loop vectorization to hybrid SLP statements. */
11556 if (PURE_SLP_STMT (stmt_info
))
11559 if (dump_enabled_p ())
11560 dump_printf_loc (MSG_NOTE
, vect_location
, "transform statement.\n");
11562 if (vect_transform_stmt (loop_vinfo
, stmt_info
, gsi
, NULL
, NULL
))
11563 *seen_store
= stmt_info
;
11568 /* Helper function to pass to simplify_replace_tree to enable replacing tree's
11569 in the hash_map with its corresponding values. */
11572 find_in_mapping (tree t
, void *context
)
11574 hash_map
<tree
,tree
>* mapping
= (hash_map
<tree
, tree
>*) context
;
11576 tree
*value
= mapping
->get (t
);
11577 return value
? *value
: t
;
11580 /* Update EPILOGUE's loop_vec_info. EPILOGUE was constructed as a copy of the
11581 original loop that has now been vectorized.
11583 The inits of the data_references need to be advanced with the number of
11584 iterations of the main loop. This has been computed in vect_do_peeling and
11585 is stored in parameter ADVANCE. We first restore the data_references
11586 initial offset with the values recored in ORIG_DRS_INIT.
11588 Since the loop_vec_info of this EPILOGUE was constructed for the original
11589 loop, its stmt_vec_infos all point to the original statements. These need
11590 to be updated to point to their corresponding copies as well as the SSA_NAMES
11591 in their PATTERN_DEF_SEQs and RELATED_STMTs.
11593 The data_reference's connections also need to be updated. Their
11594 corresponding dr_vec_info need to be reconnected to the EPILOGUE's
11595 stmt_vec_infos, their statements need to point to their corresponding copy,
11596 if they are gather loads or scatter stores then their reference needs to be
11597 updated to point to its corresponding copy. */
11600 update_epilogue_loop_vinfo (class loop
*epilogue
, tree advance
)
11602 loop_vec_info epilogue_vinfo
= loop_vec_info_for_loop (epilogue
);
11603 auto_vec
<gimple
*> stmt_worklist
;
11604 hash_map
<tree
,tree
> mapping
;
11605 gimple
*orig_stmt
, *new_stmt
;
11606 gimple_stmt_iterator epilogue_gsi
;
11607 gphi_iterator epilogue_phi_gsi
;
11608 stmt_vec_info stmt_vinfo
= NULL
, related_vinfo
;
11609 basic_block
*epilogue_bbs
= get_loop_body (epilogue
);
11612 free (LOOP_VINFO_BBS (epilogue_vinfo
));
11613 LOOP_VINFO_BBS (epilogue_vinfo
) = epilogue_bbs
;
11615 /* Advance data_reference's with the number of iterations of the previous
11616 loop and its prologue. */
11617 vect_update_inits_of_drs (epilogue_vinfo
, advance
, PLUS_EXPR
);
11620 /* The EPILOGUE loop is a copy of the original loop so they share the same
11621 gimple UIDs. In this loop we update the loop_vec_info of the EPILOGUE to
11622 point to the copied statements. We also create a mapping of all LHS' in
11623 the original loop and all the LHS' in the EPILOGUE and create worklists to
11624 update teh STMT_VINFO_PATTERN_DEF_SEQs and STMT_VINFO_RELATED_STMTs. */
11625 for (unsigned i
= 0; i
< epilogue
->num_nodes
; ++i
)
11627 for (epilogue_phi_gsi
= gsi_start_phis (epilogue_bbs
[i
]);
11628 !gsi_end_p (epilogue_phi_gsi
); gsi_next (&epilogue_phi_gsi
))
11630 new_stmt
= epilogue_phi_gsi
.phi ();
11632 gcc_assert (gimple_uid (new_stmt
) > 0);
11634 = epilogue_vinfo
->stmt_vec_infos
[gimple_uid (new_stmt
) - 1];
11636 orig_stmt
= STMT_VINFO_STMT (stmt_vinfo
);
11637 STMT_VINFO_STMT (stmt_vinfo
) = new_stmt
;
11639 mapping
.put (gimple_phi_result (orig_stmt
),
11640 gimple_phi_result (new_stmt
));
11641 /* PHI nodes can not have patterns or related statements. */
11642 gcc_assert (STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) == NULL
11643 && STMT_VINFO_RELATED_STMT (stmt_vinfo
) == NULL
);
11646 for (epilogue_gsi
= gsi_start_bb (epilogue_bbs
[i
]);
11647 !gsi_end_p (epilogue_gsi
); gsi_next (&epilogue_gsi
))
11649 new_stmt
= gsi_stmt (epilogue_gsi
);
11650 if (is_gimple_debug (new_stmt
))
11653 gcc_assert (gimple_uid (new_stmt
) > 0);
11655 = epilogue_vinfo
->stmt_vec_infos
[gimple_uid (new_stmt
) - 1];
11657 orig_stmt
= STMT_VINFO_STMT (stmt_vinfo
);
11658 STMT_VINFO_STMT (stmt_vinfo
) = new_stmt
;
11660 if (tree old_lhs
= gimple_get_lhs (orig_stmt
))
11661 mapping
.put (old_lhs
, gimple_get_lhs (new_stmt
));
11663 if (STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
))
11665 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
);
11666 for (gimple_stmt_iterator gsi
= gsi_start (seq
);
11667 !gsi_end_p (gsi
); gsi_next (&gsi
))
11668 stmt_worklist
.safe_push (gsi_stmt (gsi
));
11671 related_vinfo
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
11672 if (related_vinfo
!= NULL
&& related_vinfo
!= stmt_vinfo
)
11674 gimple
*stmt
= STMT_VINFO_STMT (related_vinfo
);
11675 stmt_worklist
.safe_push (stmt
);
11676 /* Set BB such that the assert in
11677 'get_initial_def_for_reduction' is able to determine that
11678 the BB of the related stmt is inside this loop. */
11679 gimple_set_bb (stmt
,
11680 gimple_bb (new_stmt
));
11681 related_vinfo
= STMT_VINFO_RELATED_STMT (related_vinfo
);
11682 gcc_assert (related_vinfo
== NULL
11683 || related_vinfo
== stmt_vinfo
);
11688 /* The PATTERN_DEF_SEQs and RELATED_STMTs in the epilogue were constructed
11689 using the original main loop and thus need to be updated to refer to the
11690 cloned variables used in the epilogue. */
11691 for (unsigned i
= 0; i
< stmt_worklist
.length (); ++i
)
11693 gimple
*stmt
= stmt_worklist
[i
];
11696 for (unsigned j
= 1; j
< gimple_num_ops (stmt
); ++j
)
11698 tree op
= gimple_op (stmt
, j
);
11699 if ((new_op
= mapping
.get(op
)))
11700 gimple_set_op (stmt
, j
, *new_op
);
11703 /* PR92429: The last argument of simplify_replace_tree disables
11704 folding when replacing arguments. This is required as
11705 otherwise you might end up with different statements than the
11706 ones analyzed in vect_loop_analyze, leading to different
11708 op
= simplify_replace_tree (op
, NULL_TREE
, NULL_TREE
,
11709 &find_in_mapping
, &mapping
, false);
11710 gimple_set_op (stmt
, j
, op
);
11715 struct data_reference
*dr
;
11716 vec
<data_reference_p
> datarefs
= LOOP_VINFO_DATAREFS (epilogue_vinfo
);
11717 FOR_EACH_VEC_ELT (datarefs
, i
, dr
)
11719 orig_stmt
= DR_STMT (dr
);
11720 gcc_assert (gimple_uid (orig_stmt
) > 0);
11721 stmt_vinfo
= epilogue_vinfo
->stmt_vec_infos
[gimple_uid (orig_stmt
) - 1];
11722 /* Data references for gather loads and scatter stores do not use the
11723 updated offset we set using ADVANCE. Instead we have to make sure the
11724 reference in the data references point to the corresponding copy of
11725 the original in the epilogue. Make sure to update both
11726 gather/scatters recognized by dataref analysis and also other
11727 refs that get_load_store_type classified as VMAT_GATHER_SCATTER. */
11728 auto vstmt_vinfo
= vect_stmt_to_vectorize (stmt_vinfo
);
11729 if (STMT_VINFO_MEMORY_ACCESS_TYPE (vstmt_vinfo
) == VMAT_GATHER_SCATTER
11730 || STMT_VINFO_GATHER_SCATTER_P (vstmt_vinfo
))
11733 = simplify_replace_tree (DR_REF (dr
), NULL_TREE
, NULL_TREE
,
11734 &find_in_mapping
, &mapping
);
11735 DR_BASE_ADDRESS (dr
)
11736 = simplify_replace_tree (DR_BASE_ADDRESS (dr
), NULL_TREE
, NULL_TREE
,
11737 &find_in_mapping
, &mapping
);
11739 DR_STMT (dr
) = STMT_VINFO_STMT (stmt_vinfo
);
11740 stmt_vinfo
->dr_aux
.stmt
= stmt_vinfo
;
11743 epilogue_vinfo
->shared
->datarefs_copy
.release ();
11744 epilogue_vinfo
->shared
->save_datarefs ();
11747 /* When vectorizing early break statements instructions that happen before
11748 the early break in the current BB need to be moved to after the early
11749 break. This function deals with that and assumes that any validity
11750 checks has already been performed.
11752 While moving the instructions if it encounters a VUSE or VDEF it then
11753 corrects the VUSES as it moves the statements along. GDEST is the location
11754 in which to insert the new statements. */
11757 move_early_exit_stmts (loop_vec_info loop_vinfo
)
11759 DUMP_VECT_SCOPE ("move_early_exit_stmts");
11761 if (LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo
).is_empty ())
11764 /* Move all stmts that need moving. */
11765 basic_block dest_bb
= LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo
);
11766 gimple_stmt_iterator dest_gsi
= gsi_after_labels (dest_bb
);
11768 tree last_seen_vuse
= NULL_TREE
;
11769 for (gimple
*stmt
: LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo
))
11771 /* We have to update crossed degenerate virtual PHIs. Simply
11773 if (gphi
*vphi
= dyn_cast
<gphi
*> (stmt
))
11775 tree vdef
= gimple_phi_result (vphi
);
11776 tree vuse
= gimple_phi_arg_def (vphi
, 0);
11777 imm_use_iterator iter
;
11778 use_operand_p use_p
;
11780 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, vdef
)
11782 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11783 SET_USE (use_p
, vuse
);
11785 auto gsi
= gsi_for_stmt (stmt
);
11786 remove_phi_node (&gsi
, true);
11787 last_seen_vuse
= vuse
;
11791 /* Check to see if statement is still required for vect or has been
11793 auto stmt_info
= loop_vinfo
->lookup_stmt (stmt
);
11797 if (dump_enabled_p ())
11798 dump_printf_loc (MSG_NOTE
, vect_location
, "moving stmt %G", stmt
);
11800 gimple_stmt_iterator stmt_gsi
= gsi_for_stmt (stmt
);
11801 gsi_move_before (&stmt_gsi
, &dest_gsi
, GSI_NEW_STMT
);
11802 last_seen_vuse
= gimple_vuse (stmt
);
11805 /* Update all the stmts with their new reaching VUSES. */
11806 for (auto p
: LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo
))
11808 if (dump_enabled_p ())
11809 dump_printf_loc (MSG_NOTE
, vect_location
,
11810 "updating vuse to %T for load %G",
11811 last_seen_vuse
, p
);
11812 gimple_set_vuse (p
, last_seen_vuse
);
11816 /* And update the LC PHIs on exits. */
11817 for (edge e
: get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo
)))
11818 if (!dominated_by_p (CDI_DOMINATORS
, e
->src
, dest_bb
))
11819 if (gphi
*phi
= get_virtual_phi (e
->dest
))
11820 SET_PHI_ARG_DEF_ON_EDGE (phi
, e
, last_seen_vuse
);
11823 /* Function vect_transform_loop.
11825 The analysis phase has determined that the loop is vectorizable.
11826 Vectorize the loop - created vectorized stmts to replace the scalar
11827 stmts in the loop, and update the loop exit condition.
11828 Returns scalar epilogue loop if any. */
11831 vect_transform_loop (loop_vec_info loop_vinfo
, gimple
*loop_vectorized_call
)
11833 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
11834 class loop
*epilogue
= NULL
;
11835 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
11836 int nbbs
= loop
->num_nodes
;
11838 tree niters_vector
= NULL_TREE
;
11839 tree step_vector
= NULL_TREE
;
11840 tree niters_vector_mult_vf
= NULL_TREE
;
11841 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
11842 unsigned int lowest_vf
= constant_lower_bound (vf
);
11844 bool check_profitability
= false;
11846 bool flat
= maybe_flat_loop_profile (loop
);
11848 DUMP_VECT_SCOPE ("vec_transform_loop");
11850 loop_vinfo
->shared
->check_datarefs ();
11852 /* Use the more conservative vectorization threshold. If the number
11853 of iterations is constant assume the cost check has been performed
11854 by our caller. If the threshold makes all loops profitable that
11855 run at least the (estimated) vectorization factor number of times
11856 checking is pointless, too. */
11857 th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
11858 if (vect_apply_runtime_profitability_check_p (loop_vinfo
))
11860 if (dump_enabled_p ())
11861 dump_printf_loc (MSG_NOTE
, vect_location
,
11862 "Profitability threshold is %d loop iterations.\n",
11864 check_profitability
= true;
11867 /* Make sure there exists a single-predecessor exit bb. Do this before
11869 edge e
= LOOP_VINFO_IV_EXIT (loop_vinfo
);
11870 if (! single_pred_p (e
->dest
) && !LOOP_VINFO_EARLY_BREAKS (loop_vinfo
))
11872 split_loop_exit_edge (e
, true);
11873 if (dump_enabled_p ())
11874 dump_printf (MSG_NOTE
, "split exit edge\n");
11877 /* Version the loop first, if required, so the profitability check
11880 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
11883 = vect_loop_versioning (loop_vinfo
, loop_vectorized_call
);
11884 sloop
->force_vectorize
= false;
11885 check_profitability
= false;
11888 /* Make sure there exists a single-predecessor exit bb also on the
11889 scalar loop copy. Do this after versioning but before peeling
11890 so CFG structure is fine for both scalar and if-converted loop
11891 to make slpeel_duplicate_current_defs_from_edges face matched
11892 loop closed PHI nodes on the exit. */
11893 if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
))
11895 e
= LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo
);
11896 if (! single_pred_p (e
->dest
))
11898 split_loop_exit_edge (e
, true);
11899 if (dump_enabled_p ())
11900 dump_printf (MSG_NOTE
, "split exit edge of scalar loop\n");
11904 tree niters
= vect_build_loop_niters (loop_vinfo
);
11905 LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
) = niters
;
11906 tree nitersm1
= unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo
));
11907 bool niters_no_overflow
= loop_niters_no_overflow (loop_vinfo
);
11909 drs_init_vec orig_drs_init
;
11911 epilogue
= vect_do_peeling (loop_vinfo
, niters
, nitersm1
, &niters_vector
,
11912 &step_vector
, &niters_vector_mult_vf
, th
,
11913 check_profitability
, niters_no_overflow
,
11915 if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
)
11916 && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo
).initialized_p ())
11918 /* Ifcvt duplicates loop preheader, loop body and produces an basic
11919 block after loop exit. We need to scale all that. */
11920 basic_block preheader
11921 = loop_preheader_edge (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
))->src
;
11923 = preheader
->count
.apply_probability
11924 (LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo
));
11925 scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
),
11926 LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo
));
11927 LOOP_VINFO_SCALAR_IV_EXIT (loop_vinfo
)->dest
->count
= preheader
->count
;
11930 if (niters_vector
== NULL_TREE
)
11932 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
11933 && !LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
11934 && known_eq (lowest_vf
, vf
))
11937 = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
)),
11938 LOOP_VINFO_INT_NITERS (loop_vinfo
) / lowest_vf
);
11939 step_vector
= build_one_cst (TREE_TYPE (niters
));
11941 else if (vect_use_loop_mask_for_alignment_p (loop_vinfo
))
11942 vect_gen_vector_loop_niters (loop_vinfo
, niters
, &niters_vector
,
11943 &step_vector
, niters_no_overflow
);
11945 /* vect_do_peeling subtracted the number of peeled prologue
11946 iterations from LOOP_VINFO_NITERS. */
11947 vect_gen_vector_loop_niters (loop_vinfo
, LOOP_VINFO_NITERS (loop_vinfo
),
11948 &niters_vector
, &step_vector
,
11949 niters_no_overflow
);
11952 /* 1) Make sure the loop header has exactly two entries
11953 2) Make sure we have a preheader basic block. */
11955 gcc_assert (EDGE_COUNT (loop
->header
->preds
) == 2);
11957 split_edge (loop_preheader_edge (loop
));
11959 if (vect_use_loop_mask_for_alignment_p (loop_vinfo
))
11960 /* This will deal with any possible peeling. */
11961 vect_prepare_for_masked_peels (loop_vinfo
);
11963 /* Handle any code motion that we need to for early-break vectorization after
11964 we've done peeling but just before we start vectorizing. */
11965 if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo
))
11966 move_early_exit_stmts (loop_vinfo
);
11968 /* Schedule the SLP instances first, then handle loop vectorization
11970 if (!loop_vinfo
->slp_instances
.is_empty ())
11972 DUMP_VECT_SCOPE ("scheduling SLP instances");
11973 vect_schedule_slp (loop_vinfo
, LOOP_VINFO_SLP_INSTANCES (loop_vinfo
));
11976 /* FORNOW: the vectorizer supports only loops which body consist
11977 of one basic block (header + empty latch). When the vectorizer will
11978 support more involved loop forms, the order by which the BBs are
11979 traversed need to be reconsidered. */
11981 for (i
= 0; i
< nbbs
; i
++)
11983 basic_block bb
= bbs
[i
];
11984 stmt_vec_info stmt_info
;
11986 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
11989 gphi
*phi
= si
.phi ();
11990 if (dump_enabled_p ())
11991 dump_printf_loc (MSG_NOTE
, vect_location
,
11992 "------>vectorizing phi: %G", (gimple
*) phi
);
11993 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
11997 if (MAY_HAVE_DEBUG_BIND_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
11998 vect_loop_kill_debug_uses (loop
, stmt_info
);
12000 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
12001 && !STMT_VINFO_LIVE_P (stmt_info
))
12004 if (STMT_VINFO_VECTYPE (stmt_info
)
12006 (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info
)), vf
))
12007 && dump_enabled_p ())
12008 dump_printf_loc (MSG_NOTE
, vect_location
, "multiple-types.\n");
12010 if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
12011 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
12012 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
12013 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
12014 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_first_order_recurrence
12015 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_internal_def
)
12016 && ! PURE_SLP_STMT (stmt_info
))
12018 if (dump_enabled_p ())
12019 dump_printf_loc (MSG_NOTE
, vect_location
, "transform phi.\n");
12020 vect_transform_stmt (loop_vinfo
, stmt_info
, NULL
, NULL
, NULL
);
12024 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
12027 gphi
*phi
= si
.phi ();
12028 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
12032 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
12033 && !STMT_VINFO_LIVE_P (stmt_info
))
12036 if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
12037 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
12038 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
12039 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
12040 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_internal_def
12041 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_first_order_recurrence
)
12042 && ! PURE_SLP_STMT (stmt_info
))
12043 maybe_set_vectorized_backedge_value (loop_vinfo
, stmt_info
);
12046 for (gimple_stmt_iterator si
= gsi_start_bb (bb
);
12049 stmt
= gsi_stmt (si
);
12050 /* During vectorization remove existing clobber stmts. */
12051 if (gimple_clobber_p (stmt
))
12053 unlink_stmt_vdef (stmt
);
12054 gsi_remove (&si
, true);
12055 release_defs (stmt
);
12059 /* Ignore vector stmts created in the outer loop. */
12060 stmt_info
= loop_vinfo
->lookup_stmt (stmt
);
12062 /* vector stmts created in the outer-loop during vectorization of
12063 stmts in an inner-loop may not have a stmt_info, and do not
12064 need to be vectorized. */
12065 stmt_vec_info seen_store
= NULL
;
12068 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
12070 gimple
*def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
12071 for (gimple_stmt_iterator subsi
= gsi_start (def_seq
);
12072 !gsi_end_p (subsi
); gsi_next (&subsi
))
12074 stmt_vec_info pat_stmt_info
12075 = loop_vinfo
->lookup_stmt (gsi_stmt (subsi
));
12076 vect_transform_loop_stmt (loop_vinfo
, pat_stmt_info
,
12079 stmt_vec_info pat_stmt_info
12080 = STMT_VINFO_RELATED_STMT (stmt_info
);
12081 if (vect_transform_loop_stmt (loop_vinfo
, pat_stmt_info
,
12083 maybe_set_vectorized_backedge_value (loop_vinfo
,
12088 if (vect_transform_loop_stmt (loop_vinfo
, stmt_info
, &si
,
12090 maybe_set_vectorized_backedge_value (loop_vinfo
,
12097 if (STMT_VINFO_GROUPED_ACCESS (seen_store
))
12098 /* Interleaving. If IS_STORE is TRUE, the
12099 vectorization of the interleaving chain was
12100 completed - free all the stores in the chain. */
12101 vect_remove_stores (loop_vinfo
,
12102 DR_GROUP_FIRST_ELEMENT (seen_store
));
12104 /* Free the attached stmt_vec_info and remove the stmt. */
12105 loop_vinfo
->remove_stmt (stmt_info
);
12110 /* Stub out scalar statements that must not survive vectorization.
12111 Doing this here helps with grouped statements, or statements that
12112 are involved in patterns. */
12113 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
12114 !gsi_end_p (gsi
); gsi_next (&gsi
))
12116 gcall
*call
= dyn_cast
<gcall
*> (gsi_stmt (gsi
));
12117 if (!call
|| !gimple_call_internal_p (call
))
12119 internal_fn ifn
= gimple_call_internal_fn (call
);
12120 if (ifn
== IFN_MASK_LOAD
)
12122 tree lhs
= gimple_get_lhs (call
);
12123 if (!VECTOR_TYPE_P (TREE_TYPE (lhs
)))
12125 tree zero
= build_zero_cst (TREE_TYPE (lhs
));
12126 gimple
*new_stmt
= gimple_build_assign (lhs
, zero
);
12127 gsi_replace (&gsi
, new_stmt
, true);
12130 else if (conditional_internal_fn_code (ifn
) != ERROR_MARK
)
12132 tree lhs
= gimple_get_lhs (call
);
12133 if (!VECTOR_TYPE_P (TREE_TYPE (lhs
)))
12136 = gimple_call_arg (call
, gimple_call_num_args (call
) - 1);
12137 gimple
*new_stmt
= gimple_build_assign (lhs
, else_arg
);
12138 gsi_replace (&gsi
, new_stmt
, true);
12142 } /* BBs in loop */
12144 /* The vectorization factor is always > 1, so if we use an IV increment of 1.
12145 a zero NITERS becomes a nonzero NITERS_VECTOR. */
12146 if (integer_onep (step_vector
))
12147 niters_no_overflow
= true;
12148 vect_set_loop_condition (loop
, LOOP_VINFO_IV_EXIT (loop_vinfo
), loop_vinfo
,
12149 niters_vector
, step_vector
, niters_vector_mult_vf
,
12150 !niters_no_overflow
);
12152 unsigned int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
12154 /* True if the final iteration might not handle a full vector's
12155 worth of scalar iterations. */
12156 bool final_iter_may_be_partial
12157 = LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
)
12158 || LOOP_VINFO_EARLY_BREAKS (loop_vinfo
);
12160 /* +1 to convert latch counts to loop iteration counts. */
12161 int bias_for_lowest
= 1;
12163 /* When we are peeling for gaps then we take away one scalar iteration
12164 from the vector loop. Thus we can adjust the upper bound by one
12165 scalar iteration. But only when we know the bound applies to the
12166 IV exit test which might not be true when we have multiple exits. */
12167 if (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo
))
12168 bias_for_lowest
-= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) ? 1 : 0;
12170 int bias_for_assumed
= bias_for_lowest
;
12171 int alignment_npeels
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
12172 if (alignment_npeels
&& LOOP_VINFO_USING_PARTIAL_VECTORS_P (loop_vinfo
))
12174 /* When the amount of peeling is known at compile time, the first
12175 iteration will have exactly alignment_npeels active elements.
12176 In the worst case it will have at least one. */
12177 int min_first_active
= (alignment_npeels
> 0 ? alignment_npeels
: 1);
12178 bias_for_lowest
+= lowest_vf
- min_first_active
;
12179 bias_for_assumed
+= assumed_vf
- min_first_active
;
12181 /* In these calculations the "- 1" converts loop iteration counts
12182 back to latch counts. */
12183 if (loop
->any_upper_bound
)
12185 loop_vec_info main_vinfo
= LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
);
12186 loop
->nb_iterations_upper_bound
12187 = (final_iter_may_be_partial
12188 ? wi::udiv_ceil (loop
->nb_iterations_upper_bound
+ bias_for_lowest
,
12190 : wi::udiv_floor (loop
->nb_iterations_upper_bound
+ bias_for_lowest
,
12193 /* Both peeling for alignment and peeling for gaps can end up
12194 with the scalar epilogue running for more than VF-1 iterations. */
12195 && !main_vinfo
->peeling_for_alignment
12196 && !main_vinfo
->peeling_for_gaps
)
12198 unsigned int bound
;
12199 poly_uint64 main_iters
12200 = upper_bound (LOOP_VINFO_VECT_FACTOR (main_vinfo
),
12201 LOOP_VINFO_COST_MODEL_THRESHOLD (main_vinfo
));
12203 = upper_bound (main_iters
,
12204 LOOP_VINFO_VERSIONING_THRESHOLD (main_vinfo
));
12205 if (can_div_away_from_zero_p (main_iters
,
12206 LOOP_VINFO_VECT_FACTOR (loop_vinfo
),
12208 loop
->nb_iterations_upper_bound
12209 = wi::umin ((bound_wide_int
) (bound
- 1),
12210 loop
->nb_iterations_upper_bound
);
12213 if (loop
->any_likely_upper_bound
)
12214 loop
->nb_iterations_likely_upper_bound
12215 = (final_iter_may_be_partial
12216 ? wi::udiv_ceil (loop
->nb_iterations_likely_upper_bound
12217 + bias_for_lowest
, lowest_vf
) - 1
12218 : wi::udiv_floor (loop
->nb_iterations_likely_upper_bound
12219 + bias_for_lowest
, lowest_vf
) - 1);
12220 if (loop
->any_estimate
)
12221 loop
->nb_iterations_estimate
12222 = (final_iter_may_be_partial
12223 ? wi::udiv_ceil (loop
->nb_iterations_estimate
+ bias_for_assumed
,
12225 : wi::udiv_floor (loop
->nb_iterations_estimate
+ bias_for_assumed
,
12227 scale_profile_for_vect_loop (loop
, LOOP_VINFO_IV_EXIT (loop_vinfo
),
12230 if (dump_enabled_p ())
12232 if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
12234 dump_printf_loc (MSG_NOTE
, vect_location
,
12235 "LOOP VECTORIZED\n");
12237 dump_printf_loc (MSG_NOTE
, vect_location
,
12238 "OUTER LOOP VECTORIZED\n");
12239 dump_printf (MSG_NOTE
, "\n");
12242 dump_printf_loc (MSG_NOTE
, vect_location
,
12243 "LOOP EPILOGUE VECTORIZED (MODE=%s)\n",
12244 GET_MODE_NAME (loop_vinfo
->vector_mode
));
12247 /* Loops vectorized with a variable factor won't benefit from
12248 unrolling/peeling. */
12249 if (!vf
.is_constant ())
12252 if (dump_enabled_p ())
12253 dump_printf_loc (MSG_NOTE
, vect_location
, "Disabling unrolling due to"
12254 " variable-length vectorization factor\n");
12256 /* Free SLP instances here because otherwise stmt reference counting
12258 slp_instance instance
;
12259 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), i
, instance
)
12260 vect_free_slp_instance (instance
);
12261 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).release ();
12262 /* Clear-up safelen field since its value is invalid after vectorization
12263 since vectorized loop can have loop-carried dependencies. */
12268 update_epilogue_loop_vinfo (epilogue
, advance
);
12270 epilogue
->simduid
= loop
->simduid
;
12271 epilogue
->force_vectorize
= loop
->force_vectorize
;
12272 epilogue
->dont_vectorize
= false;
12278 /* The code below is trying to perform simple optimization - revert
12279 if-conversion for masked stores, i.e. if the mask of a store is zero
12280 do not perform it and all stored value producers also if possible.
12282 for (i=0; i<n; i++)
12288 this transformation will produce the following semi-hammock:
12290 if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
12292 vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
12293 vect__12.22_172 = vect__11.19_170 + vect_cst__171;
12294 MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
12295 vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
12296 vect__19.28_184 = vect__18.25_182 + vect_cst__183;
12297 MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
12302 optimize_mask_stores (class loop
*loop
)
12304 basic_block
*bbs
= get_loop_body (loop
);
12305 unsigned nbbs
= loop
->num_nodes
;
12308 class loop
*bb_loop
;
12309 gimple_stmt_iterator gsi
;
12311 auto_vec
<gimple
*> worklist
;
12312 auto_purge_vect_location sentinel
;
12314 vect_location
= find_loop_location (loop
);
12315 /* Pick up all masked stores in loop if any. */
12316 for (i
= 0; i
< nbbs
; i
++)
12319 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
12322 stmt
= gsi_stmt (gsi
);
12323 if (gimple_call_internal_p (stmt
, IFN_MASK_STORE
))
12324 worklist
.safe_push (stmt
);
12329 if (worklist
.is_empty ())
12332 /* Loop has masked stores. */
12333 while (!worklist
.is_empty ())
12335 gimple
*last
, *last_store
;
12338 basic_block store_bb
, join_bb
;
12339 gimple_stmt_iterator gsi_to
;
12340 tree vdef
, new_vdef
;
12345 last
= worklist
.pop ();
12346 mask
= gimple_call_arg (last
, 2);
12347 bb
= gimple_bb (last
);
12348 /* Create then_bb and if-then structure in CFG, then_bb belongs to
12349 the same loop as if_bb. It could be different to LOOP when two
12350 level loop-nest is vectorized and mask_store belongs to the inner
12352 e
= split_block (bb
, last
);
12353 bb_loop
= bb
->loop_father
;
12354 gcc_assert (loop
== bb_loop
|| flow_loop_nested_p (loop
, bb_loop
));
12356 store_bb
= create_empty_bb (bb
);
12357 add_bb_to_loop (store_bb
, bb_loop
);
12358 e
->flags
= EDGE_TRUE_VALUE
;
12359 efalse
= make_edge (bb
, store_bb
, EDGE_FALSE_VALUE
);
12360 /* Put STORE_BB to likely part. */
12361 efalse
->probability
= profile_probability::likely ();
12362 e
->probability
= efalse
->probability
.invert ();
12363 store_bb
->count
= efalse
->count ();
12364 make_single_succ_edge (store_bb
, join_bb
, EDGE_FALLTHRU
);
12365 if (dom_info_available_p (CDI_DOMINATORS
))
12366 set_immediate_dominator (CDI_DOMINATORS
, store_bb
, bb
);
12367 if (dump_enabled_p ())
12368 dump_printf_loc (MSG_NOTE
, vect_location
,
12369 "Create new block %d to sink mask stores.",
12371 /* Create vector comparison with boolean result. */
12372 vectype
= TREE_TYPE (mask
);
12373 zero
= build_zero_cst (vectype
);
12374 stmt
= gimple_build_cond (EQ_EXPR
, mask
, zero
, NULL_TREE
, NULL_TREE
);
12375 gsi
= gsi_last_bb (bb
);
12376 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
12377 /* Create new PHI node for vdef of the last masked store:
12378 .MEM_2 = VDEF <.MEM_1>
12379 will be converted to
12380 .MEM.3 = VDEF <.MEM_1>
12381 and new PHI node will be created in join bb
12382 .MEM_2 = PHI <.MEM_1, .MEM_3>
12384 vdef
= gimple_vdef (last
);
12385 new_vdef
= make_ssa_name (gimple_vop (cfun
), last
);
12386 gimple_set_vdef (last
, new_vdef
);
12387 phi
= create_phi_node (vdef
, join_bb
);
12388 add_phi_arg (phi
, new_vdef
, EDGE_SUCC (store_bb
, 0), UNKNOWN_LOCATION
);
12390 /* Put all masked stores with the same mask to STORE_BB if possible. */
12393 gimple_stmt_iterator gsi_from
;
12394 gimple
*stmt1
= NULL
;
12396 /* Move masked store to STORE_BB. */
12398 gsi
= gsi_for_stmt (last
);
12400 /* Shift GSI to the previous stmt for further traversal. */
12402 gsi_to
= gsi_start_bb (store_bb
);
12403 gsi_move_before (&gsi_from
, &gsi_to
);
12404 /* Setup GSI_TO to the non-empty block start. */
12405 gsi_to
= gsi_start_bb (store_bb
);
12406 if (dump_enabled_p ())
12407 dump_printf_loc (MSG_NOTE
, vect_location
,
12408 "Move stmt to created bb\n%G", last
);
12409 /* Move all stored value producers if possible. */
12410 while (!gsi_end_p (gsi
))
12413 imm_use_iterator imm_iter
;
12414 use_operand_p use_p
;
12417 /* Skip debug statements. */
12418 if (is_gimple_debug (gsi_stmt (gsi
)))
12423 stmt1
= gsi_stmt (gsi
);
12424 /* Do not consider statements writing to memory or having
12425 volatile operand. */
12426 if (gimple_vdef (stmt1
)
12427 || gimple_has_volatile_ops (stmt1
))
12431 lhs
= gimple_get_lhs (stmt1
);
12435 /* LHS of vectorized stmt must be SSA_NAME. */
12436 if (TREE_CODE (lhs
) != SSA_NAME
)
12439 if (!VECTOR_TYPE_P (TREE_TYPE (lhs
)))
12441 /* Remove dead scalar statement. */
12442 if (has_zero_uses (lhs
))
12444 gsi_remove (&gsi_from
, true);
12449 /* Check that LHS does not have uses outside of STORE_BB. */
12451 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
12454 use_stmt
= USE_STMT (use_p
);
12455 if (is_gimple_debug (use_stmt
))
12457 if (gimple_bb (use_stmt
) != store_bb
)
12466 if (gimple_vuse (stmt1
)
12467 && gimple_vuse (stmt1
) != gimple_vuse (last_store
))
12470 /* Can move STMT1 to STORE_BB. */
12471 if (dump_enabled_p ())
12472 dump_printf_loc (MSG_NOTE
, vect_location
,
12473 "Move stmt to created bb\n%G", stmt1
);
12474 gsi_move_before (&gsi_from
, &gsi_to
);
12475 /* Shift GSI_TO for further insertion. */
12476 gsi_prev (&gsi_to
);
12478 /* Put other masked stores with the same mask to STORE_BB. */
12479 if (worklist
.is_empty ()
12480 || gimple_call_arg (worklist
.last (), 2) != mask
12481 || worklist
.last () != stmt1
)
12483 last
= worklist
.pop ();
12485 add_phi_arg (phi
, gimple_vuse (last_store
), e
, UNKNOWN_LOCATION
);
12489 /* Decide whether it is possible to use a zero-based induction variable
12490 when vectorizing LOOP_VINFO with partial vectors. If it is, return
12491 the value that the induction variable must be able to hold in order
12492 to ensure that the rgroups eventually have no active vector elements.
12493 Return -1 otherwise. */
12496 vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo
)
12498 tree niters_skip
= LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
);
12499 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
12500 unsigned HOST_WIDE_INT max_vf
= vect_max_vf (loop_vinfo
);
12502 /* Calculate the value that the induction variable must be able
12503 to hit in order to ensure that we end the loop with an all-false mask.
12504 This involves adding the maximum number of inactive trailing scalar
12506 widest_int iv_limit
= -1;
12507 if (max_loop_iterations (loop
, &iv_limit
))
12511 /* Add the maximum number of skipped iterations to the
12512 maximum iteration count. */
12513 if (TREE_CODE (niters_skip
) == INTEGER_CST
)
12514 iv_limit
+= wi::to_widest (niters_skip
);
12516 iv_limit
+= max_vf
- 1;
12518 else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
))
12519 /* Make a conservatively-correct assumption. */
12520 iv_limit
+= max_vf
- 1;
12522 /* IV_LIMIT is the maximum number of latch iterations, which is also
12523 the maximum in-range IV value. Round this value down to the previous
12524 vector alignment boundary and then add an extra full iteration. */
12525 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
12526 iv_limit
= (iv_limit
& -(int) known_alignment (vf
)) + max_vf
;
12531 /* For the given rgroup_controls RGC, check whether an induction variable
12532 would ever hit a value that produces a set of all-false masks or zero
12533 lengths before wrapping around. Return true if it's possible to wrap
12534 around before hitting the desirable value, otherwise return false. */
12537 vect_rgroup_iv_might_wrap_p (loop_vec_info loop_vinfo
, rgroup_controls
*rgc
)
12539 widest_int iv_limit
= vect_iv_limit_for_partial_vectors (loop_vinfo
);
12541 if (iv_limit
== -1)
12544 tree compare_type
= LOOP_VINFO_RGROUP_COMPARE_TYPE (loop_vinfo
);
12545 unsigned int compare_precision
= TYPE_PRECISION (compare_type
);
12546 unsigned nitems
= rgc
->max_nscalars_per_iter
* rgc
->factor
;
12548 if (wi::min_precision (iv_limit
* nitems
, UNSIGNED
) > compare_precision
)