2 Copyright (C) 2003-2013 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
29 #include "basic-block.h"
30 #include "gimple-pretty-print.h"
31 #include "tree-flow.h"
32 #include "tree-pass.h"
38 #include "diagnostic-core.h"
39 #include "tree-chrec.h"
40 #include "tree-scalar-evolution.h"
41 #include "tree-vectorizer.h"
44 /* Loop Vectorization Pass.
46 This pass tries to vectorize loops.
48 For example, the vectorizer transforms the following simple loop:
50 short a[N]; short b[N]; short c[N]; int i;
56 as if it was manually vectorized by rewriting the source code into:
58 typedef int __attribute__((mode(V8HI))) v8hi;
59 short a[N]; short b[N]; short c[N]; int i;
60 v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
63 for (i=0; i<N/8; i++){
70 The main entry to this pass is vectorize_loops(), in which
71 the vectorizer applies a set of analyses on a given set of loops,
72 followed by the actual vectorization transformation for the loops that
73 had successfully passed the analysis phase.
74 Throughout this pass we make a distinction between two types of
75 data: scalars (which are represented by SSA_NAMES), and memory references
76 ("data-refs"). These two types of data require different handling both
77 during analysis and transformation. The types of data-refs that the
78 vectorizer currently supports are ARRAY_REFS which base is an array DECL
79 (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
80 accesses are required to have a simple (consecutive) access pattern.
84 The driver for the analysis phase is vect_analyze_loop().
85 It applies a set of analyses, some of which rely on the scalar evolution
86 analyzer (scev) developed by Sebastian Pop.
88 During the analysis phase the vectorizer records some information
89 per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
90 loop, as well as general information about the loop as a whole, which is
91 recorded in a "loop_vec_info" struct attached to each loop.
95 The loop transformation phase scans all the stmts in the loop, and
96 creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
97 the loop that needs to be vectorized. It inserts the vector code sequence
98 just before the scalar stmt S, and records a pointer to the vector code
99 in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
100 attached to S). This pointer will be used for the vectorization of following
101 stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
102 otherwise, we rely on dead code elimination for removing it.
104 For example, say stmt S1 was vectorized into stmt VS1:
107 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
110 To vectorize stmt S2, the vectorizer first finds the stmt that defines
111 the operand 'b' (S1), and gets the relevant vector def 'vb' from the
112 vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
113 resulting sequence would be:
116 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
118 S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
120 Operands that are not SSA_NAMEs, are data-refs that appear in
121 load/store operations (like 'x[i]' in S1), and are handled differently.
125 Currently the only target specific information that is used is the
126 size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
127 Targets that can support different sizes of vectors, for now will need
128 to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
129 flexibility will be added in the future.
131 Since we only vectorize operations which vector form can be
132 expressed using existing tree codes, to verify that an operation is
133 supported, the vectorizer checks the relevant optab at the relevant
134 machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
135 the value found is CODE_FOR_nothing, then there's no target support, and
136 we can't vectorize the stmt.
138 For additional information on this project see:
139 http://gcc.gnu.org/projects/tree-ssa/vectorization.html
142 static void vect_estimate_min_profitable_iters (loop_vec_info
, int *, int *);
144 /* Function vect_determine_vectorization_factor
146 Determine the vectorization factor (VF). VF is the number of data elements
147 that are operated upon in parallel in a single iteration of the vectorized
148 loop. For example, when vectorizing a loop that operates on 4byte elements,
149 on a target with vector size (VS) 16byte, the VF is set to 4, since 4
150 elements can fit in a single vector register.
152 We currently support vectorization of loops in which all types operated upon
153 are of the same size. Therefore this function currently sets VF according to
154 the size of the types operated upon, and fails if there are multiple sizes
157 VF is also the factor by which the loop iterations are strip-mined, e.g.:
164 for (i=0; i<N; i+=VF){
165 a[i:VF] = b[i:VF] + c[i:VF];
170 vect_determine_vectorization_factor (loop_vec_info loop_vinfo
)
172 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
173 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
174 int nbbs
= loop
->num_nodes
;
175 gimple_stmt_iterator si
;
176 unsigned int vectorization_factor
= 0;
181 stmt_vec_info stmt_info
;
184 gimple stmt
, pattern_stmt
= NULL
;
185 gimple_seq pattern_def_seq
= NULL
;
186 gimple_stmt_iterator pattern_def_si
= gsi_none ();
187 bool analyze_pattern_stmt
= false;
189 if (dump_enabled_p ())
190 dump_printf_loc (MSG_NOTE
, vect_location
,
191 "=== vect_determine_vectorization_factor ===");
193 for (i
= 0; i
< nbbs
; i
++)
195 basic_block bb
= bbs
[i
];
197 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
200 stmt_info
= vinfo_for_stmt (phi
);
201 if (dump_enabled_p ())
203 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining phi: ");
204 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
207 gcc_assert (stmt_info
);
209 if (STMT_VINFO_RELEVANT_P (stmt_info
))
211 gcc_assert (!STMT_VINFO_VECTYPE (stmt_info
));
212 scalar_type
= TREE_TYPE (PHI_RESULT (phi
));
214 if (dump_enabled_p ())
216 dump_printf_loc (MSG_NOTE
, vect_location
,
217 "get vectype for scalar type: ");
218 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
221 vectype
= get_vectype_for_scalar_type (scalar_type
);
224 if (dump_enabled_p ())
226 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
227 "not vectorized: unsupported "
229 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
234 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
236 if (dump_enabled_p ())
238 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
239 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
242 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
243 if (dump_enabled_p ())
244 dump_printf_loc (MSG_NOTE
, vect_location
, "nunits = %d", nunits
);
246 if (!vectorization_factor
247 || (nunits
> vectorization_factor
))
248 vectorization_factor
= nunits
;
252 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
) || analyze_pattern_stmt
;)
256 if (analyze_pattern_stmt
)
259 stmt
= gsi_stmt (si
);
261 stmt_info
= vinfo_for_stmt (stmt
);
263 if (dump_enabled_p ())
265 dump_printf_loc (MSG_NOTE
, vect_location
,
266 "==> examining statement: ");
267 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
270 gcc_assert (stmt_info
);
272 /* Skip stmts which do not need to be vectorized. */
273 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
274 && !STMT_VINFO_LIVE_P (stmt_info
))
276 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
277 && (pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
))
278 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
279 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
282 stmt_info
= vinfo_for_stmt (pattern_stmt
);
283 if (dump_enabled_p ())
285 dump_printf_loc (MSG_NOTE
, vect_location
,
286 "==> examining pattern statement: ");
287 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
292 if (dump_enabled_p ())
293 dump_printf_loc (MSG_NOTE
, vect_location
, "skip.");
298 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
299 && (pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
))
300 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
301 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
302 analyze_pattern_stmt
= true;
304 /* If a pattern statement has def stmts, analyze them too. */
305 if (is_pattern_stmt_p (stmt_info
))
307 if (pattern_def_seq
== NULL
)
309 pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
310 pattern_def_si
= gsi_start (pattern_def_seq
);
312 else if (!gsi_end_p (pattern_def_si
))
313 gsi_next (&pattern_def_si
);
314 if (pattern_def_seq
!= NULL
)
316 gimple pattern_def_stmt
= NULL
;
317 stmt_vec_info pattern_def_stmt_info
= NULL
;
319 while (!gsi_end_p (pattern_def_si
))
321 pattern_def_stmt
= gsi_stmt (pattern_def_si
);
322 pattern_def_stmt_info
323 = vinfo_for_stmt (pattern_def_stmt
);
324 if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info
)
325 || STMT_VINFO_LIVE_P (pattern_def_stmt_info
))
327 gsi_next (&pattern_def_si
);
330 if (!gsi_end_p (pattern_def_si
))
332 if (dump_enabled_p ())
334 dump_printf_loc (MSG_NOTE
, vect_location
,
335 "==> examining pattern def stmt: ");
336 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
,
337 pattern_def_stmt
, 0);
340 stmt
= pattern_def_stmt
;
341 stmt_info
= pattern_def_stmt_info
;
345 pattern_def_si
= gsi_none ();
346 analyze_pattern_stmt
= false;
350 analyze_pattern_stmt
= false;
353 if (gimple_get_lhs (stmt
) == NULL_TREE
)
355 if (dump_enabled_p ())
357 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
358 "not vectorized: irregular stmt.");
359 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
,
365 if (VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))))
367 if (dump_enabled_p ())
369 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
370 "not vectorized: vector stmt in loop:");
371 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
376 if (STMT_VINFO_VECTYPE (stmt_info
))
378 /* The only case when a vectype had been already set is for stmts
379 that contain a dataref, or for "pattern-stmts" (stmts
380 generated by the vectorizer to represent/replace a certain
382 gcc_assert (STMT_VINFO_DATA_REF (stmt_info
)
383 || is_pattern_stmt_p (stmt_info
)
384 || !gsi_end_p (pattern_def_si
));
385 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
389 gcc_assert (!STMT_VINFO_DATA_REF (stmt_info
));
390 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
391 if (dump_enabled_p ())
393 dump_printf_loc (MSG_NOTE
, vect_location
,
394 "get vectype for scalar type: ");
395 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
397 vectype
= get_vectype_for_scalar_type (scalar_type
);
400 if (dump_enabled_p ())
402 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
403 "not vectorized: unsupported "
405 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
411 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
414 /* The vectorization factor is according to the smallest
415 scalar type (or the largest vector size, but we only
416 support one vector size per loop). */
417 scalar_type
= vect_get_smallest_scalar_type (stmt
, &dummy
,
419 if (dump_enabled_p ())
421 dump_printf_loc (MSG_NOTE
, vect_location
,
422 "get vectype for scalar type: ");
423 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
425 vf_vectype
= get_vectype_for_scalar_type (scalar_type
);
428 if (dump_enabled_p ())
430 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
431 "not vectorized: unsupported data-type ");
432 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
438 if ((GET_MODE_SIZE (TYPE_MODE (vectype
))
439 != GET_MODE_SIZE (TYPE_MODE (vf_vectype
))))
441 if (dump_enabled_p ())
443 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
444 "not vectorized: different sized vector "
445 "types in statement, ");
446 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
448 dump_printf (MSG_MISSED_OPTIMIZATION
, " and ");
449 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
455 if (dump_enabled_p ())
457 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
458 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vf_vectype
);
461 nunits
= TYPE_VECTOR_SUBPARTS (vf_vectype
);
462 if (dump_enabled_p ())
463 dump_printf_loc (MSG_NOTE
, vect_location
, "nunits = %d", nunits
);
464 if (!vectorization_factor
465 || (nunits
> vectorization_factor
))
466 vectorization_factor
= nunits
;
468 if (!analyze_pattern_stmt
&& gsi_end_p (pattern_def_si
))
470 pattern_def_seq
= NULL
;
476 /* TODO: Analyze cost. Decide if worth while to vectorize. */
477 if (dump_enabled_p ())
478 dump_printf_loc (MSG_NOTE
, vect_location
, "vectorization factor = %d",
479 vectorization_factor
);
480 if (vectorization_factor
<= 1)
482 if (dump_enabled_p ())
483 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
484 "not vectorized: unsupported data-type");
487 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
493 /* Function vect_is_simple_iv_evolution.
495 FORNOW: A simple evolution of an induction variables in the loop is
496 considered a polynomial evolution with constant step. */
499 vect_is_simple_iv_evolution (unsigned loop_nb
, tree access_fn
, tree
* init
,
504 tree evolution_part
= evolution_part_in_loop_num (access_fn
, loop_nb
);
506 /* When there is no evolution in this loop, the evolution function
508 if (evolution_part
== NULL_TREE
)
511 /* When the evolution is a polynomial of degree >= 2
512 the evolution function is not "simple". */
513 if (tree_is_chrec (evolution_part
))
516 step_expr
= evolution_part
;
517 init_expr
= unshare_expr (initial_condition_in_loop_num (access_fn
, loop_nb
));
519 if (dump_enabled_p ())
521 dump_printf_loc (MSG_NOTE
, vect_location
, "step: ");
522 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, step_expr
);
523 dump_printf (MSG_NOTE
, ", init: ");
524 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, init_expr
);
530 if (TREE_CODE (step_expr
) != INTEGER_CST
)
532 if (dump_enabled_p ())
533 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
541 /* Function vect_analyze_scalar_cycles_1.
543 Examine the cross iteration def-use cycles of scalar variables
544 in LOOP. LOOP_VINFO represents the loop that is now being
545 considered for vectorization (can be LOOP, or an outer-loop
549 vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo
, struct loop
*loop
)
551 basic_block bb
= loop
->header
;
553 vec
<gimple
> worklist
;
554 worklist
.create (64);
555 gimple_stmt_iterator gsi
;
558 if (dump_enabled_p ())
559 dump_printf_loc (MSG_NOTE
, vect_location
,
560 "=== vect_analyze_scalar_cycles ===");
562 /* First - identify all inductions. Reduction detection assumes that all the
563 inductions have been identified, therefore, this order must not be
565 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
567 gimple phi
= gsi_stmt (gsi
);
568 tree access_fn
= NULL
;
569 tree def
= PHI_RESULT (phi
);
570 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (phi
);
572 if (dump_enabled_p ())
574 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: ");
575 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
578 /* Skip virtual phi's. The data dependences that are associated with
579 virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
580 if (virtual_operand_p (def
))
583 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_unknown_def_type
;
585 /* Analyze the evolution function. */
586 access_fn
= analyze_scalar_evolution (loop
, def
);
589 STRIP_NOPS (access_fn
);
590 if (dump_enabled_p ())
592 dump_printf_loc (MSG_NOTE
, vect_location
,
593 "Access function of PHI: ");
594 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, access_fn
);
596 STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
)
597 = evolution_part_in_loop_num (access_fn
, loop
->num
);
601 || !vect_is_simple_iv_evolution (loop
->num
, access_fn
, &dumy
, &dumy
))
603 worklist
.safe_push (phi
);
607 gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
) != NULL_TREE
);
609 if (dump_enabled_p ())
610 dump_printf_loc (MSG_NOTE
, vect_location
, "Detected induction.");
611 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_induction_def
;
615 /* Second - identify all reductions and nested cycles. */
616 while (worklist
.length () > 0)
618 gimple phi
= worklist
.pop ();
619 tree def
= PHI_RESULT (phi
);
620 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (phi
);
624 if (dump_enabled_p ())
626 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: ");
627 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
630 gcc_assert (!virtual_operand_p (def
)
631 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_unknown_def_type
);
633 nested_cycle
= (loop
!= LOOP_VINFO_LOOP (loop_vinfo
));
634 reduc_stmt
= vect_force_simple_reduction (loop_vinfo
, phi
, !nested_cycle
,
640 if (dump_enabled_p ())
641 dump_printf_loc (MSG_NOTE
, vect_location
,
642 "Detected double reduction.");
644 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_double_reduction_def
;
645 STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt
)) =
646 vect_double_reduction_def
;
652 if (dump_enabled_p ())
653 dump_printf_loc (MSG_NOTE
, vect_location
,
654 "Detected vectorizable nested cycle.");
656 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_nested_cycle
;
657 STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt
)) =
662 if (dump_enabled_p ())
663 dump_printf_loc (MSG_NOTE
, vect_location
,
664 "Detected reduction.");
666 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_reduction_def
;
667 STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt
)) =
669 /* Store the reduction cycles for possible vectorization in
671 LOOP_VINFO_REDUCTIONS (loop_vinfo
).safe_push (reduc_stmt
);
676 if (dump_enabled_p ())
677 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
678 "Unknown def-use cycle pattern.");
685 /* Function vect_analyze_scalar_cycles.
687 Examine the cross iteration def-use cycles of scalar variables, by
688 analyzing the loop-header PHIs of scalar variables. Classify each
689 cycle as one of the following: invariant, induction, reduction, unknown.
690 We do that for the loop represented by LOOP_VINFO, and also to its
691 inner-loop, if exists.
692 Examples for scalar cycles:
707 vect_analyze_scalar_cycles (loop_vec_info loop_vinfo
)
709 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
711 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
);
713 /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
714 Reductions in such inner-loop therefore have different properties than
715 the reductions in the nest that gets vectorized:
716 1. When vectorized, they are executed in the same order as in the original
717 scalar loop, so we can't change the order of computation when
719 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
720 current checks are too strict. */
723 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
->inner
);
726 /* Function vect_get_loop_niters.
728 Determine how many iterations the loop is executed.
729 If an expression that represents the number of iterations
730 can be constructed, place it in NUMBER_OF_ITERATIONS.
731 Return the loop exit condition. */
734 vect_get_loop_niters (struct loop
*loop
, tree
*number_of_iterations
)
738 if (dump_enabled_p ())
739 dump_printf_loc (MSG_NOTE
, vect_location
,
740 "=== get_loop_niters ===");
741 niters
= number_of_exit_cond_executions (loop
);
743 if (niters
!= NULL_TREE
744 && niters
!= chrec_dont_know
)
746 *number_of_iterations
= niters
;
748 if (dump_enabled_p ())
750 dump_printf_loc (MSG_NOTE
, vect_location
, "==> get_loop_niters:");
751 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, *number_of_iterations
);
755 return get_loop_exit_condition (loop
);
759 /* Function bb_in_loop_p
761 Used as predicate for dfs order traversal of the loop bbs. */
764 bb_in_loop_p (const_basic_block bb
, const void *data
)
766 const struct loop
*const loop
= (const struct loop
*)data
;
767 if (flow_bb_inside_loop_p (loop
, bb
))
773 /* Function new_loop_vec_info.
775 Create and initialize a new loop_vec_info struct for LOOP, as well as
776 stmt_vec_info structs for all the stmts in LOOP. */
779 new_loop_vec_info (struct loop
*loop
)
783 gimple_stmt_iterator si
;
784 unsigned int i
, nbbs
;
786 res
= (loop_vec_info
) xcalloc (1, sizeof (struct _loop_vec_info
));
787 LOOP_VINFO_LOOP (res
) = loop
;
789 bbs
= get_loop_body (loop
);
791 /* Create/Update stmt_info for all stmts in the loop. */
792 for (i
= 0; i
< loop
->num_nodes
; i
++)
794 basic_block bb
= bbs
[i
];
796 /* BBs in a nested inner-loop will have been already processed (because
797 we will have called vect_analyze_loop_form for any nested inner-loop).
798 Therefore, for stmts in an inner-loop we just want to update the
799 STMT_VINFO_LOOP_VINFO field of their stmt_info to point to the new
800 loop_info of the outer-loop we are currently considering to vectorize
801 (instead of the loop_info of the inner-loop).
802 For stmts in other BBs we need to create a stmt_info from scratch. */
803 if (bb
->loop_father
!= loop
)
806 gcc_assert (loop
->inner
&& bb
->loop_father
== loop
->inner
);
807 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
809 gimple phi
= gsi_stmt (si
);
810 stmt_vec_info stmt_info
= vinfo_for_stmt (phi
);
811 loop_vec_info inner_loop_vinfo
=
812 STMT_VINFO_LOOP_VINFO (stmt_info
);
813 gcc_assert (loop
->inner
== LOOP_VINFO_LOOP (inner_loop_vinfo
));
814 STMT_VINFO_LOOP_VINFO (stmt_info
) = res
;
816 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
818 gimple stmt
= gsi_stmt (si
);
819 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
820 loop_vec_info inner_loop_vinfo
=
821 STMT_VINFO_LOOP_VINFO (stmt_info
);
822 gcc_assert (loop
->inner
== LOOP_VINFO_LOOP (inner_loop_vinfo
));
823 STMT_VINFO_LOOP_VINFO (stmt_info
) = res
;
828 /* bb in current nest. */
829 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
831 gimple phi
= gsi_stmt (si
);
832 gimple_set_uid (phi
, 0);
833 set_vinfo_for_stmt (phi
, new_stmt_vec_info (phi
, res
, NULL
));
836 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
838 gimple stmt
= gsi_stmt (si
);
839 gimple_set_uid (stmt
, 0);
840 set_vinfo_for_stmt (stmt
, new_stmt_vec_info (stmt
, res
, NULL
));
845 /* CHECKME: We want to visit all BBs before their successors (except for
846 latch blocks, for which this assertion wouldn't hold). In the simple
847 case of the loop forms we allow, a dfs order of the BBs would the same
848 as reversed postorder traversal, so we are safe. */
851 bbs
= XCNEWVEC (basic_block
, loop
->num_nodes
);
852 nbbs
= dfs_enumerate_from (loop
->header
, 0, bb_in_loop_p
,
853 bbs
, loop
->num_nodes
, loop
);
854 gcc_assert (nbbs
== loop
->num_nodes
);
856 LOOP_VINFO_BBS (res
) = bbs
;
857 LOOP_VINFO_NITERS (res
) = NULL
;
858 LOOP_VINFO_NITERS_UNCHANGED (res
) = NULL
;
859 LOOP_VINFO_COST_MODEL_MIN_ITERS (res
) = 0;
860 LOOP_VINFO_VECTORIZABLE_P (res
) = 0;
861 LOOP_PEELING_FOR_ALIGNMENT (res
) = 0;
862 LOOP_VINFO_VECT_FACTOR (res
) = 0;
863 LOOP_VINFO_LOOP_NEST (res
).create (3);
864 LOOP_VINFO_DATAREFS (res
).create (10);
865 LOOP_VINFO_DDRS (res
).create (10 * 10);
866 LOOP_VINFO_UNALIGNED_DR (res
) = NULL
;
867 LOOP_VINFO_MAY_MISALIGN_STMTS (res
).create (
868 PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS
));
869 LOOP_VINFO_MAY_ALIAS_DDRS (res
).create (
870 PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS
));
871 LOOP_VINFO_GROUPED_STORES (res
).create (10);
872 LOOP_VINFO_REDUCTIONS (res
).create (10);
873 LOOP_VINFO_REDUCTION_CHAINS (res
).create (10);
874 LOOP_VINFO_SLP_INSTANCES (res
).create (10);
875 LOOP_VINFO_SLP_UNROLLING_FACTOR (res
) = 1;
876 LOOP_VINFO_PEELING_HTAB (res
) = NULL
;
877 LOOP_VINFO_TARGET_COST_DATA (res
) = init_cost (loop
);
878 LOOP_VINFO_PEELING_FOR_GAPS (res
) = false;
879 LOOP_VINFO_OPERANDS_SWAPPED (res
) = false;
885 /* Function destroy_loop_vec_info.
887 Free LOOP_VINFO struct, as well as all the stmt_vec_info structs of all the
888 stmts in the loop. */
891 destroy_loop_vec_info (loop_vec_info loop_vinfo
, bool clean_stmts
)
896 gimple_stmt_iterator si
;
898 vec
<slp_instance
> slp_instances
;
899 slp_instance instance
;
905 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
907 bbs
= LOOP_VINFO_BBS (loop_vinfo
);
908 nbbs
= clean_stmts
? loop
->num_nodes
: 0;
909 swapped
= LOOP_VINFO_OPERANDS_SWAPPED (loop_vinfo
);
911 for (j
= 0; j
< nbbs
; j
++)
913 basic_block bb
= bbs
[j
];
914 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
915 free_stmt_vec_info (gsi_stmt (si
));
917 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); )
919 gimple stmt
= gsi_stmt (si
);
921 /* We may have broken canonical form by moving a constant
922 into RHS1 of a commutative op. Fix such occurrences. */
923 if (swapped
&& is_gimple_assign (stmt
))
925 enum tree_code code
= gimple_assign_rhs_code (stmt
);
927 if ((code
== PLUS_EXPR
928 || code
== POINTER_PLUS_EXPR
929 || code
== MULT_EXPR
)
930 && CONSTANT_CLASS_P (gimple_assign_rhs1 (stmt
)))
931 swap_tree_operands (stmt
,
932 gimple_assign_rhs1_ptr (stmt
),
933 gimple_assign_rhs2_ptr (stmt
));
936 /* Free stmt_vec_info. */
937 free_stmt_vec_info (stmt
);
942 free (LOOP_VINFO_BBS (loop_vinfo
));
943 free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo
));
944 free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo
));
945 LOOP_VINFO_LOOP_NEST (loop_vinfo
).release ();
946 LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
).release ();
947 LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo
).release ();
948 slp_instances
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
);
949 FOR_EACH_VEC_ELT (slp_instances
, j
, instance
)
950 vect_free_slp_instance (instance
);
952 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).release ();
953 LOOP_VINFO_GROUPED_STORES (loop_vinfo
).release ();
954 LOOP_VINFO_REDUCTIONS (loop_vinfo
).release ();
955 LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
).release ();
957 if (LOOP_VINFO_PEELING_HTAB (loop_vinfo
))
958 htab_delete (LOOP_VINFO_PEELING_HTAB (loop_vinfo
));
960 destroy_cost_data (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
));
967 /* Function vect_analyze_loop_1.
969 Apply a set of analyses on LOOP, and create a loop_vec_info struct
970 for it. The different analyses will record information in the
971 loop_vec_info struct. This is a subset of the analyses applied in
972 vect_analyze_loop, to be applied on an inner-loop nested in the loop
973 that is now considered for (outer-loop) vectorization. */
976 vect_analyze_loop_1 (struct loop
*loop
)
978 loop_vec_info loop_vinfo
;
980 if (dump_enabled_p ())
981 dump_printf_loc (MSG_NOTE
, vect_location
,
982 "===== analyze_loop_nest_1 =====");
984 /* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
986 loop_vinfo
= vect_analyze_loop_form (loop
);
989 if (dump_enabled_p ())
990 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
991 "bad inner-loop form.");
999 /* Function vect_analyze_loop_form.
1001 Verify that certain CFG restrictions hold, including:
1002 - the loop has a pre-header
1003 - the loop has a single entry and exit
1004 - the loop exit condition is simple enough, and the number of iterations
1005 can be analyzed (a countable loop). */
1008 vect_analyze_loop_form (struct loop
*loop
)
1010 loop_vec_info loop_vinfo
;
1012 tree number_of_iterations
= NULL
;
1013 loop_vec_info inner_loop_vinfo
= NULL
;
1015 if (dump_enabled_p ())
1016 dump_printf_loc (MSG_NOTE
, vect_location
,
1017 "=== vect_analyze_loop_form ===");
1019 /* Different restrictions apply when we are considering an inner-most loop,
1020 vs. an outer (nested) loop.
1021 (FORNOW. May want to relax some of these restrictions in the future). */
1025 /* Inner-most loop. We currently require that the number of BBs is
1026 exactly 2 (the header and latch). Vectorizable inner-most loops
1037 if (loop
->num_nodes
!= 2)
1039 if (dump_enabled_p ())
1040 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1041 "not vectorized: control flow in loop.");
1045 if (empty_block_p (loop
->header
))
1047 if (dump_enabled_p ())
1048 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1049 "not vectorized: empty loop.");
1055 struct loop
*innerloop
= loop
->inner
;
1058 /* Nested loop. We currently require that the loop is doubly-nested,
1059 contains a single inner loop, and the number of BBs is exactly 5.
1060 Vectorizable outer-loops look like this:
1072 The inner-loop has the properties expected of inner-most loops
1073 as described above. */
1075 if ((loop
->inner
)->inner
|| (loop
->inner
)->next
)
1077 if (dump_enabled_p ())
1078 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1079 "not vectorized: multiple nested loops.");
1083 /* Analyze the inner-loop. */
1084 inner_loop_vinfo
= vect_analyze_loop_1 (loop
->inner
);
1085 if (!inner_loop_vinfo
)
1087 if (dump_enabled_p ())
1088 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1089 "not vectorized: Bad inner loop.");
1093 if (!expr_invariant_in_loop_p (loop
,
1094 LOOP_VINFO_NITERS (inner_loop_vinfo
)))
1096 if (dump_enabled_p ())
1097 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1098 "not vectorized: inner-loop count not invariant.");
1099 destroy_loop_vec_info (inner_loop_vinfo
, true);
1103 if (loop
->num_nodes
!= 5)
1105 if (dump_enabled_p ())
1106 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1107 "not vectorized: control flow in loop.");
1108 destroy_loop_vec_info (inner_loop_vinfo
, true);
1112 gcc_assert (EDGE_COUNT (innerloop
->header
->preds
) == 2);
1113 entryedge
= EDGE_PRED (innerloop
->header
, 0);
1114 if (EDGE_PRED (innerloop
->header
, 0)->src
== innerloop
->latch
)
1115 entryedge
= EDGE_PRED (innerloop
->header
, 1);
1117 if (entryedge
->src
!= loop
->header
1118 || !single_exit (innerloop
)
1119 || single_exit (innerloop
)->dest
!= EDGE_PRED (loop
->latch
, 0)->src
)
1121 if (dump_enabled_p ())
1122 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1123 "not vectorized: unsupported outerloop form.");
1124 destroy_loop_vec_info (inner_loop_vinfo
, true);
1128 if (dump_enabled_p ())
1129 dump_printf_loc (MSG_NOTE
, vect_location
,
1130 "Considering outer-loop vectorization.");
1133 if (!single_exit (loop
)
1134 || EDGE_COUNT (loop
->header
->preds
) != 2)
1136 if (dump_enabled_p ())
1138 if (!single_exit (loop
))
1139 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1140 "not vectorized: multiple exits.");
1141 else if (EDGE_COUNT (loop
->header
->preds
) != 2)
1142 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1143 "not vectorized: too many incoming edges.");
1145 if (inner_loop_vinfo
)
1146 destroy_loop_vec_info (inner_loop_vinfo
, true);
1150 /* We assume that the loop exit condition is at the end of the loop. i.e,
1151 that the loop is represented as a do-while (with a proper if-guard
1152 before the loop if needed), where the loop header contains all the
1153 executable statements, and the latch is empty. */
1154 if (!empty_block_p (loop
->latch
)
1155 || !gimple_seq_empty_p (phi_nodes (loop
->latch
)))
1157 if (dump_enabled_p ())
1158 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1159 "not vectorized: latch block not empty.");
1160 if (inner_loop_vinfo
)
1161 destroy_loop_vec_info (inner_loop_vinfo
, true);
1165 /* Make sure there exists a single-predecessor exit bb: */
1166 if (!single_pred_p (single_exit (loop
)->dest
))
1168 edge e
= single_exit (loop
);
1169 if (!(e
->flags
& EDGE_ABNORMAL
))
1171 split_loop_exit_edge (e
);
1172 if (dump_enabled_p ())
1173 dump_printf (MSG_NOTE
, "split exit edge.");
1177 if (dump_enabled_p ())
1178 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1179 "not vectorized: abnormal loop exit edge.");
1180 if (inner_loop_vinfo
)
1181 destroy_loop_vec_info (inner_loop_vinfo
, true);
1186 loop_cond
= vect_get_loop_niters (loop
, &number_of_iterations
);
1189 if (dump_enabled_p ())
1190 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1191 "not vectorized: complicated exit condition.");
1192 if (inner_loop_vinfo
)
1193 destroy_loop_vec_info (inner_loop_vinfo
, true);
1197 if (!number_of_iterations
)
1199 if (dump_enabled_p ())
1200 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1201 "not vectorized: number of iterations cannot be "
1203 if (inner_loop_vinfo
)
1204 destroy_loop_vec_info (inner_loop_vinfo
, true);
1208 if (chrec_contains_undetermined (number_of_iterations
))
1210 if (dump_enabled_p ())
1211 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1212 "Infinite number of iterations.");
1213 if (inner_loop_vinfo
)
1214 destroy_loop_vec_info (inner_loop_vinfo
, true);
1218 if (!NITERS_KNOWN_P (number_of_iterations
))
1220 if (dump_enabled_p ())
1222 dump_printf_loc (MSG_NOTE
, vect_location
,
1223 "Symbolic number of iterations is ");
1224 dump_generic_expr (MSG_NOTE
, TDF_DETAILS
, number_of_iterations
);
1227 else if (TREE_INT_CST_LOW (number_of_iterations
) == 0)
1229 if (dump_enabled_p ())
1230 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1231 "not vectorized: number of iterations = 0.");
1232 if (inner_loop_vinfo
)
1233 destroy_loop_vec_info (inner_loop_vinfo
, true);
1237 loop_vinfo
= new_loop_vec_info (loop
);
1238 LOOP_VINFO_NITERS (loop_vinfo
) = number_of_iterations
;
1239 LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
) = number_of_iterations
;
1241 STMT_VINFO_TYPE (vinfo_for_stmt (loop_cond
)) = loop_exit_ctrl_vec_info_type
;
1243 /* CHECKME: May want to keep it around it in the future. */
1244 if (inner_loop_vinfo
)
1245 destroy_loop_vec_info (inner_loop_vinfo
, false);
1247 gcc_assert (!loop
->aux
);
1248 loop
->aux
= loop_vinfo
;
1253 /* Function vect_analyze_loop_operations.
1255 Scan the loop stmts and make sure they are all vectorizable. */
1258 vect_analyze_loop_operations (loop_vec_info loop_vinfo
, bool slp
)
1260 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1261 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1262 int nbbs
= loop
->num_nodes
;
1263 gimple_stmt_iterator si
;
1264 unsigned int vectorization_factor
= 0;
1267 stmt_vec_info stmt_info
;
1268 bool need_to_vectorize
= false;
1269 int min_profitable_iters
;
1270 int min_scalar_loop_bound
;
1272 bool only_slp_in_loop
= true, ok
;
1273 HOST_WIDE_INT max_niter
;
1274 HOST_WIDE_INT estimated_niter
;
1275 int min_profitable_estimate
;
1277 if (dump_enabled_p ())
1278 dump_printf_loc (MSG_NOTE
, vect_location
,
1279 "=== vect_analyze_loop_operations ===");
1281 gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo
));
1282 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1285 /* If all the stmts in the loop can be SLPed, we perform only SLP, and
1286 vectorization factor of the loop is the unrolling factor required by
1287 the SLP instances. If that unrolling factor is 1, we say, that we
1288 perform pure SLP on loop - cross iteration parallelism is not
1290 for (i
= 0; i
< nbbs
; i
++)
1292 basic_block bb
= bbs
[i
];
1293 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1295 gimple stmt
= gsi_stmt (si
);
1296 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1297 gcc_assert (stmt_info
);
1298 if ((STMT_VINFO_RELEVANT_P (stmt_info
)
1299 || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
1300 && !PURE_SLP_STMT (stmt_info
))
1301 /* STMT needs both SLP and loop-based vectorization. */
1302 only_slp_in_loop
= false;
1306 if (only_slp_in_loop
)
1307 vectorization_factor
= LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
);
1309 vectorization_factor
= least_common_multiple (vectorization_factor
,
1310 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
));
1312 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
1313 if (dump_enabled_p ())
1314 dump_printf_loc (MSG_NOTE
, vect_location
,
1315 "Updating vectorization factor to %d ",
1316 vectorization_factor
);
1319 for (i
= 0; i
< nbbs
; i
++)
1321 basic_block bb
= bbs
[i
];
1323 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
1325 phi
= gsi_stmt (si
);
1328 stmt_info
= vinfo_for_stmt (phi
);
1329 if (dump_enabled_p ())
1331 dump_printf_loc (MSG_NOTE
, vect_location
, "examining phi: ");
1332 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
1335 /* Inner-loop loop-closed exit phi in outer-loop vectorization
1336 (i.e., a phi in the tail of the outer-loop). */
1337 if (! is_loop_header_bb_p (bb
))
1339 /* FORNOW: we currently don't support the case that these phis
1340 are not used in the outerloop (unless it is double reduction,
1341 i.e., this phi is vect_reduction_def), cause this case
1342 requires to actually do something here. */
1343 if ((!STMT_VINFO_RELEVANT_P (stmt_info
)
1344 || STMT_VINFO_LIVE_P (stmt_info
))
1345 && STMT_VINFO_DEF_TYPE (stmt_info
)
1346 != vect_double_reduction_def
)
1348 if (dump_enabled_p ())
1349 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1350 "Unsupported loop-closed phi in "
1355 /* If PHI is used in the outer loop, we check that its operand
1356 is defined in the inner loop. */
1357 if (STMT_VINFO_RELEVANT_P (stmt_info
))
1362 if (gimple_phi_num_args (phi
) != 1)
1365 phi_op
= PHI_ARG_DEF (phi
, 0);
1366 if (TREE_CODE (phi_op
) != SSA_NAME
)
1369 op_def_stmt
= SSA_NAME_DEF_STMT (phi_op
);
1371 || !flow_bb_inside_loop_p (loop
, gimple_bb (op_def_stmt
))
1372 || !vinfo_for_stmt (op_def_stmt
))
1375 if (STMT_VINFO_RELEVANT (vinfo_for_stmt (op_def_stmt
))
1376 != vect_used_in_outer
1377 && STMT_VINFO_RELEVANT (vinfo_for_stmt (op_def_stmt
))
1378 != vect_used_in_outer_by_reduction
)
1385 gcc_assert (stmt_info
);
1387 if (STMT_VINFO_LIVE_P (stmt_info
))
1389 /* FORNOW: not yet supported. */
1390 if (dump_enabled_p ())
1391 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1392 "not vectorized: value used after loop.");
1396 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_scope
1397 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_induction_def
)
1399 /* A scalar-dependence cycle that we don't support. */
1400 if (dump_enabled_p ())
1401 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1402 "not vectorized: scalar dependence cycle.");
1406 if (STMT_VINFO_RELEVANT_P (stmt_info
))
1408 need_to_vectorize
= true;
1409 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
)
1410 ok
= vectorizable_induction (phi
, NULL
, NULL
);
1415 if (dump_enabled_p ())
1417 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1418 "not vectorized: relevant phi not "
1420 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, phi
, 0);
1426 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1428 gimple stmt
= gsi_stmt (si
);
1429 if (!vect_analyze_stmt (stmt
, &need_to_vectorize
, NULL
))
1434 /* All operations in the loop are either irrelevant (deal with loop
1435 control, or dead), or only used outside the loop and can be moved
1436 out of the loop (e.g. invariants, inductions). The loop can be
1437 optimized away by scalar optimizations. We're better off not
1438 touching this loop. */
1439 if (!need_to_vectorize
)
1441 if (dump_enabled_p ())
1442 dump_printf_loc (MSG_NOTE
, vect_location
,
1443 "All the computation can be taken out of the loop.");
1444 if (dump_enabled_p ())
1445 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1446 "not vectorized: redundant loop. no profit to "
1451 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) && dump_enabled_p ())
1452 dump_printf_loc (MSG_NOTE
, vect_location
,
1453 "vectorization_factor = %d, niters = "
1454 HOST_WIDE_INT_PRINT_DEC
, vectorization_factor
,
1455 LOOP_VINFO_INT_NITERS (loop_vinfo
));
1457 if ((LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1458 && (LOOP_VINFO_INT_NITERS (loop_vinfo
) < vectorization_factor
))
1459 || ((max_niter
= max_stmt_executions_int (loop
)) != -1
1460 && (unsigned HOST_WIDE_INT
) max_niter
< vectorization_factor
))
1462 if (dump_enabled_p ())
1463 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1464 "not vectorized: iteration count too small.");
1465 if (dump_enabled_p ())
1466 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1467 "not vectorized: iteration count smaller than "
1468 "vectorization factor.");
1472 /* Analyze cost. Decide if worth while to vectorize. */
1474 /* Once VF is set, SLP costs should be updated since the number of created
1475 vector stmts depends on VF. */
1476 vect_update_slp_costs_according_to_vf (loop_vinfo
);
1478 vect_estimate_min_profitable_iters (loop_vinfo
, &min_profitable_iters
,
1479 &min_profitable_estimate
);
1480 LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo
) = min_profitable_iters
;
1482 if (min_profitable_iters
< 0)
1484 if (dump_enabled_p ())
1485 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1486 "not vectorized: vectorization not profitable.");
1487 if (dump_enabled_p ())
1488 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1489 "not vectorized: vector version will never be "
1494 min_scalar_loop_bound
= ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND
)
1495 * vectorization_factor
) - 1);
1498 /* Use the cost model only if it is more conservative than user specified
1501 th
= (unsigned) min_scalar_loop_bound
;
1502 if (min_profitable_iters
1503 && (!min_scalar_loop_bound
1504 || min_profitable_iters
> min_scalar_loop_bound
))
1505 th
= (unsigned) min_profitable_iters
;
1507 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1508 && LOOP_VINFO_INT_NITERS (loop_vinfo
) <= th
)
1510 if (dump_enabled_p ())
1511 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1512 "not vectorized: vectorization not profitable.");
1513 if (dump_enabled_p ())
1514 dump_printf_loc (MSG_NOTE
, vect_location
,
1515 "not vectorized: iteration count smaller than user "
1516 "specified loop bound parameter or minimum profitable "
1517 "iterations (whichever is more conservative).");
1521 if ((estimated_niter
= estimated_stmt_executions_int (loop
)) != -1
1522 && ((unsigned HOST_WIDE_INT
) estimated_niter
1523 <= MAX (th
, (unsigned)min_profitable_estimate
)))
1525 if (dump_enabled_p ())
1526 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1527 "not vectorized: estimated iteration count too "
1529 if (dump_enabled_p ())
1530 dump_printf_loc (MSG_NOTE
, vect_location
,
1531 "not vectorized: estimated iteration count smaller "
1532 "than specified loop bound parameter or minimum "
1533 "profitable iterations (whichever is more "
1538 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1539 || LOOP_VINFO_INT_NITERS (loop_vinfo
) % vectorization_factor
!= 0
1540 || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
1542 if (dump_enabled_p ())
1543 dump_printf_loc (MSG_NOTE
, vect_location
, "epilog loop required.");
1544 if (!vect_can_advance_ivs_p (loop_vinfo
))
1546 if (dump_enabled_p ())
1547 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1548 "not vectorized: can't create epilog loop 1.");
1551 if (!slpeel_can_duplicate_loop_p (loop
, single_exit (loop
)))
1553 if (dump_enabled_p ())
1554 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1555 "not vectorized: can't create epilog loop 2.");
1564 /* Function vect_analyze_loop_2.
1566 Apply a set of analyses on LOOP, and create a loop_vec_info struct
1567 for it. The different analyses will record information in the
1568 loop_vec_info struct. */
1570 vect_analyze_loop_2 (loop_vec_info loop_vinfo
)
1572 bool ok
, slp
= false;
1573 int max_vf
= MAX_VECTORIZATION_FACTOR
;
1576 /* Find all data references in the loop (which correspond to vdefs/vuses)
1577 and analyze their evolution in the loop. Also adjust the minimal
1578 vectorization factor according to the loads and stores.
1580 FORNOW: Handle only simple, array references, which
1581 alignment can be forced, and aligned pointer-references. */
1583 ok
= vect_analyze_data_refs (loop_vinfo
, NULL
, &min_vf
);
1586 if (dump_enabled_p ())
1587 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1588 "bad data references.");
1592 /* Classify all cross-iteration scalar data-flow cycles.
1593 Cross-iteration cycles caused by virtual phis are analyzed separately. */
1595 vect_analyze_scalar_cycles (loop_vinfo
);
1597 vect_pattern_recog (loop_vinfo
, NULL
);
1599 /* Data-flow analysis to detect stmts that do not need to be vectorized. */
1601 ok
= vect_mark_stmts_to_be_vectorized (loop_vinfo
);
1604 if (dump_enabled_p ())
1605 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1606 "unexpected pattern.");
1610 /* Analyze data dependences between the data-refs in the loop
1611 and adjust the maximum vectorization factor according to
1613 FORNOW: fail at the first data dependence that we encounter. */
1615 ok
= vect_analyze_data_ref_dependences (loop_vinfo
, NULL
, &max_vf
);
1619 if (dump_enabled_p ())
1620 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1621 "bad data dependence.");
1625 ok
= vect_determine_vectorization_factor (loop_vinfo
);
1628 if (dump_enabled_p ())
1629 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1630 "can't determine vectorization factor.");
1633 if (max_vf
< LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
1635 if (dump_enabled_p ())
1636 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1637 "bad data dependence.");
1641 /* Analyze the alignment of the data-refs in the loop.
1642 Fail if a data reference is found that cannot be vectorized. */
1644 ok
= vect_analyze_data_refs_alignment (loop_vinfo
, NULL
);
1647 if (dump_enabled_p ())
1648 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1649 "bad data alignment.");
1653 /* Analyze the access patterns of the data-refs in the loop (consecutive,
1654 complex, etc.). FORNOW: Only handle consecutive access pattern. */
1656 ok
= vect_analyze_data_ref_accesses (loop_vinfo
, NULL
);
1659 if (dump_enabled_p ())
1660 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1661 "bad data access.");
1665 /* Prune the list of ddrs to be tested at run-time by versioning for alias.
1666 It is important to call pruning after vect_analyze_data_ref_accesses,
1667 since we use grouping information gathered by interleaving analysis. */
1668 ok
= vect_prune_runtime_alias_test_list (loop_vinfo
);
1671 if (dump_enabled_p ())
1672 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1673 "too long list of versioning for alias "
1678 /* This pass will decide on using loop versioning and/or loop peeling in
1679 order to enhance the alignment of data references in the loop. */
1681 ok
= vect_enhance_data_refs_alignment (loop_vinfo
);
1684 if (dump_enabled_p ())
1685 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1686 "bad data alignment.");
1690 /* Check the SLP opportunities in the loop, analyze and build SLP trees. */
1691 ok
= vect_analyze_slp (loop_vinfo
, NULL
);
1694 /* Decide which possible SLP instances to SLP. */
1695 slp
= vect_make_slp_decision (loop_vinfo
);
1697 /* Find stmts that need to be both vectorized and SLPed. */
1698 vect_detect_hybrid_slp (loop_vinfo
);
1703 /* Scan all the operations in the loop and make sure they are
1706 ok
= vect_analyze_loop_operations (loop_vinfo
, slp
);
1709 if (dump_enabled_p ())
1710 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1711 "bad operation or unsupported loop bound.");
1718 /* Function vect_analyze_loop.
1720 Apply a set of analyses on LOOP, and create a loop_vec_info struct
1721 for it. The different analyses will record information in the
1722 loop_vec_info struct. */
1724 vect_analyze_loop (struct loop
*loop
)
1726 loop_vec_info loop_vinfo
;
1727 unsigned int vector_sizes
;
1729 /* Autodetect first vector size we try. */
1730 current_vector_size
= 0;
1731 vector_sizes
= targetm
.vectorize
.autovectorize_vector_sizes ();
1733 if (dump_enabled_p ())
1734 dump_printf_loc (MSG_NOTE
, vect_location
,
1735 "===== analyze_loop_nest =====");
1737 if (loop_outer (loop
)
1738 && loop_vec_info_for_loop (loop_outer (loop
))
1739 && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop
))))
1741 if (dump_enabled_p ())
1742 dump_printf_loc (MSG_NOTE
, vect_location
,
1743 "outer-loop already vectorized.");
1749 /* Check the CFG characteristics of the loop (nesting, entry/exit). */
1750 loop_vinfo
= vect_analyze_loop_form (loop
);
1753 if (dump_enabled_p ())
1754 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1759 if (vect_analyze_loop_2 (loop_vinfo
))
1761 LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
) = 1;
1766 destroy_loop_vec_info (loop_vinfo
, true);
1768 vector_sizes
&= ~current_vector_size
;
1769 if (vector_sizes
== 0
1770 || current_vector_size
== 0)
1773 /* Try the next biggest vector size. */
1774 current_vector_size
= 1 << floor_log2 (vector_sizes
);
1775 if (dump_enabled_p ())
1776 dump_printf_loc (MSG_NOTE
, vect_location
,
1777 "***** Re-trying analysis with "
1778 "vector size %d\n", current_vector_size
);
1783 /* Function reduction_code_for_scalar_code
1786 CODE - tree_code of a reduction operations.
1789 REDUC_CODE - the corresponding tree-code to be used to reduce the
1790 vector of partial results into a single scalar result (which
1791 will also reside in a vector) or ERROR_MARK if the operation is
1792 a supported reduction operation, but does not have such tree-code.
1794 Return FALSE if CODE currently cannot be vectorized as reduction. */
1797 reduction_code_for_scalar_code (enum tree_code code
,
1798 enum tree_code
*reduc_code
)
1803 *reduc_code
= REDUC_MAX_EXPR
;
1807 *reduc_code
= REDUC_MIN_EXPR
;
1811 *reduc_code
= REDUC_PLUS_EXPR
;
1819 *reduc_code
= ERROR_MARK
;
1828 /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
1829 STMT is printed with a message MSG. */
1832 report_vect_op (int msg_type
, gimple stmt
, const char *msg
)
1834 dump_printf_loc (msg_type
, vect_location
, "%s", msg
);
1835 dump_gimple_stmt (msg_type
, TDF_SLIM
, stmt
, 0);
1839 /* Detect SLP reduction of the form:
1849 PHI is the reduction phi node (#a1 = phi <a5, a0> above)
1850 FIRST_STMT is the first reduction stmt in the chain
1851 (a2 = operation (a1)).
1853 Return TRUE if a reduction chain was detected. */
1856 vect_is_slp_reduction (loop_vec_info loop_info
, gimple phi
, gimple first_stmt
)
1858 struct loop
*loop
= (gimple_bb (phi
))->loop_father
;
1859 struct loop
*vect_loop
= LOOP_VINFO_LOOP (loop_info
);
1860 enum tree_code code
;
1861 gimple current_stmt
= NULL
, loop_use_stmt
= NULL
, first
, next_stmt
;
1862 stmt_vec_info use_stmt_info
, current_stmt_info
;
1864 imm_use_iterator imm_iter
;
1865 use_operand_p use_p
;
1866 int nloop_uses
, size
= 0, n_out_of_loop_uses
;
1869 if (loop
!= vect_loop
)
1872 lhs
= PHI_RESULT (phi
);
1873 code
= gimple_assign_rhs_code (first_stmt
);
1877 n_out_of_loop_uses
= 0;
1878 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
1880 gimple use_stmt
= USE_STMT (use_p
);
1881 if (is_gimple_debug (use_stmt
))
1884 use_stmt
= USE_STMT (use_p
);
1886 /* Check if we got back to the reduction phi. */
1887 if (use_stmt
== phi
)
1889 loop_use_stmt
= use_stmt
;
1894 if (flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
1896 if (vinfo_for_stmt (use_stmt
)
1897 && !STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
1899 loop_use_stmt
= use_stmt
;
1904 n_out_of_loop_uses
++;
1906 /* There are can be either a single use in the loop or two uses in
1908 if (nloop_uses
> 1 || (n_out_of_loop_uses
&& nloop_uses
))
1915 /* We reached a statement with no loop uses. */
1916 if (nloop_uses
== 0)
1919 /* This is a loop exit phi, and we haven't reached the reduction phi. */
1920 if (gimple_code (loop_use_stmt
) == GIMPLE_PHI
)
1923 if (!is_gimple_assign (loop_use_stmt
)
1924 || code
!= gimple_assign_rhs_code (loop_use_stmt
)
1925 || !flow_bb_inside_loop_p (loop
, gimple_bb (loop_use_stmt
)))
1928 /* Insert USE_STMT into reduction chain. */
1929 use_stmt_info
= vinfo_for_stmt (loop_use_stmt
);
1932 current_stmt_info
= vinfo_for_stmt (current_stmt
);
1933 GROUP_NEXT_ELEMENT (current_stmt_info
) = loop_use_stmt
;
1934 GROUP_FIRST_ELEMENT (use_stmt_info
)
1935 = GROUP_FIRST_ELEMENT (current_stmt_info
);
1938 GROUP_FIRST_ELEMENT (use_stmt_info
) = loop_use_stmt
;
1940 lhs
= gimple_assign_lhs (loop_use_stmt
);
1941 current_stmt
= loop_use_stmt
;
1945 if (!found
|| loop_use_stmt
!= phi
|| size
< 2)
1948 /* Swap the operands, if needed, to make the reduction operand be the second
1950 lhs
= PHI_RESULT (phi
);
1951 next_stmt
= GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt
));
1954 if (gimple_assign_rhs2 (next_stmt
) == lhs
)
1956 tree op
= gimple_assign_rhs1 (next_stmt
);
1957 gimple def_stmt
= NULL
;
1959 if (TREE_CODE (op
) == SSA_NAME
)
1960 def_stmt
= SSA_NAME_DEF_STMT (op
);
1962 /* Check that the other def is either defined in the loop
1963 ("vect_internal_def"), or it's an induction (defined by a
1964 loop-header phi-node). */
1966 && gimple_bb (def_stmt
)
1967 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
1968 && (is_gimple_assign (def_stmt
)
1969 || is_gimple_call (def_stmt
)
1970 || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
))
1971 == vect_induction_def
1972 || (gimple_code (def_stmt
) == GIMPLE_PHI
1973 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
))
1974 == vect_internal_def
1975 && !is_loop_header_bb_p (gimple_bb (def_stmt
)))))
1977 lhs
= gimple_assign_lhs (next_stmt
);
1978 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
1986 tree op
= gimple_assign_rhs2 (next_stmt
);
1987 gimple def_stmt
= NULL
;
1989 if (TREE_CODE (op
) == SSA_NAME
)
1990 def_stmt
= SSA_NAME_DEF_STMT (op
);
1992 /* Check that the other def is either defined in the loop
1993 ("vect_internal_def"), or it's an induction (defined by a
1994 loop-header phi-node). */
1996 && gimple_bb (def_stmt
)
1997 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
1998 && (is_gimple_assign (def_stmt
)
1999 || is_gimple_call (def_stmt
)
2000 || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
))
2001 == vect_induction_def
2002 || (gimple_code (def_stmt
) == GIMPLE_PHI
2003 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
))
2004 == vect_internal_def
2005 && !is_loop_header_bb_p (gimple_bb (def_stmt
)))))
2007 if (dump_enabled_p ())
2009 dump_printf_loc (MSG_NOTE
, vect_location
, "swapping oprnds: ");
2010 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, next_stmt
, 0);
2013 swap_tree_operands (next_stmt
,
2014 gimple_assign_rhs1_ptr (next_stmt
),
2015 gimple_assign_rhs2_ptr (next_stmt
));
2016 update_stmt (next_stmt
);
2018 if (CONSTANT_CLASS_P (gimple_assign_rhs1 (next_stmt
)))
2019 LOOP_VINFO_OPERANDS_SWAPPED (loop_info
) = true;
2025 lhs
= gimple_assign_lhs (next_stmt
);
2026 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
2029 /* Save the chain for further analysis in SLP detection. */
2030 first
= GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt
));
2031 LOOP_VINFO_REDUCTION_CHAINS (loop_info
).safe_push (first
);
2032 GROUP_SIZE (vinfo_for_stmt (first
)) = size
;
2038 /* Function vect_is_simple_reduction_1
2040 (1) Detect a cross-iteration def-use cycle that represents a simple
2041 reduction computation. We look for the following pattern:
2046 a2 = operation (a3, a1)
2049 1. operation is commutative and associative and it is safe to
2050 change the order of the computation (if CHECK_REDUCTION is true)
2051 2. no uses for a2 in the loop (a2 is used out of the loop)
2052 3. no uses of a1 in the loop besides the reduction operation
2053 4. no uses of a1 outside the loop.
2055 Conditions 1,4 are tested here.
2056 Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
2058 (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
2059 nested cycles, if CHECK_REDUCTION is false.
2061 (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
2065 inner loop (def of a3)
2068 If MODIFY is true it tries also to rework the code in-place to enable
2069 detection of more reduction patterns. For the time being we rewrite
2070 "res -= RHS" into "rhs += -RHS" when it seems worthwhile.
2074 vect_is_simple_reduction_1 (loop_vec_info loop_info
, gimple phi
,
2075 bool check_reduction
, bool *double_reduc
,
2078 struct loop
*loop
= (gimple_bb (phi
))->loop_father
;
2079 struct loop
*vect_loop
= LOOP_VINFO_LOOP (loop_info
);
2080 edge latch_e
= loop_latch_edge (loop
);
2081 tree loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
2082 gimple def_stmt
, def1
= NULL
, def2
= NULL
;
2083 enum tree_code orig_code
, code
;
2084 tree op1
, op2
, op3
= NULL_TREE
, op4
= NULL_TREE
;
2088 imm_use_iterator imm_iter
;
2089 use_operand_p use_p
;
2092 *double_reduc
= false;
2094 /* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
2095 otherwise, we assume outer loop vectorization. */
2096 gcc_assert ((check_reduction
&& loop
== vect_loop
)
2097 || (!check_reduction
&& flow_loop_nested_p (vect_loop
, loop
)));
2099 name
= PHI_RESULT (phi
);
2100 /* ??? If there are no uses of the PHI result the inner loop reduction
2101 won't be detected as possibly double-reduction by vectorizable_reduction
2102 because that tries to walk the PHI arg from the preheader edge which
2103 can be constant. See PR60382. */
2104 if (has_zero_uses (name
))
2107 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, name
)
2109 gimple use_stmt
= USE_STMT (use_p
);
2110 if (is_gimple_debug (use_stmt
))
2113 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
2115 if (dump_enabled_p ())
2116 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2117 "intermediate value used outside loop.");
2122 if (vinfo_for_stmt (use_stmt
)
2123 && !is_pattern_stmt_p (vinfo_for_stmt (use_stmt
)))
2127 if (dump_enabled_p ())
2128 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2129 "reduction used in loop.");
2134 if (TREE_CODE (loop_arg
) != SSA_NAME
)
2136 if (dump_enabled_p ())
2138 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2139 "reduction: not ssa_name: ");
2140 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, loop_arg
);
2145 def_stmt
= SSA_NAME_DEF_STMT (loop_arg
);
2148 if (dump_enabled_p ())
2149 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2150 "reduction: no def_stmt.");
2154 if (!is_gimple_assign (def_stmt
) && gimple_code (def_stmt
) != GIMPLE_PHI
)
2156 if (dump_enabled_p ())
2157 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
2161 if (is_gimple_assign (def_stmt
))
2163 name
= gimple_assign_lhs (def_stmt
);
2168 name
= PHI_RESULT (def_stmt
);
2173 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, name
)
2175 gimple use_stmt
= USE_STMT (use_p
);
2176 if (is_gimple_debug (use_stmt
))
2178 if (flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
))
2179 && vinfo_for_stmt (use_stmt
)
2180 && !is_pattern_stmt_p (vinfo_for_stmt (use_stmt
)))
2184 if (dump_enabled_p ())
2185 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2186 "reduction used in loop.");
2191 /* If DEF_STMT is a phi node itself, we expect it to have a single argument
2192 defined in the inner loop. */
2195 op1
= PHI_ARG_DEF (def_stmt
, 0);
2197 if (gimple_phi_num_args (def_stmt
) != 1
2198 || TREE_CODE (op1
) != SSA_NAME
)
2200 if (dump_enabled_p ())
2201 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2202 "unsupported phi node definition.");
2207 def1
= SSA_NAME_DEF_STMT (op1
);
2208 if (gimple_bb (def1
)
2209 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
2211 && flow_bb_inside_loop_p (loop
->inner
, gimple_bb (def1
))
2212 && is_gimple_assign (def1
))
2214 if (dump_enabled_p ())
2215 report_vect_op (MSG_NOTE
, def_stmt
,
2216 "detected double reduction: ");
2218 *double_reduc
= true;
2225 code
= orig_code
= gimple_assign_rhs_code (def_stmt
);
2227 /* We can handle "res -= x[i]", which is non-associative by
2228 simply rewriting this into "res += -x[i]". Avoid changing
2229 gimple instruction for the first simple tests and only do this
2230 if we're allowed to change code at all. */
2231 if (code
== MINUS_EXPR
2233 && (op1
= gimple_assign_rhs1 (def_stmt
))
2234 && TREE_CODE (op1
) == SSA_NAME
2235 && SSA_NAME_DEF_STMT (op1
) == phi
)
2239 && (!commutative_tree_code (code
) || !associative_tree_code (code
)))
2241 if (dump_enabled_p ())
2242 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2243 "reduction: not commutative/associative: ");
2247 if (get_gimple_rhs_class (code
) != GIMPLE_BINARY_RHS
)
2249 if (code
!= COND_EXPR
)
2251 if (dump_enabled_p ())
2252 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2253 "reduction: not binary operation: ");
2258 op3
= gimple_assign_rhs1 (def_stmt
);
2259 if (COMPARISON_CLASS_P (op3
))
2261 op4
= TREE_OPERAND (op3
, 1);
2262 op3
= TREE_OPERAND (op3
, 0);
2265 op1
= gimple_assign_rhs2 (def_stmt
);
2266 op2
= gimple_assign_rhs3 (def_stmt
);
2268 if (TREE_CODE (op1
) != SSA_NAME
&& TREE_CODE (op2
) != SSA_NAME
)
2270 if (dump_enabled_p ())
2271 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2272 "reduction: uses not ssa_names: ");
2279 op1
= gimple_assign_rhs1 (def_stmt
);
2280 op2
= gimple_assign_rhs2 (def_stmt
);
2282 if (TREE_CODE (op1
) != SSA_NAME
&& TREE_CODE (op2
) != SSA_NAME
)
2284 if (dump_enabled_p ())
2285 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2286 "reduction: uses not ssa_names: ");
2292 type
= TREE_TYPE (gimple_assign_lhs (def_stmt
));
2293 if ((TREE_CODE (op1
) == SSA_NAME
2294 && !types_compatible_p (type
,TREE_TYPE (op1
)))
2295 || (TREE_CODE (op2
) == SSA_NAME
2296 && !types_compatible_p (type
, TREE_TYPE (op2
)))
2297 || (op3
&& TREE_CODE (op3
) == SSA_NAME
2298 && !types_compatible_p (type
, TREE_TYPE (op3
)))
2299 || (op4
&& TREE_CODE (op4
) == SSA_NAME
2300 && !types_compatible_p (type
, TREE_TYPE (op4
))))
2302 if (dump_enabled_p ())
2304 dump_printf_loc (MSG_NOTE
, vect_location
,
2305 "reduction: multiple types: operation type: ");
2306 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, type
);
2307 dump_printf (MSG_NOTE
, ", operands types: ");
2308 dump_generic_expr (MSG_NOTE
, TDF_SLIM
,
2310 dump_printf (MSG_NOTE
, ",");
2311 dump_generic_expr (MSG_NOTE
, TDF_SLIM
,
2315 dump_printf (MSG_NOTE
, ",");
2316 dump_generic_expr (MSG_NOTE
, TDF_SLIM
,
2322 dump_printf (MSG_NOTE
, ",");
2323 dump_generic_expr (MSG_NOTE
, TDF_SLIM
,
2331 /* Check that it's ok to change the order of the computation.
2332 Generally, when vectorizing a reduction we change the order of the
2333 computation. This may change the behavior of the program in some
2334 cases, so we need to check that this is ok. One exception is when
2335 vectorizing an outer-loop: the inner-loop is executed sequentially,
2336 and therefore vectorizing reductions in the inner-loop during
2337 outer-loop vectorization is safe. */
2339 /* CHECKME: check for !flag_finite_math_only too? */
2340 if (SCALAR_FLOAT_TYPE_P (type
) && !flag_associative_math
2343 /* Changing the order of operations changes the semantics. */
2344 if (dump_enabled_p ())
2345 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2346 "reduction: unsafe fp math optimization: ");
2349 else if (INTEGRAL_TYPE_P (type
) && TYPE_OVERFLOW_TRAPS (type
)
2352 /* Changing the order of operations changes the semantics. */
2353 if (dump_enabled_p ())
2354 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2355 "reduction: unsafe int math optimization: ");
2358 else if (SAT_FIXED_POINT_TYPE_P (type
) && check_reduction
)
2360 /* Changing the order of operations changes the semantics. */
2361 if (dump_enabled_p ())
2362 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2363 "reduction: unsafe fixed-point math optimization: ");
2367 /* If we detected "res -= x[i]" earlier, rewrite it into
2368 "res += -x[i]" now. If this turns out to be useless reassoc
2369 will clean it up again. */
2370 if (orig_code
== MINUS_EXPR
)
2372 tree rhs
= gimple_assign_rhs2 (def_stmt
);
2373 tree negrhs
= make_ssa_name (TREE_TYPE (rhs
), NULL
);
2374 gimple negate_stmt
= gimple_build_assign_with_ops (NEGATE_EXPR
, negrhs
,
2376 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
2377 set_vinfo_for_stmt (negate_stmt
, new_stmt_vec_info (negate_stmt
,
2379 gsi_insert_before (&gsi
, negate_stmt
, GSI_NEW_STMT
);
2380 gimple_assign_set_rhs2 (def_stmt
, negrhs
);
2381 gimple_assign_set_rhs_code (def_stmt
, PLUS_EXPR
);
2382 update_stmt (def_stmt
);
2385 /* Reduction is safe. We're dealing with one of the following:
2386 1) integer arithmetic and no trapv
2387 2) floating point arithmetic, and special flags permit this optimization
2388 3) nested cycle (i.e., outer loop vectorization). */
2389 if (TREE_CODE (op1
) == SSA_NAME
)
2390 def1
= SSA_NAME_DEF_STMT (op1
);
2392 if (TREE_CODE (op2
) == SSA_NAME
)
2393 def2
= SSA_NAME_DEF_STMT (op2
);
2395 if (code
!= COND_EXPR
2396 && ((!def1
|| gimple_nop_p (def1
)) && (!def2
|| gimple_nop_p (def2
))))
2398 if (dump_enabled_p ())
2399 report_vect_op (MSG_NOTE
, def_stmt
, "reduction: no defs for operands: ");
2403 /* Check that one def is the reduction def, defined by PHI,
2404 the other def is either defined in the loop ("vect_internal_def"),
2405 or it's an induction (defined by a loop-header phi-node). */
2407 if (def2
&& def2
== phi
2408 && (code
== COND_EXPR
2409 || !def1
|| gimple_nop_p (def1
)
2410 || (def1
&& flow_bb_inside_loop_p (loop
, gimple_bb (def1
))
2411 && (is_gimple_assign (def1
)
2412 || is_gimple_call (def1
)
2413 || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1
))
2414 == vect_induction_def
2415 || (gimple_code (def1
) == GIMPLE_PHI
2416 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1
))
2417 == vect_internal_def
2418 && !is_loop_header_bb_p (gimple_bb (def1
)))))))
2420 if (dump_enabled_p ())
2421 report_vect_op (MSG_NOTE
, def_stmt
, "detected reduction: ");
2425 if (def1
&& def1
== phi
2426 && (code
== COND_EXPR
2427 || !def2
|| gimple_nop_p (def2
)
2428 || (def2
&& flow_bb_inside_loop_p (loop
, gimple_bb (def2
))
2429 && (is_gimple_assign (def2
)
2430 || is_gimple_call (def2
)
2431 || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2
))
2432 == vect_induction_def
2433 || (gimple_code (def2
) == GIMPLE_PHI
2434 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2
))
2435 == vect_internal_def
2436 && !is_loop_header_bb_p (gimple_bb (def2
)))))))
2438 if (check_reduction
)
2440 /* Swap operands (just for simplicity - so that the rest of the code
2441 can assume that the reduction variable is always the last (second)
2443 if (dump_enabled_p ())
2444 report_vect_op (MSG_NOTE
, def_stmt
,
2445 "detected reduction: need to swap operands: ");
2447 swap_tree_operands (def_stmt
, gimple_assign_rhs1_ptr (def_stmt
),
2448 gimple_assign_rhs2_ptr (def_stmt
));
2450 if (CONSTANT_CLASS_P (gimple_assign_rhs1 (def_stmt
)))
2451 LOOP_VINFO_OPERANDS_SWAPPED (loop_info
) = true;
2455 if (dump_enabled_p ())
2456 report_vect_op (MSG_NOTE
, def_stmt
, "detected reduction: ");
2462 /* Try to find SLP reduction chain. */
2463 if (check_reduction
&& vect_is_slp_reduction (loop_info
, phi
, def_stmt
))
2465 if (dump_enabled_p ())
2466 report_vect_op (MSG_NOTE
, def_stmt
,
2467 "reduction: detected reduction chain: ");
2472 if (dump_enabled_p ())
2473 report_vect_op (MSG_MISSED_OPTIMIZATION
, def_stmt
,
2474 "reduction: unknown pattern: ");
2479 /* Wrapper around vect_is_simple_reduction_1, that won't modify code
2480 in-place. Arguments as there. */
2483 vect_is_simple_reduction (loop_vec_info loop_info
, gimple phi
,
2484 bool check_reduction
, bool *double_reduc
)
2486 return vect_is_simple_reduction_1 (loop_info
, phi
, check_reduction
,
2487 double_reduc
, false);
2490 /* Wrapper around vect_is_simple_reduction_1, which will modify code
2491 in-place if it enables detection of more reductions. Arguments
2495 vect_force_simple_reduction (loop_vec_info loop_info
, gimple phi
,
2496 bool check_reduction
, bool *double_reduc
)
2498 return vect_is_simple_reduction_1 (loop_info
, phi
, check_reduction
,
2499 double_reduc
, true);
2502 /* Calculate the cost of one scalar iteration of the loop. */
2504 vect_get_single_scalar_iteration_cost (loop_vec_info loop_vinfo
)
2506 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2507 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
2508 int nbbs
= loop
->num_nodes
, factor
, scalar_single_iter_cost
= 0;
2509 int innerloop_iters
, i
, stmt_cost
;
2511 /* Count statements in scalar loop. Using this as scalar cost for a single
2514 TODO: Add outer loop support.
2516 TODO: Consider assigning different costs to different scalar
2520 innerloop_iters
= 1;
2522 innerloop_iters
= 50; /* FIXME */
2524 for (i
= 0; i
< nbbs
; i
++)
2526 gimple_stmt_iterator si
;
2527 basic_block bb
= bbs
[i
];
2529 if (bb
->loop_father
== loop
->inner
)
2530 factor
= innerloop_iters
;
2534 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
2536 gimple stmt
= gsi_stmt (si
);
2537 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2539 if (!is_gimple_assign (stmt
) && !is_gimple_call (stmt
))
2542 /* Skip stmts that are not vectorized inside the loop. */
2544 && !STMT_VINFO_RELEVANT_P (stmt_info
)
2545 && (!STMT_VINFO_LIVE_P (stmt_info
)
2546 || !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
2547 && !STMT_VINFO_IN_PATTERN_P (stmt_info
))
2550 if (STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt
)))
2552 if (DR_IS_READ (STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt
))))
2553 stmt_cost
= vect_get_stmt_cost (scalar_load
);
2555 stmt_cost
= vect_get_stmt_cost (scalar_store
);
2558 stmt_cost
= vect_get_stmt_cost (scalar_stmt
);
2560 scalar_single_iter_cost
+= stmt_cost
* factor
;
2563 return scalar_single_iter_cost
;
2566 /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */
2568 vect_get_known_peeling_cost (loop_vec_info loop_vinfo
, int peel_iters_prologue
,
2569 int *peel_iters_epilogue
,
2570 int scalar_single_iter_cost
,
2571 stmt_vector_for_cost
*prologue_cost_vec
,
2572 stmt_vector_for_cost
*epilogue_cost_vec
)
2575 int vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2577 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
2579 *peel_iters_epilogue
= vf
/2;
2580 if (dump_enabled_p ())
2581 dump_printf_loc (MSG_NOTE
, vect_location
,
2582 "cost model: epilogue peel iters set to vf/2 "
2583 "because loop iterations are unknown .");
2585 /* If peeled iterations are known but number of scalar loop
2586 iterations are unknown, count a taken branch per peeled loop. */
2587 retval
= record_stmt_cost (prologue_cost_vec
, 2, cond_branch_taken
,
2588 NULL
, 0, vect_prologue
);
2592 int niters
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
2593 peel_iters_prologue
= niters
< peel_iters_prologue
?
2594 niters
: peel_iters_prologue
;
2595 *peel_iters_epilogue
= (niters
- peel_iters_prologue
) % vf
;
2596 /* If we need to peel for gaps, but no peeling is required, we have to
2597 peel VF iterations. */
2598 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) && !*peel_iters_epilogue
)
2599 *peel_iters_epilogue
= vf
;
2602 if (peel_iters_prologue
)
2603 retval
+= record_stmt_cost (prologue_cost_vec
,
2604 peel_iters_prologue
* scalar_single_iter_cost
,
2605 scalar_stmt
, NULL
, 0, vect_prologue
);
2606 if (*peel_iters_epilogue
)
2607 retval
+= record_stmt_cost (epilogue_cost_vec
,
2608 *peel_iters_epilogue
* scalar_single_iter_cost
,
2609 scalar_stmt
, NULL
, 0, vect_epilogue
);
2613 /* Function vect_estimate_min_profitable_iters
2615 Return the number of iterations required for the vector version of the
2616 loop to be profitable relative to the cost of the scalar version of the
2620 vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo
,
2621 int *ret_min_profitable_niters
,
2622 int *ret_min_profitable_estimate
)
2624 int min_profitable_iters
;
2625 int min_profitable_estimate
;
2626 int peel_iters_prologue
;
2627 int peel_iters_epilogue
;
2628 unsigned vec_inside_cost
= 0;
2629 int vec_outside_cost
= 0;
2630 unsigned vec_prologue_cost
= 0;
2631 unsigned vec_epilogue_cost
= 0;
2632 int scalar_single_iter_cost
= 0;
2633 int scalar_outside_cost
= 0;
2634 int vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2635 int npeel
= LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
);
2636 void *target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
2638 /* Cost model disabled. */
2639 if (unlimited_cost_model ())
2641 dump_printf_loc (MSG_NOTE
, vect_location
, "cost model disabled.");
2642 *ret_min_profitable_niters
= 0;
2643 *ret_min_profitable_estimate
= 0;
2647 /* Requires loop versioning tests to handle misalignment. */
2648 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
))
2650 /* FIXME: Make cost depend on complexity of individual check. */
2651 unsigned len
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
).length ();
2652 (void) add_stmt_cost (target_cost_data
, len
, vector_stmt
, NULL
, 0,
2654 dump_printf (MSG_NOTE
,
2655 "cost model: Adding cost of checks for loop "
2656 "versioning to treat misalignment.\n");
2659 /* Requires loop versioning with alias checks. */
2660 if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
2662 /* FIXME: Make cost depend on complexity of individual check. */
2663 unsigned len
= LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo
).length ();
2664 (void) add_stmt_cost (target_cost_data
, len
, vector_stmt
, NULL
, 0,
2666 dump_printf (MSG_NOTE
,
2667 "cost model: Adding cost of checks for loop "
2668 "versioning aliasing.\n");
2671 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
)
2672 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
2673 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
, NULL
, 0,
2676 /* Count statements in scalar loop. Using this as scalar cost for a single
2679 TODO: Add outer loop support.
2681 TODO: Consider assigning different costs to different scalar
2684 scalar_single_iter_cost
= vect_get_single_scalar_iteration_cost (loop_vinfo
);
2686 /* Add additional cost for the peeled instructions in prologue and epilogue
2689 FORNOW: If we don't know the value of peel_iters for prologue or epilogue
2690 at compile-time - we assume it's vf/2 (the worst would be vf-1).
2692 TODO: Build an expression that represents peel_iters for prologue and
2693 epilogue to be used in a run-time test. */
2697 peel_iters_prologue
= vf
/2;
2698 dump_printf (MSG_NOTE
, "cost model: "
2699 "prologue peel iters set to vf/2.");
2701 /* If peeling for alignment is unknown, loop bound of main loop becomes
2703 peel_iters_epilogue
= vf
/2;
2704 dump_printf (MSG_NOTE
, "cost model: "
2705 "epilogue peel iters set to vf/2 because "
2706 "peeling for alignment is unknown.");
2708 /* If peeled iterations are unknown, count a taken branch and a not taken
2709 branch per peeled loop. Even if scalar loop iterations are known,
2710 vector iterations are not known since peeled prologue iterations are
2711 not known. Hence guards remain the same. */
2712 (void) add_stmt_cost (target_cost_data
, 2, cond_branch_taken
,
2713 NULL
, 0, vect_prologue
);
2714 (void) add_stmt_cost (target_cost_data
, 2, cond_branch_not_taken
,
2715 NULL
, 0, vect_prologue
);
2716 /* FORNOW: Don't attempt to pass individual scalar instructions to
2717 the model; just assume linear cost for scalar iterations. */
2718 (void) add_stmt_cost (target_cost_data
,
2719 peel_iters_prologue
* scalar_single_iter_cost
,
2720 scalar_stmt
, NULL
, 0, vect_prologue
);
2721 (void) add_stmt_cost (target_cost_data
,
2722 peel_iters_epilogue
* scalar_single_iter_cost
,
2723 scalar_stmt
, NULL
, 0, vect_epilogue
);
2727 stmt_vector_for_cost prologue_cost_vec
, epilogue_cost_vec
;
2728 stmt_info_for_cost
*si
;
2730 void *data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
2732 prologue_cost_vec
.create (2);
2733 epilogue_cost_vec
.create (2);
2734 peel_iters_prologue
= npeel
;
2736 (void) vect_get_known_peeling_cost (loop_vinfo
, peel_iters_prologue
,
2737 &peel_iters_epilogue
,
2738 scalar_single_iter_cost
,
2740 &epilogue_cost_vec
);
2742 FOR_EACH_VEC_ELT (prologue_cost_vec
, j
, si
)
2744 struct _stmt_vec_info
*stmt_info
2745 = si
->stmt
? vinfo_for_stmt (si
->stmt
) : NULL
;
2746 (void) add_stmt_cost (data
, si
->count
, si
->kind
, stmt_info
,
2747 si
->misalign
, vect_prologue
);
2750 FOR_EACH_VEC_ELT (epilogue_cost_vec
, j
, si
)
2752 struct _stmt_vec_info
*stmt_info
2753 = si
->stmt
? vinfo_for_stmt (si
->stmt
) : NULL
;
2754 (void) add_stmt_cost (data
, si
->count
, si
->kind
, stmt_info
,
2755 si
->misalign
, vect_epilogue
);
2758 prologue_cost_vec
.release ();
2759 epilogue_cost_vec
.release ();
2762 /* FORNOW: The scalar outside cost is incremented in one of the
2765 1. The vectorizer checks for alignment and aliasing and generates
2766 a condition that allows dynamic vectorization. A cost model
2767 check is ANDED with the versioning condition. Hence scalar code
2768 path now has the added cost of the versioning check.
2770 if (cost > th & versioning_check)
2773 Hence run-time scalar is incremented by not-taken branch cost.
2775 2. The vectorizer then checks if a prologue is required. If the
2776 cost model check was not done before during versioning, it has to
2777 be done before the prologue check.
2780 prologue = scalar_iters
2785 if (prologue == num_iters)
2788 Hence the run-time scalar cost is incremented by a taken branch,
2789 plus a not-taken branch, plus a taken branch cost.
2791 3. The vectorizer then checks if an epilogue is required. If the
2792 cost model check was not done before during prologue check, it
2793 has to be done with the epilogue check.
2799 if (prologue == num_iters)
2802 if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
2805 Hence the run-time scalar cost should be incremented by 2 taken
2808 TODO: The back end may reorder the BBS's differently and reverse
2809 conditions/branch directions. Change the estimates below to
2810 something more reasonable. */
2812 /* If the number of iterations is known and we do not do versioning, we can
2813 decide whether to vectorize at compile time. Hence the scalar version
2814 do not carry cost model guard costs. */
2815 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
2816 || LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
)
2817 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
2819 /* Cost model check occurs at versioning. */
2820 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
)
2821 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
2822 scalar_outside_cost
+= vect_get_stmt_cost (cond_branch_not_taken
);
2825 /* Cost model check occurs at prologue generation. */
2826 if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
) < 0)
2827 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
)
2828 + vect_get_stmt_cost (cond_branch_not_taken
);
2829 /* Cost model check occurs at epilogue generation. */
2831 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
);
2835 /* Complete the target-specific cost calculations. */
2836 finish_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
), &vec_prologue_cost
,
2837 &vec_inside_cost
, &vec_epilogue_cost
);
2839 vec_outside_cost
= (int)(vec_prologue_cost
+ vec_epilogue_cost
);
2841 /* Calculate number of iterations required to make the vector version
2842 profitable, relative to the loop bodies only. The following condition
2844 SIC * niters + SOC > VIC * ((niters-PL_ITERS-EP_ITERS)/VF) + VOC
2846 SIC = scalar iteration cost, VIC = vector iteration cost,
2847 VOC = vector outside cost, VF = vectorization factor,
2848 PL_ITERS = prologue iterations, EP_ITERS= epilogue iterations
2849 SOC = scalar outside cost for run time cost model check. */
2851 if ((scalar_single_iter_cost
* vf
) > (int) vec_inside_cost
)
2853 if (vec_outside_cost
<= 0)
2854 min_profitable_iters
= 1;
2857 min_profitable_iters
= ((vec_outside_cost
- scalar_outside_cost
) * vf
2858 - vec_inside_cost
* peel_iters_prologue
2859 - vec_inside_cost
* peel_iters_epilogue
)
2860 / ((scalar_single_iter_cost
* vf
)
2863 if ((scalar_single_iter_cost
* vf
* min_profitable_iters
)
2864 <= (((int) vec_inside_cost
* min_profitable_iters
)
2865 + (((int) vec_outside_cost
- scalar_outside_cost
) * vf
)))
2866 min_profitable_iters
++;
2869 /* vector version will never be profitable. */
2872 if (dump_enabled_p ())
2873 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2874 "cost model: the vector iteration cost = %d "
2875 "divided by the scalar iteration cost = %d "
2876 "is greater or equal to the vectorization factor = %d.",
2877 vec_inside_cost
, scalar_single_iter_cost
, vf
);
2878 *ret_min_profitable_niters
= -1;
2879 *ret_min_profitable_estimate
= -1;
2883 if (dump_enabled_p ())
2885 dump_printf_loc (MSG_NOTE
, vect_location
, "Cost model analysis: \n");
2886 dump_printf (MSG_NOTE
, " Vector inside of loop cost: %d\n",
2888 dump_printf (MSG_NOTE
, " Vector prologue cost: %d\n",
2890 dump_printf (MSG_NOTE
, " Vector epilogue cost: %d\n",
2892 dump_printf (MSG_NOTE
, " Scalar iteration cost: %d\n",
2893 scalar_single_iter_cost
);
2894 dump_printf (MSG_NOTE
, " Scalar outside cost: %d\n",
2895 scalar_outside_cost
);
2896 dump_printf (MSG_NOTE
, " Vector outside cost: %d\n",
2898 dump_printf (MSG_NOTE
, " prologue iterations: %d\n",
2899 peel_iters_prologue
);
2900 dump_printf (MSG_NOTE
, " epilogue iterations: %d\n",
2901 peel_iters_epilogue
);
2902 dump_printf (MSG_NOTE
,
2903 " Calculated minimum iters for profitability: %d\n",
2904 min_profitable_iters
);
2907 min_profitable_iters
=
2908 min_profitable_iters
< vf
? vf
: min_profitable_iters
;
2910 /* Because the condition we create is:
2911 if (niters <= min_profitable_iters)
2912 then skip the vectorized loop. */
2913 min_profitable_iters
--;
2915 if (dump_enabled_p ())
2916 dump_printf_loc (MSG_NOTE
, vect_location
,
2917 " Runtime profitability threshold = %d\n", min_profitable_iters
);
2919 *ret_min_profitable_niters
= min_profitable_iters
;
2921 /* Calculate number of iterations required to make the vector version
2922 profitable, relative to the loop bodies only.
2924 Non-vectorized variant is SIC * niters and it must win over vector
2925 variant on the expected loop trip count. The following condition must hold true:
2926 SIC * niters > VIC * ((niters-PL_ITERS-EP_ITERS)/VF) + VOC + SOC */
2928 if (vec_outside_cost
<= 0)
2929 min_profitable_estimate
= 1;
2932 min_profitable_estimate
= ((vec_outside_cost
+ scalar_outside_cost
) * vf
2933 - vec_inside_cost
* peel_iters_prologue
2934 - vec_inside_cost
* peel_iters_epilogue
)
2935 / ((scalar_single_iter_cost
* vf
)
2938 min_profitable_estimate
--;
2939 min_profitable_estimate
= MAX (min_profitable_estimate
, min_profitable_iters
);
2940 if (dump_enabled_p ())
2941 dump_printf_loc (MSG_NOTE
, vect_location
,
2942 " Static estimate profitability threshold = %d\n",
2943 min_profitable_iters
);
2945 *ret_min_profitable_estimate
= min_profitable_estimate
;
2949 /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
2950 functions. Design better to avoid maintenance issues. */
2952 /* Function vect_model_reduction_cost.
2954 Models cost for a reduction operation, including the vector ops
2955 generated within the strip-mine loop, the initial definition before
2956 the loop, and the epilogue code that must be generated. */
2959 vect_model_reduction_cost (stmt_vec_info stmt_info
, enum tree_code reduc_code
,
2962 int prologue_cost
= 0, epilogue_cost
= 0;
2963 enum tree_code code
;
2966 gimple stmt
, orig_stmt
;
2968 enum machine_mode mode
;
2969 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2970 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2971 void *target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
2973 /* Cost of reduction op inside loop. */
2974 unsigned inside_cost
= add_stmt_cost (target_cost_data
, ncopies
, vector_stmt
,
2975 stmt_info
, 0, vect_body
);
2976 stmt
= STMT_VINFO_STMT (stmt_info
);
2978 switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt
)))
2980 case GIMPLE_SINGLE_RHS
:
2981 gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt
)) == ternary_op
);
2982 reduction_op
= TREE_OPERAND (gimple_assign_rhs1 (stmt
), 2);
2984 case GIMPLE_UNARY_RHS
:
2985 reduction_op
= gimple_assign_rhs1 (stmt
);
2987 case GIMPLE_BINARY_RHS
:
2988 reduction_op
= gimple_assign_rhs2 (stmt
);
2990 case GIMPLE_TERNARY_RHS
:
2991 reduction_op
= gimple_assign_rhs3 (stmt
);
2997 vectype
= get_vectype_for_scalar_type (TREE_TYPE (reduction_op
));
3000 if (dump_enabled_p ())
3002 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3003 "unsupported data-type ");
3004 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
3005 TREE_TYPE (reduction_op
));
3010 mode
= TYPE_MODE (vectype
);
3011 orig_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
3014 orig_stmt
= STMT_VINFO_STMT (stmt_info
);
3016 code
= gimple_assign_rhs_code (orig_stmt
);
3018 /* Add in cost for initial definition. */
3019 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, scalar_to_vec
,
3020 stmt_info
, 0, vect_prologue
);
3022 /* Determine cost of epilogue code.
3024 We have a reduction operator that will reduce the vector in one statement.
3025 Also requires scalar extract. */
3027 if (!nested_in_vect_loop_p (loop
, orig_stmt
))
3029 if (reduc_code
!= ERROR_MARK
)
3031 epilogue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
3032 stmt_info
, 0, vect_epilogue
);
3033 epilogue_cost
+= add_stmt_cost (target_cost_data
, 1, vec_to_scalar
,
3034 stmt_info
, 0, vect_epilogue
);
3038 int vec_size_in_bits
= tree_low_cst (TYPE_SIZE (vectype
), 1);
3040 TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt
)));
3041 int element_bitsize
= tree_low_cst (bitsize
, 1);
3042 int nelements
= vec_size_in_bits
/ element_bitsize
;
3044 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
3046 /* We have a whole vector shift available. */
3047 if (VECTOR_MODE_P (mode
)
3048 && optab_handler (optab
, mode
) != CODE_FOR_nothing
3049 && optab_handler (vec_shr_optab
, mode
) != CODE_FOR_nothing
)
3051 /* Final reduction via vector shifts and the reduction operator.
3052 Also requires scalar extract. */
3053 epilogue_cost
+= add_stmt_cost (target_cost_data
,
3054 exact_log2 (nelements
) * 2,
3055 vector_stmt
, stmt_info
, 0,
3057 epilogue_cost
+= add_stmt_cost (target_cost_data
, 1,
3058 vec_to_scalar
, stmt_info
, 0,
3062 /* Use extracts and reduction op for final reduction. For N
3063 elements, we have N extracts and N-1 reduction ops. */
3064 epilogue_cost
+= add_stmt_cost (target_cost_data
,
3065 nelements
+ nelements
- 1,
3066 vector_stmt
, stmt_info
, 0,
3071 if (dump_enabled_p ())
3072 dump_printf (MSG_NOTE
,
3073 "vect_model_reduction_cost: inside_cost = %d, "
3074 "prologue_cost = %d, epilogue_cost = %d .", inside_cost
,
3075 prologue_cost
, epilogue_cost
);
3081 /* Function vect_model_induction_cost.
3083 Models cost for induction operations. */
3086 vect_model_induction_cost (stmt_vec_info stmt_info
, int ncopies
)
3088 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3089 void *target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
3090 unsigned inside_cost
, prologue_cost
;
3092 /* loop cost for vec_loop. */
3093 inside_cost
= add_stmt_cost (target_cost_data
, ncopies
, vector_stmt
,
3094 stmt_info
, 0, vect_body
);
3096 /* prologue cost for vec_init and vec_step. */
3097 prologue_cost
= add_stmt_cost (target_cost_data
, 2, scalar_to_vec
,
3098 stmt_info
, 0, vect_prologue
);
3100 if (dump_enabled_p ())
3101 dump_printf_loc (MSG_NOTE
, vect_location
,
3102 "vect_model_induction_cost: inside_cost = %d, "
3103 "prologue_cost = %d .", inside_cost
, prologue_cost
);
3107 /* Function get_initial_def_for_induction
3110 STMT - a stmt that performs an induction operation in the loop.
3111 IV_PHI - the initial value of the induction variable
3114 Return a vector variable, initialized with the first VF values of
3115 the induction variable. E.g., for an iv with IV_PHI='X' and
3116 evolution S, for a vector of 4 units, we want to return:
3117 [X, X + S, X + 2*S, X + 3*S]. */
3120 get_initial_def_for_induction (gimple iv_phi
)
3122 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (iv_phi
);
3123 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
3124 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
3127 edge pe
= loop_preheader_edge (loop
);
3128 struct loop
*iv_loop
;
3130 tree new_vec
, vec_init
, vec_step
, t
;
3133 gimple init_stmt
, induction_phi
, new_stmt
;
3134 tree induc_def
, vec_def
, vec_dest
;
3135 tree init_expr
, step_expr
;
3136 int vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
3140 stmt_vec_info phi_info
= vinfo_for_stmt (iv_phi
);
3141 bool nested_in_vect_loop
= false;
3142 gimple_seq stmts
= NULL
;
3143 imm_use_iterator imm_iter
;
3144 use_operand_p use_p
;
3148 gimple_stmt_iterator si
;
3149 basic_block bb
= gimple_bb (iv_phi
);
3153 /* Is phi in an inner-loop, while vectorizing an enclosing outer-loop? */
3154 if (nested_in_vect_loop_p (loop
, iv_phi
))
3156 nested_in_vect_loop
= true;
3157 iv_loop
= loop
->inner
;
3161 gcc_assert (iv_loop
== (gimple_bb (iv_phi
))->loop_father
);
3163 latch_e
= loop_latch_edge (iv_loop
);
3164 loop_arg
= PHI_ARG_DEF_FROM_EDGE (iv_phi
, latch_e
);
3166 step_expr
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info
);
3167 gcc_assert (step_expr
!= NULL_TREE
);
3169 pe
= loop_preheader_edge (iv_loop
);
3170 init_expr
= PHI_ARG_DEF_FROM_EDGE (iv_phi
,
3171 loop_preheader_edge (iv_loop
));
3173 vectype
= get_vectype_for_scalar_type (TREE_TYPE (init_expr
));
3174 resvectype
= get_vectype_for_scalar_type (TREE_TYPE (PHI_RESULT (iv_phi
)));
3175 gcc_assert (vectype
);
3176 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
3177 ncopies
= vf
/ nunits
;
3179 gcc_assert (phi_info
);
3180 gcc_assert (ncopies
>= 1);
3182 /* Convert the step to the desired type. */
3183 step_expr
= force_gimple_operand (fold_convert (TREE_TYPE (vectype
),
3185 &stmts
, true, NULL_TREE
);
3188 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3189 gcc_assert (!new_bb
);
3192 /* Find the first insertion point in the BB. */
3193 si
= gsi_after_labels (bb
);
3195 /* Create the vector that holds the initial_value of the induction. */
3196 if (nested_in_vect_loop
)
3198 /* iv_loop is nested in the loop to be vectorized. init_expr had already
3199 been created during vectorization of previous stmts. We obtain it
3200 from the STMT_VINFO_VEC_STMT of the defining stmt. */
3201 vec_init
= vect_get_vec_def_for_operand (init_expr
, iv_phi
, NULL
);
3202 /* If the initial value is not of proper type, convert it. */
3203 if (!useless_type_conversion_p (vectype
, TREE_TYPE (vec_init
)))
3205 new_stmt
= gimple_build_assign_with_ops
3207 vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_"),
3208 build1 (VIEW_CONVERT_EXPR
, vectype
, vec_init
), NULL_TREE
);
3209 vec_init
= make_ssa_name (gimple_assign_lhs (new_stmt
), new_stmt
);
3210 gimple_assign_set_lhs (new_stmt
, vec_init
);
3211 new_bb
= gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop
),
3213 gcc_assert (!new_bb
);
3214 set_vinfo_for_stmt (new_stmt
,
3215 new_stmt_vec_info (new_stmt
, loop_vinfo
, NULL
));
3220 vec
<constructor_elt
, va_gc
> *v
;
3222 /* iv_loop is the loop to be vectorized. Create:
3223 vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr) */
3224 new_var
= vect_get_new_vect_var (TREE_TYPE (vectype
),
3225 vect_scalar_var
, "var_");
3226 new_name
= force_gimple_operand (fold_convert (TREE_TYPE (vectype
),
3228 &stmts
, false, new_var
);
3231 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3232 gcc_assert (!new_bb
);
3235 vec_alloc (v
, nunits
);
3236 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, new_name
);
3237 for (i
= 1; i
< nunits
; i
++)
3239 /* Create: new_name_i = new_name + step_expr */
3240 init_stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, new_var
,
3241 new_name
, step_expr
);
3242 new_name
= make_ssa_name (new_var
, init_stmt
);
3243 gimple_assign_set_lhs (init_stmt
, new_name
);
3245 new_bb
= gsi_insert_on_edge_immediate (pe
, init_stmt
);
3246 gcc_assert (!new_bb
);
3248 if (dump_enabled_p ())
3250 dump_printf_loc (MSG_NOTE
, vect_location
,
3251 "created new init_stmt: ");
3252 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, init_stmt
, 0);
3254 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, new_name
);
3256 /* Create a vector from [new_name_0, new_name_1, ..., new_name_nunits-1] */
3257 new_vec
= build_constructor (vectype
, v
);
3258 vec_init
= vect_init_vector (iv_phi
, new_vec
, vectype
, NULL
);
3262 /* Create the vector that holds the step of the induction. */
3263 if (nested_in_vect_loop
)
3264 /* iv_loop is nested in the loop to be vectorized. Generate:
3265 vec_step = [S, S, S, S] */
3266 new_name
= step_expr
;
3269 /* iv_loop is the loop to be vectorized. Generate:
3270 vec_step = [VF*S, VF*S, VF*S, VF*S] */
3271 expr
= build_int_cst (TREE_TYPE (step_expr
), vf
);
3272 new_name
= fold_build2 (MULT_EXPR
, TREE_TYPE (step_expr
),
3276 t
= unshare_expr (new_name
);
3277 gcc_assert (CONSTANT_CLASS_P (new_name
));
3278 stepvectype
= get_vectype_for_scalar_type (TREE_TYPE (new_name
));
3279 gcc_assert (stepvectype
);
3280 new_vec
= build_vector_from_val (stepvectype
, t
);
3281 vec_step
= vect_init_vector (iv_phi
, new_vec
, stepvectype
, NULL
);
3284 /* Create the following def-use cycle:
3289 vec_iv = PHI <vec_init, vec_loop>
3293 vec_loop = vec_iv + vec_step; */
3295 /* Create the induction-phi that defines the induction-operand. */
3296 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_");
3297 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
3298 set_vinfo_for_stmt (induction_phi
,
3299 new_stmt_vec_info (induction_phi
, loop_vinfo
, NULL
));
3300 induc_def
= PHI_RESULT (induction_phi
);
3302 /* Create the iv update inside the loop */
3303 new_stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, vec_dest
,
3304 induc_def
, vec_step
);
3305 vec_def
= make_ssa_name (vec_dest
, new_stmt
);
3306 gimple_assign_set_lhs (new_stmt
, vec_def
);
3307 gsi_insert_before (&si
, new_stmt
, GSI_SAME_STMT
);
3308 set_vinfo_for_stmt (new_stmt
, new_stmt_vec_info (new_stmt
, loop_vinfo
,
3311 /* Set the arguments of the phi node: */
3312 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
3313 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
3317 /* In case that vectorization factor (VF) is bigger than the number
3318 of elements that we can fit in a vectype (nunits), we have to generate
3319 more than one vector stmt - i.e - we need to "unroll" the
3320 vector stmt by a factor VF/nunits. For more details see documentation
3321 in vectorizable_operation. */
3325 stmt_vec_info prev_stmt_vinfo
;
3326 /* FORNOW. This restriction should be relaxed. */
3327 gcc_assert (!nested_in_vect_loop
);
3329 /* Create the vector that holds the step of the induction. */
3330 expr
= build_int_cst (TREE_TYPE (step_expr
), nunits
);
3331 new_name
= fold_build2 (MULT_EXPR
, TREE_TYPE (step_expr
),
3333 t
= unshare_expr (new_name
);
3334 gcc_assert (CONSTANT_CLASS_P (new_name
));
3335 new_vec
= build_vector_from_val (stepvectype
, t
);
3336 vec_step
= vect_init_vector (iv_phi
, new_vec
, stepvectype
, NULL
);
3338 vec_def
= induc_def
;
3339 prev_stmt_vinfo
= vinfo_for_stmt (induction_phi
);
3340 for (i
= 1; i
< ncopies
; i
++)
3342 /* vec_i = vec_prev + vec_step */
3343 new_stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, vec_dest
,
3345 vec_def
= make_ssa_name (vec_dest
, new_stmt
);
3346 gimple_assign_set_lhs (new_stmt
, vec_def
);
3348 gsi_insert_before (&si
, new_stmt
, GSI_SAME_STMT
);
3349 if (!useless_type_conversion_p (resvectype
, vectype
))
3351 new_stmt
= gimple_build_assign_with_ops
3353 vect_get_new_vect_var (resvectype
, vect_simple_var
,
3355 build1 (VIEW_CONVERT_EXPR
, resvectype
,
3356 gimple_assign_lhs (new_stmt
)), NULL_TREE
);
3357 gimple_assign_set_lhs (new_stmt
,
3359 (gimple_assign_lhs (new_stmt
), new_stmt
));
3360 gsi_insert_before (&si
, new_stmt
, GSI_SAME_STMT
);
3362 set_vinfo_for_stmt (new_stmt
,
3363 new_stmt_vec_info (new_stmt
, loop_vinfo
, NULL
));
3364 STMT_VINFO_RELATED_STMT (prev_stmt_vinfo
) = new_stmt
;
3365 prev_stmt_vinfo
= vinfo_for_stmt (new_stmt
);
3369 if (nested_in_vect_loop
)
3371 /* Find the loop-closed exit-phi of the induction, and record
3372 the final vector of induction results: */
3374 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, loop_arg
)
3376 if (!flow_bb_inside_loop_p (iv_loop
, gimple_bb (USE_STMT (use_p
))))
3378 exit_phi
= USE_STMT (use_p
);
3384 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (exit_phi
);
3385 /* FORNOW. Currently not supporting the case that an inner-loop induction
3386 is not used in the outer-loop (i.e. only outside the outer-loop). */
3387 gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo
)
3388 && !STMT_VINFO_LIVE_P (stmt_vinfo
));
3390 STMT_VINFO_VEC_STMT (stmt_vinfo
) = new_stmt
;
3391 if (dump_enabled_p ())
3393 dump_printf_loc (MSG_NOTE
, vect_location
,
3394 "vector of inductions after inner-loop:");
3395 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
3401 if (dump_enabled_p ())
3403 dump_printf_loc (MSG_NOTE
, vect_location
,
3404 "transform induction: created def-use cycle: ");
3405 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, induction_phi
, 0);
3406 dump_printf (MSG_NOTE
, "\n");
3407 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
,
3408 SSA_NAME_DEF_STMT (vec_def
), 0);
3411 STMT_VINFO_VEC_STMT (phi_info
) = induction_phi
;
3412 if (!useless_type_conversion_p (resvectype
, vectype
))
3414 new_stmt
= gimple_build_assign_with_ops
3416 vect_get_new_vect_var (resvectype
, vect_simple_var
, "vec_iv_"),
3417 build1 (VIEW_CONVERT_EXPR
, resvectype
, induc_def
), NULL_TREE
);
3418 induc_def
= make_ssa_name (gimple_assign_lhs (new_stmt
), new_stmt
);
3419 gimple_assign_set_lhs (new_stmt
, induc_def
);
3420 si
= gsi_after_labels (bb
);
3421 gsi_insert_before (&si
, new_stmt
, GSI_SAME_STMT
);
3422 set_vinfo_for_stmt (new_stmt
,
3423 new_stmt_vec_info (new_stmt
, loop_vinfo
, NULL
));
3424 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_stmt
))
3425 = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (induction_phi
));
3432 /* Function get_initial_def_for_reduction
3435 STMT - a stmt that performs a reduction operation in the loop.
3436 INIT_VAL - the initial value of the reduction variable
3439 ADJUSTMENT_DEF - a tree that holds a value to be added to the final result
3440 of the reduction (used for adjusting the epilog - see below).
3441 Return a vector variable, initialized according to the operation that STMT
3442 performs. This vector will be used as the initial value of the
3443 vector of partial results.
3445 Option1 (adjust in epilog): Initialize the vector as follows:
3446 add/bit or/xor: [0,0,...,0,0]
3447 mult/bit and: [1,1,...,1,1]
3448 min/max/cond_expr: [init_val,init_val,..,init_val,init_val]
3449 and when necessary (e.g. add/mult case) let the caller know
3450 that it needs to adjust the result by init_val.
3452 Option2: Initialize the vector as follows:
3453 add/bit or/xor: [init_val,0,0,...,0]
3454 mult/bit and: [init_val,1,1,...,1]
3455 min/max/cond_expr: [init_val,init_val,...,init_val]
3456 and no adjustments are needed.
3458 For example, for the following code:
3464 STMT is 's = s + a[i]', and the reduction variable is 's'.
3465 For a vector of 4 units, we want to return either [0,0,0,init_val],
3466 or [0,0,0,0] and let the caller know that it needs to adjust
3467 the result at the end by 'init_val'.
3469 FORNOW, we are using the 'adjust in epilog' scheme, because this way the
3470 initialization vector is simpler (same element in all entries), if
3471 ADJUSTMENT_DEF is not NULL, and Option2 otherwise.
3473 A cost model should help decide between these two schemes. */
3476 get_initial_def_for_reduction (gimple stmt
, tree init_val
,
3477 tree
*adjustment_def
)
3479 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
3480 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
3481 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
3482 tree scalar_type
= TREE_TYPE (init_val
);
3483 tree vectype
= get_vectype_for_scalar_type (scalar_type
);
3485 enum tree_code code
= gimple_assign_rhs_code (stmt
);
3490 bool nested_in_vect_loop
= false;
3492 REAL_VALUE_TYPE real_init_val
= dconst0
;
3493 int int_init_val
= 0;
3494 gimple def_stmt
= NULL
;
3496 gcc_assert (vectype
);
3497 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
3499 gcc_assert (POINTER_TYPE_P (scalar_type
) || INTEGRAL_TYPE_P (scalar_type
)
3500 || SCALAR_FLOAT_TYPE_P (scalar_type
));
3502 if (nested_in_vect_loop_p (loop
, stmt
))
3503 nested_in_vect_loop
= true;
3505 gcc_assert (loop
== (gimple_bb (stmt
))->loop_father
);
3507 /* In case of double reduction we only create a vector variable to be put
3508 in the reduction phi node. The actual statement creation is done in
3509 vect_create_epilog_for_reduction. */
3510 if (adjustment_def
&& nested_in_vect_loop
3511 && TREE_CODE (init_val
) == SSA_NAME
3512 && (def_stmt
= SSA_NAME_DEF_STMT (init_val
))
3513 && gimple_code (def_stmt
) == GIMPLE_PHI
3514 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
3515 && vinfo_for_stmt (def_stmt
)
3516 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
))
3517 == vect_double_reduction_def
)
3519 *adjustment_def
= NULL
;
3520 return vect_create_destination_var (init_val
, vectype
);
3523 if (TREE_CONSTANT (init_val
))
3525 if (SCALAR_FLOAT_TYPE_P (scalar_type
))
3526 init_value
= build_real (scalar_type
, TREE_REAL_CST (init_val
));
3528 init_value
= build_int_cst (scalar_type
, TREE_INT_CST_LOW (init_val
));
3531 init_value
= init_val
;
3535 case WIDEN_SUM_EXPR
:
3543 /* ADJUSMENT_DEF is NULL when called from
3544 vect_create_epilog_for_reduction to vectorize double reduction. */
3547 if (nested_in_vect_loop
)
3548 *adjustment_def
= vect_get_vec_def_for_operand (init_val
, stmt
,
3551 *adjustment_def
= init_val
;
3554 if (code
== MULT_EXPR
)
3556 real_init_val
= dconst1
;
3560 if (code
== BIT_AND_EXPR
)
3563 if (SCALAR_FLOAT_TYPE_P (scalar_type
))
3564 def_for_init
= build_real (scalar_type
, real_init_val
);
3566 def_for_init
= build_int_cst (scalar_type
, int_init_val
);
3568 /* Create a vector of '0' or '1' except the first element. */
3569 elts
= XALLOCAVEC (tree
, nunits
);
3570 for (i
= nunits
- 2; i
>= 0; --i
)
3571 elts
[i
+ 1] = def_for_init
;
3573 /* Option1: the first element is '0' or '1' as well. */
3576 elts
[0] = def_for_init
;
3577 init_def
= build_vector (vectype
, elts
);
3581 /* Option2: the first element is INIT_VAL. */
3583 if (TREE_CONSTANT (init_val
))
3584 init_def
= build_vector (vectype
, elts
);
3587 vec
<constructor_elt
, va_gc
> *v
;
3588 vec_alloc (v
, nunits
);
3589 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, init_val
);
3590 for (i
= 1; i
< nunits
; ++i
)
3591 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, elts
[i
]);
3592 init_def
= build_constructor (vectype
, v
);
3602 *adjustment_def
= NULL_TREE
;
3603 init_def
= vect_get_vec_def_for_operand (init_val
, stmt
, NULL
);
3607 init_def
= build_vector_from_val (vectype
, init_value
);
3618 /* Function vect_create_epilog_for_reduction
3620 Create code at the loop-epilog to finalize the result of a reduction
3623 VECT_DEFS is list of vector of partial results, i.e., the lhs's of vector
3624 reduction statements.
3625 STMT is the scalar reduction stmt that is being vectorized.
3626 NCOPIES is > 1 in case the vectorization factor (VF) is bigger than the
3627 number of elements that we can fit in a vectype (nunits). In this case
3628 we have to generate more than one vector stmt - i.e - we need to "unroll"
3629 the vector stmt by a factor VF/nunits. For more details see documentation
3630 in vectorizable_operation.
3631 REDUC_CODE is the tree-code for the epilog reduction.
3632 REDUCTION_PHIS is a list of the phi-nodes that carry the reduction
3634 REDUC_INDEX is the index of the operand in the right hand side of the
3635 statement that is defined by REDUCTION_PHI.
3636 DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
3637 SLP_NODE is an SLP node containing a group of reduction statements. The
3638 first one in this group is STMT.
3641 1. Creates the reduction def-use cycles: sets the arguments for
3643 The loop-entry argument is the vectorized initial-value of the reduction.
3644 The loop-latch argument is taken from VECT_DEFS - the vector of partial
3646 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
3647 by applying the operation specified by REDUC_CODE if available, or by
3648 other means (whole-vector shifts or a scalar loop).
3649 The function also creates a new phi node at the loop exit to preserve
3650 loop-closed form, as illustrated below.
3652 The flow at the entry to this function:
3655 vec_def = phi <null, null> # REDUCTION_PHI
3656 VECT_DEF = vector_stmt # vectorized form of STMT
3657 s_loop = scalar_stmt # (scalar) STMT
3659 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
3663 The above is transformed by this function into:
3666 vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
3667 VECT_DEF = vector_stmt # vectorized form of STMT
3668 s_loop = scalar_stmt # (scalar) STMT
3670 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
3671 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
3672 v_out2 = reduce <v_out1>
3673 s_out3 = extract_field <v_out2, 0>
3674 s_out4 = adjust_result <s_out3>
3680 vect_create_epilog_for_reduction (vec
<tree
> vect_defs
, gimple stmt
,
3681 int ncopies
, enum tree_code reduc_code
,
3682 vec
<gimple
> reduction_phis
,
3683 int reduc_index
, bool double_reduc
,
3686 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3687 stmt_vec_info prev_phi_info
;
3689 enum machine_mode mode
;
3690 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3691 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
), *outer_loop
= NULL
;
3692 basic_block exit_bb
;
3695 gimple new_phi
= NULL
, phi
;
3696 gimple_stmt_iterator exit_gsi
;
3698 tree new_temp
= NULL_TREE
, new_dest
, new_name
, new_scalar_dest
;
3699 gimple epilog_stmt
= NULL
;
3700 enum tree_code code
= gimple_assign_rhs_code (stmt
);
3702 tree bitsize
, bitpos
;
3703 tree adjustment_def
= NULL
;
3704 tree vec_initial_def
= NULL
;
3705 tree reduction_op
, expr
, def
;
3706 tree orig_name
, scalar_result
;
3707 imm_use_iterator imm_iter
, phi_imm_iter
;
3708 use_operand_p use_p
, phi_use_p
;
3709 bool extract_scalar_result
= false;
3710 gimple use_stmt
, orig_stmt
, reduction_phi
= NULL
;
3711 bool nested_in_vect_loop
= false;
3712 vec
<gimple
> new_phis
= vNULL
;
3713 vec
<gimple
> inner_phis
= vNULL
;
3714 enum vect_def_type dt
= vect_unknown_def_type
;
3716 vec
<tree
> scalar_results
= vNULL
;
3717 unsigned int group_size
= 1, k
, ratio
;
3718 vec
<tree
> vec_initial_defs
= vNULL
;
3720 bool slp_reduc
= false;
3721 tree new_phi_result
;
3722 gimple inner_phi
= NULL
;
3725 group_size
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
3727 if (nested_in_vect_loop_p (loop
, stmt
))
3731 nested_in_vect_loop
= true;
3732 gcc_assert (!slp_node
);
3735 switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt
)))
3737 case GIMPLE_SINGLE_RHS
:
3738 gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt
))
3740 reduction_op
= TREE_OPERAND (gimple_assign_rhs1 (stmt
), reduc_index
);
3742 case GIMPLE_UNARY_RHS
:
3743 reduction_op
= gimple_assign_rhs1 (stmt
);
3745 case GIMPLE_BINARY_RHS
:
3746 reduction_op
= reduc_index
?
3747 gimple_assign_rhs2 (stmt
) : gimple_assign_rhs1 (stmt
);
3749 case GIMPLE_TERNARY_RHS
:
3750 reduction_op
= gimple_op (stmt
, reduc_index
+ 1);
3756 vectype
= get_vectype_for_scalar_type (TREE_TYPE (reduction_op
));
3757 gcc_assert (vectype
);
3758 mode
= TYPE_MODE (vectype
);
3760 /* 1. Create the reduction def-use cycle:
3761 Set the arguments of REDUCTION_PHIS, i.e., transform
3764 vec_def = phi <null, null> # REDUCTION_PHI
3765 VECT_DEF = vector_stmt # vectorized form of STMT
3771 vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
3772 VECT_DEF = vector_stmt # vectorized form of STMT
3775 (in case of SLP, do it for all the phis). */
3777 /* Get the loop-entry arguments. */
3779 vect_get_vec_defs (reduction_op
, NULL_TREE
, stmt
, &vec_initial_defs
,
3780 NULL
, slp_node
, reduc_index
);
3783 vec_initial_defs
.create (1);
3784 /* For the case of reduction, vect_get_vec_def_for_operand returns
3785 the scalar def before the loop, that defines the initial value
3786 of the reduction variable. */
3787 vec_initial_def
= vect_get_vec_def_for_operand (reduction_op
, stmt
,
3789 vec_initial_defs
.quick_push (vec_initial_def
);
3792 /* Set phi nodes arguments. */
3793 FOR_EACH_VEC_ELT (reduction_phis
, i
, phi
)
3795 tree vec_init_def
, def
;
3797 vec_init_def
= force_gimple_operand (vec_initial_defs
[i
], &stmts
,
3799 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
3801 for (j
= 0; j
< ncopies
; j
++)
3803 /* Set the loop-entry arg of the reduction-phi. */
3804 add_phi_arg (phi
, vec_init_def
, loop_preheader_edge (loop
),
3807 /* Set the loop-latch arg for the reduction-phi. */
3809 def
= vect_get_vec_def_for_stmt_copy (vect_unknown_def_type
, def
);
3811 add_phi_arg (phi
, def
, loop_latch_edge (loop
), UNKNOWN_LOCATION
);
3813 if (dump_enabled_p ())
3815 dump_printf_loc (MSG_NOTE
, vect_location
,
3816 "transform reduction: created def-use cycle: ");
3817 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
3818 dump_printf (MSG_NOTE
, "\n");
3819 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, SSA_NAME_DEF_STMT (def
), 0);
3822 phi
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi
));
3826 vec_initial_defs
.release ();
3828 /* 2. Create epilog code.
3829 The reduction epilog code operates across the elements of the vector
3830 of partial results computed by the vectorized loop.
3831 The reduction epilog code consists of:
3833 step 1: compute the scalar result in a vector (v_out2)
3834 step 2: extract the scalar result (s_out3) from the vector (v_out2)
3835 step 3: adjust the scalar result (s_out3) if needed.
3837 Step 1 can be accomplished using one the following three schemes:
3838 (scheme 1) using reduc_code, if available.
3839 (scheme 2) using whole-vector shifts, if available.
3840 (scheme 3) using a scalar loop. In this case steps 1+2 above are
3843 The overall epilog code looks like this:
3845 s_out0 = phi <s_loop> # original EXIT_PHI
3846 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
3847 v_out2 = reduce <v_out1> # step 1
3848 s_out3 = extract_field <v_out2, 0> # step 2
3849 s_out4 = adjust_result <s_out3> # step 3
3851 (step 3 is optional, and steps 1 and 2 may be combined).
3852 Lastly, the uses of s_out0 are replaced by s_out4. */
3855 /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
3856 v_out1 = phi <VECT_DEF>
3857 Store them in NEW_PHIS. */
3859 exit_bb
= single_exit (loop
)->dest
;
3860 prev_phi_info
= NULL
;
3861 new_phis
.create (vect_defs
.length ());
3862 FOR_EACH_VEC_ELT (vect_defs
, i
, def
)
3864 for (j
= 0; j
< ncopies
; j
++)
3866 tree new_def
= copy_ssa_name (def
, NULL
);
3867 phi
= create_phi_node (new_def
, exit_bb
);
3868 set_vinfo_for_stmt (phi
, new_stmt_vec_info (phi
, loop_vinfo
, NULL
));
3870 new_phis
.quick_push (phi
);
3873 def
= vect_get_vec_def_for_stmt_copy (dt
, def
);
3874 STMT_VINFO_RELATED_STMT (prev_phi_info
) = phi
;
3877 SET_PHI_ARG_DEF (phi
, single_exit (loop
)->dest_idx
, def
);
3878 prev_phi_info
= vinfo_for_stmt (phi
);
3882 /* The epilogue is created for the outer-loop, i.e., for the loop being
3883 vectorized. Create exit phis for the outer loop. */
3887 exit_bb
= single_exit (loop
)->dest
;
3888 inner_phis
.create (vect_defs
.length ());
3889 FOR_EACH_VEC_ELT (new_phis
, i
, phi
)
3891 tree new_result
= copy_ssa_name (PHI_RESULT (phi
), NULL
);
3892 gimple outer_phi
= create_phi_node (new_result
, exit_bb
);
3893 SET_PHI_ARG_DEF (outer_phi
, single_exit (loop
)->dest_idx
,
3895 set_vinfo_for_stmt (outer_phi
, new_stmt_vec_info (outer_phi
,
3897 inner_phis
.quick_push (phi
);
3898 new_phis
[i
] = outer_phi
;
3899 prev_phi_info
= vinfo_for_stmt (outer_phi
);
3900 while (STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi
)))
3902 phi
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi
));
3903 new_result
= copy_ssa_name (PHI_RESULT (phi
), NULL
);
3904 outer_phi
= create_phi_node (new_result
, exit_bb
);
3905 SET_PHI_ARG_DEF (outer_phi
, single_exit (loop
)->dest_idx
,
3907 set_vinfo_for_stmt (outer_phi
, new_stmt_vec_info (outer_phi
,
3909 STMT_VINFO_RELATED_STMT (prev_phi_info
) = outer_phi
;
3910 prev_phi_info
= vinfo_for_stmt (outer_phi
);
3915 exit_gsi
= gsi_after_labels (exit_bb
);
3917 /* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
3918 (i.e. when reduc_code is not available) and in the final adjustment
3919 code (if needed). Also get the original scalar reduction variable as
3920 defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
3921 represents a reduction pattern), the tree-code and scalar-def are
3922 taken from the original stmt that the pattern-stmt (STMT) replaces.
3923 Otherwise (it is a regular reduction) - the tree-code and scalar-def
3924 are taken from STMT. */
3926 orig_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
3929 /* Regular reduction */
3934 /* Reduction pattern */
3935 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (orig_stmt
);
3936 gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
));
3937 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo
) == stmt
);
3940 code
= gimple_assign_rhs_code (orig_stmt
);
3941 /* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore,
3942 partial results are added and not subtracted. */
3943 if (code
== MINUS_EXPR
)
3946 scalar_dest
= gimple_assign_lhs (orig_stmt
);
3947 scalar_type
= TREE_TYPE (scalar_dest
);
3948 scalar_results
.create (group_size
);
3949 new_scalar_dest
= vect_create_destination_var (scalar_dest
, NULL
);
3950 bitsize
= TYPE_SIZE (scalar_type
);
3952 /* In case this is a reduction in an inner-loop while vectorizing an outer
3953 loop - we don't need to extract a single scalar result at the end of the
3954 inner-loop (unless it is double reduction, i.e., the use of reduction is
3955 outside the outer-loop). The final vector of partial results will be used
3956 in the vectorized outer-loop, or reduced to a scalar result at the end of
3958 if (nested_in_vect_loop
&& !double_reduc
)
3959 goto vect_finalize_reduction
;
3961 /* SLP reduction without reduction chain, e.g.,
3965 b2 = operation (b1) */
3966 slp_reduc
= (slp_node
&& !GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt
)));
3968 /* In case of reduction chain, e.g.,
3971 a3 = operation (a2),
3973 we may end up with more than one vector result. Here we reduce them to
3975 if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt
)))
3977 tree first_vect
= PHI_RESULT (new_phis
[0]);
3979 gimple new_vec_stmt
= NULL
;
3981 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3982 for (k
= 1; k
< new_phis
.length (); k
++)
3984 gimple next_phi
= new_phis
[k
];
3985 tree second_vect
= PHI_RESULT (next_phi
);
3987 tmp
= build2 (code
, vectype
, first_vect
, second_vect
);
3988 new_vec_stmt
= gimple_build_assign (vec_dest
, tmp
);
3989 first_vect
= make_ssa_name (vec_dest
, new_vec_stmt
);
3990 gimple_assign_set_lhs (new_vec_stmt
, first_vect
);
3991 gsi_insert_before (&exit_gsi
, new_vec_stmt
, GSI_SAME_STMT
);
3994 new_phi_result
= first_vect
;
3997 new_phis
.truncate (0);
3998 new_phis
.safe_push (new_vec_stmt
);
4002 new_phi_result
= PHI_RESULT (new_phis
[0]);
4004 /* 2.3 Create the reduction code, using one of the three schemes described
4005 above. In SLP we simply need to extract all the elements from the
4006 vector (without reducing them), so we use scalar shifts. */
4007 if (reduc_code
!= ERROR_MARK
&& !slp_reduc
)
4011 /*** Case 1: Create:
4012 v_out2 = reduc_expr <v_out1> */
4014 if (dump_enabled_p ())
4015 dump_printf_loc (MSG_NOTE
, vect_location
,
4016 "Reduce using direct vector reduction.");
4018 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4019 tmp
= build1 (reduc_code
, vectype
, new_phi_result
);
4020 epilog_stmt
= gimple_build_assign (vec_dest
, tmp
);
4021 new_temp
= make_ssa_name (vec_dest
, epilog_stmt
);
4022 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4023 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4025 extract_scalar_result
= true;
4029 enum tree_code shift_code
= ERROR_MARK
;
4030 bool have_whole_vector_shift
= true;
4032 int element_bitsize
= tree_low_cst (bitsize
, 1);
4033 int vec_size_in_bits
= tree_low_cst (TYPE_SIZE (vectype
), 1);
4036 if (optab_handler (vec_shr_optab
, mode
) != CODE_FOR_nothing
)
4037 shift_code
= VEC_RSHIFT_EXPR
;
4039 have_whole_vector_shift
= false;
4041 /* Regardless of whether we have a whole vector shift, if we're
4042 emulating the operation via tree-vect-generic, we don't want
4043 to use it. Only the first round of the reduction is likely
4044 to still be profitable via emulation. */
4045 /* ??? It might be better to emit a reduction tree code here, so that
4046 tree-vect-generic can expand the first round via bit tricks. */
4047 if (!VECTOR_MODE_P (mode
))
4048 have_whole_vector_shift
= false;
4051 optab optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4052 if (optab_handler (optab
, mode
) == CODE_FOR_nothing
)
4053 have_whole_vector_shift
= false;
4056 if (have_whole_vector_shift
&& !slp_reduc
)
4058 /*** Case 2: Create:
4059 for (offset = VS/2; offset >= element_size; offset/=2)
4061 Create: va' = vec_shift <va, offset>
4062 Create: va = vop <va, va'>
4065 if (dump_enabled_p ())
4066 dump_printf_loc (MSG_NOTE
, vect_location
,
4067 "Reduce using vector shifts");
4069 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4070 new_temp
= new_phi_result
;
4071 for (bit_offset
= vec_size_in_bits
/2;
4072 bit_offset
>= element_bitsize
;
4075 tree bitpos
= size_int (bit_offset
);
4077 epilog_stmt
= gimple_build_assign_with_ops (shift_code
,
4078 vec_dest
, new_temp
, bitpos
);
4079 new_name
= make_ssa_name (vec_dest
, epilog_stmt
);
4080 gimple_assign_set_lhs (epilog_stmt
, new_name
);
4081 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4083 epilog_stmt
= gimple_build_assign_with_ops (code
, vec_dest
,
4084 new_name
, new_temp
);
4085 new_temp
= make_ssa_name (vec_dest
, epilog_stmt
);
4086 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4087 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4090 extract_scalar_result
= true;
4096 /*** Case 3: Create:
4097 s = extract_field <v_out2, 0>
4098 for (offset = element_size;
4099 offset < vector_size;
4100 offset += element_size;)
4102 Create: s' = extract_field <v_out2, offset>
4103 Create: s = op <s, s'> // For non SLP cases
4106 if (dump_enabled_p ())
4107 dump_printf_loc (MSG_NOTE
, vect_location
,
4108 "Reduce using scalar code. ");
4110 vec_size_in_bits
= tree_low_cst (TYPE_SIZE (vectype
), 1);
4111 FOR_EACH_VEC_ELT (new_phis
, i
, new_phi
)
4113 if (gimple_code (new_phi
) == GIMPLE_PHI
)
4114 vec_temp
= PHI_RESULT (new_phi
);
4116 vec_temp
= gimple_assign_lhs (new_phi
);
4117 rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vec_temp
, bitsize
,
4119 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
4120 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
4121 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4122 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4124 /* In SLP we don't need to apply reduction operation, so we just
4125 collect s' values in SCALAR_RESULTS. */
4127 scalar_results
.safe_push (new_temp
);
4129 for (bit_offset
= element_bitsize
;
4130 bit_offset
< vec_size_in_bits
;
4131 bit_offset
+= element_bitsize
)
4133 tree bitpos
= bitsize_int (bit_offset
);
4134 tree rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vec_temp
,
4137 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
4138 new_name
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
4139 gimple_assign_set_lhs (epilog_stmt
, new_name
);
4140 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4144 /* In SLP we don't need to apply reduction operation, so
4145 we just collect s' values in SCALAR_RESULTS. */
4146 new_temp
= new_name
;
4147 scalar_results
.safe_push (new_name
);
4151 epilog_stmt
= gimple_build_assign_with_ops (code
,
4152 new_scalar_dest
, new_name
, new_temp
);
4153 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
4154 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4155 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4160 /* The only case where we need to reduce scalar results in SLP, is
4161 unrolling. If the size of SCALAR_RESULTS is greater than
4162 GROUP_SIZE, we reduce them combining elements modulo
4166 tree res
, first_res
, new_res
;
4169 /* Reduce multiple scalar results in case of SLP unrolling. */
4170 for (j
= group_size
; scalar_results
.iterate (j
, &res
);
4173 first_res
= scalar_results
[j
% group_size
];
4174 new_stmt
= gimple_build_assign_with_ops (code
,
4175 new_scalar_dest
, first_res
, res
);
4176 new_res
= make_ssa_name (new_scalar_dest
, new_stmt
);
4177 gimple_assign_set_lhs (new_stmt
, new_res
);
4178 gsi_insert_before (&exit_gsi
, new_stmt
, GSI_SAME_STMT
);
4179 scalar_results
[j
% group_size
] = new_res
;
4183 /* Not SLP - we have one scalar to keep in SCALAR_RESULTS. */
4184 scalar_results
.safe_push (new_temp
);
4186 extract_scalar_result
= false;
4190 /* 2.4 Extract the final scalar result. Create:
4191 s_out3 = extract_field <v_out2, bitpos> */
4193 if (extract_scalar_result
)
4197 if (dump_enabled_p ())
4198 dump_printf_loc (MSG_NOTE
, vect_location
,
4199 "extract scalar result");
4201 if (BYTES_BIG_ENDIAN
)
4202 bitpos
= size_binop (MULT_EXPR
,
4203 bitsize_int (TYPE_VECTOR_SUBPARTS (vectype
) - 1),
4204 TYPE_SIZE (scalar_type
));
4206 bitpos
= bitsize_zero_node
;
4208 rhs
= build3 (BIT_FIELD_REF
, scalar_type
, new_temp
, bitsize
, bitpos
);
4209 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
4210 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
4211 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4212 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4213 scalar_results
.safe_push (new_temp
);
4216 vect_finalize_reduction
:
4221 /* 2.5 Adjust the final result by the initial value of the reduction
4222 variable. (When such adjustment is not needed, then
4223 'adjustment_def' is zero). For example, if code is PLUS we create:
4224 new_temp = loop_exit_def + adjustment_def */
4228 gcc_assert (!slp_reduc
);
4229 if (nested_in_vect_loop
)
4231 new_phi
= new_phis
[0];
4232 gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def
)) == VECTOR_TYPE
);
4233 expr
= build2 (code
, vectype
, PHI_RESULT (new_phi
), adjustment_def
);
4234 new_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4238 new_temp
= scalar_results
[0];
4239 gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def
)) != VECTOR_TYPE
);
4240 expr
= build2 (code
, scalar_type
, new_temp
, adjustment_def
);
4241 new_dest
= vect_create_destination_var (scalar_dest
, scalar_type
);
4244 epilog_stmt
= gimple_build_assign (new_dest
, expr
);
4245 new_temp
= make_ssa_name (new_dest
, epilog_stmt
);
4246 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
4247 SSA_NAME_DEF_STMT (new_temp
) = epilog_stmt
;
4248 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4249 if (nested_in_vect_loop
)
4251 set_vinfo_for_stmt (epilog_stmt
,
4252 new_stmt_vec_info (epilog_stmt
, loop_vinfo
,
4254 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (epilog_stmt
)) =
4255 STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_phi
));
4258 scalar_results
.quick_push (new_temp
);
4260 scalar_results
[0] = new_temp
;
4263 scalar_results
[0] = new_temp
;
4265 new_phis
[0] = epilog_stmt
;
4268 /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
4269 phis with new adjusted scalar results, i.e., replace use <s_out0>
4274 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
4275 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
4276 v_out2 = reduce <v_out1>
4277 s_out3 = extract_field <v_out2, 0>
4278 s_out4 = adjust_result <s_out3>
4285 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
4286 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
4287 v_out2 = reduce <v_out1>
4288 s_out3 = extract_field <v_out2, 0>
4289 s_out4 = adjust_result <s_out3>
4294 /* In SLP reduction chain we reduce vector results into one vector if
4295 necessary, hence we set here GROUP_SIZE to 1. SCALAR_DEST is the LHS of
4296 the last stmt in the reduction chain, since we are looking for the loop
4298 if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt
)))
4300 scalar_dest
= gimple_assign_lhs (
4301 SLP_TREE_SCALAR_STMTS (slp_node
)[group_size
- 1]);
4305 /* In SLP we may have several statements in NEW_PHIS and REDUCTION_PHIS (in
4306 case that GROUP_SIZE is greater than vectorization factor). Therefore, we
4307 need to match SCALAR_RESULTS with corresponding statements. The first
4308 (GROUP_SIZE / number of new vector stmts) scalar results correspond to
4309 the first vector stmt, etc.
4310 (RATIO is equal to (GROUP_SIZE / number of new vector stmts)). */
4311 if (group_size
> new_phis
.length ())
4313 ratio
= group_size
/ new_phis
.length ();
4314 gcc_assert (!(group_size
% new_phis
.length ()));
4319 for (k
= 0; k
< group_size
; k
++)
4323 epilog_stmt
= new_phis
[k
/ ratio
];
4324 reduction_phi
= reduction_phis
[k
/ ratio
];
4326 inner_phi
= inner_phis
[k
/ ratio
];
4331 gimple current_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[k
];
4333 orig_stmt
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (current_stmt
));
4334 /* SLP statements can't participate in patterns. */
4335 gcc_assert (!orig_stmt
);
4336 scalar_dest
= gimple_assign_lhs (current_stmt
);
4340 /* Find the loop-closed-use at the loop exit of the original scalar
4341 result. (The reduction result is expected to have two immediate uses -
4342 one at the latch block, and one at the loop exit). */
4343 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
4344 if (!flow_bb_inside_loop_p (loop
, gimple_bb (USE_STMT (use_p
)))
4345 && !is_gimple_debug (USE_STMT (use_p
)))
4346 phis
.safe_push (USE_STMT (use_p
));
4348 /* While we expect to have found an exit_phi because of loop-closed-ssa
4349 form we can end up without one if the scalar cycle is dead. */
4351 FOR_EACH_VEC_ELT (phis
, i
, exit_phi
)
4355 stmt_vec_info exit_phi_vinfo
= vinfo_for_stmt (exit_phi
);
4358 /* FORNOW. Currently not supporting the case that an inner-loop
4359 reduction is not used in the outer-loop (but only outside the
4360 outer-loop), unless it is double reduction. */
4361 gcc_assert ((STMT_VINFO_RELEVANT_P (exit_phi_vinfo
)
4362 && !STMT_VINFO_LIVE_P (exit_phi_vinfo
))
4365 STMT_VINFO_VEC_STMT (exit_phi_vinfo
) = epilog_stmt
;
4367 || STMT_VINFO_DEF_TYPE (exit_phi_vinfo
)
4368 != vect_double_reduction_def
)
4371 /* Handle double reduction:
4373 stmt1: s1 = phi <s0, s2> - double reduction phi (outer loop)
4374 stmt2: s3 = phi <s1, s4> - (regular) reduc phi (inner loop)
4375 stmt3: s4 = use (s3) - (regular) reduc stmt (inner loop)
4376 stmt4: s2 = phi <s4> - double reduction stmt (outer loop)
4378 At that point the regular reduction (stmt2 and stmt3) is
4379 already vectorized, as well as the exit phi node, stmt4.
4380 Here we vectorize the phi node of double reduction, stmt1, and
4381 update all relevant statements. */
4383 /* Go through all the uses of s2 to find double reduction phi
4384 node, i.e., stmt1 above. */
4385 orig_name
= PHI_RESULT (exit_phi
);
4386 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, orig_name
)
4388 stmt_vec_info use_stmt_vinfo
;
4389 stmt_vec_info new_phi_vinfo
;
4390 tree vect_phi_init
, preheader_arg
, vect_phi_res
, init_def
;
4391 basic_block bb
= gimple_bb (use_stmt
);
4394 /* Check that USE_STMT is really double reduction phi
4396 if (gimple_code (use_stmt
) != GIMPLE_PHI
4397 || gimple_phi_num_args (use_stmt
) != 2
4398 || bb
->loop_father
!= outer_loop
)
4400 use_stmt_vinfo
= vinfo_for_stmt (use_stmt
);
4402 || STMT_VINFO_DEF_TYPE (use_stmt_vinfo
)
4403 != vect_double_reduction_def
)
4406 /* Create vector phi node for double reduction:
4407 vs1 = phi <vs0, vs2>
4408 vs1 was created previously in this function by a call to
4409 vect_get_vec_def_for_operand and is stored in
4411 vs2 is defined by INNER_PHI, the vectorized EXIT_PHI;
4412 vs0 is created here. */
4414 /* Create vector phi node. */
4415 vect_phi
= create_phi_node (vec_initial_def
, bb
);
4416 new_phi_vinfo
= new_stmt_vec_info (vect_phi
,
4417 loop_vec_info_for_loop (outer_loop
), NULL
);
4418 set_vinfo_for_stmt (vect_phi
, new_phi_vinfo
);
4420 /* Create vs0 - initial def of the double reduction phi. */
4421 preheader_arg
= PHI_ARG_DEF_FROM_EDGE (use_stmt
,
4422 loop_preheader_edge (outer_loop
));
4423 init_def
= get_initial_def_for_reduction (stmt
,
4424 preheader_arg
, NULL
);
4425 vect_phi_init
= vect_init_vector (use_stmt
, init_def
,
4428 /* Update phi node arguments with vs0 and vs2. */
4429 add_phi_arg (vect_phi
, vect_phi_init
,
4430 loop_preheader_edge (outer_loop
),
4432 add_phi_arg (vect_phi
, PHI_RESULT (inner_phi
),
4433 loop_latch_edge (outer_loop
), UNKNOWN_LOCATION
);
4434 if (dump_enabled_p ())
4436 dump_printf_loc (MSG_NOTE
, vect_location
,
4437 "created double reduction phi node: ");
4438 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vect_phi
, 0);
4441 vect_phi_res
= PHI_RESULT (vect_phi
);
4443 /* Replace the use, i.e., set the correct vs1 in the regular
4444 reduction phi node. FORNOW, NCOPIES is always 1, so the
4445 loop is redundant. */
4446 use
= reduction_phi
;
4447 for (j
= 0; j
< ncopies
; j
++)
4449 edge pr_edge
= loop_preheader_edge (loop
);
4450 SET_PHI_ARG_DEF (use
, pr_edge
->dest_idx
, vect_phi_res
);
4451 use
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use
));
4458 if (nested_in_vect_loop
)
4467 /* Find the loop-closed-use at the loop exit of the original scalar
4468 result. (The reduction result is expected to have two immediate uses,
4469 one at the latch block, and one at the loop exit). For double
4470 reductions we are looking for exit phis of the outer loop. */
4471 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
4473 if (!flow_bb_inside_loop_p (loop
, gimple_bb (USE_STMT (use_p
))))
4475 if (!is_gimple_debug (USE_STMT (use_p
)))
4476 phis
.safe_push (USE_STMT (use_p
));
4480 if (double_reduc
&& gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
)
4482 tree phi_res
= PHI_RESULT (USE_STMT (use_p
));
4484 FOR_EACH_IMM_USE_FAST (phi_use_p
, phi_imm_iter
, phi_res
)
4486 if (!flow_bb_inside_loop_p (loop
,
4487 gimple_bb (USE_STMT (phi_use_p
)))
4488 && !is_gimple_debug (USE_STMT (phi_use_p
)))
4489 phis
.safe_push (USE_STMT (phi_use_p
));
4495 FOR_EACH_VEC_ELT (phis
, i
, exit_phi
)
4497 /* Replace the uses: */
4498 orig_name
= PHI_RESULT (exit_phi
);
4499 scalar_result
= scalar_results
[k
];
4500 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, orig_name
)
4501 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
4502 SET_USE (use_p
, scalar_result
);
4508 scalar_results
.release ();
4509 inner_phis
.release ();
4510 new_phis
.release ();
4514 /* Function vectorizable_reduction.
4516 Check if STMT performs a reduction operation that can be vectorized.
4517 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4518 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
4519 Return FALSE if not a vectorizable STMT, TRUE otherwise.
4521 This function also handles reduction idioms (patterns) that have been
4522 recognized in advance during vect_pattern_recog. In this case, STMT may be
4524 X = pattern_expr (arg0, arg1, ..., X)
4525 and it's STMT_VINFO_RELATED_STMT points to the last stmt in the original
4526 sequence that had been detected and replaced by the pattern-stmt (STMT).
4528 In some cases of reduction patterns, the type of the reduction variable X is
4529 different than the type of the other arguments of STMT.
4530 In such cases, the vectype that is used when transforming STMT into a vector
4531 stmt is different than the vectype that is used to determine the
4532 vectorization factor, because it consists of a different number of elements
4533 than the actual number of elements that are being operated upon in parallel.
4535 For example, consider an accumulation of shorts into an int accumulator.
4536 On some targets it's possible to vectorize this pattern operating on 8
4537 shorts at a time (hence, the vectype for purposes of determining the
4538 vectorization factor should be V8HI); on the other hand, the vectype that
4539 is used to create the vector form is actually V4SI (the type of the result).
4541 Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
4542 indicates what is the actual level of parallelism (V8HI in the example), so
4543 that the right vectorization factor would be derived. This vectype
4544 corresponds to the type of arguments to the reduction stmt, and should *NOT*
4545 be used to create the vectorized stmt. The right vectype for the vectorized
4546 stmt is obtained from the type of the result X:
4547 get_vectype_for_scalar_type (TREE_TYPE (X))
4549 This means that, contrary to "regular" reductions (or "regular" stmts in
4550 general), the following equation:
4551 STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (TREE_TYPE (X))
4552 does *NOT* necessarily hold for reduction patterns. */
4555 vectorizable_reduction (gimple stmt
, gimple_stmt_iterator
*gsi
,
4556 gimple
*vec_stmt
, slp_tree slp_node
)
4560 tree loop_vec_def0
= NULL_TREE
, loop_vec_def1
= NULL_TREE
;
4561 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4562 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4563 tree vectype_in
= NULL_TREE
;
4564 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4565 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
4566 enum tree_code code
, orig_code
, epilog_reduc_code
;
4567 enum machine_mode vec_mode
;
4569 optab optab
, reduc_optab
;
4570 tree new_temp
= NULL_TREE
;
4573 enum vect_def_type dt
;
4574 gimple new_phi
= NULL
;
4578 stmt_vec_info orig_stmt_info
;
4579 tree expr
= NULL_TREE
;
4583 stmt_vec_info prev_stmt_info
, prev_phi_info
;
4584 bool single_defuse_cycle
= false;
4585 tree reduc_def
= NULL_TREE
;
4586 gimple new_stmt
= NULL
;
4589 bool nested_cycle
= false, found_nested_cycle_def
= false;
4590 gimple reduc_def_stmt
= NULL
;
4591 /* The default is that the reduction variable is the last in statement. */
4592 int reduc_index
= 2;
4593 bool double_reduc
= false, dummy
;
4595 struct loop
* def_stmt_loop
, *outer_loop
= NULL
;
4597 gimple def_arg_stmt
;
4598 vec
<tree
> vec_oprnds0
= vNULL
;
4599 vec
<tree
> vec_oprnds1
= vNULL
;
4600 vec
<tree
> vect_defs
= vNULL
;
4601 vec
<gimple
> phis
= vNULL
;
4603 tree def0
, def1
, tem
, op0
, op1
= NULL_TREE
;
4605 /* In case of reduction chain we switch to the first stmt in the chain, but
4606 we don't update STMT_INFO, since only the last stmt is marked as reduction
4607 and has reduction properties. */
4608 if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt
)))
4609 stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
4611 if (nested_in_vect_loop_p (loop
, stmt
))
4615 nested_cycle
= true;
4618 /* 1. Is vectorizable reduction? */
4619 /* Not supportable if the reduction variable is used in the loop, unless
4620 it's a reduction chain. */
4621 if (STMT_VINFO_RELEVANT (stmt_info
) > vect_used_in_outer
4622 && !GROUP_FIRST_ELEMENT (stmt_info
))
4625 /* Reductions that are not used even in an enclosing outer-loop,
4626 are expected to be "live" (used out of the loop). */
4627 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_unused_in_scope
4628 && !STMT_VINFO_LIVE_P (stmt_info
))
4631 /* Make sure it was already recognized as a reduction computation. */
4632 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_reduction_def
4633 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_nested_cycle
)
4636 /* 2. Has this been recognized as a reduction pattern?
4638 Check if STMT represents a pattern that has been recognized
4639 in earlier analysis stages. For stmts that represent a pattern,
4640 the STMT_VINFO_RELATED_STMT field records the last stmt in
4641 the original sequence that constitutes the pattern. */
4643 orig_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
4646 orig_stmt_info
= vinfo_for_stmt (orig_stmt
);
4647 gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info
));
4648 gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info
));
4651 /* 3. Check the operands of the operation. The first operands are defined
4652 inside the loop body. The last operand is the reduction variable,
4653 which is defined by the loop-header-phi. */
4655 gcc_assert (is_gimple_assign (stmt
));
4658 switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt
)))
4660 case GIMPLE_SINGLE_RHS
:
4661 op_type
= TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt
));
4662 if (op_type
== ternary_op
)
4664 tree rhs
= gimple_assign_rhs1 (stmt
);
4665 ops
[0] = TREE_OPERAND (rhs
, 0);
4666 ops
[1] = TREE_OPERAND (rhs
, 1);
4667 ops
[2] = TREE_OPERAND (rhs
, 2);
4668 code
= TREE_CODE (rhs
);
4674 case GIMPLE_BINARY_RHS
:
4675 code
= gimple_assign_rhs_code (stmt
);
4676 op_type
= TREE_CODE_LENGTH (code
);
4677 gcc_assert (op_type
== binary_op
);
4678 ops
[0] = gimple_assign_rhs1 (stmt
);
4679 ops
[1] = gimple_assign_rhs2 (stmt
);
4682 case GIMPLE_TERNARY_RHS
:
4683 code
= gimple_assign_rhs_code (stmt
);
4684 op_type
= TREE_CODE_LENGTH (code
);
4685 gcc_assert (op_type
== ternary_op
);
4686 ops
[0] = gimple_assign_rhs1 (stmt
);
4687 ops
[1] = gimple_assign_rhs2 (stmt
);
4688 ops
[2] = gimple_assign_rhs3 (stmt
);
4691 case GIMPLE_UNARY_RHS
:
4698 if (code
== COND_EXPR
&& slp_node
)
4701 scalar_dest
= gimple_assign_lhs (stmt
);
4702 scalar_type
= TREE_TYPE (scalar_dest
);
4703 if (!POINTER_TYPE_P (scalar_type
) && !INTEGRAL_TYPE_P (scalar_type
)
4704 && !SCALAR_FLOAT_TYPE_P (scalar_type
))
4707 /* Do not try to vectorize bit-precision reductions. */
4708 if ((TYPE_PRECISION (scalar_type
)
4709 != GET_MODE_PRECISION (TYPE_MODE (scalar_type
))))
4712 /* All uses but the last are expected to be defined in the loop.
4713 The last use is the reduction variable. In case of nested cycle this
4714 assumption is not true: we use reduc_index to record the index of the
4715 reduction variable. */
4716 for (i
= 0; i
< op_type
- 1; i
++)
4718 /* The condition of COND_EXPR is checked in vectorizable_condition(). */
4719 if (i
== 0 && code
== COND_EXPR
)
4722 is_simple_use
= vect_is_simple_use_1 (ops
[i
], stmt
, loop_vinfo
, NULL
,
4723 &def_stmt
, &def
, &dt
, &tem
);
4726 gcc_assert (is_simple_use
);
4728 if (dt
!= vect_internal_def
4729 && dt
!= vect_external_def
4730 && dt
!= vect_constant_def
4731 && dt
!= vect_induction_def
4732 && !(dt
== vect_nested_cycle
&& nested_cycle
))
4735 if (dt
== vect_nested_cycle
)
4737 found_nested_cycle_def
= true;
4738 reduc_def_stmt
= def_stmt
;
4743 is_simple_use
= vect_is_simple_use_1 (ops
[i
], stmt
, loop_vinfo
, NULL
,
4744 &def_stmt
, &def
, &dt
, &tem
);
4747 gcc_assert (is_simple_use
);
4748 if (!(dt
== vect_reduction_def
4749 || dt
== vect_nested_cycle
4750 || ((dt
== vect_internal_def
|| dt
== vect_external_def
4751 || dt
== vect_constant_def
|| dt
== vect_induction_def
)
4752 && nested_cycle
&& found_nested_cycle_def
)))
4754 /* For pattern recognized stmts, orig_stmt might be a reduction,
4755 but some helper statements for the pattern might not, or
4756 might be COND_EXPRs with reduction uses in the condition. */
4757 gcc_assert (orig_stmt
);
4760 if (!found_nested_cycle_def
)
4761 reduc_def_stmt
= def_stmt
;
4763 gcc_assert (gimple_code (reduc_def_stmt
) == GIMPLE_PHI
);
4765 gcc_assert (orig_stmt
== vect_is_simple_reduction (loop_vinfo
,
4771 gimple tmp
= vect_is_simple_reduction (loop_vinfo
, reduc_def_stmt
,
4772 !nested_cycle
, &dummy
);
4773 /* We changed STMT to be the first stmt in reduction chain, hence we
4774 check that in this case the first element in the chain is STMT. */
4775 gcc_assert (stmt
== tmp
4776 || GROUP_FIRST_ELEMENT (vinfo_for_stmt (tmp
)) == stmt
);
4779 if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt
)))
4782 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4785 ncopies
= (LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
4786 / TYPE_VECTOR_SUBPARTS (vectype_in
));
4788 gcc_assert (ncopies
>= 1);
4790 vec_mode
= TYPE_MODE (vectype_in
);
4792 if (code
== COND_EXPR
)
4794 if (!vectorizable_condition (stmt
, gsi
, NULL
, ops
[reduc_index
], 0, NULL
))
4796 if (dump_enabled_p ())
4797 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4798 "unsupported condition in reduction");
4805 /* 4. Supportable by target? */
4807 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
4808 || code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
4810 /* Shifts and rotates are only supported by vectorizable_shifts,
4811 not vectorizable_reduction. */
4812 if (dump_enabled_p ())
4813 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4814 "unsupported shift or rotation.");
4818 /* 4.1. check support for the operation in the loop */
4819 optab
= optab_for_tree_code (code
, vectype_in
, optab_default
);
4822 if (dump_enabled_p ())
4823 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4829 if (optab_handler (optab
, vec_mode
) == CODE_FOR_nothing
)
4831 if (dump_enabled_p ())
4832 dump_printf (MSG_NOTE
, "op not supported by target.");
4834 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4835 || LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
4836 < vect_min_worthwhile_factor (code
))
4839 if (dump_enabled_p ())
4840 dump_printf (MSG_NOTE
, "proceeding using word mode.");
4843 /* Worthwhile without SIMD support? */
4844 if (!VECTOR_MODE_P (TYPE_MODE (vectype_in
))
4845 && LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
4846 < vect_min_worthwhile_factor (code
))
4848 if (dump_enabled_p ())
4849 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4850 "not worthwhile without SIMD support.");
4856 /* 4.2. Check support for the epilog operation.
4858 If STMT represents a reduction pattern, then the type of the
4859 reduction variable may be different than the type of the rest
4860 of the arguments. For example, consider the case of accumulation
4861 of shorts into an int accumulator; The original code:
4862 S1: int_a = (int) short_a;
4863 orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
4866 STMT: int_acc = widen_sum <short_a, int_acc>
4869 1. The tree-code that is used to create the vector operation in the
4870 epilog code (that reduces the partial results) is not the
4871 tree-code of STMT, but is rather the tree-code of the original
4872 stmt from the pattern that STMT is replacing. I.e, in the example
4873 above we want to use 'widen_sum' in the loop, but 'plus' in the
4875 2. The type (mode) we use to check available target support
4876 for the vector operation to be created in the *epilog*, is
4877 determined by the type of the reduction variable (in the example
4878 above we'd check this: optab_handler (plus_optab, vect_int_mode])).
4879 However the type (mode) we use to check available target support
4880 for the vector operation to be created *inside the loop*, is
4881 determined by the type of the other arguments to STMT (in the
4882 example we'd check this: optab_handler (widen_sum_optab,
4885 This is contrary to "regular" reductions, in which the types of all
4886 the arguments are the same as the type of the reduction variable.
4887 For "regular" reductions we can therefore use the same vector type
4888 (and also the same tree-code) when generating the epilog code and
4889 when generating the code inside the loop. */
4893 /* This is a reduction pattern: get the vectype from the type of the
4894 reduction variable, and get the tree-code from orig_stmt. */
4895 orig_code
= gimple_assign_rhs_code (orig_stmt
);
4896 gcc_assert (vectype_out
);
4897 vec_mode
= TYPE_MODE (vectype_out
);
4901 /* Regular reduction: use the same vectype and tree-code as used for
4902 the vector code inside the loop can be used for the epilog code. */
4908 def_bb
= gimple_bb (reduc_def_stmt
);
4909 def_stmt_loop
= def_bb
->loop_father
;
4910 def_arg
= PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt
,
4911 loop_preheader_edge (def_stmt_loop
));
4912 if (TREE_CODE (def_arg
) == SSA_NAME
4913 && (def_arg_stmt
= SSA_NAME_DEF_STMT (def_arg
))
4914 && gimple_code (def_arg_stmt
) == GIMPLE_PHI
4915 && flow_bb_inside_loop_p (outer_loop
, gimple_bb (def_arg_stmt
))
4916 && vinfo_for_stmt (def_arg_stmt
)
4917 && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_arg_stmt
))
4918 == vect_double_reduction_def
)
4919 double_reduc
= true;
4922 epilog_reduc_code
= ERROR_MARK
;
4923 if (reduction_code_for_scalar_code (orig_code
, &epilog_reduc_code
))
4925 reduc_optab
= optab_for_tree_code (epilog_reduc_code
, vectype_out
,
4929 if (dump_enabled_p ())
4930 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4931 "no optab for reduction.");
4933 epilog_reduc_code
= ERROR_MARK
;
4937 && optab_handler (reduc_optab
, vec_mode
) == CODE_FOR_nothing
)
4939 if (dump_enabled_p ())
4940 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4941 "reduc op not supported by target.");
4943 epilog_reduc_code
= ERROR_MARK
;
4948 if (!nested_cycle
|| double_reduc
)
4950 if (dump_enabled_p ())
4951 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4952 "no reduc code for scalar code.");
4958 if (double_reduc
&& ncopies
> 1)
4960 if (dump_enabled_p ())
4961 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4962 "multiple types in double reduction");
4967 /* In case of widenning multiplication by a constant, we update the type
4968 of the constant to be the type of the other operand. We check that the
4969 constant fits the type in the pattern recognition pass. */
4970 if (code
== DOT_PROD_EXPR
4971 && !types_compatible_p (TREE_TYPE (ops
[0]), TREE_TYPE (ops
[1])))
4973 if (TREE_CODE (ops
[0]) == INTEGER_CST
)
4974 ops
[0] = fold_convert (TREE_TYPE (ops
[1]), ops
[0]);
4975 else if (TREE_CODE (ops
[1]) == INTEGER_CST
)
4976 ops
[1] = fold_convert (TREE_TYPE (ops
[0]), ops
[1]);
4979 if (dump_enabled_p ())
4980 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4981 "invalid types in dot-prod");
4987 if (!vec_stmt
) /* transformation not required. */
4989 if (!vect_model_reduction_cost (stmt_info
, epilog_reduc_code
, ncopies
))
4991 STMT_VINFO_TYPE (stmt_info
) = reduc_vec_info_type
;
4997 if (dump_enabled_p ())
4998 dump_printf_loc (MSG_NOTE
, vect_location
, "transform reduction.");
5000 /* FORNOW: Multiple types are not supported for condition. */
5001 if (code
== COND_EXPR
)
5002 gcc_assert (ncopies
== 1);
5004 /* Create the destination vector */
5005 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
5007 /* In case the vectorization factor (VF) is bigger than the number
5008 of elements that we can fit in a vectype (nunits), we have to generate
5009 more than one vector stmt - i.e - we need to "unroll" the
5010 vector stmt by a factor VF/nunits. For more details see documentation
5011 in vectorizable_operation. */
5013 /* If the reduction is used in an outer loop we need to generate
5014 VF intermediate results, like so (e.g. for ncopies=2):
5019 (i.e. we generate VF results in 2 registers).
5020 In this case we have a separate def-use cycle for each copy, and therefore
5021 for each copy we get the vector def for the reduction variable from the
5022 respective phi node created for this copy.
5024 Otherwise (the reduction is unused in the loop nest), we can combine
5025 together intermediate results, like so (e.g. for ncopies=2):
5029 (i.e. we generate VF/2 results in a single register).
5030 In this case for each copy we get the vector def for the reduction variable
5031 from the vectorized reduction operation generated in the previous iteration.
5034 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_unused_in_scope
)
5036 single_defuse_cycle
= true;
5040 epilog_copies
= ncopies
;
5042 prev_stmt_info
= NULL
;
5043 prev_phi_info
= NULL
;
5046 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5047 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype_out
)
5048 == TYPE_VECTOR_SUBPARTS (vectype_in
));
5053 vec_oprnds0
.create (1);
5054 if (op_type
== ternary_op
)
5055 vec_oprnds1
.create (1);
5058 phis
.create (vec_num
);
5059 vect_defs
.create (vec_num
);
5061 vect_defs
.quick_push (NULL_TREE
);
5063 for (j
= 0; j
< ncopies
; j
++)
5065 if (j
== 0 || !single_defuse_cycle
)
5067 for (i
= 0; i
< vec_num
; i
++)
5069 /* Create the reduction-phi that defines the reduction
5071 new_phi
= create_phi_node (vec_dest
, loop
->header
);
5072 set_vinfo_for_stmt (new_phi
,
5073 new_stmt_vec_info (new_phi
, loop_vinfo
,
5075 if (j
== 0 || slp_node
)
5076 phis
.quick_push (new_phi
);
5080 if (code
== COND_EXPR
)
5082 gcc_assert (!slp_node
);
5083 vectorizable_condition (stmt
, gsi
, vec_stmt
,
5084 PHI_RESULT (phis
[0]),
5086 /* Multiple types are not supported for condition. */
5093 op0
= ops
[!reduc_index
];
5094 if (op_type
== ternary_op
)
5096 if (reduc_index
== 0)
5103 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
5107 loop_vec_def0
= vect_get_vec_def_for_operand (ops
[!reduc_index
],
5109 vec_oprnds0
.quick_push (loop_vec_def0
);
5110 if (op_type
== ternary_op
)
5112 loop_vec_def1
= vect_get_vec_def_for_operand (op1
, stmt
,
5114 vec_oprnds1
.quick_push (loop_vec_def1
);
5122 enum vect_def_type dt
;
5126 vect_is_simple_use (ops
[!reduc_index
], stmt
, loop_vinfo
, NULL
,
5127 &dummy_stmt
, &dummy
, &dt
);
5128 loop_vec_def0
= vect_get_vec_def_for_stmt_copy (dt
,
5130 vec_oprnds0
[0] = loop_vec_def0
;
5131 if (op_type
== ternary_op
)
5133 vect_is_simple_use (op1
, stmt
, loop_vinfo
, NULL
, &dummy_stmt
,
5135 loop_vec_def1
= vect_get_vec_def_for_stmt_copy (dt
,
5137 vec_oprnds1
[0] = loop_vec_def1
;
5141 if (single_defuse_cycle
)
5142 reduc_def
= gimple_assign_lhs (new_stmt
);
5144 STMT_VINFO_RELATED_STMT (prev_phi_info
) = new_phi
;
5147 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, def0
)
5150 reduc_def
= PHI_RESULT (phis
[i
]);
5153 if (!single_defuse_cycle
|| j
== 0)
5154 reduc_def
= PHI_RESULT (new_phi
);
5157 def1
= ((op_type
== ternary_op
)
5158 ? vec_oprnds1
[i
] : NULL
);
5159 if (op_type
== binary_op
)
5161 if (reduc_index
== 0)
5162 expr
= build2 (code
, vectype_out
, reduc_def
, def0
);
5164 expr
= build2 (code
, vectype_out
, def0
, reduc_def
);
5168 if (reduc_index
== 0)
5169 expr
= build3 (code
, vectype_out
, reduc_def
, def0
, def1
);
5172 if (reduc_index
== 1)
5173 expr
= build3 (code
, vectype_out
, def0
, reduc_def
, def1
);
5175 expr
= build3 (code
, vectype_out
, def0
, def1
, reduc_def
);
5179 new_stmt
= gimple_build_assign (vec_dest
, expr
);
5180 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5181 gimple_assign_set_lhs (new_stmt
, new_temp
);
5182 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5186 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
5187 vect_defs
.quick_push (new_temp
);
5190 vect_defs
[0] = new_temp
;
5197 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5199 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5201 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5202 prev_phi_info
= vinfo_for_stmt (new_phi
);
5205 /* Finalize the reduction-phi (set its arguments) and create the
5206 epilog reduction code. */
5207 if ((!single_defuse_cycle
|| code
== COND_EXPR
) && !slp_node
)
5209 new_temp
= gimple_assign_lhs (*vec_stmt
);
5210 vect_defs
[0] = new_temp
;
5213 vect_create_epilog_for_reduction (vect_defs
, stmt
, epilog_copies
,
5214 epilog_reduc_code
, phis
, reduc_index
,
5215 double_reduc
, slp_node
);
5218 vect_defs
.release ();
5219 vec_oprnds0
.release ();
5220 vec_oprnds1
.release ();
5225 /* Function vect_min_worthwhile_factor.
5227 For a loop where we could vectorize the operation indicated by CODE,
5228 return the minimum vectorization factor that makes it worthwhile
5229 to use generic vectors. */
5231 vect_min_worthwhile_factor (enum tree_code code
)
5252 /* Function vectorizable_induction
5254 Check if PHI performs an induction computation that can be vectorized.
5255 If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
5256 phi to replace it, put it in VEC_STMT, and add it to the same basic block.
5257 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5260 vectorizable_induction (gimple phi
, gimple_stmt_iterator
*gsi ATTRIBUTE_UNUSED
,
5263 stmt_vec_info stmt_info
= vinfo_for_stmt (phi
);
5264 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5265 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5266 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5267 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5268 int ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5271 gcc_assert (ncopies
>= 1);
5272 /* FORNOW. These restrictions should be relaxed. */
5273 if (nested_in_vect_loop_p (loop
, phi
))
5275 imm_use_iterator imm_iter
;
5276 use_operand_p use_p
;
5283 if (dump_enabled_p ())
5284 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5285 "multiple types in nested loop.");
5290 latch_e
= loop_latch_edge (loop
->inner
);
5291 loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
5292 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, loop_arg
)
5294 if (!flow_bb_inside_loop_p (loop
->inner
,
5295 gimple_bb (USE_STMT (use_p
))))
5297 exit_phi
= USE_STMT (use_p
);
5303 stmt_vec_info exit_phi_vinfo
= vinfo_for_stmt (exit_phi
);
5304 if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo
)
5305 && !STMT_VINFO_LIVE_P (exit_phi_vinfo
)))
5307 if (dump_enabled_p ())
5308 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5309 "inner-loop induction only used outside "
5310 "of the outer vectorized loop.");
5316 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
5319 /* FORNOW: SLP not supported. */
5320 if (STMT_SLP_TYPE (stmt_info
))
5323 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
);
5325 if (gimple_code (phi
) != GIMPLE_PHI
)
5328 if (!vec_stmt
) /* transformation not required. */
5330 STMT_VINFO_TYPE (stmt_info
) = induc_vec_info_type
;
5331 if (dump_enabled_p ())
5332 dump_printf_loc (MSG_NOTE
, vect_location
,
5333 "=== vectorizable_induction ===");
5334 vect_model_induction_cost (stmt_info
, ncopies
);
5340 if (dump_enabled_p ())
5341 dump_printf_loc (MSG_NOTE
, vect_location
, "transform induction phi.");
5343 vec_def
= get_initial_def_for_induction (phi
);
5344 *vec_stmt
= SSA_NAME_DEF_STMT (vec_def
);
5348 /* Function vectorizable_live_operation.
5350 STMT computes a value that is used outside the loop. Check if
5351 it can be supported. */
5354 vectorizable_live_operation (gimple stmt
,
5355 gimple_stmt_iterator
*gsi ATTRIBUTE_UNUSED
,
5356 gimple
*vec_stmt ATTRIBUTE_UNUSED
)
5358 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5359 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5360 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5366 enum vect_def_type dt
;
5367 enum tree_code code
;
5368 enum gimple_rhs_class rhs_class
;
5370 gcc_assert (STMT_VINFO_LIVE_P (stmt_info
));
5372 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
)
5375 if (!is_gimple_assign (stmt
))
5378 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
5381 /* FORNOW. CHECKME. */
5382 if (nested_in_vect_loop_p (loop
, stmt
))
5385 code
= gimple_assign_rhs_code (stmt
);
5386 op_type
= TREE_CODE_LENGTH (code
);
5387 rhs_class
= get_gimple_rhs_class (code
);
5388 gcc_assert (rhs_class
!= GIMPLE_UNARY_RHS
|| op_type
== unary_op
);
5389 gcc_assert (rhs_class
!= GIMPLE_BINARY_RHS
|| op_type
== binary_op
);
5391 /* FORNOW: support only if all uses are invariant. This means
5392 that the scalar operations can remain in place, unvectorized.
5393 The original last scalar value that they compute will be used. */
5395 for (i
= 0; i
< op_type
; i
++)
5397 if (rhs_class
== GIMPLE_SINGLE_RHS
)
5398 op
= TREE_OPERAND (gimple_op (stmt
, 1), i
);
5400 op
= gimple_op (stmt
, i
+ 1);
5402 && !vect_is_simple_use (op
, stmt
, loop_vinfo
, NULL
, &def_stmt
, &def
,
5405 if (dump_enabled_p ())
5406 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5411 if (dt
!= vect_external_def
&& dt
!= vect_constant_def
)
5415 /* No transformation is required for the cases we currently support. */
5419 /* Kill any debug uses outside LOOP of SSA names defined in STMT. */
5422 vect_loop_kill_debug_uses (struct loop
*loop
, gimple stmt
)
5424 ssa_op_iter op_iter
;
5425 imm_use_iterator imm_iter
;
5426 def_operand_p def_p
;
5429 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
5431 FOR_EACH_IMM_USE_STMT (ustmt
, imm_iter
, DEF_FROM_PTR (def_p
))
5435 if (!is_gimple_debug (ustmt
))
5438 bb
= gimple_bb (ustmt
);
5440 if (!flow_bb_inside_loop_p (loop
, bb
))
5442 if (gimple_debug_bind_p (ustmt
))
5444 if (dump_enabled_p ())
5445 dump_printf_loc (MSG_NOTE
, vect_location
,
5446 "killing debug use");
5448 gimple_debug_bind_reset_value (ustmt
);
5449 update_stmt (ustmt
);
5458 /* Function vect_transform_loop.
5460 The analysis phase has determined that the loop is vectorizable.
5461 Vectorize the loop - created vectorized stmts to replace the scalar
5462 stmts in the loop, and update the loop exit condition. */
5465 vect_transform_loop (loop_vec_info loop_vinfo
)
5467 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5468 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
5469 int nbbs
= loop
->num_nodes
;
5470 gimple_stmt_iterator si
;
5473 int vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5475 bool slp_scheduled
= false;
5476 unsigned int nunits
;
5477 gimple stmt
, pattern_stmt
;
5478 gimple_seq pattern_def_seq
= NULL
;
5479 gimple_stmt_iterator pattern_def_si
= gsi_none ();
5480 bool transform_pattern_stmt
= false;
5481 bool check_profitability
= false;
5483 /* Record number of iterations before we started tampering with the profile. */
5484 gcov_type expected_iterations
= expected_loop_iterations_unbounded (loop
);
5486 if (dump_enabled_p ())
5487 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vec_transform_loop ===");
5489 /* If profile is inprecise, we have chance to fix it up. */
5490 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
5491 expected_iterations
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
5493 /* Use the more conservative vectorization threshold. If the number
5494 of iterations is constant assume the cost check has been performed
5495 by our caller. If the threshold makes all loops profitable that
5496 run at least the vectorization factor number of times checking
5497 is pointless, too. */
5498 th
= ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND
)
5499 * LOOP_VINFO_VECT_FACTOR (loop_vinfo
)) - 1);
5500 th
= MAX (th
, LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo
));
5501 if (th
>= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) - 1
5502 && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
5504 if (dump_enabled_p ())
5505 dump_printf_loc (MSG_NOTE
, vect_location
,
5506 "Profitability threshold is %d loop iterations.", th
);
5507 check_profitability
= true;
5510 /* Peel the loop if there are data refs with unknown alignment.
5511 Only one data ref with unknown store is allowed. */
5513 if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
))
5515 vect_do_peeling_for_alignment (loop_vinfo
, th
, check_profitability
);
5516 check_profitability
= false;
5519 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
)
5520 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
5522 vect_loop_versioning (loop_vinfo
, th
, check_profitability
);
5523 check_profitability
= false;
5526 /* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
5527 compile time constant), or it is a constant that doesn't divide by the
5528 vectorization factor, then an epilog loop needs to be created.
5529 We therefore duplicate the loop: the original loop will be vectorized,
5530 and will compute the first (n/VF) iterations. The second copy of the loop
5531 will remain scalar and will compute the remaining (n%VF) iterations.
5532 (VF is the vectorization factor). */
5534 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
5535 || (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
5536 && LOOP_VINFO_INT_NITERS (loop_vinfo
) % vectorization_factor
!= 0)
5537 || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
5538 vect_do_peeling_for_loop_bound (loop_vinfo
, &ratio
,
5539 th
, check_profitability
);
5541 ratio
= build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
)),
5542 LOOP_VINFO_INT_NITERS (loop_vinfo
) / vectorization_factor
);
5544 /* 1) Make sure the loop header has exactly two entries
5545 2) Make sure we have a preheader basic block. */
5547 gcc_assert (EDGE_COUNT (loop
->header
->preds
) == 2);
5549 split_edge (loop_preheader_edge (loop
));
5551 /* FORNOW: the vectorizer supports only loops which body consist
5552 of one basic block (header + empty latch). When the vectorizer will
5553 support more involved loop forms, the order by which the BBs are
5554 traversed need to be reconsidered. */
5556 for (i
= 0; i
< nbbs
; i
++)
5558 basic_block bb
= bbs
[i
];
5559 stmt_vec_info stmt_info
;
5562 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
5564 phi
= gsi_stmt (si
);
5565 if (dump_enabled_p ())
5567 dump_printf_loc (MSG_NOTE
, vect_location
,
5568 "------>vectorizing phi: ");
5569 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
5571 stmt_info
= vinfo_for_stmt (phi
);
5575 if (MAY_HAVE_DEBUG_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
5576 vect_loop_kill_debug_uses (loop
, phi
);
5578 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
5579 && !STMT_VINFO_LIVE_P (stmt_info
))
5582 if (STMT_VINFO_VECTYPE (stmt_info
)
5583 && (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info
))
5584 != (unsigned HOST_WIDE_INT
) vectorization_factor
)
5585 && dump_enabled_p ())
5586 dump_printf_loc (MSG_NOTE
, vect_location
, "multiple-types.");
5588 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
)
5590 if (dump_enabled_p ())
5591 dump_printf_loc (MSG_NOTE
, vect_location
, "transform phi.");
5592 vect_transform_stmt (phi
, NULL
, NULL
, NULL
, NULL
);
5596 pattern_stmt
= NULL
;
5597 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
) || transform_pattern_stmt
;)
5601 if (transform_pattern_stmt
)
5602 stmt
= pattern_stmt
;
5604 stmt
= gsi_stmt (si
);
5606 if (dump_enabled_p ())
5608 dump_printf_loc (MSG_NOTE
, vect_location
,
5609 "------>vectorizing statement: ");
5610 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
5613 stmt_info
= vinfo_for_stmt (stmt
);
5615 /* vector stmts created in the outer-loop during vectorization of
5616 stmts in an inner-loop may not have a stmt_info, and do not
5617 need to be vectorized. */
5624 if (MAY_HAVE_DEBUG_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
5625 vect_loop_kill_debug_uses (loop
, stmt
);
5627 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
5628 && !STMT_VINFO_LIVE_P (stmt_info
))
5630 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
5631 && (pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
))
5632 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
5633 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
5635 stmt
= pattern_stmt
;
5636 stmt_info
= vinfo_for_stmt (stmt
);
5644 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
5645 && (pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
))
5646 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
5647 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
5648 transform_pattern_stmt
= true;
5650 /* If pattern statement has def stmts, vectorize them too. */
5651 if (is_pattern_stmt_p (stmt_info
))
5653 if (pattern_def_seq
== NULL
)
5655 pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
5656 pattern_def_si
= gsi_start (pattern_def_seq
);
5658 else if (!gsi_end_p (pattern_def_si
))
5659 gsi_next (&pattern_def_si
);
5660 if (pattern_def_seq
!= NULL
)
5662 gimple pattern_def_stmt
= NULL
;
5663 stmt_vec_info pattern_def_stmt_info
= NULL
;
5665 while (!gsi_end_p (pattern_def_si
))
5667 pattern_def_stmt
= gsi_stmt (pattern_def_si
);
5668 pattern_def_stmt_info
5669 = vinfo_for_stmt (pattern_def_stmt
);
5670 if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info
)
5671 || STMT_VINFO_LIVE_P (pattern_def_stmt_info
))
5673 gsi_next (&pattern_def_si
);
5676 if (!gsi_end_p (pattern_def_si
))
5678 if (dump_enabled_p ())
5680 dump_printf_loc (MSG_NOTE
, vect_location
,
5681 "==> vectorizing pattern def "
5683 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
,
5684 pattern_def_stmt
, 0);
5687 stmt
= pattern_def_stmt
;
5688 stmt_info
= pattern_def_stmt_info
;
5692 pattern_def_si
= gsi_none ();
5693 transform_pattern_stmt
= false;
5697 transform_pattern_stmt
= false;
5700 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
));
5701 nunits
= (unsigned int) TYPE_VECTOR_SUBPARTS (
5702 STMT_VINFO_VECTYPE (stmt_info
));
5703 if (!STMT_SLP_TYPE (stmt_info
)
5704 && nunits
!= (unsigned int) vectorization_factor
5705 && dump_enabled_p ())
5706 /* For SLP VF is set according to unrolling factor, and not to
5707 vector size, hence for SLP this print is not valid. */
5708 dump_printf_loc (MSG_NOTE
, vect_location
,
5711 /* SLP. Schedule all the SLP instances when the first SLP stmt is
5713 if (STMT_SLP_TYPE (stmt_info
))
5717 slp_scheduled
= true;
5719 if (dump_enabled_p ())
5720 dump_printf_loc (MSG_NOTE
, vect_location
,
5721 "=== scheduling SLP instances ===");
5723 vect_schedule_slp (loop_vinfo
, NULL
);
5726 /* Hybrid SLP stmts must be vectorized in addition to SLP. */
5727 if (!vinfo_for_stmt (stmt
) || PURE_SLP_STMT (stmt_info
))
5729 if (!transform_pattern_stmt
&& gsi_end_p (pattern_def_si
))
5731 pattern_def_seq
= NULL
;
5738 /* -------- vectorize statement ------------ */
5739 if (dump_enabled_p ())
5740 dump_printf_loc (MSG_NOTE
, vect_location
, "transform statement.");
5742 grouped_store
= false;
5743 is_store
= vect_transform_stmt (stmt
, &si
, &grouped_store
, NULL
, NULL
);
5746 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5748 /* Interleaving. If IS_STORE is TRUE, the vectorization of the
5749 interleaving chain was completed - free all the stores in
5752 vect_remove_stores (GROUP_FIRST_ELEMENT (stmt_info
));
5757 /* Free the attached stmt_vec_info and remove the stmt. */
5758 gimple store
= gsi_stmt (si
);
5759 free_stmt_vec_info (store
);
5760 unlink_stmt_vdef (store
);
5761 gsi_remove (&si
, true);
5762 release_defs (store
);
5767 if (!transform_pattern_stmt
&& gsi_end_p (pattern_def_si
))
5769 pattern_def_seq
= NULL
;
5775 slpeel_make_loop_iterate_ntimes (loop
, ratio
);
5777 /* Reduce loop iterations by the vectorization factor. */
5778 scale_loop_profile (loop
, RDIV (REG_BR_PROB_BASE
, vectorization_factor
),
5779 expected_iterations
/ vectorization_factor
);
5780 loop
->nb_iterations_upper_bound
5781 = loop
->nb_iterations_upper_bound
.udiv (double_int::from_uhwi (vectorization_factor
),
5783 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
5784 && loop
->nb_iterations_upper_bound
!= double_int_zero
)
5785 loop
->nb_iterations_upper_bound
= loop
->nb_iterations_upper_bound
- double_int_one
;
5786 if (loop
->any_estimate
)
5788 loop
->nb_iterations_estimate
5789 = loop
->nb_iterations_estimate
.udiv (double_int::from_uhwi (vectorization_factor
),
5791 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
5792 && loop
->nb_iterations_estimate
!= double_int_zero
)
5793 loop
->nb_iterations_estimate
= loop
->nb_iterations_estimate
- double_int_one
;
5796 /* The memory tags and pointers in vectorized statements need to
5797 have their SSA forms updated. FIXME, why can't this be delayed
5798 until all the loops have been transformed? */
5799 update_ssa (TODO_update_ssa
);
5801 if (dump_enabled_p ())
5802 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
, "LOOP VECTORIZED.");
5803 if (loop
->inner
&& dump_enabled_p ())
5804 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
,
5805 "OUTER LOOP VECTORIZED.");