2 Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
22 /* Loop Vectorization Pass.
24 This pass tries to vectorize loops. This first implementation focuses on
25 simple inner-most loops, with no conditional control flow, and a set of
26 simple operations which vector form can be expressed using existing
27 tree codes (PLUS, MULT etc).
29 For example, the vectorizer transforms the following simple loop:
31 short a[N]; short b[N]; short c[N]; int i;
37 as if it was manually vectorized by rewriting the source code into:
39 typedef int __attribute__((mode(V8HI))) v8hi;
40 short a[N]; short b[N]; short c[N]; int i;
41 v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
44 for (i=0; i<N/8; i++){
51 The main entry to this pass is vectorize_loops(), in which
52 the vectorizer applies a set of analyses on a given set of loops,
53 followed by the actual vectorization transformation for the loops that
54 had successfully passed the analysis phase.
56 Throughout this pass we make a distinction between two types of
57 data: scalars (which are represented by SSA_NAMES), and memory references
58 ("data-refs"). These two types of data require different handling both
59 during analysis and transformation. The types of data-refs that the
60 vectorizer currently supports are ARRAY_REFS which base is an array DECL
61 (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
62 accesses are required to have a simple (consecutive) access pattern.
66 The driver for the analysis phase is vect_analyze_loop_nest().
67 It applies a set of analyses, some of which rely on the scalar evolution
68 analyzer (scev) developed by Sebastian Pop.
70 During the analysis phase the vectorizer records some information
71 per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
72 loop, as well as general information about the loop as a whole, which is
73 recorded in a "loop_vec_info" struct attached to each loop.
77 The loop transformation phase scans all the stmts in the loop, and
78 creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
79 the loop that needs to be vectorized. It insert the vector code sequence
80 just before the scalar stmt S, and records a pointer to the vector code
81 in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
82 attached to S). This pointer will be used for the vectorization of following
83 stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
84 otherwise, we rely on dead code elimination for removing it.
86 For example, say stmt S1 was vectorized into stmt VS1:
89 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
92 To vectorize stmt S2, the vectorizer first finds the stmt that defines
93 the operand 'b' (S1), and gets the relevant vector def 'vb' from the
94 vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
95 resulting sequence would be:
98 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
100 S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
102 Operands that are not SSA_NAMEs, are data-refs that appear in
103 load/store operations (like 'x[i]' in S1), and are handled differently.
107 Currently the only target specific information that is used is the
108 size of the vector (in bytes) - "UNITS_PER_SIMD_WORD". Targets that can
109 support different sizes of vectors, for now will need to specify one value
110 for "UNITS_PER_SIMD_WORD". More flexibility will be added in the future.
112 Since we only vectorize operations which vector form can be
113 expressed using existing tree codes, to verify that an operation is
114 supported, the vectorizer checks the relevant optab at the relevant
115 machine_mode (e.g, add_optab->handlers[(int) V8HImode].insn_code). If
116 the value found is CODE_FOR_nothing, then there's no target support, and
117 we can't vectorize the stmt.
119 For additional information on this project see:
120 http://gcc.gnu.org/projects/tree-ssa/vectorization.html
125 #include "coretypes.h"
131 #include "basic-block.h"
132 #include "diagnostic.h"
133 #include "tree-flow.h"
134 #include "tree-dump.h"
137 #include "cfglayout.h"
143 #include "tree-chrec.h"
144 #include "tree-data-ref.h"
145 #include "tree-scalar-evolution.h"
147 #include "tree-vectorizer.h"
148 #include "tree-pass.h"
150 /*************************************************************************
151 Simple Loop Peeling Utilities
152 *************************************************************************/
153 static void slpeel_update_phis_for_duplicate_loop
154 (struct loop
*, struct loop
*, bool after
);
155 static void slpeel_update_phi_nodes_for_guard1
156 (edge
, struct loop
*, bool, basic_block
*, bitmap
*);
157 static void slpeel_update_phi_nodes_for_guard2
158 (edge
, struct loop
*, bool, basic_block
*);
159 static edge
slpeel_add_loop_guard (basic_block
, tree
, basic_block
, basic_block
);
161 static void rename_use_op (use_operand_p
);
162 static void rename_variables_in_bb (basic_block
);
163 static void rename_variables_in_loop (struct loop
*);
165 /*************************************************************************
166 General Vectorization Utilities
167 *************************************************************************/
168 static void vect_set_dump_settings (void);
170 /* vect_dump will be set to stderr or dump_file if exist. */
173 /* vect_verbosity_level set to an invalid value
174 to mark that it's uninitialized. */
175 enum verbosity_levels vect_verbosity_level
= MAX_VERBOSITY_LEVEL
;
178 static LOC vect_loop_location
;
180 /* Bitmap of virtual variables to be renamed. */
181 bitmap vect_memsyms_to_rename
;
183 /*************************************************************************
184 Simple Loop Peeling Utilities
186 Utilities to support loop peeling for vectorization purposes.
187 *************************************************************************/
190 /* Renames the use *OP_P. */
193 rename_use_op (use_operand_p op_p
)
197 if (TREE_CODE (USE_FROM_PTR (op_p
)) != SSA_NAME
)
200 new_name
= get_current_def (USE_FROM_PTR (op_p
));
202 /* Something defined outside of the loop. */
206 /* An ordinary ssa name defined in the loop. */
208 SET_USE (op_p
, new_name
);
212 /* Renames the variables in basic block BB. */
215 rename_variables_in_bb (basic_block bb
)
218 block_stmt_iterator bsi
;
224 struct loop
*loop
= bb
->loop_father
;
226 for (bsi
= bsi_start (bb
); !bsi_end_p (bsi
); bsi_next (&bsi
))
228 stmt
= bsi_stmt (bsi
);
229 FOR_EACH_SSA_USE_OPERAND (use_p
, stmt
, iter
, SSA_OP_ALL_USES
)
230 rename_use_op (use_p
);
233 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
235 if (!flow_bb_inside_loop_p (loop
, e
->dest
))
237 for (phi
= phi_nodes (e
->dest
); phi
; phi
= PHI_CHAIN (phi
))
238 rename_use_op (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
));
243 /* Renames variables in new generated LOOP. */
246 rename_variables_in_loop (struct loop
*loop
)
251 bbs
= get_loop_body (loop
);
253 for (i
= 0; i
< loop
->num_nodes
; i
++)
254 rename_variables_in_bb (bbs
[i
]);
260 /* Update the PHI nodes of NEW_LOOP.
262 NEW_LOOP is a duplicate of ORIG_LOOP.
263 AFTER indicates whether NEW_LOOP executes before or after ORIG_LOOP:
264 AFTER is true if NEW_LOOP executes after ORIG_LOOP, and false if it
265 executes before it. */
268 slpeel_update_phis_for_duplicate_loop (struct loop
*orig_loop
,
269 struct loop
*new_loop
, bool after
)
272 tree phi_new
, phi_orig
;
274 edge orig_loop_latch
= loop_latch_edge (orig_loop
);
275 edge orig_entry_e
= loop_preheader_edge (orig_loop
);
276 edge new_loop_exit_e
= single_exit (new_loop
);
277 edge new_loop_entry_e
= loop_preheader_edge (new_loop
);
278 edge entry_arg_e
= (after
? orig_loop_latch
: orig_entry_e
);
281 step 1. For each loop-header-phi:
282 Add the first phi argument for the phi in NEW_LOOP
283 (the one associated with the entry of NEW_LOOP)
285 step 2. For each loop-header-phi:
286 Add the second phi argument for the phi in NEW_LOOP
287 (the one associated with the latch of NEW_LOOP)
289 step 3. Update the phis in the successor block of NEW_LOOP.
291 case 1: NEW_LOOP was placed before ORIG_LOOP:
292 The successor block of NEW_LOOP is the header of ORIG_LOOP.
293 Updating the phis in the successor block can therefore be done
294 along with the scanning of the loop header phis, because the
295 header blocks of ORIG_LOOP and NEW_LOOP have exactly the same
296 phi nodes, organized in the same order.
298 case 2: NEW_LOOP was placed after ORIG_LOOP:
299 The successor block of NEW_LOOP is the original exit block of
300 ORIG_LOOP - the phis to be updated are the loop-closed-ssa phis.
301 We postpone updating these phis to a later stage (when
302 loop guards are added).
306 /* Scan the phis in the headers of the old and new loops
307 (they are organized in exactly the same order). */
309 for (phi_new
= phi_nodes (new_loop
->header
),
310 phi_orig
= phi_nodes (orig_loop
->header
);
312 phi_new
= PHI_CHAIN (phi_new
), phi_orig
= PHI_CHAIN (phi_orig
))
315 def
= PHI_ARG_DEF_FROM_EDGE (phi_orig
, entry_arg_e
);
316 add_phi_arg (phi_new
, def
, new_loop_entry_e
);
319 def
= PHI_ARG_DEF_FROM_EDGE (phi_orig
, orig_loop_latch
);
320 if (TREE_CODE (def
) != SSA_NAME
)
323 new_ssa_name
= get_current_def (def
);
326 /* This only happens if there are no definitions
327 inside the loop. use the phi_result in this case. */
328 new_ssa_name
= PHI_RESULT (phi_new
);
331 /* An ordinary ssa name defined in the loop. */
332 add_phi_arg (phi_new
, new_ssa_name
, loop_latch_edge (new_loop
));
334 /* step 3 (case 1). */
337 gcc_assert (new_loop_exit_e
== orig_entry_e
);
338 SET_PHI_ARG_DEF (phi_orig
,
339 new_loop_exit_e
->dest_idx
,
346 /* Update PHI nodes for a guard of the LOOP.
349 - LOOP, GUARD_EDGE: LOOP is a loop for which we added guard code that
350 controls whether LOOP is to be executed. GUARD_EDGE is the edge that
351 originates from the guard-bb, skips LOOP and reaches the (unique) exit
352 bb of LOOP. This loop-exit-bb is an empty bb with one successor.
353 We denote this bb NEW_MERGE_BB because before the guard code was added
354 it had a single predecessor (the LOOP header), and now it became a merge
355 point of two paths - the path that ends with the LOOP exit-edge, and
356 the path that ends with GUARD_EDGE.
357 - NEW_EXIT_BB: New basic block that is added by this function between LOOP
358 and NEW_MERGE_BB. It is used to place loop-closed-ssa-form exit-phis.
360 ===> The CFG before the guard-code was added:
363 if (exit_loop) goto update_bb
364 else goto LOOP_header_bb
367 ==> The CFG after the guard-code was added:
369 if (LOOP_guard_condition) goto new_merge_bb
370 else goto LOOP_header_bb
373 if (exit_loop_condition) goto new_merge_bb
374 else goto LOOP_header_bb
379 ==> The CFG after this function:
381 if (LOOP_guard_condition) goto new_merge_bb
382 else goto LOOP_header_bb
385 if (exit_loop_condition) goto new_exit_bb
386 else goto LOOP_header_bb
393 1. creates and updates the relevant phi nodes to account for the new
394 incoming edge (GUARD_EDGE) into NEW_MERGE_BB. This involves:
395 1.1. Create phi nodes at NEW_MERGE_BB.
396 1.2. Update the phi nodes at the successor of NEW_MERGE_BB (denoted
397 UPDATE_BB). UPDATE_BB was the exit-bb of LOOP before NEW_MERGE_BB
398 2. preserves loop-closed-ssa-form by creating the required phi nodes
399 at the exit of LOOP (i.e, in NEW_EXIT_BB).
401 There are two flavors to this function:
403 slpeel_update_phi_nodes_for_guard1:
404 Here the guard controls whether we enter or skip LOOP, where LOOP is a
405 prolog_loop (loop1 below), and the new phis created in NEW_MERGE_BB are
406 for variables that have phis in the loop header.
408 slpeel_update_phi_nodes_for_guard2:
409 Here the guard controls whether we enter or skip LOOP, where LOOP is an
410 epilog_loop (loop2 below), and the new phis created in NEW_MERGE_BB are
411 for variables that have phis in the loop exit.
413 I.E., the overall structure is:
416 guard1 (goto loop1/merg1_bb)
419 guard2 (goto merge1_bb/merge2_bb)
426 slpeel_update_phi_nodes_for_guard1 takes care of creating phis in
427 loop1_exit_bb and merge1_bb. These are entry phis (phis for the vars
428 that have phis in loop1->header).
430 slpeel_update_phi_nodes_for_guard2 takes care of creating phis in
431 loop2_exit_bb and merge2_bb. These are exit phis (phis for the vars
432 that have phis in next_bb). It also adds some of these phis to
435 slpeel_update_phi_nodes_for_guard1 is always called before
436 slpeel_update_phi_nodes_for_guard2. They are both needed in order
437 to create correct data-flow and loop-closed-ssa-form.
439 Generally slpeel_update_phi_nodes_for_guard1 creates phis for variables
440 that change between iterations of a loop (and therefore have a phi-node
441 at the loop entry), whereas slpeel_update_phi_nodes_for_guard2 creates
442 phis for variables that are used out of the loop (and therefore have
443 loop-closed exit phis). Some variables may be both updated between
444 iterations and used after the loop. This is why in loop1_exit_bb we
445 may need both entry_phis (created by slpeel_update_phi_nodes_for_guard1)
446 and exit phis (created by slpeel_update_phi_nodes_for_guard2).
448 - IS_NEW_LOOP: if IS_NEW_LOOP is true, then LOOP is a newly created copy of
449 an original loop. i.e., we have:
452 guard_bb (goto LOOP/new_merge)
458 If IS_NEW_LOOP is false, then LOOP is an original loop, in which case we
462 guard_bb (goto LOOP/new_merge)
468 The SSA names defined in the original loop have a current
469 reaching definition that that records the corresponding new
470 ssa-name used in the new duplicated loop copy.
473 /* Function slpeel_update_phi_nodes_for_guard1
476 - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.
477 - DEFS - a bitmap of ssa names to mark new names for which we recorded
480 In the context of the overall structure, we have:
483 guard1 (goto loop1/merg1_bb)
486 guard2 (goto merge1_bb/merge2_bb)
493 For each name updated between loop iterations (i.e - for each name that has
494 an entry (loop-header) phi in LOOP) we create a new phi in:
495 1. merge1_bb (to account for the edge from guard1)
496 2. loop1_exit_bb (an exit-phi to keep LOOP in loop-closed form)
500 slpeel_update_phi_nodes_for_guard1 (edge guard_edge
, struct loop
*loop
,
501 bool is_new_loop
, basic_block
*new_exit_bb
,
504 tree orig_phi
, new_phi
;
505 tree update_phi
, update_phi2
;
506 tree guard_arg
, loop_arg
;
507 basic_block new_merge_bb
= guard_edge
->dest
;
508 edge e
= EDGE_SUCC (new_merge_bb
, 0);
509 basic_block update_bb
= e
->dest
;
510 basic_block orig_bb
= loop
->header
;
512 tree current_new_name
;
515 /* Create new bb between loop and new_merge_bb. */
516 *new_exit_bb
= split_edge (single_exit (loop
));
518 new_exit_e
= EDGE_SUCC (*new_exit_bb
, 0);
520 for (orig_phi
= phi_nodes (orig_bb
), update_phi
= phi_nodes (update_bb
);
521 orig_phi
&& update_phi
;
522 orig_phi
= PHI_CHAIN (orig_phi
), update_phi
= PHI_CHAIN (update_phi
))
524 /* Virtual phi; Mark it for renaming. We actually want to call
525 mar_sym_for_renaming, but since all ssa renaming datastructures
526 are going to be freed before we get to call ssa_upate, we just
527 record this name for now in a bitmap, and will mark it for
529 name
= PHI_RESULT (orig_phi
);
530 if (!is_gimple_reg (SSA_NAME_VAR (name
)))
531 bitmap_set_bit (vect_memsyms_to_rename
, DECL_UID (SSA_NAME_VAR (name
)));
533 /** 1. Handle new-merge-point phis **/
535 /* 1.1. Generate new phi node in NEW_MERGE_BB: */
536 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
539 /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge
540 of LOOP. Set the two phi args in NEW_PHI for these edges: */
541 loop_arg
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, EDGE_SUCC (loop
->latch
, 0));
542 guard_arg
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, loop_preheader_edge (loop
));
544 add_phi_arg (new_phi
, loop_arg
, new_exit_e
);
545 add_phi_arg (new_phi
, guard_arg
, guard_edge
);
547 /* 1.3. Update phi in successor block. */
548 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == loop_arg
549 || PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == guard_arg
);
550 SET_PHI_ARG_DEF (update_phi
, e
->dest_idx
, PHI_RESULT (new_phi
));
551 update_phi2
= new_phi
;
554 /** 2. Handle loop-closed-ssa-form phis **/
556 if (!is_gimple_reg (PHI_RESULT (orig_phi
)))
559 /* 2.1. Generate new phi node in NEW_EXIT_BB: */
560 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
563 /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop. */
564 add_phi_arg (new_phi
, loop_arg
, single_exit (loop
));
566 /* 2.3. Update phi in successor of NEW_EXIT_BB: */
567 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, new_exit_e
) == loop_arg
);
568 SET_PHI_ARG_DEF (update_phi2
, new_exit_e
->dest_idx
, PHI_RESULT (new_phi
));
570 /* 2.4. Record the newly created name with set_current_def.
571 We want to find a name such that
572 name = get_current_def (orig_loop_name)
573 and to set its current definition as follows:
574 set_current_def (name, new_phi_name)
576 If LOOP is a new loop then loop_arg is already the name we're
577 looking for. If LOOP is the original loop, then loop_arg is
578 the orig_loop_name and the relevant name is recorded in its
579 current reaching definition. */
581 current_new_name
= loop_arg
;
584 current_new_name
= get_current_def (loop_arg
);
585 /* current_def is not available only if the variable does not
586 change inside the loop, in which case we also don't care
587 about recording a current_def for it because we won't be
588 trying to create loop-exit-phis for it. */
589 if (!current_new_name
)
592 gcc_assert (get_current_def (current_new_name
) == NULL_TREE
);
594 set_current_def (current_new_name
, PHI_RESULT (new_phi
));
595 bitmap_set_bit (*defs
, SSA_NAME_VERSION (current_new_name
));
598 set_phi_nodes (new_merge_bb
, phi_reverse (phi_nodes (new_merge_bb
)));
602 /* Function slpeel_update_phi_nodes_for_guard2
605 - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.
607 In the context of the overall structure, we have:
610 guard1 (goto loop1/merg1_bb)
613 guard2 (goto merge1_bb/merge2_bb)
620 For each name used out side the loop (i.e - for each name that has an exit
621 phi in next_bb) we create a new phi in:
622 1. merge2_bb (to account for the edge from guard_bb)
623 2. loop2_exit_bb (an exit-phi to keep LOOP in loop-closed form)
624 3. guard2 bb (an exit phi to keep the preceding loop in loop-closed form),
625 if needed (if it wasn't handled by slpeel_update_phis_nodes_for_phi1).
629 slpeel_update_phi_nodes_for_guard2 (edge guard_edge
, struct loop
*loop
,
630 bool is_new_loop
, basic_block
*new_exit_bb
)
632 tree orig_phi
, new_phi
;
633 tree update_phi
, update_phi2
;
634 tree guard_arg
, loop_arg
;
635 basic_block new_merge_bb
= guard_edge
->dest
;
636 edge e
= EDGE_SUCC (new_merge_bb
, 0);
637 basic_block update_bb
= e
->dest
;
639 tree orig_def
, orig_def_new_name
;
640 tree new_name
, new_name2
;
643 /* Create new bb between loop and new_merge_bb. */
644 *new_exit_bb
= split_edge (single_exit (loop
));
646 new_exit_e
= EDGE_SUCC (*new_exit_bb
, 0);
648 for (update_phi
= phi_nodes (update_bb
); update_phi
;
649 update_phi
= PHI_CHAIN (update_phi
))
651 orig_phi
= update_phi
;
652 orig_def
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, e
);
653 /* This loop-closed-phi actually doesn't represent a use
654 out of the loop - the phi arg is a constant. */
655 if (TREE_CODE (orig_def
) != SSA_NAME
)
657 orig_def_new_name
= get_current_def (orig_def
);
660 /** 1. Handle new-merge-point phis **/
662 /* 1.1. Generate new phi node in NEW_MERGE_BB: */
663 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
666 /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge
667 of LOOP. Set the two PHI args in NEW_PHI for these edges: */
669 new_name2
= NULL_TREE
;
670 if (orig_def_new_name
)
672 new_name
= orig_def_new_name
;
673 /* Some variables have both loop-entry-phis and loop-exit-phis.
674 Such variables were given yet newer names by phis placed in
675 guard_bb by slpeel_update_phi_nodes_for_guard1. I.e:
676 new_name2 = get_current_def (get_current_def (orig_name)). */
677 new_name2
= get_current_def (new_name
);
682 guard_arg
= orig_def
;
687 guard_arg
= new_name
;
691 guard_arg
= new_name2
;
693 add_phi_arg (new_phi
, loop_arg
, new_exit_e
);
694 add_phi_arg (new_phi
, guard_arg
, guard_edge
);
696 /* 1.3. Update phi in successor block. */
697 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == orig_def
);
698 SET_PHI_ARG_DEF (update_phi
, e
->dest_idx
, PHI_RESULT (new_phi
));
699 update_phi2
= new_phi
;
702 /** 2. Handle loop-closed-ssa-form phis **/
704 /* 2.1. Generate new phi node in NEW_EXIT_BB: */
705 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
708 /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop. */
709 add_phi_arg (new_phi
, loop_arg
, single_exit (loop
));
711 /* 2.3. Update phi in successor of NEW_EXIT_BB: */
712 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, new_exit_e
) == loop_arg
);
713 SET_PHI_ARG_DEF (update_phi2
, new_exit_e
->dest_idx
, PHI_RESULT (new_phi
));
716 /** 3. Handle loop-closed-ssa-form phis for first loop **/
718 /* 3.1. Find the relevant names that need an exit-phi in
719 GUARD_BB, i.e. names for which
720 slpeel_update_phi_nodes_for_guard1 had not already created a
721 phi node. This is the case for names that are used outside
722 the loop (and therefore need an exit phi) but are not updated
723 across loop iterations (and therefore don't have a
726 slpeel_update_phi_nodes_for_guard1 is responsible for
727 creating loop-exit phis in GUARD_BB for names that have a
728 loop-header-phi. When such a phi is created we also record
729 the new name in its current definition. If this new name
730 exists, then guard_arg was set to this new name (see 1.2
731 above). Therefore, if guard_arg is not this new name, this
732 is an indication that an exit-phi in GUARD_BB was not yet
733 created, so we take care of it here. */
734 if (guard_arg
== new_name2
)
738 /* 3.2. Generate new phi node in GUARD_BB: */
739 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
742 /* 3.3. GUARD_BB has one incoming edge: */
743 gcc_assert (EDGE_COUNT (guard_edge
->src
->preds
) == 1);
744 add_phi_arg (new_phi
, arg
, EDGE_PRED (guard_edge
->src
, 0));
746 /* 3.4. Update phi in successor of GUARD_BB: */
747 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, guard_edge
)
749 SET_PHI_ARG_DEF (update_phi2
, guard_edge
->dest_idx
, PHI_RESULT (new_phi
));
752 set_phi_nodes (new_merge_bb
, phi_reverse (phi_nodes (new_merge_bb
)));
756 /* Make the LOOP iterate NITERS times. This is done by adding a new IV
757 that starts at zero, increases by one and its limit is NITERS.
759 Assumption: the exit-condition of LOOP is the last stmt in the loop. */
762 slpeel_make_loop_iterate_ntimes (struct loop
*loop
, tree niters
)
764 tree indx_before_incr
, indx_after_incr
, cond_stmt
, cond
;
766 edge exit_edge
= single_exit (loop
);
767 block_stmt_iterator loop_cond_bsi
;
768 block_stmt_iterator incr_bsi
;
770 tree begin_label
= tree_block_label (loop
->latch
);
771 tree exit_label
= tree_block_label (single_exit (loop
)->dest
);
772 tree init
= build_int_cst (TREE_TYPE (niters
), 0);
773 tree step
= build_int_cst (TREE_TYPE (niters
), 1);
778 orig_cond
= get_loop_exit_condition (loop
);
779 gcc_assert (orig_cond
);
780 loop_cond_bsi
= bsi_for_stmt (orig_cond
);
782 standard_iv_increment_position (loop
, &incr_bsi
, &insert_after
);
783 create_iv (init
, step
, NULL_TREE
, loop
,
784 &incr_bsi
, insert_after
, &indx_before_incr
, &indx_after_incr
);
786 if (exit_edge
->flags
& EDGE_TRUE_VALUE
) /* 'then' edge exits the loop. */
788 cond
= build2 (GE_EXPR
, boolean_type_node
, indx_after_incr
, niters
);
789 then_label
= build1 (GOTO_EXPR
, void_type_node
, exit_label
);
790 else_label
= build1 (GOTO_EXPR
, void_type_node
, begin_label
);
792 else /* 'then' edge loops back. */
794 cond
= build2 (LT_EXPR
, boolean_type_node
, indx_after_incr
, niters
);
795 then_label
= build1 (GOTO_EXPR
, void_type_node
, begin_label
);
796 else_label
= build1 (GOTO_EXPR
, void_type_node
, exit_label
);
799 cond_stmt
= build3 (COND_EXPR
, TREE_TYPE (orig_cond
), cond
,
800 then_label
, else_label
);
801 bsi_insert_before (&loop_cond_bsi
, cond_stmt
, BSI_SAME_STMT
);
803 /* Remove old loop exit test: */
804 bsi_remove (&loop_cond_bsi
, true);
806 loop_loc
= find_loop_location (loop
);
807 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
809 if (loop_loc
!= UNKNOWN_LOC
)
810 fprintf (dump_file
, "\nloop at %s:%d: ",
811 LOC_FILE (loop_loc
), LOC_LINE (loop_loc
));
812 print_generic_expr (dump_file
, cond_stmt
, TDF_SLIM
);
815 loop
->nb_iterations
= niters
;
819 /* Given LOOP this function generates a new copy of it and puts it
820 on E which is either the entry or exit of LOOP. */
823 slpeel_tree_duplicate_loop_to_edge_cfg (struct loop
*loop
, edge e
)
825 struct loop
*new_loop
;
826 basic_block
*new_bbs
, *bbs
;
829 basic_block exit_dest
;
833 at_exit
= (e
== single_exit (loop
));
834 if (!at_exit
&& e
!= loop_preheader_edge (loop
))
837 bbs
= get_loop_body (loop
);
839 /* Check whether duplication is possible. */
840 if (!can_copy_bbs_p (bbs
, loop
->num_nodes
))
846 /* Generate new loop structure. */
847 new_loop
= duplicate_loop (loop
, loop
->outer
);
854 exit_dest
= single_exit (loop
)->dest
;
855 was_imm_dom
= (get_immediate_dominator (CDI_DOMINATORS
,
856 exit_dest
) == loop
->header
?
859 new_bbs
= XNEWVEC (basic_block
, loop
->num_nodes
);
861 exit
= single_exit (loop
);
862 copy_bbs (bbs
, loop
->num_nodes
, new_bbs
,
863 &exit
, 1, &new_exit
, NULL
,
865 set_single_exit (new_loop
, new_exit
);
867 /* Duplicating phi args at exit bbs as coming
868 also from exit of duplicated loop. */
869 for (phi
= phi_nodes (exit_dest
); phi
; phi
= PHI_CHAIN (phi
))
871 phi_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, single_exit (loop
));
874 edge new_loop_exit_edge
;
876 if (EDGE_SUCC (new_loop
->header
, 0)->dest
== new_loop
->latch
)
877 new_loop_exit_edge
= EDGE_SUCC (new_loop
->header
, 1);
879 new_loop_exit_edge
= EDGE_SUCC (new_loop
->header
, 0);
881 add_phi_arg (phi
, phi_arg
, new_loop_exit_edge
);
885 if (at_exit
) /* Add the loop copy at exit. */
887 redirect_edge_and_branch_force (e
, new_loop
->header
);
888 set_immediate_dominator (CDI_DOMINATORS
, new_loop
->header
, e
->src
);
890 set_immediate_dominator (CDI_DOMINATORS
, exit_dest
, new_loop
->header
);
892 else /* Add the copy at entry. */
895 edge entry_e
= loop_preheader_edge (loop
);
896 basic_block preheader
= entry_e
->src
;
898 if (!flow_bb_inside_loop_p (new_loop
,
899 EDGE_SUCC (new_loop
->header
, 0)->dest
))
900 new_exit_e
= EDGE_SUCC (new_loop
->header
, 0);
902 new_exit_e
= EDGE_SUCC (new_loop
->header
, 1);
904 redirect_edge_and_branch_force (new_exit_e
, loop
->header
);
905 set_immediate_dominator (CDI_DOMINATORS
, loop
->header
,
908 /* We have to add phi args to the loop->header here as coming
909 from new_exit_e edge. */
910 for (phi
= phi_nodes (loop
->header
); phi
; phi
= PHI_CHAIN (phi
))
912 phi_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, entry_e
);
914 add_phi_arg (phi
, phi_arg
, new_exit_e
);
917 redirect_edge_and_branch_force (entry_e
, new_loop
->header
);
918 set_immediate_dominator (CDI_DOMINATORS
, new_loop
->header
, preheader
);
928 /* Given the condition statement COND, put it as the last statement
929 of GUARD_BB; EXIT_BB is the basic block to skip the loop;
930 Assumes that this is the single exit of the guarded loop.
931 Returns the skip edge. */
934 slpeel_add_loop_guard (basic_block guard_bb
, tree cond
, basic_block exit_bb
,
937 block_stmt_iterator bsi
;
939 tree cond_stmt
, then_label
, else_label
;
941 enter_e
= EDGE_SUCC (guard_bb
, 0);
942 enter_e
->flags
&= ~EDGE_FALLTHRU
;
943 enter_e
->flags
|= EDGE_FALSE_VALUE
;
944 bsi
= bsi_last (guard_bb
);
946 then_label
= build1 (GOTO_EXPR
, void_type_node
,
947 tree_block_label (exit_bb
));
948 else_label
= build1 (GOTO_EXPR
, void_type_node
,
949 tree_block_label (enter_e
->dest
));
950 cond_stmt
= build3 (COND_EXPR
, void_type_node
, cond
,
951 then_label
, else_label
);
952 bsi_insert_after (&bsi
, cond_stmt
, BSI_NEW_STMT
);
953 /* Add new edge to connect guard block to the merge/loop-exit block. */
954 new_e
= make_edge (guard_bb
, exit_bb
, EDGE_TRUE_VALUE
);
955 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, dom_bb
);
960 /* This function verifies that the following restrictions apply to LOOP:
962 (2) it consists of exactly 2 basic blocks - header, and an empty latch.
963 (3) it is single entry, single exit
964 (4) its exit condition is the last stmt in the header
965 (5) E is the entry/exit edge of LOOP.
969 slpeel_can_duplicate_loop_p (struct loop
*loop
, edge e
)
971 edge exit_e
= single_exit (loop
);
972 edge entry_e
= loop_preheader_edge (loop
);
973 tree orig_cond
= get_loop_exit_condition (loop
);
974 block_stmt_iterator loop_exit_bsi
= bsi_last (exit_e
->src
);
976 if (need_ssa_update_p ())
980 /* All loops have an outer scope; the only case loop->outer is NULL is for
981 the function itself. */
983 || loop
->num_nodes
!= 2
984 || !empty_block_p (loop
->latch
)
985 || !single_exit (loop
)
986 /* Verify that new loop exit condition can be trivially modified. */
987 || (!orig_cond
|| orig_cond
!= bsi_stmt (loop_exit_bsi
))
988 || (e
!= exit_e
&& e
!= entry_e
))
994 #ifdef ENABLE_CHECKING
996 slpeel_verify_cfg_after_peeling (struct loop
*first_loop
,
997 struct loop
*second_loop
)
999 basic_block loop1_exit_bb
= single_exit (first_loop
)->dest
;
1000 basic_block loop2_entry_bb
= loop_preheader_edge (second_loop
)->src
;
1001 basic_block loop1_entry_bb
= loop_preheader_edge (first_loop
)->src
;
1003 /* A guard that controls whether the second_loop is to be executed or skipped
1004 is placed in first_loop->exit. first_loopt->exit therefore has two
1005 successors - one is the preheader of second_loop, and the other is a bb
1008 gcc_assert (EDGE_COUNT (loop1_exit_bb
->succs
) == 2);
1010 /* 1. Verify that one of the successors of first_loopt->exit is the preheader
1013 /* The preheader of new_loop is expected to have two predecessors:
1014 first_loop->exit and the block that precedes first_loop. */
1016 gcc_assert (EDGE_COUNT (loop2_entry_bb
->preds
) == 2
1017 && ((EDGE_PRED (loop2_entry_bb
, 0)->src
== loop1_exit_bb
1018 && EDGE_PRED (loop2_entry_bb
, 1)->src
== loop1_entry_bb
)
1019 || (EDGE_PRED (loop2_entry_bb
, 1)->src
== loop1_exit_bb
1020 && EDGE_PRED (loop2_entry_bb
, 0)->src
== loop1_entry_bb
)));
1022 /* Verify that the other successor of first_loopt->exit is after the
1028 /* Function slpeel_tree_peel_loop_to_edge.
1030 Peel the first (last) iterations of LOOP into a new prolog (epilog) loop
1031 that is placed on the entry (exit) edge E of LOOP. After this transformation
1032 we have two loops one after the other - first-loop iterates FIRST_NITERS
1033 times, and second-loop iterates the remainder NITERS - FIRST_NITERS times.
1036 - LOOP: the loop to be peeled.
1037 - E: the exit or entry edge of LOOP.
1038 If it is the entry edge, we peel the first iterations of LOOP. In this
1039 case first-loop is LOOP, and second-loop is the newly created loop.
1040 If it is the exit edge, we peel the last iterations of LOOP. In this
1041 case, first-loop is the newly created loop, and second-loop is LOOP.
1042 - NITERS: the number of iterations that LOOP iterates.
1043 - FIRST_NITERS: the number of iterations that the first-loop should iterate.
1044 - UPDATE_FIRST_LOOP_COUNT: specified whether this function is responsible
1045 for updating the loop bound of the first-loop to FIRST_NITERS. If it
1046 is false, the caller of this function may want to take care of this
1047 (this can be useful if we don't want new stmts added to first-loop).
1050 The function returns a pointer to the new loop-copy, or NULL if it failed
1051 to perform the transformation.
1053 The function generates two if-then-else guards: one before the first loop,
1054 and the other before the second loop:
1056 if (FIRST_NITERS == 0) then skip the first loop,
1057 and go directly to the second loop.
1058 The second guard is:
1059 if (FIRST_NITERS == NITERS) then skip the second loop.
1061 FORNOW only simple loops are supported (see slpeel_can_duplicate_loop_p).
1062 FORNOW the resulting code will not be in loop-closed-ssa form.
1066 slpeel_tree_peel_loop_to_edge (struct loop
*loop
,
1067 edge e
, tree first_niters
,
1068 tree niters
, bool update_first_loop_count
)
1070 struct loop
*new_loop
= NULL
, *first_loop
, *second_loop
;
1074 basic_block bb_before_second_loop
, bb_after_second_loop
;
1075 basic_block bb_before_first_loop
;
1076 basic_block bb_between_loops
;
1077 basic_block new_exit_bb
;
1078 edge exit_e
= single_exit (loop
);
1081 if (!slpeel_can_duplicate_loop_p (loop
, e
))
1084 /* We have to initialize cfg_hooks. Then, when calling
1085 cfg_hooks->split_edge, the function tree_split_edge
1086 is actually called and, when calling cfg_hooks->duplicate_block,
1087 the function tree_duplicate_bb is called. */
1088 tree_register_cfg_hooks ();
1091 /* 1. Generate a copy of LOOP and put it on E (E is the entry/exit of LOOP).
1092 Resulting CFG would be:
1105 if (!(new_loop
= slpeel_tree_duplicate_loop_to_edge_cfg (loop
, e
)))
1107 loop_loc
= find_loop_location (loop
);
1108 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1110 if (loop_loc
!= UNKNOWN_LOC
)
1111 fprintf (dump_file
, "\n%s:%d: note: ",
1112 LOC_FILE (loop_loc
), LOC_LINE (loop_loc
));
1113 fprintf (dump_file
, "tree_duplicate_loop_to_edge_cfg failed.\n");
1120 /* NEW_LOOP was placed after LOOP. */
1122 second_loop
= new_loop
;
1126 /* NEW_LOOP was placed before LOOP. */
1127 first_loop
= new_loop
;
1131 definitions
= ssa_names_to_replace ();
1132 slpeel_update_phis_for_duplicate_loop (loop
, new_loop
, e
== exit_e
);
1133 rename_variables_in_loop (new_loop
);
1136 /* 2. Add the guard that controls whether the first loop is executed.
1137 Resulting CFG would be:
1139 bb_before_first_loop:
1140 if (FIRST_NITERS == 0) GOTO bb_before_second_loop
1147 bb_before_second_loop:
1156 bb_before_first_loop
= split_edge (loop_preheader_edge (first_loop
));
1157 bb_before_second_loop
= split_edge (single_exit (first_loop
));
1160 fold_build2 (LE_EXPR
, boolean_type_node
, first_niters
,
1161 build_int_cst (TREE_TYPE (first_niters
), 0));
1162 skip_e
= slpeel_add_loop_guard (bb_before_first_loop
, pre_condition
,
1163 bb_before_second_loop
, bb_before_first_loop
);
1164 slpeel_update_phi_nodes_for_guard1 (skip_e
, first_loop
,
1165 first_loop
== new_loop
,
1166 &new_exit_bb
, &definitions
);
1169 /* 3. Add the guard that controls whether the second loop is executed.
1170 Resulting CFG would be:
1172 bb_before_first_loop:
1173 if (FIRST_NITERS == 0) GOTO bb_before_second_loop (skip first loop)
1181 if (FIRST_NITERS == NITERS) GOTO bb_after_second_loop (skip second loop)
1182 GOTO bb_before_second_loop
1184 bb_before_second_loop:
1190 bb_after_second_loop:
1195 bb_between_loops
= new_exit_bb
;
1196 bb_after_second_loop
= split_edge (single_exit (second_loop
));
1199 fold_build2 (EQ_EXPR
, boolean_type_node
, first_niters
, niters
);
1200 skip_e
= slpeel_add_loop_guard (bb_between_loops
, pre_condition
,
1201 bb_after_second_loop
, bb_before_first_loop
);
1202 slpeel_update_phi_nodes_for_guard2 (skip_e
, second_loop
,
1203 second_loop
== new_loop
, &new_exit_bb
);
1205 /* 4. Make first-loop iterate FIRST_NITERS times, if requested.
1207 if (update_first_loop_count
)
1208 slpeel_make_loop_iterate_ntimes (first_loop
, first_niters
);
1210 BITMAP_FREE (definitions
);
1211 delete_update_ssa ();
1216 /* Function vect_get_loop_location.
1218 Extract the location of the loop in the source code.
1219 If the loop is not well formed for vectorization, an estimated
1220 location is calculated.
1221 Return the loop location if succeed and NULL if not. */
1224 find_loop_location (struct loop
*loop
)
1226 tree node
= NULL_TREE
;
1228 block_stmt_iterator si
;
1233 node
= get_loop_exit_condition (loop
);
1235 if (node
&& CAN_HAVE_LOCATION_P (node
) && EXPR_HAS_LOCATION (node
)
1236 && EXPR_FILENAME (node
) && EXPR_LINENO (node
))
1237 return EXPR_LOC (node
);
1239 /* If we got here the loop is probably not "well formed",
1240 try to estimate the loop location */
1247 for (si
= bsi_start (bb
); !bsi_end_p (si
); bsi_next (&si
))
1249 node
= bsi_stmt (si
);
1250 if (node
&& CAN_HAVE_LOCATION_P (node
) && EXPR_HAS_LOCATION (node
))
1251 return EXPR_LOC (node
);
1258 /*************************************************************************
1259 Vectorization Debug Information.
1260 *************************************************************************/
1262 /* Function vect_set_verbosity_level.
1264 Called from toplev.c upon detection of the
1265 -ftree-vectorizer-verbose=N option. */
1268 vect_set_verbosity_level (const char *val
)
1273 if (vl
< MAX_VERBOSITY_LEVEL
)
1274 vect_verbosity_level
= vl
;
1276 vect_verbosity_level
= MAX_VERBOSITY_LEVEL
- 1;
1280 /* Function vect_set_dump_settings.
1282 Fix the verbosity level of the vectorizer if the
1283 requested level was not set explicitly using the flag
1284 -ftree-vectorizer-verbose=N.
1285 Decide where to print the debugging information (dump_file/stderr).
1286 If the user defined the verbosity level, but there is no dump file,
1287 print to stderr, otherwise print to the dump file. */
1290 vect_set_dump_settings (void)
1292 vect_dump
= dump_file
;
1294 /* Check if the verbosity level was defined by the user: */
1295 if (vect_verbosity_level
!= MAX_VERBOSITY_LEVEL
)
1297 /* If there is no dump file, print to stderr. */
1303 /* User didn't specify verbosity level: */
1304 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1305 vect_verbosity_level
= REPORT_DETAILS
;
1306 else if (dump_file
&& (dump_flags
& TDF_STATS
))
1307 vect_verbosity_level
= REPORT_UNVECTORIZED_LOOPS
;
1309 vect_verbosity_level
= REPORT_NONE
;
1311 gcc_assert (dump_file
|| vect_verbosity_level
== REPORT_NONE
);
1315 /* Function debug_loop_details.
1317 For vectorization debug dumps. */
1320 vect_print_dump_info (enum verbosity_levels vl
)
1322 if (vl
> vect_verbosity_level
)
1325 if (!current_function_decl
|| !vect_dump
)
1328 if (vect_loop_location
== UNKNOWN_LOC
)
1329 fprintf (vect_dump
, "\n%s:%d: note: ",
1330 DECL_SOURCE_FILE (current_function_decl
),
1331 DECL_SOURCE_LINE (current_function_decl
));
1333 fprintf (vect_dump
, "\n%s:%d: note: ",
1334 LOC_FILE (vect_loop_location
), LOC_LINE (vect_loop_location
));
1340 /*************************************************************************
1341 Vectorization Utilities.
1342 *************************************************************************/
1344 /* Function new_stmt_vec_info.
1346 Create and initialize a new stmt_vec_info struct for STMT. */
1349 new_stmt_vec_info (tree stmt
, loop_vec_info loop_vinfo
)
1352 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
1354 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
1355 STMT_VINFO_STMT (res
) = stmt
;
1356 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
1357 STMT_VINFO_RELEVANT (res
) = 0;
1358 STMT_VINFO_LIVE_P (res
) = false;
1359 STMT_VINFO_VECTYPE (res
) = NULL
;
1360 STMT_VINFO_VEC_STMT (res
) = NULL
;
1361 STMT_VINFO_IN_PATTERN_P (res
) = false;
1362 STMT_VINFO_RELATED_STMT (res
) = NULL
;
1363 STMT_VINFO_DATA_REF (res
) = NULL
;
1364 if (TREE_CODE (stmt
) == PHI_NODE
)
1365 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
1367 STMT_VINFO_DEF_TYPE (res
) = vect_loop_def
;
1368 STMT_VINFO_SAME_ALIGN_REFS (res
) = VEC_alloc (dr_p
, heap
, 5);
1369 DR_GROUP_FIRST_DR (res
) = NULL_TREE
;
1370 DR_GROUP_NEXT_DR (res
) = NULL_TREE
;
1371 DR_GROUP_SIZE (res
) = 0;
1372 DR_GROUP_STORE_COUNT (res
) = 0;
1373 DR_GROUP_GAP (res
) = 0;
1374 DR_GROUP_SAME_DR_STMT (res
) = NULL_TREE
;
1380 /* Function new_loop_vec_info.
1382 Create and initialize a new loop_vec_info struct for LOOP, as well as
1383 stmt_vec_info structs for all the stmts in LOOP. */
1386 new_loop_vec_info (struct loop
*loop
)
1390 block_stmt_iterator si
;
1393 res
= (loop_vec_info
) xcalloc (1, sizeof (struct _loop_vec_info
));
1395 bbs
= get_loop_body (loop
);
1397 /* Create stmt_info for all stmts in the loop. */
1398 for (i
= 0; i
< loop
->num_nodes
; i
++)
1400 basic_block bb
= bbs
[i
];
1403 for (phi
= phi_nodes (bb
); phi
; phi
= PHI_CHAIN (phi
))
1405 stmt_ann_t ann
= get_stmt_ann (phi
);
1406 set_stmt_info (ann
, new_stmt_vec_info (phi
, res
));
1409 for (si
= bsi_start (bb
); !bsi_end_p (si
); bsi_next (&si
))
1411 tree stmt
= bsi_stmt (si
);
1414 ann
= stmt_ann (stmt
);
1415 set_stmt_info (ann
, new_stmt_vec_info (stmt
, res
));
1419 LOOP_VINFO_LOOP (res
) = loop
;
1420 LOOP_VINFO_BBS (res
) = bbs
;
1421 LOOP_VINFO_EXIT_COND (res
) = NULL
;
1422 LOOP_VINFO_NITERS (res
) = NULL
;
1423 LOOP_VINFO_VECTORIZABLE_P (res
) = 0;
1424 LOOP_PEELING_FOR_ALIGNMENT (res
) = 0;
1425 LOOP_VINFO_VECT_FACTOR (res
) = 0;
1426 LOOP_VINFO_DATAREFS (res
) = VEC_alloc (data_reference_p
, heap
, 10);
1427 LOOP_VINFO_DDRS (res
) = VEC_alloc (ddr_p
, heap
, 10 * 10);
1428 LOOP_VINFO_UNALIGNED_DR (res
) = NULL
;
1429 LOOP_VINFO_MAY_MISALIGN_STMTS (res
)
1430 = VEC_alloc (tree
, heap
, PARAM_VALUE (PARAM_VECT_MAX_VERSION_CHECKS
));
1436 /* Function destroy_loop_vec_info.
1438 Free LOOP_VINFO struct, as well as all the stmt_vec_info structs of all the
1439 stmts in the loop. */
1442 destroy_loop_vec_info (loop_vec_info loop_vinfo
)
1447 block_stmt_iterator si
;
1453 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1455 bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1456 nbbs
= loop
->num_nodes
;
1458 for (j
= 0; j
< nbbs
; j
++)
1460 basic_block bb
= bbs
[j
];
1462 stmt_vec_info stmt_info
;
1464 for (phi
= phi_nodes (bb
); phi
; phi
= PHI_CHAIN (phi
))
1466 stmt_ann_t ann
= stmt_ann (phi
);
1468 stmt_info
= vinfo_for_stmt (phi
);
1470 set_stmt_info (ann
, NULL
);
1473 for (si
= bsi_start (bb
); !bsi_end_p (si
); )
1475 tree stmt
= bsi_stmt (si
);
1476 stmt_ann_t ann
= stmt_ann (stmt
);
1477 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1481 /* Check if this is a "pattern stmt" (introduced by the
1482 vectorizer during the pattern recognition pass). */
1483 bool remove_stmt_p
= false;
1484 tree orig_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
1487 stmt_vec_info orig_stmt_info
= vinfo_for_stmt (orig_stmt
);
1489 && STMT_VINFO_IN_PATTERN_P (orig_stmt_info
))
1490 remove_stmt_p
= true;
1493 /* Free stmt_vec_info. */
1494 VEC_free (dr_p
, heap
, STMT_VINFO_SAME_ALIGN_REFS (stmt_info
));
1496 set_stmt_info (ann
, NULL
);
1498 /* Remove dead "pattern stmts". */
1500 bsi_remove (&si
, true);
1506 free (LOOP_VINFO_BBS (loop_vinfo
));
1507 free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo
));
1508 free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo
));
1509 VEC_free (tree
, heap
, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
));
1515 /* Function vect_force_dr_alignment_p.
1517 Returns whether the alignment of a DECL can be forced to be aligned
1518 on ALIGNMENT bit boundary. */
1521 vect_can_force_dr_alignment_p (tree decl
, unsigned int alignment
)
1523 if (TREE_CODE (decl
) != VAR_DECL
)
1526 if (DECL_EXTERNAL (decl
))
1529 if (TREE_ASM_WRITTEN (decl
))
1532 if (TREE_STATIC (decl
))
1533 return (alignment
<= MAX_OFILE_ALIGNMENT
);
1535 /* This is not 100% correct. The absolute correct stack alignment
1536 is STACK_BOUNDARY. We're supposed to hope, but not assume, that
1537 PREFERRED_STACK_BOUNDARY is honored by all translation units.
1538 However, until someone implements forced stack alignment, SSE
1539 isn't really usable without this. */
1540 return (alignment
<= PREFERRED_STACK_BOUNDARY
);
1544 /* Function get_vectype_for_scalar_type.
1546 Returns the vector type corresponding to SCALAR_TYPE as supported
1550 get_vectype_for_scalar_type (tree scalar_type
)
1552 enum machine_mode inner_mode
= TYPE_MODE (scalar_type
);
1553 int nbytes
= GET_MODE_SIZE (inner_mode
);
1557 if (nbytes
== 0 || nbytes
>= UNITS_PER_SIMD_WORD
)
1560 /* FORNOW: Only a single vector size per target (UNITS_PER_SIMD_WORD)
1562 nunits
= UNITS_PER_SIMD_WORD
/ nbytes
;
1564 vectype
= build_vector_type (scalar_type
, nunits
);
1565 if (vect_print_dump_info (REPORT_DETAILS
))
1567 fprintf (vect_dump
, "get vectype with %d units of type ", nunits
);
1568 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
1574 if (vect_print_dump_info (REPORT_DETAILS
))
1576 fprintf (vect_dump
, "vectype: ");
1577 print_generic_expr (vect_dump
, vectype
, TDF_SLIM
);
1580 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1581 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
1583 if (vect_print_dump_info (REPORT_DETAILS
))
1584 fprintf (vect_dump
, "mode not supported by target.");
1592 /* Function vect_supportable_dr_alignment
1594 Return whether the data reference DR is supported with respect to its
1597 enum dr_alignment_support
1598 vect_supportable_dr_alignment (struct data_reference
*dr
)
1600 tree vectype
= STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr
)));
1601 enum machine_mode mode
= (int) TYPE_MODE (vectype
);
1603 if (aligned_access_p (dr
))
1606 /* Possibly unaligned access. */
1608 if (DR_IS_READ (dr
))
1610 if (vec_realign_load_optab
->handlers
[mode
].insn_code
!= CODE_FOR_nothing
1611 && (!targetm
.vectorize
.builtin_mask_for_load
1612 || targetm
.vectorize
.builtin_mask_for_load ()))
1613 return dr_unaligned_software_pipeline
;
1615 if (movmisalign_optab
->handlers
[mode
].insn_code
!= CODE_FOR_nothing
)
1616 /* Can't software pipeline the loads, but can at least do them. */
1617 return dr_unaligned_supported
;
1621 return dr_unaligned_unsupported
;
1625 /* Function vect_is_simple_use.
1628 LOOP - the loop that is being vectorized.
1629 OPERAND - operand of a stmt in LOOP.
1630 DEF - the defining stmt in case OPERAND is an SSA_NAME.
1632 Returns whether a stmt with OPERAND can be vectorized.
1633 Supportable operands are constants, loop invariants, and operands that are
1634 defined by the current iteration of the loop. Unsupportable operands are
1635 those that are defined by a previous iteration of the loop (as is the case
1636 in reduction/induction computations). */
1639 vect_is_simple_use (tree operand
, loop_vec_info loop_vinfo
, tree
*def_stmt
,
1640 tree
*def
, enum vect_def_type
*dt
)
1643 stmt_vec_info stmt_vinfo
;
1644 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1646 *def_stmt
= NULL_TREE
;
1649 if (vect_print_dump_info (REPORT_DETAILS
))
1651 fprintf (vect_dump
, "vect_is_simple_use: operand ");
1652 print_generic_expr (vect_dump
, operand
, TDF_SLIM
);
1655 if (TREE_CODE (operand
) == INTEGER_CST
|| TREE_CODE (operand
) == REAL_CST
)
1657 *dt
= vect_constant_def
;
1661 if (TREE_CODE (operand
) != SSA_NAME
)
1663 if (vect_print_dump_info (REPORT_DETAILS
))
1664 fprintf (vect_dump
, "not ssa-name.");
1668 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
1669 if (*def_stmt
== NULL_TREE
)
1671 if (vect_print_dump_info (REPORT_DETAILS
))
1672 fprintf (vect_dump
, "no def_stmt.");
1676 if (vect_print_dump_info (REPORT_DETAILS
))
1678 fprintf (vect_dump
, "def_stmt: ");
1679 print_generic_expr (vect_dump
, *def_stmt
, TDF_SLIM
);
1682 /* empty stmt is expected only in case of a function argument.
1683 (Otherwise - we expect a phi_node or a GIMPLE_MODIFY_STMT). */
1684 if (IS_EMPTY_STMT (*def_stmt
))
1686 tree arg
= TREE_OPERAND (*def_stmt
, 0);
1687 if (TREE_CODE (arg
) == INTEGER_CST
|| TREE_CODE (arg
) == REAL_CST
)
1690 *dt
= vect_invariant_def
;
1694 if (vect_print_dump_info (REPORT_DETAILS
))
1695 fprintf (vect_dump
, "Unexpected empty stmt.");
1699 bb
= bb_for_stmt (*def_stmt
);
1700 if (!flow_bb_inside_loop_p (loop
, bb
))
1701 *dt
= vect_invariant_def
;
1704 stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
1705 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
1708 if (*dt
== vect_unknown_def_type
)
1710 if (vect_print_dump_info (REPORT_DETAILS
))
1711 fprintf (vect_dump
, "Unsupported pattern.");
1715 /* stmts inside the loop that have been identified as performing
1716 a reduction operation cannot have uses in the loop. */
1717 if (*dt
== vect_reduction_def
&& TREE_CODE (*def_stmt
) != PHI_NODE
)
1719 if (vect_print_dump_info (REPORT_DETAILS
))
1720 fprintf (vect_dump
, "reduction used in loop.");
1724 if (vect_print_dump_info (REPORT_DETAILS
))
1725 fprintf (vect_dump
, "type of def: %d.",*dt
);
1727 switch (TREE_CODE (*def_stmt
))
1730 *def
= PHI_RESULT (*def_stmt
);
1731 gcc_assert (*dt
== vect_induction_def
|| *dt
== vect_reduction_def
1732 || *dt
== vect_invariant_def
);
1735 case GIMPLE_MODIFY_STMT
:
1736 *def
= GIMPLE_STMT_OPERAND (*def_stmt
, 0);
1737 gcc_assert (*dt
== vect_loop_def
|| *dt
== vect_invariant_def
);
1741 if (vect_print_dump_info (REPORT_DETAILS
))
1742 fprintf (vect_dump
, "unsupported defining stmt: ");
1746 if (*dt
== vect_induction_def
)
1748 if (vect_print_dump_info (REPORT_DETAILS
))
1749 fprintf (vect_dump
, "induction not supported.");
1757 /* Function supportable_widening_operation
1759 Check whether an operation represented by the code CODE is a
1760 widening operation that is supported by the target platform in
1761 vector form (i.e., when operating on arguments of type VECTYPE).
1763 The two kinds of widening operations we currently support are
1764 NOP and WIDEN_MULT. This function checks if these operations
1765 are supported by the target platform either directly (via vector
1766 tree-codes), or via target builtins.
1769 - CODE1 and CODE2 are codes of vector operations to be used when
1770 vectorizing the operation, if available.
1771 - DECL1 and DECL2 are decls of target builtin functions to be used
1772 when vectorizing the operation, if available. In this case,
1773 CODE1 and CODE2 are CALL_EXPR. */
1776 supportable_widening_operation (enum tree_code code
, tree stmt
, tree vectype
,
1777 tree
*decl1
, tree
*decl2
,
1778 enum tree_code
*code1
, enum tree_code
*code2
)
1780 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1782 enum machine_mode vec_mode
;
1783 enum insn_code icode1
, icode2
;
1784 optab optab1
, optab2
;
1785 tree expr
= GIMPLE_STMT_OPERAND (stmt
, 1);
1786 tree type
= TREE_TYPE (expr
);
1787 tree wide_vectype
= get_vectype_for_scalar_type (type
);
1788 enum tree_code c1
, c2
;
1790 /* The result of a vectorized widening operation usually requires two vectors
1791 (because the widened results do not fit int one vector). The generated
1792 vector results would normally be expected to be generated in the same
1793 order as in the original scalar computation. i.e. if 8 results are
1794 generated in each vector iteration, they are to be organized as follows:
1795 vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
1797 However, in the special case that the result of the widening operation is
1798 used in a reduction computation only, the order doesn't matter (because
1799 when vectorizing a reduction we change the order of the computation).
1800 Some targets can take advantage of this and generate more efficient code.
1801 For example, targets like Altivec, that support widen_mult using a sequence
1802 of {mult_even,mult_odd} generate the following vectors:
1803 vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8]. */
1805 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
)
1811 && code
== WIDEN_MULT_EXPR
1812 && targetm
.vectorize
.builtin_mul_widen_even
1813 && targetm
.vectorize
.builtin_mul_widen_even (vectype
)
1814 && targetm
.vectorize
.builtin_mul_widen_odd
1815 && targetm
.vectorize
.builtin_mul_widen_odd (vectype
))
1817 if (vect_print_dump_info (REPORT_DETAILS
))
1818 fprintf (vect_dump
, "Unordered widening operation detected.");
1820 *code1
= *code2
= CALL_EXPR
;
1821 *decl1
= targetm
.vectorize
.builtin_mul_widen_even (vectype
);
1822 *decl2
= targetm
.vectorize
.builtin_mul_widen_odd (vectype
);
1828 case WIDEN_MULT_EXPR
:
1829 if (BYTES_BIG_ENDIAN
)
1831 c1
= VEC_WIDEN_MULT_HI_EXPR
;
1832 c2
= VEC_WIDEN_MULT_LO_EXPR
;
1836 c2
= VEC_WIDEN_MULT_HI_EXPR
;
1837 c1
= VEC_WIDEN_MULT_LO_EXPR
;
1842 if (BYTES_BIG_ENDIAN
)
1844 c1
= VEC_UNPACK_HI_EXPR
;
1845 c2
= VEC_UNPACK_LO_EXPR
;
1849 c2
= VEC_UNPACK_HI_EXPR
;
1850 c1
= VEC_UNPACK_LO_EXPR
;
1860 optab1
= optab_for_tree_code (c1
, vectype
);
1861 optab2
= optab_for_tree_code (c2
, vectype
);
1863 if (!optab1
|| !optab2
)
1866 vec_mode
= TYPE_MODE (vectype
);
1867 if ((icode1
= optab1
->handlers
[(int) vec_mode
].insn_code
) == CODE_FOR_nothing
1868 || insn_data
[icode1
].operand
[0].mode
!= TYPE_MODE (wide_vectype
)
1869 || (icode2
= optab2
->handlers
[(int) vec_mode
].insn_code
)
1871 || insn_data
[icode2
].operand
[0].mode
!= TYPE_MODE (wide_vectype
))
1878 /* Function reduction_code_for_scalar_code
1881 CODE - tree_code of a reduction operations.
1884 REDUC_CODE - the corresponding tree-code to be used to reduce the
1885 vector of partial results into a single scalar result (which
1886 will also reside in a vector).
1888 Return TRUE if a corresponding REDUC_CODE was found, FALSE otherwise. */
1891 reduction_code_for_scalar_code (enum tree_code code
,
1892 enum tree_code
*reduc_code
)
1897 *reduc_code
= REDUC_MAX_EXPR
;
1901 *reduc_code
= REDUC_MIN_EXPR
;
1905 *reduc_code
= REDUC_PLUS_EXPR
;
1914 /* Function vect_is_simple_reduction
1916 Detect a cross-iteration def-use cucle that represents a simple
1917 reduction computation. We look for the following pattern:
1922 a2 = operation (a3, a1)
1925 1. operation is commutative and associative and it is safe to
1926 change the order of the computation.
1927 2. no uses for a2 in the loop (a2 is used out of the loop)
1928 3. no uses of a1 in the loop besides the reduction operation.
1930 Condition 1 is tested here.
1931 Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized. */
1934 vect_is_simple_reduction (struct loop
*loop
, tree phi
)
1936 edge latch_e
= loop_latch_edge (loop
);
1937 tree loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
1938 tree def_stmt
, def1
, def2
;
1939 enum tree_code code
;
1941 tree operation
, op1
, op2
;
1944 if (TREE_CODE (loop_arg
) != SSA_NAME
)
1946 if (vect_print_dump_info (REPORT_DETAILS
))
1948 fprintf (vect_dump
, "reduction: not ssa_name: ");
1949 print_generic_expr (vect_dump
, loop_arg
, TDF_SLIM
);
1954 def_stmt
= SSA_NAME_DEF_STMT (loop_arg
);
1957 if (vect_print_dump_info (REPORT_DETAILS
))
1958 fprintf (vect_dump
, "reduction: no def_stmt.");
1962 if (TREE_CODE (def_stmt
) != GIMPLE_MODIFY_STMT
)
1964 if (vect_print_dump_info (REPORT_DETAILS
))
1966 print_generic_expr (vect_dump
, def_stmt
, TDF_SLIM
);
1971 operation
= GIMPLE_STMT_OPERAND (def_stmt
, 1);
1972 code
= TREE_CODE (operation
);
1973 if (!commutative_tree_code (code
) || !associative_tree_code (code
))
1975 if (vect_print_dump_info (REPORT_DETAILS
))
1977 fprintf (vect_dump
, "reduction: not commutative/associative: ");
1978 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
1983 op_type
= TREE_CODE_LENGTH (code
);
1984 if (op_type
!= binary_op
)
1986 if (vect_print_dump_info (REPORT_DETAILS
))
1988 fprintf (vect_dump
, "reduction: not binary operation: ");
1989 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
1994 op1
= TREE_OPERAND (operation
, 0);
1995 op2
= TREE_OPERAND (operation
, 1);
1996 if (TREE_CODE (op1
) != SSA_NAME
|| TREE_CODE (op2
) != SSA_NAME
)
1998 if (vect_print_dump_info (REPORT_DETAILS
))
2000 fprintf (vect_dump
, "reduction: uses not ssa_names: ");
2001 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2006 /* Check that it's ok to change the order of the computation. */
2007 type
= TREE_TYPE (operation
);
2008 if (TYPE_MAIN_VARIANT (type
) != TYPE_MAIN_VARIANT (TREE_TYPE (op1
))
2009 || TYPE_MAIN_VARIANT (type
) != TYPE_MAIN_VARIANT (TREE_TYPE (op2
)))
2011 if (vect_print_dump_info (REPORT_DETAILS
))
2013 fprintf (vect_dump
, "reduction: multiple types: operation type: ");
2014 print_generic_expr (vect_dump
, type
, TDF_SLIM
);
2015 fprintf (vect_dump
, ", operands types: ");
2016 print_generic_expr (vect_dump
, TREE_TYPE (op1
), TDF_SLIM
);
2017 fprintf (vect_dump
, ",");
2018 print_generic_expr (vect_dump
, TREE_TYPE (op2
), TDF_SLIM
);
2023 /* CHECKME: check for !flag_finite_math_only too? */
2024 if (SCALAR_FLOAT_TYPE_P (type
) && !flag_unsafe_math_optimizations
)
2026 /* Changing the order of operations changes the semantics. */
2027 if (vect_print_dump_info (REPORT_DETAILS
))
2029 fprintf (vect_dump
, "reduction: unsafe fp math optimization: ");
2030 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2034 else if (INTEGRAL_TYPE_P (type
) && !TYPE_UNSIGNED (type
) && flag_trapv
)
2036 /* Changing the order of operations changes the semantics. */
2037 if (vect_print_dump_info (REPORT_DETAILS
))
2039 fprintf (vect_dump
, "reduction: unsafe int math optimization: ");
2040 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2045 /* reduction is safe. we're dealing with one of the following:
2046 1) integer arithmetic and no trapv
2047 2) floating point arithmetic, and special flags permit this optimization.
2049 def1
= SSA_NAME_DEF_STMT (op1
);
2050 def2
= SSA_NAME_DEF_STMT (op2
);
2053 if (vect_print_dump_info (REPORT_DETAILS
))
2055 fprintf (vect_dump
, "reduction: no defs for operands: ");
2056 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2061 if (TREE_CODE (def1
) == GIMPLE_MODIFY_STMT
2062 && flow_bb_inside_loop_p (loop
, bb_for_stmt (def1
))
2065 if (vect_print_dump_info (REPORT_DETAILS
))
2067 fprintf (vect_dump
, "detected reduction:");
2068 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2072 else if (TREE_CODE (def2
) == GIMPLE_MODIFY_STMT
2073 && flow_bb_inside_loop_p (loop
, bb_for_stmt (def2
))
2076 /* Swap operands (just for simplicity - so that the rest of the code
2077 can assume that the reduction variable is always the last (second)
2079 if (vect_print_dump_info (REPORT_DETAILS
))
2081 fprintf (vect_dump
, "detected reduction: need to swap operands:");
2082 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2084 swap_tree_operands (def_stmt
, &TREE_OPERAND (operation
, 0),
2085 &TREE_OPERAND (operation
, 1));
2090 if (vect_print_dump_info (REPORT_DETAILS
))
2092 fprintf (vect_dump
, "reduction: unknown pattern.");
2093 print_generic_expr (vect_dump
, operation
, TDF_SLIM
);
2100 /* Function vect_is_simple_iv_evolution.
2102 FORNOW: A simple evolution of an induction variables in the loop is
2103 considered a polynomial evolution with constant step. */
2106 vect_is_simple_iv_evolution (unsigned loop_nb
, tree access_fn
, tree
* init
,
2112 tree evolution_part
= evolution_part_in_loop_num (access_fn
, loop_nb
);
2114 /* When there is no evolution in this loop, the evolution function
2116 if (evolution_part
== NULL_TREE
)
2119 /* When the evolution is a polynomial of degree >= 2
2120 the evolution function is not "simple". */
2121 if (tree_is_chrec (evolution_part
))
2124 step_expr
= evolution_part
;
2125 init_expr
= unshare_expr (initial_condition_in_loop_num (access_fn
,
2128 if (vect_print_dump_info (REPORT_DETAILS
))
2130 fprintf (vect_dump
, "step: ");
2131 print_generic_expr (vect_dump
, step_expr
, TDF_SLIM
);
2132 fprintf (vect_dump
, ", init: ");
2133 print_generic_expr (vect_dump
, init_expr
, TDF_SLIM
);
2139 if (TREE_CODE (step_expr
) != INTEGER_CST
)
2141 if (vect_print_dump_info (REPORT_DETAILS
))
2142 fprintf (vect_dump
, "step unknown.");
2150 /* Function vectorize_loops.
2152 Entry Point to loop vectorization phase. */
2155 vectorize_loops (void)
2158 unsigned int num_vectorized_loops
= 0;
2159 unsigned int vect_loops_num
;
2163 /* Fix the verbosity level if not defined explicitly by the user. */
2164 vect_set_dump_settings ();
2166 /* Allocate the bitmap that records which virtual variables that
2167 need to be renamed. */
2168 vect_memsyms_to_rename
= BITMAP_ALLOC (NULL
);
2170 /* ----------- Analyze loops. ----------- */
2172 /* If some loop was duplicated, it gets bigger number
2173 than all previously defined loops. This fact allows us to run
2174 only over initial loops skipping newly generated ones. */
2175 vect_loops_num
= number_of_loops ();
2176 FOR_EACH_LOOP (li
, loop
, LI_ONLY_OLD
)
2178 loop_vec_info loop_vinfo
;
2180 vect_loop_location
= find_loop_location (loop
);
2181 loop_vinfo
= vect_analyze_loop (loop
);
2182 loop
->aux
= loop_vinfo
;
2184 if (!loop_vinfo
|| !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
))
2187 vect_transform_loop (loop_vinfo
);
2188 num_vectorized_loops
++;
2190 vect_loop_location
= UNKNOWN_LOC
;
2192 if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS
))
2193 fprintf (vect_dump
, "vectorized %u loops in function.\n",
2194 num_vectorized_loops
);
2196 /* ----------- Finalize. ----------- */
2198 BITMAP_FREE (vect_memsyms_to_rename
);
2200 for (i
= 1; i
< vect_loops_num
; i
++)
2202 loop_vec_info loop_vinfo
;
2204 loop
= get_loop (i
);
2207 loop_vinfo
= loop
->aux
;
2208 destroy_loop_vec_info (loop_vinfo
);
2212 return num_vectorized_loops
> 0 ? TODO_cleanup_cfg
: 0;