1 /* SLP - Basic Block Vectorization
2 Copyright (C) 2007, 2008, 2009 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "diagnostic.h"
32 #include "tree-flow.h"
33 #include "tree-dump.h"
35 #include "cfglayout.h"
39 #include "tree-vectorizer.h"
41 /* Extract the location of the basic block in the source code.
42 Return the basic block location if succeed and NULL if not. */
45 find_bb_location (basic_block bb
)
48 gimple_stmt_iterator si
;
53 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
56 if (gimple_location (stmt
) != UNKNOWN_LOC
)
57 return gimple_location (stmt
);
64 /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
67 vect_free_slp_tree (slp_tree node
)
72 if (SLP_TREE_LEFT (node
))
73 vect_free_slp_tree (SLP_TREE_LEFT (node
));
75 if (SLP_TREE_RIGHT (node
))
76 vect_free_slp_tree (SLP_TREE_RIGHT (node
));
78 VEC_free (gimple
, heap
, SLP_TREE_SCALAR_STMTS (node
));
80 if (SLP_TREE_VEC_STMTS (node
))
81 VEC_free (gimple
, heap
, SLP_TREE_VEC_STMTS (node
));
87 /* Free the memory allocated for the SLP instance. */
90 vect_free_slp_instance (slp_instance instance
)
92 vect_free_slp_tree (SLP_INSTANCE_TREE (instance
));
93 VEC_free (int, heap
, SLP_INSTANCE_LOAD_PERMUTATION (instance
));
94 VEC_free (slp_tree
, heap
, SLP_INSTANCE_LOADS (instance
));
98 /* Get the defs for the rhs of STMT (collect them in DEF_STMTS0/1), check that
99 they are of a legal type and that they match the defs of the first stmt of
100 the SLP group (stored in FIRST_STMT_...). */
103 vect_get_and_check_slp_defs (loop_vec_info loop_vinfo
, bb_vec_info bb_vinfo
,
104 slp_tree slp_node
, gimple stmt
,
105 VEC (gimple
, heap
) **def_stmts0
,
106 VEC (gimple
, heap
) **def_stmts1
,
107 enum vect_def_type
*first_stmt_dt0
,
108 enum vect_def_type
*first_stmt_dt1
,
109 tree
*first_stmt_def0_type
,
110 tree
*first_stmt_def1_type
,
111 tree
*first_stmt_const_oprnd
,
112 int ncopies_for_cost
,
113 bool *pattern0
, bool *pattern1
)
116 unsigned int i
, number_of_oprnds
;
119 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
120 stmt_vec_info stmt_info
=
121 vinfo_for_stmt (VEC_index (gimple
, SLP_TREE_SCALAR_STMTS (slp_node
), 0));
122 enum gimple_rhs_class rhs_class
;
123 struct loop
*loop
= NULL
;
126 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
128 rhs_class
= get_gimple_rhs_class (gimple_assign_rhs_code (stmt
));
129 number_of_oprnds
= gimple_num_ops (stmt
) - 1; /* RHS only */
131 for (i
= 0; i
< number_of_oprnds
; i
++)
133 oprnd
= gimple_op (stmt
, i
+ 1);
135 if (!vect_is_simple_use (oprnd
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
,
137 || (!def_stmt
&& dt
[i
] != vect_constant_def
))
139 if (vect_print_dump_info (REPORT_SLP
))
141 fprintf (vect_dump
, "Build SLP failed: can't find def for ");
142 print_generic_expr (vect_dump
, oprnd
, TDF_SLIM
);
148 /* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
149 from the pattern. Check that all the stmts of the node are in the
151 if (loop
&& def_stmt
&& gimple_bb (def_stmt
)
152 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
153 && vinfo_for_stmt (def_stmt
)
154 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt
)))
156 if (!*first_stmt_dt0
)
160 if (i
== 1 && !*first_stmt_dt1
)
162 else if ((i
== 0 && !*pattern0
) || (i
== 1 && !*pattern1
))
164 if (vect_print_dump_info (REPORT_DETAILS
))
166 fprintf (vect_dump
, "Build SLP failed: some of the stmts"
167 " are in a pattern, and others are not ");
168 print_generic_expr (vect_dump
, oprnd
, TDF_SLIM
);
175 def_stmt
= STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt
));
176 dt
[i
] = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt
));
178 if (*dt
== vect_unknown_def_type
)
180 if (vect_print_dump_info (REPORT_DETAILS
))
181 fprintf (vect_dump
, "Unsupported pattern.");
185 switch (gimple_code (def_stmt
))
188 def
= gimple_phi_result (def_stmt
);
192 def
= gimple_assign_lhs (def_stmt
);
196 if (vect_print_dump_info (REPORT_DETAILS
))
197 fprintf (vect_dump
, "unsupported defining stmt: ");
202 if (!*first_stmt_dt0
)
204 /* op0 of the first stmt of the group - store its info. */
205 *first_stmt_dt0
= dt
[i
];
207 *first_stmt_def0_type
= TREE_TYPE (def
);
209 *first_stmt_const_oprnd
= oprnd
;
211 /* Analyze costs (for the first stmt of the group only). */
212 if (rhs_class
!= GIMPLE_SINGLE_RHS
)
213 /* Not memory operation (we don't call this functions for loads). */
214 vect_model_simple_cost (stmt_info
, ncopies_for_cost
, dt
, slp_node
);
217 vect_model_store_cost (stmt_info
, ncopies_for_cost
, dt
[0], slp_node
);
222 if (!*first_stmt_dt1
&& i
== 1)
224 /* op1 of the first stmt of the group - store its info. */
225 *first_stmt_dt1
= dt
[i
];
227 *first_stmt_def1_type
= TREE_TYPE (def
);
230 /* We assume that the stmt contains only one constant
231 operand. We fail otherwise, to be on the safe side. */
232 if (*first_stmt_const_oprnd
)
234 if (vect_print_dump_info (REPORT_SLP
))
235 fprintf (vect_dump
, "Build SLP failed: two constant "
239 *first_stmt_const_oprnd
= oprnd
;
244 /* Not first stmt of the group, check that the def-stmt/s match
245 the def-stmt/s of the first stmt. */
247 && (*first_stmt_dt0
!= dt
[i
]
248 || (*first_stmt_def0_type
&& def
249 && *first_stmt_def0_type
!= TREE_TYPE (def
))))
251 && (*first_stmt_dt1
!= dt
[i
]
252 || (*first_stmt_def1_type
&& def
253 && *first_stmt_def1_type
!= TREE_TYPE (def
))))
255 && TREE_TYPE (*first_stmt_const_oprnd
)
256 != TREE_TYPE (oprnd
)))
258 if (vect_print_dump_info (REPORT_SLP
))
259 fprintf (vect_dump
, "Build SLP failed: different types ");
266 /* Check the types of the definitions. */
269 case vect_constant_def
:
270 case vect_external_def
:
273 case vect_internal_def
:
275 VEC_safe_push (gimple
, heap
, *def_stmts0
, def_stmt
);
277 VEC_safe_push (gimple
, heap
, *def_stmts1
, def_stmt
);
281 /* FORNOW: Not supported. */
282 if (vect_print_dump_info (REPORT_SLP
))
284 fprintf (vect_dump
, "Build SLP failed: illegal type of def ");
285 print_generic_expr (vect_dump
, def
, TDF_SLIM
);
296 /* Recursively build an SLP tree starting from NODE.
297 Fail (and return FALSE) if def-stmts are not isomorphic, require data
298 permutation or are of unsupported types of operation. Otherwise, return
302 vect_build_slp_tree (loop_vec_info loop_vinfo
, bb_vec_info bb_vinfo
,
303 slp_tree
*node
, unsigned int group_size
,
304 int *inside_cost
, int *outside_cost
,
305 int ncopies_for_cost
, unsigned int *max_nunits
,
306 VEC (int, heap
) **load_permutation
,
307 VEC (slp_tree
, heap
) **loads
,
308 unsigned int vectorization_factor
)
310 VEC (gimple
, heap
) *def_stmts0
= VEC_alloc (gimple
, heap
, group_size
);
311 VEC (gimple
, heap
) *def_stmts1
= VEC_alloc (gimple
, heap
, group_size
);
313 VEC (gimple
, heap
) *stmts
= SLP_TREE_SCALAR_STMTS (*node
);
314 gimple stmt
= VEC_index (gimple
, stmts
, 0);
315 enum vect_def_type first_stmt_dt0
= vect_uninitialized_def
;
316 enum vect_def_type first_stmt_dt1
= vect_uninitialized_def
;
317 enum tree_code first_stmt_code
= ERROR_MARK
, rhs_code
;
318 tree first_stmt_def1_type
= NULL_TREE
, first_stmt_def0_type
= NULL_TREE
;
320 bool stop_recursion
= false, need_same_oprnds
= false;
321 tree vectype
, scalar_type
, first_op1
= NULL_TREE
;
322 unsigned int ncopies
;
325 enum machine_mode optab_op2_mode
;
326 enum machine_mode vec_mode
;
327 tree first_stmt_const_oprnd
= NULL_TREE
;
328 struct data_reference
*first_dr
;
329 bool pattern0
= false, pattern1
= false;
331 bool permutation
= false;
332 unsigned int load_place
;
335 /* For every stmt in NODE find its def stmt/s. */
336 for (i
= 0; VEC_iterate (gimple
, stmts
, i
, stmt
); i
++)
338 if (vect_print_dump_info (REPORT_SLP
))
340 fprintf (vect_dump
, "Build SLP for ");
341 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
344 lhs
= gimple_get_lhs (stmt
);
345 if (lhs
== NULL_TREE
)
347 if (vect_print_dump_info (REPORT_SLP
))
350 "Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL");
351 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
357 scalar_type
= vect_get_smallest_scalar_type (stmt
, &dummy
, &dummy
);
358 vectype
= get_vectype_for_scalar_type (scalar_type
);
361 if (vect_print_dump_info (REPORT_SLP
))
363 fprintf (vect_dump
, "Build SLP failed: unsupported data-type ");
364 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
369 ncopies
= vectorization_factor
/ TYPE_VECTOR_SUBPARTS (vectype
);
372 if (vect_print_dump_info (REPORT_SLP
))
373 fprintf (vect_dump
, "SLP with multiple types ");
375 /* FORNOW: multiple types are unsupported in BB SLP. */
380 /* In case of multiple types we need to detect the smallest type. */
381 if (*max_nunits
< TYPE_VECTOR_SUBPARTS (vectype
))
382 *max_nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
384 if (is_gimple_call (stmt
))
385 rhs_code
= CALL_EXPR
;
387 rhs_code
= gimple_assign_rhs_code (stmt
);
389 /* Check the operation. */
392 first_stmt_code
= rhs_code
;
394 /* Shift arguments should be equal in all the packed stmts for a
395 vector shift with scalar shift operand. */
396 if (rhs_code
== LSHIFT_EXPR
|| rhs_code
== RSHIFT_EXPR
397 || rhs_code
== LROTATE_EXPR
398 || rhs_code
== RROTATE_EXPR
)
400 vec_mode
= TYPE_MODE (vectype
);
402 /* First see if we have a vector/vector shift. */
403 optab
= optab_for_tree_code (rhs_code
, vectype
,
407 || (optab
->handlers
[(int) vec_mode
].insn_code
408 == CODE_FOR_nothing
))
410 /* No vector/vector shift, try for a vector/scalar shift. */
411 optab
= optab_for_tree_code (rhs_code
, vectype
,
416 if (vect_print_dump_info (REPORT_SLP
))
417 fprintf (vect_dump
, "Build SLP failed: no optab.");
420 icode
= (int) optab
->handlers
[(int) vec_mode
].insn_code
;
421 if (icode
== CODE_FOR_nothing
)
423 if (vect_print_dump_info (REPORT_SLP
))
424 fprintf (vect_dump
, "Build SLP failed: "
425 "op not supported by target.");
428 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
429 if (!VECTOR_MODE_P (optab_op2_mode
))
431 need_same_oprnds
= true;
432 first_op1
= gimple_assign_rhs2 (stmt
);
439 if (first_stmt_code
!= rhs_code
440 && (first_stmt_code
!= IMAGPART_EXPR
441 || rhs_code
!= REALPART_EXPR
)
442 && (first_stmt_code
!= REALPART_EXPR
443 || rhs_code
!= IMAGPART_EXPR
))
445 if (vect_print_dump_info (REPORT_SLP
))
448 "Build SLP failed: different operation in stmt ");
449 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
456 && !operand_equal_p (first_op1
, gimple_assign_rhs2 (stmt
), 0))
458 if (vect_print_dump_info (REPORT_SLP
))
461 "Build SLP failed: different shift arguments in ");
462 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
469 /* Strided store or load. */
470 if (STMT_VINFO_STRIDED_ACCESS (vinfo_for_stmt (stmt
)))
472 if (REFERENCE_CLASS_P (lhs
))
475 if (!vect_get_and_check_slp_defs (loop_vinfo
, bb_vinfo
, *node
,
476 stmt
, &def_stmts0
, &def_stmts1
,
479 &first_stmt_def0_type
,
480 &first_stmt_def1_type
,
481 &first_stmt_const_oprnd
,
483 &pattern0
, &pattern1
))
489 /* FORNOW: Check that there is no gap between the loads. */
490 if ((DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt
)) == stmt
491 && DR_GROUP_GAP (vinfo_for_stmt (stmt
)) != 0)
492 || (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt
)) != stmt
493 && DR_GROUP_GAP (vinfo_for_stmt (stmt
)) != 1))
495 if (vect_print_dump_info (REPORT_SLP
))
497 fprintf (vect_dump
, "Build SLP failed: strided "
499 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
505 /* Check that the size of interleaved loads group is not
506 greater than the SLP group size. */
507 if (DR_GROUP_SIZE (vinfo_for_stmt (stmt
))
508 > ncopies
* group_size
)
510 if (vect_print_dump_info (REPORT_SLP
))
512 fprintf (vect_dump
, "Build SLP failed: the number of "
513 "interleaved loads is greater than"
514 " the SLP group size ");
515 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
521 first_load
= DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt
));
523 if (first_load
== stmt
)
525 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt
));
526 if (vect_supportable_dr_alignment (first_dr
)
527 == dr_unaligned_unsupported
)
529 if (vect_print_dump_info (REPORT_SLP
))
531 fprintf (vect_dump
, "Build SLP failed: unsupported "
533 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
539 /* Analyze costs (for the first stmt in the group). */
540 vect_model_load_cost (vinfo_for_stmt (stmt
),
541 ncopies_for_cost
, *node
);
544 /* Store the place of this load in the interleaving chain. In
545 case that permutation is needed we later decide if a specific
546 permutation is supported. */
547 load_place
= vect_get_place_in_interleaving_chain (stmt
,
552 VEC_safe_push (int, heap
, *load_permutation
, load_place
);
554 /* We stop the tree when we reach a group of loads. */
555 stop_recursion
= true;
558 } /* Strided access. */
561 if (TREE_CODE_CLASS (rhs_code
) == tcc_reference
)
563 /* Not strided load. */
564 if (vect_print_dump_info (REPORT_SLP
))
566 fprintf (vect_dump
, "Build SLP failed: not strided load ");
567 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
570 /* FORNOW: Not strided loads are not supported. */
574 /* Not memory operation. */
575 if (TREE_CODE_CLASS (rhs_code
) != tcc_binary
576 && TREE_CODE_CLASS (rhs_code
) != tcc_unary
)
578 if (vect_print_dump_info (REPORT_SLP
))
580 fprintf (vect_dump
, "Build SLP failed: operation");
581 fprintf (vect_dump
, " unsupported ");
582 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
588 /* Find the def-stmts. */
589 if (!vect_get_and_check_slp_defs (loop_vinfo
, bb_vinfo
, *node
, stmt
,
590 &def_stmts0
, &def_stmts1
,
591 &first_stmt_dt0
, &first_stmt_dt1
,
592 &first_stmt_def0_type
,
593 &first_stmt_def1_type
,
594 &first_stmt_const_oprnd
,
596 &pattern0
, &pattern1
))
601 /* Add the costs of the node to the overall instance costs. */
602 *inside_cost
+= SLP_TREE_INSIDE_OF_LOOP_COST (*node
);
603 *outside_cost
+= SLP_TREE_OUTSIDE_OF_LOOP_COST (*node
);
605 /* Strided loads were reached - stop the recursion. */
610 VEC_safe_push (slp_tree
, heap
, *loads
, *node
);
611 *inside_cost
+= TARG_VEC_PERMUTE_COST
* group_size
;
617 /* Create SLP_TREE nodes for the definition node/s. */
618 if (first_stmt_dt0
== vect_internal_def
)
620 slp_tree left_node
= XNEW (struct _slp_tree
);
621 SLP_TREE_SCALAR_STMTS (left_node
) = def_stmts0
;
622 SLP_TREE_VEC_STMTS (left_node
) = NULL
;
623 SLP_TREE_LEFT (left_node
) = NULL
;
624 SLP_TREE_RIGHT (left_node
) = NULL
;
625 SLP_TREE_OUTSIDE_OF_LOOP_COST (left_node
) = 0;
626 SLP_TREE_INSIDE_OF_LOOP_COST (left_node
) = 0;
627 if (!vect_build_slp_tree (loop_vinfo
, bb_vinfo
, &left_node
, group_size
,
628 inside_cost
, outside_cost
, ncopies_for_cost
,
629 max_nunits
, load_permutation
, loads
,
630 vectorization_factor
))
633 SLP_TREE_LEFT (*node
) = left_node
;
636 if (first_stmt_dt1
== vect_internal_def
)
638 slp_tree right_node
= XNEW (struct _slp_tree
);
639 SLP_TREE_SCALAR_STMTS (right_node
) = def_stmts1
;
640 SLP_TREE_VEC_STMTS (right_node
) = NULL
;
641 SLP_TREE_LEFT (right_node
) = NULL
;
642 SLP_TREE_RIGHT (right_node
) = NULL
;
643 SLP_TREE_OUTSIDE_OF_LOOP_COST (right_node
) = 0;
644 SLP_TREE_INSIDE_OF_LOOP_COST (right_node
) = 0;
645 if (!vect_build_slp_tree (loop_vinfo
, bb_vinfo
, &right_node
, group_size
,
646 inside_cost
, outside_cost
, ncopies_for_cost
,
647 max_nunits
, load_permutation
, loads
,
648 vectorization_factor
))
651 SLP_TREE_RIGHT (*node
) = right_node
;
659 vect_print_slp_tree (slp_tree node
)
667 fprintf (vect_dump
, "node ");
668 for (i
= 0; VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (node
), i
, stmt
); i
++)
670 fprintf (vect_dump
, "\n\tstmt %d ", i
);
671 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
673 fprintf (vect_dump
, "\n");
675 vect_print_slp_tree (SLP_TREE_LEFT (node
));
676 vect_print_slp_tree (SLP_TREE_RIGHT (node
));
680 /* Mark the tree rooted at NODE with MARK (PURE_SLP or HYBRID).
681 If MARK is HYBRID, it refers to a specific stmt in NODE (the stmt at index
682 J). Otherwise, MARK is PURE_SLP and J is -1, which indicates that all the
683 stmts in NODE are to be marked. */
686 vect_mark_slp_stmts (slp_tree node
, enum slp_vect_type mark
, int j
)
694 for (i
= 0; VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (node
), i
, stmt
); i
++)
696 STMT_SLP_TYPE (vinfo_for_stmt (stmt
)) = mark
;
698 vect_mark_slp_stmts (SLP_TREE_LEFT (node
), mark
, j
);
699 vect_mark_slp_stmts (SLP_TREE_RIGHT (node
), mark
, j
);
703 /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
706 vect_mark_slp_stmts_relevant (slp_tree node
)
710 stmt_vec_info stmt_info
;
715 for (i
= 0; VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (node
), i
, stmt
); i
++)
717 stmt_info
= vinfo_for_stmt (stmt
);
718 gcc_assert (!STMT_VINFO_RELEVANT (stmt_info
)
719 || STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_scope
);
720 STMT_VINFO_RELEVANT (stmt_info
) = vect_used_in_scope
;
723 vect_mark_slp_stmts_relevant (SLP_TREE_LEFT (node
));
724 vect_mark_slp_stmts_relevant (SLP_TREE_RIGHT (node
));
728 /* Check if the permutation required by the SLP INSTANCE is supported.
729 Reorganize the SLP nodes stored in SLP_INSTANCE_LOADS if needed. */
732 vect_supported_slp_permutation_p (slp_instance instance
)
734 slp_tree node
= VEC_index (slp_tree
, SLP_INSTANCE_LOADS (instance
), 0);
735 gimple stmt
= VEC_index (gimple
, SLP_TREE_SCALAR_STMTS (node
), 0);
736 gimple first_load
= DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt
));
737 VEC (slp_tree
, heap
) *sorted_loads
= NULL
;
739 slp_tree
*tmp_loads
= NULL
;
740 int group_size
= SLP_INSTANCE_GROUP_SIZE (instance
), i
, j
;
743 /* FORNOW: The only supported loads permutation is loads from the same
744 location in all the loads in the node, when the data-refs in
745 nodes of LOADS constitute an interleaving chain.
746 Sort the nodes according to the order of accesses in the chain. */
747 tmp_loads
= (slp_tree
*) xmalloc (sizeof (slp_tree
) * group_size
);
749 VEC_iterate (int, SLP_INSTANCE_LOAD_PERMUTATION (instance
), i
, index
)
750 && VEC_iterate (slp_tree
, SLP_INSTANCE_LOADS (instance
), j
, load
);
751 i
+= group_size
, j
++)
753 gimple scalar_stmt
= VEC_index (gimple
, SLP_TREE_SCALAR_STMTS (load
), 0);
754 /* Check that the loads are all in the same interleaving chain. */
755 if (DR_GROUP_FIRST_DR (vinfo_for_stmt (scalar_stmt
)) != first_load
)
757 if (vect_print_dump_info (REPORT_DETAILS
))
759 fprintf (vect_dump
, "Build SLP failed: unsupported data "
761 print_gimple_stmt (vect_dump
, scalar_stmt
, 0, TDF_SLIM
);
768 tmp_loads
[index
] = load
;
771 sorted_loads
= VEC_alloc (slp_tree
, heap
, group_size
);
772 for (i
= 0; i
< group_size
; i
++)
773 VEC_safe_push (slp_tree
, heap
, sorted_loads
, tmp_loads
[i
]);
775 VEC_free (slp_tree
, heap
, SLP_INSTANCE_LOADS (instance
));
776 SLP_INSTANCE_LOADS (instance
) = sorted_loads
;
779 if (!vect_transform_slp_perm_load (stmt
, NULL
, NULL
,
780 SLP_INSTANCE_UNROLLING_FACTOR (instance
),
788 /* Check if the required load permutation is supported.
789 LOAD_PERMUTATION contains a list of indices of the loads.
790 In SLP this permutation is relative to the order of strided stores that are
791 the base of the SLP instance. */
794 vect_supported_load_permutation_p (slp_instance slp_instn
, int group_size
,
795 VEC (int, heap
) *load_permutation
)
797 int i
= 0, j
, prev
= -1, next
, k
;
800 /* FORNOW: permutations are only supported in SLP. */
804 if (vect_print_dump_info (REPORT_SLP
))
806 fprintf (vect_dump
, "Load permutation ");
807 for (i
= 0; VEC_iterate (int, load_permutation
, i
, next
); i
++)
808 fprintf (vect_dump
, "%d ", next
);
811 /* FORNOW: the only supported permutation is 0..01..1.. of length equal to
812 GROUP_SIZE and where each sequence of same drs is of GROUP_SIZE length as
814 if (VEC_length (int, load_permutation
)
815 != (unsigned int) (group_size
* group_size
))
819 for (j
= 0; j
< group_size
; j
++)
821 for (i
= j
* group_size
, k
= 0;
822 VEC_iterate (int, load_permutation
, i
, next
) && k
< group_size
;
825 if (i
!= j
* group_size
&& next
!= prev
)
835 if (supported
&& i
== group_size
* group_size
836 && vect_supported_slp_permutation_p (slp_instn
))
843 /* Find the first load in the loop that belongs to INSTANCE.
844 When loads are in several SLP nodes, there can be a case in which the first
845 load does not appear in the first SLP node to be transformed, causing
846 incorrect order of statements. Since we generate all the loads together,
847 they must be inserted before the first load of the SLP instance and not
848 before the first load of the first node of the instance. */
850 vect_find_first_load_in_slp_instance (slp_instance instance
)
854 gimple first_load
= NULL
, load
;
857 VEC_iterate (slp_tree
, SLP_INSTANCE_LOADS (instance
), i
, load_node
);
860 VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (load_node
), j
, load
);
862 first_load
= get_earlier_stmt (load
, first_load
);
868 /* Analyze an SLP instance starting from a group of strided stores. Call
869 vect_build_slp_tree to build a tree of packed stmts if possible.
870 Return FALSE if it's impossible to SLP any stmt in the loop. */
873 vect_analyze_slp_instance (loop_vec_info loop_vinfo
, bb_vec_info bb_vinfo
,
876 slp_instance new_instance
;
877 slp_tree node
= XNEW (struct _slp_tree
);
878 unsigned int group_size
= DR_GROUP_SIZE (vinfo_for_stmt (stmt
));
879 unsigned int unrolling_factor
= 1, nunits
;
880 tree vectype
, scalar_type
;
882 unsigned int vectorization_factor
= 0;
883 int inside_cost
= 0, outside_cost
= 0, ncopies_for_cost
;
884 unsigned int max_nunits
= 0;
885 VEC (int, heap
) *load_permutation
;
886 VEC (slp_tree
, heap
) *loads
;
888 scalar_type
= TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (
889 vinfo_for_stmt (stmt
))));
890 vectype
= get_vectype_for_scalar_type (scalar_type
);
893 if (vect_print_dump_info (REPORT_SLP
))
895 fprintf (vect_dump
, "Build SLP failed: unsupported data-type ");
896 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
901 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
903 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
905 /* No multitypes in BB SLP. */
906 vectorization_factor
= nunits
;
908 /* Calculate the unrolling factor. */
909 unrolling_factor
= least_common_multiple (nunits
, group_size
) / group_size
;
910 if (unrolling_factor
!= 1 && !loop_vinfo
)
912 if (vect_print_dump_info (REPORT_SLP
))
913 fprintf (vect_dump
, "Build SLP failed: unrolling required in basic"
919 /* Create a node (a root of the SLP tree) for the packed strided stores. */
920 SLP_TREE_SCALAR_STMTS (node
) = VEC_alloc (gimple
, heap
, group_size
);
922 /* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
925 VEC_safe_push (gimple
, heap
, SLP_TREE_SCALAR_STMTS (node
), next
);
926 next
= DR_GROUP_NEXT_DR (vinfo_for_stmt (next
));
929 SLP_TREE_VEC_STMTS (node
) = NULL
;
930 SLP_TREE_NUMBER_OF_VEC_STMTS (node
) = 0;
931 SLP_TREE_LEFT (node
) = NULL
;
932 SLP_TREE_RIGHT (node
) = NULL
;
933 SLP_TREE_OUTSIDE_OF_LOOP_COST (node
) = 0;
934 SLP_TREE_INSIDE_OF_LOOP_COST (node
) = 0;
936 /* Calculate the number of vector stmts to create based on the unrolling
937 factor (number of vectors is 1 if NUNITS >= GROUP_SIZE, and is
938 GROUP_SIZE / NUNITS otherwise. */
939 ncopies_for_cost
= unrolling_factor
* group_size
/ nunits
;
941 load_permutation
= VEC_alloc (int, heap
, group_size
* group_size
);
942 loads
= VEC_alloc (slp_tree
, heap
, group_size
);
944 /* Build the tree for the SLP instance. */
945 if (vect_build_slp_tree (loop_vinfo
, bb_vinfo
, &node
, group_size
,
946 &inside_cost
, &outside_cost
, ncopies_for_cost
,
947 &max_nunits
, &load_permutation
, &loads
,
948 vectorization_factor
))
950 /* Create a new SLP instance. */
951 new_instance
= XNEW (struct _slp_instance
);
952 SLP_INSTANCE_TREE (new_instance
) = node
;
953 SLP_INSTANCE_GROUP_SIZE (new_instance
) = group_size
;
954 /* Calculate the unrolling factor based on the smallest type in the
956 if (max_nunits
> nunits
)
957 unrolling_factor
= least_common_multiple (max_nunits
, group_size
)
960 SLP_INSTANCE_UNROLLING_FACTOR (new_instance
) = unrolling_factor
;
961 SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance
) = outside_cost
;
962 SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance
) = inside_cost
;
963 SLP_INSTANCE_LOADS (new_instance
) = loads
;
964 SLP_INSTANCE_FIRST_LOAD_STMT (new_instance
) = NULL
;
965 SLP_INSTANCE_LOAD_PERMUTATION (new_instance
) = load_permutation
;
966 if (VEC_length (slp_tree
, loads
))
968 if (!vect_supported_load_permutation_p (new_instance
, group_size
,
971 if (vect_print_dump_info (REPORT_SLP
))
973 fprintf (vect_dump
, "Build SLP failed: unsupported load "
975 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
978 vect_free_slp_instance (new_instance
);
982 SLP_INSTANCE_FIRST_LOAD_STMT (new_instance
)
983 = vect_find_first_load_in_slp_instance (new_instance
);
986 VEC_free (int, heap
, SLP_INSTANCE_LOAD_PERMUTATION (new_instance
));
989 VEC_safe_push (slp_instance
, heap
,
990 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
),
993 VEC_safe_push (slp_instance
, heap
, BB_VINFO_SLP_INSTANCES (bb_vinfo
),
996 if (vect_print_dump_info (REPORT_SLP
))
997 vect_print_slp_tree (node
);
1002 /* Failed to SLP. */
1003 /* Free the allocated memory. */
1004 vect_free_slp_tree (node
);
1005 VEC_free (int, heap
, load_permutation
);
1006 VEC_free (slp_tree
, heap
, loads
);
1012 /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
1013 trees of packed scalar stmts if SLP is possible. */
1016 vect_analyze_slp (loop_vec_info loop_vinfo
, bb_vec_info bb_vinfo
)
1019 VEC (gimple
, heap
) *strided_stores
;
1023 if (vect_print_dump_info (REPORT_SLP
))
1024 fprintf (vect_dump
, "=== vect_analyze_slp ===");
1027 strided_stores
= LOOP_VINFO_STRIDED_STORES (loop_vinfo
);
1029 strided_stores
= BB_VINFO_STRIDED_STORES (bb_vinfo
);
1031 for (i
= 0; VEC_iterate (gimple
, strided_stores
, i
, store
); i
++)
1032 if (vect_analyze_slp_instance (loop_vinfo
, bb_vinfo
, store
))
1035 if (bb_vinfo
&& !ok
)
1037 if (vect_print_dump_info (REPORT_SLP
))
1038 fprintf (vect_dump
, "Failed to SLP the basic block.");
1047 /* For each possible SLP instance decide whether to SLP it and calculate overall
1048 unrolling factor needed to SLP the loop. */
1051 vect_make_slp_decision (loop_vec_info loop_vinfo
)
1053 unsigned int i
, unrolling_factor
= 1;
1054 VEC (slp_instance
, heap
) *slp_instances
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
);
1055 slp_instance instance
;
1056 int decided_to_slp
= 0;
1058 if (vect_print_dump_info (REPORT_SLP
))
1059 fprintf (vect_dump
, "=== vect_make_slp_decision ===");
1061 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); i
++)
1063 /* FORNOW: SLP if you can. */
1064 if (unrolling_factor
< SLP_INSTANCE_UNROLLING_FACTOR (instance
))
1065 unrolling_factor
= SLP_INSTANCE_UNROLLING_FACTOR (instance
);
1067 /* Mark all the stmts that belong to INSTANCE as PURE_SLP stmts. Later we
1068 call vect_detect_hybrid_slp () to find stmts that need hybrid SLP and
1069 loop-based vectorization. Such stmts will be marked as HYBRID. */
1070 vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance
), pure_slp
, -1);
1074 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
) = unrolling_factor
;
1076 if (decided_to_slp
&& vect_print_dump_info (REPORT_SLP
))
1077 fprintf (vect_dump
, "Decided to SLP %d instances. Unrolling factor %d",
1078 decided_to_slp
, unrolling_factor
);
1082 /* Find stmts that must be both vectorized and SLPed (since they feed stmts that
1083 can't be SLPed) in the tree rooted at NODE. Mark such stmts as HYBRID. */
1086 vect_detect_hybrid_slp_stmts (slp_tree node
)
1090 imm_use_iterator imm_iter
;
1096 for (i
= 0; VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (node
), i
, stmt
); i
++)
1097 if (PURE_SLP_STMT (vinfo_for_stmt (stmt
))
1098 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1099 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, gimple_op (stmt
, 0))
1100 if (vinfo_for_stmt (use_stmt
)
1101 && !STMT_SLP_TYPE (vinfo_for_stmt (use_stmt
))
1102 && STMT_VINFO_RELEVANT (vinfo_for_stmt (use_stmt
)))
1103 vect_mark_slp_stmts (node
, hybrid
, i
);
1105 vect_detect_hybrid_slp_stmts (SLP_TREE_LEFT (node
));
1106 vect_detect_hybrid_slp_stmts (SLP_TREE_RIGHT (node
));
1110 /* Find stmts that must be both vectorized and SLPed. */
1113 vect_detect_hybrid_slp (loop_vec_info loop_vinfo
)
1116 VEC (slp_instance
, heap
) *slp_instances
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
);
1117 slp_instance instance
;
1119 if (vect_print_dump_info (REPORT_SLP
))
1120 fprintf (vect_dump
, "=== vect_detect_hybrid_slp ===");
1122 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); i
++)
1123 vect_detect_hybrid_slp_stmts (SLP_INSTANCE_TREE (instance
));
1127 /* Create and initialize a new bb_vec_info struct for BB, as well as
1128 stmt_vec_info structs for all the stmts in it. */
1131 new_bb_vec_info (basic_block bb
)
1133 bb_vec_info res
= NULL
;
1134 gimple_stmt_iterator gsi
;
1136 res
= (bb_vec_info
) xcalloc (1, sizeof (struct _bb_vec_info
));
1137 BB_VINFO_BB (res
) = bb
;
1139 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1141 gimple stmt
= gsi_stmt (gsi
);
1142 gimple_set_uid (stmt
, 0);
1143 set_vinfo_for_stmt (stmt
, new_stmt_vec_info (stmt
, NULL
, res
));
1146 BB_VINFO_STRIDED_STORES (res
) = VEC_alloc (gimple
, heap
, 10);
1147 BB_VINFO_SLP_INSTANCES (res
) = VEC_alloc (slp_instance
, heap
, 2);
1154 /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
1155 stmts in the basic block. */
1158 destroy_bb_vec_info (bb_vec_info bb_vinfo
)
1161 gimple_stmt_iterator si
;
1166 bb
= BB_VINFO_BB (bb_vinfo
);
1168 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1170 gimple stmt
= gsi_stmt (si
);
1171 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1174 /* Free stmt_vec_info. */
1175 free_stmt_vec_info (stmt
);
1178 VEC_free (gimple
, heap
, BB_VINFO_STRIDED_STORES (bb_vinfo
));
1179 VEC_free (slp_instance
, heap
, BB_VINFO_SLP_INSTANCES (bb_vinfo
));
1185 /* Analyze statements contained in SLP tree node after recursively analyzing
1186 the subtree. Return TRUE if the operations are supported. */
1189 vect_slp_analyze_node_operations (bb_vec_info bb_vinfo
, slp_tree node
)
1198 if (!vect_slp_analyze_node_operations (bb_vinfo
, SLP_TREE_LEFT (node
))
1199 || !vect_slp_analyze_node_operations (bb_vinfo
, SLP_TREE_RIGHT (node
)))
1202 for (i
= 0; VEC_iterate (gimple
, SLP_TREE_SCALAR_STMTS (node
), i
, stmt
); i
++)
1204 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1205 gcc_assert (stmt_info
);
1206 gcc_assert (PURE_SLP_STMT (stmt_info
));
1208 if (!vect_analyze_stmt (stmt
, &dummy
, node
))
1216 /* Analyze statements in SLP instances of the basic block. Return TRUE if the
1217 operations are supported. */
1220 vect_slp_analyze_operations (bb_vec_info bb_vinfo
)
1222 VEC (slp_instance
, heap
) *slp_instances
= BB_VINFO_SLP_INSTANCES (bb_vinfo
);
1223 slp_instance instance
;
1226 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); )
1228 if (!vect_slp_analyze_node_operations (bb_vinfo
,
1229 SLP_INSTANCE_TREE (instance
)))
1231 vect_free_slp_instance (instance
);
1232 VEC_ordered_remove (slp_instance
, slp_instances
, i
);
1238 if (!VEC_length (slp_instance
, slp_instances
))
1245 /* Cheick if the basic block can be vectorized. */
1248 vect_slp_analyze_bb (basic_block bb
)
1250 bb_vec_info bb_vinfo
;
1251 VEC (ddr_p
, heap
) *ddrs
;
1252 VEC (slp_instance
, heap
) *slp_instances
;
1253 slp_instance instance
;
1255 gimple_stmt_iterator gsi
;
1257 if (vect_print_dump_info (REPORT_DETAILS
))
1258 fprintf (vect_dump
, "===vect_slp_analyze_bb===\n");
1260 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1263 if (insns
> PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB
))
1265 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1266 fprintf (vect_dump
, "not vectorized: too many instructions in basic "
1272 bb_vinfo
= new_bb_vec_info (bb
);
1276 if (!vect_analyze_data_refs (NULL
, bb_vinfo
))
1278 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1279 fprintf (vect_dump
, "not vectorized: unhandled data-ref in basic "
1282 destroy_bb_vec_info (bb_vinfo
);
1286 ddrs
= BB_VINFO_DDRS (bb_vinfo
);
1287 if (!VEC_length (ddr_p
, ddrs
))
1289 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1290 fprintf (vect_dump
, "not vectorized: not enough data-refs in basic "
1293 destroy_bb_vec_info (bb_vinfo
);
1297 if (!vect_analyze_data_refs_alignment (NULL
, bb_vinfo
))
1299 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1300 fprintf (vect_dump
, "not vectorized: bad data alignment in basic "
1303 destroy_bb_vec_info (bb_vinfo
);
1307 if (!vect_analyze_data_ref_dependences (NULL
, bb_vinfo
))
1309 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1310 fprintf (vect_dump
, "not vectorized: unhandled data dependence in basic"
1313 destroy_bb_vec_info (bb_vinfo
);
1317 if (!vect_analyze_data_ref_accesses (NULL
, bb_vinfo
))
1319 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1320 fprintf (vect_dump
, "not vectorized: unhandled data access in basic "
1323 destroy_bb_vec_info (bb_vinfo
);
1327 if (!vect_verify_datarefs_alignment (NULL
, bb_vinfo
))
1329 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1330 fprintf (vect_dump
, "not vectorized: unsupported alignment in basic "
1333 destroy_bb_vec_info (bb_vinfo
);
1337 /* Check the SLP opportunities in the basic block, analyze and build SLP
1339 if (!vect_analyze_slp (NULL
, bb_vinfo
))
1341 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1342 fprintf (vect_dump
, "not vectorized: failed to find SLP opportunities "
1343 "in basic block.\n");
1345 destroy_bb_vec_info (bb_vinfo
);
1349 slp_instances
= BB_VINFO_SLP_INSTANCES (bb_vinfo
);
1351 /* Mark all the statements that we want to vectorize as pure SLP and
1353 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); i
++)
1355 vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance
), pure_slp
, -1);
1356 vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance
));
1359 if (!vect_slp_analyze_operations (bb_vinfo
))
1361 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
1362 fprintf (vect_dump
, "not vectorized: bad operation in basic block.\n");
1364 destroy_bb_vec_info (bb_vinfo
);
1368 if (vect_print_dump_info (REPORT_DETAILS
))
1369 fprintf (vect_dump
, "Basic block will be vectorized using SLP\n");
1375 /* SLP costs are calculated according to SLP instance unrolling factor (i.e.,
1376 the number of created vector stmts depends on the unrolling factor). However,
1377 the actual number of vector stmts for every SLP node depends on VF which is
1378 set later in vect_analyze_operations(). Hence, SLP costs should be updated.
1379 In this function we assume that the inside costs calculated in
1380 vect_model_xxx_cost are linear in ncopies. */
1383 vect_update_slp_costs_according_to_vf (loop_vec_info loop_vinfo
)
1385 unsigned int i
, vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1386 VEC (slp_instance
, heap
) *slp_instances
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
);
1387 slp_instance instance
;
1389 if (vect_print_dump_info (REPORT_SLP
))
1390 fprintf (vect_dump
, "=== vect_update_slp_costs_according_to_vf ===");
1392 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); i
++)
1393 /* We assume that costs are linear in ncopies. */
1394 SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance
) *= vf
1395 / SLP_INSTANCE_UNROLLING_FACTOR (instance
);
1399 /* For constant and loop invariant defs of SLP_NODE this function returns
1400 (vector) defs (VEC_OPRNDS) that will be used in the vectorized stmts.
1401 OP_NUM determines if we gather defs for operand 0 or operand 1 of the scalar
1402 stmts. NUMBER_OF_VECTORS is the number of vector defs to create. */
1405 vect_get_constant_vectors (slp_tree slp_node
, VEC(tree
,heap
) **vec_oprnds
,
1406 unsigned int op_num
, unsigned int number_of_vectors
)
1408 VEC (gimple
, heap
) *stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
1409 gimple stmt
= VEC_index (gimple
, stmts
, 0);
1410 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1411 tree vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
1415 int j
, number_of_places_left_in_vector
;
1418 int group_size
= VEC_length (gimple
, stmts
);
1419 unsigned int vec_num
, i
;
1420 int number_of_copies
= 1;
1421 VEC (tree
, heap
) *voprnds
= VEC_alloc (tree
, heap
, number_of_vectors
);
1422 bool constant_p
, is_store
;
1424 if (STMT_VINFO_DATA_REF (stmt_vinfo
))
1427 op
= gimple_assign_rhs1 (stmt
);
1432 op
= gimple_op (stmt
, op_num
+ 1);
1435 if (CONSTANT_CLASS_P (op
))
1437 vector_type
= vectype
;
1442 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1443 gcc_assert (vector_type
);
1447 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
1449 /* NUMBER_OF_COPIES is the number of times we need to use the same values in
1450 created vectors. It is greater than 1 if unrolling is performed.
1452 For example, we have two scalar operands, s1 and s2 (e.g., group of
1453 strided accesses of size two), while NUNITS is four (i.e., four scalars
1454 of this type can be packed in a vector). The output vector will contain
1455 two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
1458 If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
1459 containing the operands.
1461 For example, NUNITS is four as before, and the group size is 8
1462 (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
1463 {s5, s6, s7, s8}. */
1465 number_of_copies
= least_common_multiple (nunits
, group_size
) / group_size
;
1467 number_of_places_left_in_vector
= nunits
;
1468 for (j
= 0; j
< number_of_copies
; j
++)
1470 for (i
= group_size
- 1; VEC_iterate (gimple
, stmts
, i
, stmt
); i
--)
1473 op
= gimple_assign_rhs1 (stmt
);
1475 op
= gimple_op (stmt
, op_num
+ 1);
1477 /* Create 'vect_ = {op0,op1,...,opn}'. */
1478 t
= tree_cons (NULL_TREE
, op
, t
);
1480 number_of_places_left_in_vector
--;
1482 if (number_of_places_left_in_vector
== 0)
1484 number_of_places_left_in_vector
= nunits
;
1487 vec_cst
= build_vector (vector_type
, t
);
1489 vec_cst
= build_constructor_from_list (vector_type
, t
);
1490 VEC_quick_push (tree
, voprnds
,
1491 vect_init_vector (stmt
, vec_cst
, vector_type
, NULL
));
1497 /* Since the vectors are created in the reverse order, we should invert
1499 vec_num
= VEC_length (tree
, voprnds
);
1500 for (j
= vec_num
- 1; j
>= 0; j
--)
1502 vop
= VEC_index (tree
, voprnds
, j
);
1503 VEC_quick_push (tree
, *vec_oprnds
, vop
);
1506 VEC_free (tree
, heap
, voprnds
);
1508 /* In case that VF is greater than the unrolling factor needed for the SLP
1509 group of stmts, NUMBER_OF_VECTORS to be created is greater than
1510 NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
1511 to replicate the vectors. */
1512 while (number_of_vectors
> VEC_length (tree
, *vec_oprnds
))
1514 for (i
= 0; VEC_iterate (tree
, *vec_oprnds
, i
, vop
) && i
< vec_num
; i
++)
1515 VEC_quick_push (tree
, *vec_oprnds
, vop
);
1520 /* Get vectorized definitions from SLP_NODE that contains corresponding
1521 vectorized def-stmts. */
1524 vect_get_slp_vect_defs (slp_tree slp_node
, VEC (tree
,heap
) **vec_oprnds
)
1527 gimple vec_def_stmt
;
1530 gcc_assert (SLP_TREE_VEC_STMTS (slp_node
));
1533 VEC_iterate (gimple
, SLP_TREE_VEC_STMTS (slp_node
), i
, vec_def_stmt
);
1536 gcc_assert (vec_def_stmt
);
1537 vec_oprnd
= gimple_get_lhs (vec_def_stmt
);
1538 VEC_quick_push (tree
, *vec_oprnds
, vec_oprnd
);
1543 /* Get vectorized definitions for SLP_NODE.
1544 If the scalar definitions are loop invariants or constants, collect them and
1545 call vect_get_constant_vectors() to create vector stmts.
1546 Otherwise, the def-stmts must be already vectorized and the vectorized stmts
1547 must be stored in the LEFT/RIGHT node of SLP_NODE, and we call
1548 vect_get_slp_vect_defs() to retrieve them.
1549 If VEC_OPRNDS1 is NULL, don't get vector defs for the second operand (from
1550 the right node. This is used when the second operand must remain scalar. */
1553 vect_get_slp_defs (slp_tree slp_node
, VEC (tree
,heap
) **vec_oprnds0
,
1554 VEC (tree
,heap
) **vec_oprnds1
)
1557 enum tree_code code
;
1558 int number_of_vects
;
1559 HOST_WIDE_INT lhs_size_unit
, rhs_size_unit
;
1561 first_stmt
= VEC_index (gimple
, SLP_TREE_SCALAR_STMTS (slp_node
), 0);
1562 /* The number of vector defs is determined by the number of vector statements
1563 in the node from which we get those statements. */
1564 if (SLP_TREE_LEFT (slp_node
))
1565 number_of_vects
= SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node
));
1568 number_of_vects
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
1569 /* Number of vector stmts was calculated according to LHS in
1570 vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
1571 necessary. See vect_get_smallest_scalar_type() for details. */
1572 vect_get_smallest_scalar_type (first_stmt
, &lhs_size_unit
,
1574 if (rhs_size_unit
!= lhs_size_unit
)
1576 number_of_vects
*= rhs_size_unit
;
1577 number_of_vects
/= lhs_size_unit
;
1581 /* Allocate memory for vectorized defs. */
1582 *vec_oprnds0
= VEC_alloc (tree
, heap
, number_of_vects
);
1584 /* SLP_NODE corresponds either to a group of stores or to a group of
1585 unary/binary operations. We don't call this function for loads. */
1586 if (SLP_TREE_LEFT (slp_node
))
1587 /* The defs are already vectorized. */
1588 vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node
), vec_oprnds0
);
1590 /* Build vectors from scalar defs. */
1591 vect_get_constant_vectors (slp_node
, vec_oprnds0
, 0, number_of_vects
);
1593 if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
)))
1594 /* Since we don't call this function with loads, this is a group of
1598 code
= gimple_assign_rhs_code (first_stmt
);
1599 if (get_gimple_rhs_class (code
) != GIMPLE_BINARY_RHS
|| !vec_oprnds1
)
1602 /* The number of vector defs is determined by the number of vector statements
1603 in the node from which we get those statements. */
1604 if (SLP_TREE_RIGHT (slp_node
))
1605 number_of_vects
= SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_RIGHT (slp_node
));
1607 number_of_vects
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
1609 *vec_oprnds1
= VEC_alloc (tree
, heap
, number_of_vects
);
1611 if (SLP_TREE_RIGHT (slp_node
))
1612 /* The defs are already vectorized. */
1613 vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node
), vec_oprnds1
);
1615 /* Build vectors from scalar defs. */
1616 vect_get_constant_vectors (slp_node
, vec_oprnds1
, 1, number_of_vects
);
1620 /* Create NCOPIES permutation statements using the mask MASK_BYTES (by
1621 building a vector of type MASK_TYPE from it) and two input vectors placed in
1622 DR_CHAIN at FIRST_VEC_INDX and SECOND_VEC_INDX for the first copy and
1623 shifting by STRIDE elements of DR_CHAIN for every copy.
1624 (STRIDE is the number of vectorized stmts for NODE divided by the number of
1626 VECT_STMTS_COUNTER specifies the index in the vectorized stmts of NODE, where
1627 the created stmts must be inserted. */
1630 vect_create_mask_and_perm (gimple stmt
, gimple next_scalar_stmt
,
1631 tree mask
, int first_vec_indx
, int second_vec_indx
,
1632 gimple_stmt_iterator
*gsi
, slp_tree node
,
1633 tree builtin_decl
, tree vectype
,
1634 VEC(tree
,heap
) *dr_chain
,
1635 int ncopies
, int vect_stmts_counter
)
1638 gimple perm_stmt
= NULL
;
1639 stmt_vec_info next_stmt_info
;
1641 tree first_vec
, second_vec
, data_ref
;
1642 VEC (tree
, heap
) *params
= NULL
;
1644 stride
= SLP_TREE_NUMBER_OF_VEC_STMTS (node
) / ncopies
;
1646 /* Initialize the vect stmts of NODE to properly insert the generated
1648 for (i
= VEC_length (gimple
, SLP_TREE_VEC_STMTS (node
));
1649 i
< (int) SLP_TREE_NUMBER_OF_VEC_STMTS (node
); i
++)
1650 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (node
), NULL
);
1652 perm_dest
= vect_create_destination_var (gimple_assign_lhs (stmt
), vectype
);
1653 for (i
= 0; i
< ncopies
; i
++)
1655 first_vec
= VEC_index (tree
, dr_chain
, first_vec_indx
);
1656 second_vec
= VEC_index (tree
, dr_chain
, second_vec_indx
);
1658 /* Build argument list for the vectorized call. */
1659 VEC_free (tree
, heap
, params
);
1660 params
= VEC_alloc (tree
, heap
, 3);
1661 VEC_quick_push (tree
, params
, first_vec
);
1662 VEC_quick_push (tree
, params
, second_vec
);
1663 VEC_quick_push (tree
, params
, mask
);
1665 /* Generate the permute statement. */
1666 perm_stmt
= gimple_build_call_vec (builtin_decl
, params
);
1667 data_ref
= make_ssa_name (perm_dest
, perm_stmt
);
1668 gimple_call_set_lhs (perm_stmt
, data_ref
);
1669 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
1671 /* Store the vector statement in NODE. */
1672 VEC_replace (gimple
, SLP_TREE_VEC_STMTS (node
),
1673 stride
* i
+ vect_stmts_counter
, perm_stmt
);
1675 first_vec_indx
+= stride
;
1676 second_vec_indx
+= stride
;
1679 /* Mark the scalar stmt as vectorized. */
1680 next_stmt_info
= vinfo_for_stmt (next_scalar_stmt
);
1681 STMT_VINFO_VEC_STMT (next_stmt_info
) = perm_stmt
;
1685 /* Given FIRST_MASK_ELEMENT - the mask element in element representation,
1686 return in CURRENT_MASK_ELEMENT its equivalent in target specific
1687 representation. Check that the mask is valid and return FALSE if not.
1688 Return TRUE in NEED_NEXT_VECTOR if the permutation requires to move to
1689 the next vector, i.e., the current first vector is not needed. */
1692 vect_get_mask_element (gimple stmt
, int first_mask_element
, int m
,
1693 int mask_nunits
, bool only_one_vec
, int index
,
1694 int *mask
, int *current_mask_element
,
1695 bool *need_next_vector
)
1698 static int number_of_mask_fixes
= 1;
1699 static bool mask_fixed
= false;
1700 static bool needs_first_vector
= false;
1702 /* Convert to target specific representation. */
1703 *current_mask_element
= first_mask_element
+ m
;
1704 /* Adjust the value in case it's a mask for second and third vectors. */
1705 *current_mask_element
-= mask_nunits
* (number_of_mask_fixes
- 1);
1707 if (*current_mask_element
< mask_nunits
)
1708 needs_first_vector
= true;
1710 /* We have only one input vector to permute but the mask accesses values in
1711 the next vector as well. */
1712 if (only_one_vec
&& *current_mask_element
>= mask_nunits
)
1714 if (vect_print_dump_info (REPORT_DETAILS
))
1716 fprintf (vect_dump
, "permutation requires at least two vectors ");
1717 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
1723 /* The mask requires the next vector. */
1724 if (*current_mask_element
>= mask_nunits
* 2)
1726 if (needs_first_vector
|| mask_fixed
)
1728 /* We either need the first vector too or have already moved to the
1729 next vector. In both cases, this permutation needs three
1731 if (vect_print_dump_info (REPORT_DETAILS
))
1733 fprintf (vect_dump
, "permutation requires at "
1734 "least three vectors ");
1735 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
1741 /* We move to the next vector, dropping the first one and working with
1742 the second and the third - we need to adjust the values of the mask
1744 *current_mask_element
-= mask_nunits
* number_of_mask_fixes
;
1746 for (i
= 0; i
< index
; i
++)
1747 mask
[i
] -= mask_nunits
* number_of_mask_fixes
;
1749 (number_of_mask_fixes
)++;
1753 *need_next_vector
= mask_fixed
;
1755 /* This was the last element of this mask. Start a new one. */
1756 if (index
== mask_nunits
- 1)
1758 number_of_mask_fixes
= 1;
1760 needs_first_vector
= false;
1767 /* Generate vector permute statements from a list of loads in DR_CHAIN.
1768 If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
1769 permute statements for SLP_NODE_INSTANCE. */
1771 vect_transform_slp_perm_load (gimple stmt
, VEC (tree
, heap
) *dr_chain
,
1772 gimple_stmt_iterator
*gsi
, int vf
,
1773 slp_instance slp_node_instance
, bool analyze_only
)
1775 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1776 tree mask_element_type
= NULL_TREE
, mask_type
;
1777 int i
, j
, k
, m
, scale
, mask_nunits
, nunits
, vec_index
= 0, scalar_index
;
1779 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
), builtin_decl
;
1780 gimple next_scalar_stmt
;
1781 int group_size
= SLP_INSTANCE_GROUP_SIZE (slp_node_instance
);
1782 int first_mask_element
;
1783 int index
, unroll_factor
, *mask
, current_mask_element
, ncopies
;
1784 bool only_one_vec
= false, need_next_vector
= false;
1785 int first_vec_index
, second_vec_index
, orig_vec_stmts_num
, vect_stmts_counter
;
1787 if (!targetm
.vectorize
.builtin_vec_perm
)
1789 if (vect_print_dump_info (REPORT_DETAILS
))
1791 fprintf (vect_dump
, "no builtin for vect permute for ");
1792 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
1798 builtin_decl
= targetm
.vectorize
.builtin_vec_perm (vectype
,
1799 &mask_element_type
);
1800 if (!builtin_decl
|| !mask_element_type
)
1802 if (vect_print_dump_info (REPORT_DETAILS
))
1804 fprintf (vect_dump
, "no builtin for vect permute for ");
1805 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
1811 mask_type
= get_vectype_for_scalar_type (mask_element_type
);
1812 mask_nunits
= TYPE_VECTOR_SUBPARTS (mask_type
);
1813 mask
= (int *) xmalloc (sizeof (int) * mask_nunits
);
1814 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1815 scale
= mask_nunits
/ nunits
;
1816 unroll_factor
= SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance
);
1818 /* The number of vector stmts to generate based only on SLP_NODE_INSTANCE
1819 unrolling factor. */
1820 orig_vec_stmts_num
= group_size
*
1821 SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance
) / nunits
;
1822 if (orig_vec_stmts_num
== 1)
1823 only_one_vec
= true;
1825 /* Number of copies is determined by the final vectorization factor
1826 relatively to SLP_NODE_INSTANCE unrolling factor. */
1827 ncopies
= vf
/ SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance
);
1829 /* Generate permutation masks for every NODE. Number of masks for each NODE
1830 is equal to GROUP_SIZE.
1831 E.g., we have a group of three nodes with three loads from the same
1832 location in each node, and the vector size is 4. I.e., we have a
1833 a0b0c0a1b1c1... sequence and we need to create the following vectors:
1834 for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
1835 for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
1838 The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9} (in target
1839 scpecific type, e.g., in bytes for Altivec.
1840 The last mask is illegal since we assume two operands for permute
1841 operation, and the mask element values can't be outside that range. Hence,
1842 the last mask must be converted into {2,5,5,5}.
1843 For the first two permutations we need the first and the second input
1844 vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
1845 we need the second and the third vectors: {b1,c1,a2,b2} and
1849 VEC_iterate (slp_tree
, SLP_INSTANCE_LOADS (slp_node_instance
),
1855 vect_stmts_counter
= 0;
1857 first_vec_index
= vec_index
++;
1859 second_vec_index
= first_vec_index
;
1861 second_vec_index
= vec_index
++;
1863 for (j
= 0; j
< unroll_factor
; j
++)
1865 for (k
= 0; k
< group_size
; k
++)
1867 first_mask_element
= (i
+ j
* group_size
) * scale
;
1868 for (m
= 0; m
< scale
; m
++)
1870 if (!vect_get_mask_element (stmt
, first_mask_element
, m
,
1871 mask_nunits
, only_one_vec
, index
, mask
,
1872 ¤t_mask_element
, &need_next_vector
))
1875 mask
[index
++] = current_mask_element
;
1878 if (index
== mask_nunits
)
1880 tree mask_vec
= NULL
;
1882 while (--index
>= 0)
1884 tree t
= build_int_cst (mask_element_type
, mask
[index
]);
1885 mask_vec
= tree_cons (NULL
, t
, mask_vec
);
1887 mask_vec
= build_vector (mask_type
, mask_vec
);
1890 if (!targetm
.vectorize
.builtin_vec_perm_ok (vectype
,
1893 if (vect_print_dump_info (REPORT_DETAILS
))
1895 fprintf (vect_dump
, "unsupported vect permute ");
1896 print_generic_expr (vect_dump
, mask_vec
, 0);
1904 if (need_next_vector
)
1906 first_vec_index
= second_vec_index
;
1907 second_vec_index
= vec_index
;
1910 next_scalar_stmt
= VEC_index (gimple
,
1911 SLP_TREE_SCALAR_STMTS (node
), scalar_index
++);
1913 vect_create_mask_and_perm (stmt
, next_scalar_stmt
,
1914 mask_vec
, first_vec_index
, second_vec_index
,
1915 gsi
, node
, builtin_decl
, vectype
, dr_chain
,
1916 ncopies
, vect_stmts_counter
++);
1929 /* Vectorize SLP instance tree in postorder. */
1932 vect_schedule_slp_instance (slp_tree node
, slp_instance instance
,
1933 unsigned int vectorization_factor
)
1936 bool strided_store
, is_store
;
1937 gimple_stmt_iterator si
;
1938 stmt_vec_info stmt_info
;
1939 unsigned int vec_stmts_size
, nunits
, group_size
;
1942 slp_tree loads_node
;
1947 vect_schedule_slp_instance (SLP_TREE_LEFT (node
), instance
,
1948 vectorization_factor
);
1949 vect_schedule_slp_instance (SLP_TREE_RIGHT (node
), instance
,
1950 vectorization_factor
);
1952 stmt
= VEC_index (gimple
, SLP_TREE_SCALAR_STMTS (node
), 0);
1953 stmt_info
= vinfo_for_stmt (stmt
);
1955 /* VECTYPE is the type of the destination. */
1956 vectype
= get_vectype_for_scalar_type (TREE_TYPE (gimple_assign_lhs (stmt
)));
1957 nunits
= (unsigned int) TYPE_VECTOR_SUBPARTS (vectype
);
1958 group_size
= SLP_INSTANCE_GROUP_SIZE (instance
);
1960 /* For each SLP instance calculate number of vector stmts to be created
1961 for the scalar stmts in each node of the SLP tree. Number of vector
1962 elements in one vector iteration is the number of scalar elements in
1963 one scalar iteration (GROUP_SIZE) multiplied by VF divided by vector
1965 vec_stmts_size
= (vectorization_factor
* group_size
) / nunits
;
1967 /* In case of load permutation we have to allocate vectorized statements for
1968 all the nodes that participate in that permutation. */
1969 if (SLP_INSTANCE_LOAD_PERMUTATION (instance
))
1972 VEC_iterate (slp_tree
, SLP_INSTANCE_LOADS (instance
), i
, loads_node
);
1975 if (!SLP_TREE_VEC_STMTS (loads_node
))
1977 SLP_TREE_VEC_STMTS (loads_node
) = VEC_alloc (gimple
, heap
,
1979 SLP_TREE_NUMBER_OF_VEC_STMTS (loads_node
) = vec_stmts_size
;
1984 if (!SLP_TREE_VEC_STMTS (node
))
1986 SLP_TREE_VEC_STMTS (node
) = VEC_alloc (gimple
, heap
, vec_stmts_size
);
1987 SLP_TREE_NUMBER_OF_VEC_STMTS (node
) = vec_stmts_size
;
1990 if (vect_print_dump_info (REPORT_DETAILS
))
1992 fprintf (vect_dump
, "------>vectorizing SLP node starting from: ");
1993 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
1996 /* Loads should be inserted before the first load. */
1997 if (SLP_INSTANCE_FIRST_LOAD_STMT (instance
)
1998 && STMT_VINFO_STRIDED_ACCESS (stmt_info
)
1999 && !REFERENCE_CLASS_P (gimple_get_lhs (stmt
)))
2000 si
= gsi_for_stmt (SLP_INSTANCE_FIRST_LOAD_STMT (instance
));
2002 si
= gsi_for_stmt (stmt
);
2004 is_store
= vect_transform_stmt (stmt
, &si
, &strided_store
, node
, instance
);
2007 if (DR_GROUP_FIRST_DR (stmt_info
))
2008 /* If IS_STORE is TRUE, the vectorization of the
2009 interleaving chain was completed - free all the stores in
2011 vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info
));
2013 /* FORNOW: SLP originates only from strided stores. */
2019 /* FORNOW: SLP originates only from strided stores. */
2025 vect_schedule_slp (loop_vec_info loop_vinfo
, bb_vec_info bb_vinfo
)
2027 VEC (slp_instance
, heap
) *slp_instances
;
2028 slp_instance instance
;
2030 bool is_store
= false;
2034 slp_instances
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
);
2035 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2039 slp_instances
= BB_VINFO_SLP_INSTANCES (bb_vinfo
);
2043 for (i
= 0; VEC_iterate (slp_instance
, slp_instances
, i
, instance
); i
++)
2045 /* Schedule the tree of INSTANCE. */
2046 is_store
= vect_schedule_slp_instance (SLP_INSTANCE_TREE (instance
),
2048 if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS
)
2049 || vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
2050 fprintf (vect_dump
, "vectorizing stmts using SLP.");
2057 /* Vectorize the basic block. */
2060 vect_slp_transform_bb (basic_block bb
)
2062 bb_vec_info bb_vinfo
= vec_info_for_bb (bb
);
2063 gimple_stmt_iterator si
;
2065 gcc_assert (bb_vinfo
);
2067 if (vect_print_dump_info (REPORT_DETAILS
))
2068 fprintf (vect_dump
, "SLPing BB\n");
2070 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
2072 gimple stmt
= gsi_stmt (si
);
2073 stmt_vec_info stmt_info
;
2075 if (vect_print_dump_info (REPORT_DETAILS
))
2077 fprintf (vect_dump
, "------>SLPing statement: ");
2078 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
2081 stmt_info
= vinfo_for_stmt (stmt
);
2082 gcc_assert (stmt_info
);
2084 /* Schedule all the SLP instances when the first SLP stmt is reached. */
2085 if (STMT_SLP_TYPE (stmt_info
))
2087 vect_schedule_slp (NULL
, bb_vinfo
);
2092 mark_sym_for_renaming (gimple_vop (cfun
));
2093 /* The memory tags and pointers in vectorized statements need to
2094 have their SSA forms updated. FIXME, why can't this be delayed
2095 until all the loops have been transformed? */
2096 update_ssa (TODO_update_ssa
);
2098 if (vect_print_dump_info (REPORT_DETAILS
))
2099 fprintf (vect_dump
, "BASIC BLOCK VECTORIZED\n");
2101 destroy_bb_vec_info (bb_vinfo
);