1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2016 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
37 #include "fold-const.h"
38 #include "stor-layout.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
44 #include "tree-ssa-loop-manip.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
50 #include "internal-fn.h"
52 /* For lang_hooks.types.type_for_mode. */
53 #include "langhooks.h"
55 /* Return the vectorized type for the given statement. */
58 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
60 return STMT_VINFO_VECTYPE (stmt_info
);
63 /* Return TRUE iff the given statement is in an inner loop relative to
64 the loop being vectorized. */
66 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
68 gimple
*stmt
= STMT_VINFO_STMT (stmt_info
);
69 basic_block bb
= gimple_bb (stmt
);
70 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
76 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
78 return (bb
->loop_father
== loop
->inner
);
81 /* Record the cost of a statement, either by directly informing the
82 target model or by saving it in a vector for later processing.
83 Return a preliminary estimate of the statement's cost. */
86 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
87 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
88 int misalign
, enum vect_cost_model_location where
)
92 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
93 stmt_info_for_cost si
= { count
, kind
,
94 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
96 body_cost_vec
->safe_push (si
);
98 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
101 return add_stmt_cost (stmt_info
->vinfo
->target_cost_data
,
102 count
, kind
, stmt_info
, misalign
, where
);
105 /* Return a variable of type ELEM_TYPE[NELEMS]. */
108 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
110 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
114 /* ARRAY is an array of vectors created by create_vector_array.
115 Return an SSA_NAME for the vector in index N. The reference
116 is part of the vectorization of STMT and the vector is associated
117 with scalar destination SCALAR_DEST. */
120 read_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
121 tree array
, unsigned HOST_WIDE_INT n
)
123 tree vect_type
, vect
, vect_name
, array_ref
;
126 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
127 vect_type
= TREE_TYPE (TREE_TYPE (array
));
128 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
129 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
130 build_int_cst (size_type_node
, n
),
131 NULL_TREE
, NULL_TREE
);
133 new_stmt
= gimple_build_assign (vect
, array_ref
);
134 vect_name
= make_ssa_name (vect
, new_stmt
);
135 gimple_assign_set_lhs (new_stmt
, vect_name
);
136 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
141 /* ARRAY is an array of vectors created by create_vector_array.
142 Emit code to store SSA_NAME VECT in index N of the array.
143 The store is part of the vectorization of STMT. */
146 write_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
147 tree array
, unsigned HOST_WIDE_INT n
)
152 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
153 build_int_cst (size_type_node
, n
),
154 NULL_TREE
, NULL_TREE
);
156 new_stmt
= gimple_build_assign (array_ref
, vect
);
157 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
160 /* PTR is a pointer to an array of type TYPE. Return a representation
161 of *PTR. The memory reference replaces those in FIRST_DR
165 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
167 tree mem_ref
, alias_ptr_type
;
169 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
170 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
171 /* Arrays have the same alignment as their type. */
172 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
176 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
178 /* Function vect_mark_relevant.
180 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
183 vect_mark_relevant (vec
<gimple
*> *worklist
, gimple
*stmt
,
184 enum vect_relevant relevant
, bool live_p
)
186 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
187 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
188 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
189 gimple
*pattern_stmt
;
191 if (dump_enabled_p ())
193 dump_printf_loc (MSG_NOTE
, vect_location
,
194 "mark relevant %d, live %d: ", relevant
, live_p
);
195 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
198 /* If this stmt is an original stmt in a pattern, we might need to mark its
199 related pattern stmt instead of the original stmt. However, such stmts
200 may have their own uses that are not in any pattern, in such cases the
201 stmt itself should be marked. */
202 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
204 /* This is the last stmt in a sequence that was detected as a
205 pattern that can potentially be vectorized. Don't mark the stmt
206 as relevant/live because it's not going to be vectorized.
207 Instead mark the pattern-stmt that replaces it. */
209 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
211 if (dump_enabled_p ())
212 dump_printf_loc (MSG_NOTE
, vect_location
,
213 "last stmt in pattern. don't mark"
214 " relevant/live.\n");
215 stmt_info
= vinfo_for_stmt (pattern_stmt
);
216 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
217 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
218 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
222 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
223 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
224 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
226 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
227 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
229 if (dump_enabled_p ())
230 dump_printf_loc (MSG_NOTE
, vect_location
,
231 "already marked relevant/live.\n");
235 worklist
->safe_push (stmt
);
239 /* Function is_simple_and_all_uses_invariant
241 Return true if STMT is simple and all uses of it are invariant. */
244 is_simple_and_all_uses_invariant (gimple
*stmt
, loop_vec_info loop_vinfo
)
250 if (!is_gimple_assign (stmt
))
253 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
255 enum vect_def_type dt
= vect_uninitialized_def
;
257 if (!vect_is_simple_use (op
, loop_vinfo
, &def_stmt
, &dt
))
259 if (dump_enabled_p ())
260 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
261 "use not simple.\n");
265 if (dt
!= vect_external_def
&& dt
!= vect_constant_def
)
271 /* Function vect_stmt_relevant_p.
273 Return true if STMT in loop that is represented by LOOP_VINFO is
274 "relevant for vectorization".
276 A stmt is considered "relevant for vectorization" if:
277 - it has uses outside the loop.
278 - it has vdefs (it alters memory).
279 - control stmts in the loop (except for the exit condition).
281 CHECKME: what other side effects would the vectorizer allow? */
284 vect_stmt_relevant_p (gimple
*stmt
, loop_vec_info loop_vinfo
,
285 enum vect_relevant
*relevant
, bool *live_p
)
287 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
289 imm_use_iterator imm_iter
;
293 *relevant
= vect_unused_in_scope
;
296 /* cond stmt other than loop exit cond. */
297 if (is_ctrl_stmt (stmt
)
298 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
299 != loop_exit_ctrl_vec_info_type
)
300 *relevant
= vect_used_in_scope
;
302 /* changing memory. */
303 if (gimple_code (stmt
) != GIMPLE_PHI
)
304 if (gimple_vdef (stmt
)
305 && !gimple_clobber_p (stmt
))
307 if (dump_enabled_p ())
308 dump_printf_loc (MSG_NOTE
, vect_location
,
309 "vec_stmt_relevant_p: stmt has vdefs.\n");
310 *relevant
= vect_used_in_scope
;
313 /* uses outside the loop. */
314 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
316 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
318 basic_block bb
= gimple_bb (USE_STMT (use_p
));
319 if (!flow_bb_inside_loop_p (loop
, bb
))
321 if (dump_enabled_p ())
322 dump_printf_loc (MSG_NOTE
, vect_location
,
323 "vec_stmt_relevant_p: used out of loop.\n");
325 if (is_gimple_debug (USE_STMT (use_p
)))
328 /* We expect all such uses to be in the loop exit phis
329 (because of loop closed form) */
330 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
331 gcc_assert (bb
== single_exit (loop
)->dest
);
338 if (*live_p
&& *relevant
== vect_unused_in_scope
339 && !is_simple_and_all_uses_invariant (stmt
, loop_vinfo
))
341 if (dump_enabled_p ())
342 dump_printf_loc (MSG_NOTE
, vect_location
,
343 "vec_stmt_relevant_p: stmt live but not relevant.\n");
344 *relevant
= vect_used_only_live
;
347 return (*live_p
|| *relevant
);
351 /* Function exist_non_indexing_operands_for_use_p
353 USE is one of the uses attached to STMT. Check if USE is
354 used in STMT for anything other than indexing an array. */
357 exist_non_indexing_operands_for_use_p (tree use
, gimple
*stmt
)
360 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
362 /* USE corresponds to some operand in STMT. If there is no data
363 reference in STMT, then any operand that corresponds to USE
364 is not indexing an array. */
365 if (!STMT_VINFO_DATA_REF (stmt_info
))
368 /* STMT has a data_ref. FORNOW this means that its of one of
372 (This should have been verified in analyze_data_refs).
374 'var' in the second case corresponds to a def, not a use,
375 so USE cannot correspond to any operands that are not used
378 Therefore, all we need to check is if STMT falls into the
379 first case, and whether var corresponds to USE. */
381 if (!gimple_assign_copy_p (stmt
))
383 if (is_gimple_call (stmt
)
384 && gimple_call_internal_p (stmt
))
385 switch (gimple_call_internal_fn (stmt
))
388 operand
= gimple_call_arg (stmt
, 3);
393 operand
= gimple_call_arg (stmt
, 2);
403 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
405 operand
= gimple_assign_rhs1 (stmt
);
406 if (TREE_CODE (operand
) != SSA_NAME
)
417 Function process_use.
420 - a USE in STMT in a loop represented by LOOP_VINFO
421 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
422 that defined USE. This is done by calling mark_relevant and passing it
423 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
424 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
428 Generally, LIVE_P and RELEVANT are used to define the liveness and
429 relevance info of the DEF_STMT of this USE:
430 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
431 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
433 - case 1: If USE is used only for address computations (e.g. array indexing),
434 which does not need to be directly vectorized, then the liveness/relevance
435 of the respective DEF_STMT is left unchanged.
436 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
437 skip DEF_STMT cause it had already been processed.
438 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
439 be modified accordingly.
441 Return true if everything is as expected. Return false otherwise. */
444 process_use (gimple
*stmt
, tree use
, loop_vec_info loop_vinfo
,
445 enum vect_relevant relevant
, vec
<gimple
*> *worklist
,
448 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
449 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
450 stmt_vec_info dstmt_vinfo
;
451 basic_block bb
, def_bb
;
453 enum vect_def_type dt
;
455 /* case 1: we are only interested in uses that need to be vectorized. Uses
456 that are used for address computation are not considered relevant. */
457 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
460 if (!vect_is_simple_use (use
, loop_vinfo
, &def_stmt
, &dt
))
462 if (dump_enabled_p ())
463 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
464 "not vectorized: unsupported use in stmt.\n");
468 if (!def_stmt
|| gimple_nop_p (def_stmt
))
471 def_bb
= gimple_bb (def_stmt
);
472 if (!flow_bb_inside_loop_p (loop
, def_bb
))
474 if (dump_enabled_p ())
475 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
479 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
480 DEF_STMT must have already been processed, because this should be the
481 only way that STMT, which is a reduction-phi, was put in the worklist,
482 as there should be no other uses for DEF_STMT in the loop. So we just
483 check that everything is as expected, and we are done. */
484 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
485 bb
= gimple_bb (stmt
);
486 if (gimple_code (stmt
) == GIMPLE_PHI
487 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
488 && gimple_code (def_stmt
) != GIMPLE_PHI
489 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
490 && bb
->loop_father
== def_bb
->loop_father
)
492 if (dump_enabled_p ())
493 dump_printf_loc (MSG_NOTE
, vect_location
,
494 "reduc-stmt defining reduc-phi in the same nest.\n");
495 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
496 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
497 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
498 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
499 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
503 /* case 3a: outer-loop stmt defining an inner-loop stmt:
504 outer-loop-header-bb:
510 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
512 if (dump_enabled_p ())
513 dump_printf_loc (MSG_NOTE
, vect_location
,
514 "outer-loop def-stmt defining inner-loop stmt.\n");
518 case vect_unused_in_scope
:
519 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
520 vect_used_in_scope
: vect_unused_in_scope
;
523 case vect_used_in_outer_by_reduction
:
524 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
525 relevant
= vect_used_by_reduction
;
528 case vect_used_in_outer
:
529 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
530 relevant
= vect_used_in_scope
;
533 case vect_used_in_scope
:
541 /* case 3b: inner-loop stmt defining an outer-loop stmt:
542 outer-loop-header-bb:
546 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
548 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
550 if (dump_enabled_p ())
551 dump_printf_loc (MSG_NOTE
, vect_location
,
552 "inner-loop def-stmt defining outer-loop stmt.\n");
556 case vect_unused_in_scope
:
557 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
558 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
559 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
562 case vect_used_by_reduction
:
563 case vect_used_only_live
:
564 relevant
= vect_used_in_outer_by_reduction
;
567 case vect_used_in_scope
:
568 relevant
= vect_used_in_outer
;
576 vect_mark_relevant (worklist
, def_stmt
, relevant
, false);
581 /* Function vect_mark_stmts_to_be_vectorized.
583 Not all stmts in the loop need to be vectorized. For example:
592 Stmt 1 and 3 do not need to be vectorized, because loop control and
593 addressing of vectorized data-refs are handled differently.
595 This pass detects such stmts. */
598 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
600 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
601 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
602 unsigned int nbbs
= loop
->num_nodes
;
603 gimple_stmt_iterator si
;
606 stmt_vec_info stmt_vinfo
;
610 enum vect_relevant relevant
;
612 if (dump_enabled_p ())
613 dump_printf_loc (MSG_NOTE
, vect_location
,
614 "=== vect_mark_stmts_to_be_vectorized ===\n");
616 auto_vec
<gimple
*, 64> worklist
;
618 /* 1. Init worklist. */
619 for (i
= 0; i
< nbbs
; i
++)
622 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
625 if (dump_enabled_p ())
627 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
628 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
631 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
632 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
);
634 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
636 stmt
= gsi_stmt (si
);
637 if (dump_enabled_p ())
639 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
640 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
643 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
644 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
);
648 /* 2. Process_worklist */
649 while (worklist
.length () > 0)
654 stmt
= worklist
.pop ();
655 if (dump_enabled_p ())
657 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
658 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
661 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
662 (DEF_STMT) as relevant/irrelevant according to the relevance property
664 stmt_vinfo
= vinfo_for_stmt (stmt
);
665 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
667 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
668 propagated as is to the DEF_STMTs of its USEs.
670 One exception is when STMT has been identified as defining a reduction
671 variable; in this case we set the relevance to vect_used_by_reduction.
672 This is because we distinguish between two kinds of relevant stmts -
673 those that are used by a reduction computation, and those that are
674 (also) used by a regular computation. This allows us later on to
675 identify stmts that are used solely by a reduction, and therefore the
676 order of the results that they produce does not have to be kept. */
678 switch (STMT_VINFO_DEF_TYPE (stmt_vinfo
))
680 case vect_reduction_def
:
681 gcc_assert (relevant
!= vect_unused_in_scope
);
682 if (relevant
!= vect_unused_in_scope
683 && relevant
!= vect_used_in_scope
684 && relevant
!= vect_used_by_reduction
685 && relevant
!= vect_used_only_live
)
687 if (dump_enabled_p ())
688 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
689 "unsupported use of reduction.\n");
694 case vect_nested_cycle
:
695 if (relevant
!= vect_unused_in_scope
696 && relevant
!= vect_used_in_outer_by_reduction
697 && relevant
!= vect_used_in_outer
)
699 if (dump_enabled_p ())
700 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
701 "unsupported use of nested cycle.\n");
707 case vect_double_reduction_def
:
708 if (relevant
!= vect_unused_in_scope
709 && relevant
!= vect_used_by_reduction
710 && relevant
!= vect_used_only_live
)
712 if (dump_enabled_p ())
713 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
714 "unsupported use of double reduction.\n");
724 if (is_pattern_stmt_p (stmt_vinfo
))
726 /* Pattern statements are not inserted into the code, so
727 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
728 have to scan the RHS or function arguments instead. */
729 if (is_gimple_assign (stmt
))
731 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
732 tree op
= gimple_assign_rhs1 (stmt
);
735 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
737 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
738 relevant
, &worklist
, false)
739 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
740 relevant
, &worklist
, false))
744 for (; i
< gimple_num_ops (stmt
); i
++)
746 op
= gimple_op (stmt
, i
);
747 if (TREE_CODE (op
) == SSA_NAME
748 && !process_use (stmt
, op
, loop_vinfo
, relevant
,
753 else if (is_gimple_call (stmt
))
755 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
757 tree arg
= gimple_call_arg (stmt
, i
);
758 if (!process_use (stmt
, arg
, loop_vinfo
, relevant
,
765 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
767 tree op
= USE_FROM_PTR (use_p
);
768 if (!process_use (stmt
, op
, loop_vinfo
, relevant
,
773 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo
))
775 gather_scatter_info gs_info
;
776 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, &gs_info
))
778 if (!process_use (stmt
, gs_info
.offset
, loop_vinfo
, relevant
,
782 } /* while worklist */
788 /* Function vect_model_simple_cost.
790 Models cost for simple operations, i.e. those that only emit ncopies of a
791 single op. Right now, this does not account for multiple insns that could
792 be generated for the single vector op. We will handle that shortly. */
795 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
796 enum vect_def_type
*dt
,
797 stmt_vector_for_cost
*prologue_cost_vec
,
798 stmt_vector_for_cost
*body_cost_vec
)
801 int inside_cost
= 0, prologue_cost
= 0;
803 /* The SLP costs were already calculated during SLP tree build. */
804 if (PURE_SLP_STMT (stmt_info
))
807 /* FORNOW: Assuming maximum 2 args per stmts. */
808 for (i
= 0; i
< 2; i
++)
809 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
810 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
811 stmt_info
, 0, vect_prologue
);
813 /* Pass the inside-of-loop statements to the target-specific cost model. */
814 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
815 stmt_info
, 0, vect_body
);
817 if (dump_enabled_p ())
818 dump_printf_loc (MSG_NOTE
, vect_location
,
819 "vect_model_simple_cost: inside_cost = %d, "
820 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
824 /* Model cost for type demotion and promotion operations. PWR is normally
825 zero for single-step promotions and demotions. It will be one if
826 two-step promotion/demotion is required, and so on. Each additional
827 step doubles the number of instructions required. */
830 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
831 enum vect_def_type
*dt
, int pwr
)
834 int inside_cost
= 0, prologue_cost
= 0;
835 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
836 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
837 void *target_cost_data
;
839 /* The SLP costs were already calculated during SLP tree build. */
840 if (PURE_SLP_STMT (stmt_info
))
844 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
846 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
848 for (i
= 0; i
< pwr
+ 1; i
++)
850 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
852 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
853 vec_promote_demote
, stmt_info
, 0,
857 /* FORNOW: Assuming maximum 2 args per stmts. */
858 for (i
= 0; i
< 2; i
++)
859 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
860 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
861 stmt_info
, 0, vect_prologue
);
863 if (dump_enabled_p ())
864 dump_printf_loc (MSG_NOTE
, vect_location
,
865 "vect_model_promotion_demotion_cost: inside_cost = %d, "
866 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
869 /* Function vect_model_store_cost
871 Models cost for stores. In the case of grouped accesses, one access
872 has the overhead of the grouped access attributed to it. */
875 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
876 bool store_lanes_p
, enum vect_def_type dt
,
878 stmt_vector_for_cost
*prologue_cost_vec
,
879 stmt_vector_for_cost
*body_cost_vec
)
881 unsigned int inside_cost
= 0, prologue_cost
= 0;
882 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
883 gimple
*first_stmt
= STMT_VINFO_STMT (stmt_info
);
884 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
886 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
887 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
888 stmt_info
, 0, vect_prologue
);
890 /* Grouped stores update all elements in the group at once,
891 so we want the DR for the first statement. */
892 if (!slp_node
&& grouped_access_p
)
894 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
895 dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
898 /* True if we should include any once-per-group costs as well as
899 the cost of the statement itself. For SLP we only get called
900 once per group anyhow. */
901 bool first_stmt_p
= (first_stmt
== STMT_VINFO_STMT (stmt_info
));
903 /* We assume that the cost of a single store-lanes instruction is
904 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
905 access is instead being provided by a permute-and-store operation,
906 include the cost of the permutes.
908 For SLP, the caller has already counted the permutation, if any. */
912 && !STMT_VINFO_STRIDED_P (stmt_info
)
915 /* Uses a high and low interleave or shuffle operations for each
917 int group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
918 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
919 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
920 stmt_info
, 0, vect_body
);
922 if (dump_enabled_p ())
923 dump_printf_loc (MSG_NOTE
, vect_location
,
924 "vect_model_store_cost: strided group_size = %d .\n",
928 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
929 /* Costs of the stores. */
930 if (STMT_VINFO_STRIDED_P (stmt_info
) && !slp_node
)
932 /* N scalar stores plus extracting the elements. */
933 inside_cost
+= record_stmt_cost (body_cost_vec
,
934 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
935 scalar_store
, stmt_info
, 0, vect_body
);
938 vect_get_store_cost (dr
, ncopies
, &inside_cost
, body_cost_vec
);
940 if (STMT_VINFO_STRIDED_P (stmt_info
))
941 inside_cost
+= record_stmt_cost (body_cost_vec
,
942 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
943 vec_to_scalar
, stmt_info
, 0, vect_body
);
945 if (dump_enabled_p ())
946 dump_printf_loc (MSG_NOTE
, vect_location
,
947 "vect_model_store_cost: inside_cost = %d, "
948 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
952 /* Calculate cost of DR's memory access. */
954 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
955 unsigned int *inside_cost
,
956 stmt_vector_for_cost
*body_cost_vec
)
958 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
959 gimple
*stmt
= DR_STMT (dr
);
960 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
962 switch (alignment_support_scheme
)
966 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
967 vector_store
, stmt_info
, 0,
970 if (dump_enabled_p ())
971 dump_printf_loc (MSG_NOTE
, vect_location
,
972 "vect_model_store_cost: aligned.\n");
976 case dr_unaligned_supported
:
978 /* Here, we assign an additional cost for the unaligned store. */
979 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
980 unaligned_store
, stmt_info
,
981 DR_MISALIGNMENT (dr
), vect_body
);
982 if (dump_enabled_p ())
983 dump_printf_loc (MSG_NOTE
, vect_location
,
984 "vect_model_store_cost: unaligned supported by "
989 case dr_unaligned_unsupported
:
991 *inside_cost
= VECT_MAX_COST
;
993 if (dump_enabled_p ())
994 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
995 "vect_model_store_cost: unsupported access.\n");
1005 /* Function vect_model_load_cost
1007 Models cost for loads. In the case of grouped accesses, one access has
1008 the overhead of the grouped access attributed to it. Since unaligned
1009 accesses are supported for loads, we also account for the costs of the
1010 access scheme chosen. */
1013 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1014 bool load_lanes_p
, slp_tree slp_node
,
1015 stmt_vector_for_cost
*prologue_cost_vec
,
1016 stmt_vector_for_cost
*body_cost_vec
)
1018 gimple
*first_stmt
= STMT_VINFO_STMT (stmt_info
);
1019 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1020 unsigned int inside_cost
= 0, prologue_cost
= 0;
1021 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
1023 /* Grouped loads read all elements in the group at once,
1024 so we want the DR for the first statement. */
1025 if (!slp_node
&& grouped_access_p
)
1027 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1028 dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1031 /* True if we should include any once-per-group costs as well as
1032 the cost of the statement itself. For SLP we only get called
1033 once per group anyhow. */
1034 bool first_stmt_p
= (first_stmt
== STMT_VINFO_STMT (stmt_info
));
1036 /* We assume that the cost of a single load-lanes instruction is
1037 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1038 access is instead being provided by a load-and-permute operation,
1039 include the cost of the permutes.
1041 For SLP, the caller has already counted the permutation, if any. */
1042 if (grouped_access_p
1045 && !STMT_VINFO_STRIDED_P (stmt_info
)
1048 /* Uses an even and odd extract operations or shuffle operations
1049 for each needed permute. */
1050 int group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
1051 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1052 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1053 stmt_info
, 0, vect_body
);
1055 if (dump_enabled_p ())
1056 dump_printf_loc (MSG_NOTE
, vect_location
,
1057 "vect_model_load_cost: strided group_size = %d .\n",
1061 /* The loads themselves. */
1062 if (STMT_VINFO_STRIDED_P (stmt_info
) && !slp_node
)
1064 /* N scalar loads plus gathering them into a vector. */
1065 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1066 inside_cost
+= record_stmt_cost (body_cost_vec
,
1067 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1068 scalar_load
, stmt_info
, 0, vect_body
);
1071 vect_get_load_cost (dr
, ncopies
, first_stmt_p
,
1072 &inside_cost
, &prologue_cost
,
1073 prologue_cost_vec
, body_cost_vec
, true);
1074 if (STMT_VINFO_STRIDED_P (stmt_info
))
1075 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1076 stmt_info
, 0, vect_body
);
1078 if (dump_enabled_p ())
1079 dump_printf_loc (MSG_NOTE
, vect_location
,
1080 "vect_model_load_cost: inside_cost = %d, "
1081 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1085 /* Calculate cost of DR's memory access. */
1087 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1088 bool add_realign_cost
, unsigned int *inside_cost
,
1089 unsigned int *prologue_cost
,
1090 stmt_vector_for_cost
*prologue_cost_vec
,
1091 stmt_vector_for_cost
*body_cost_vec
,
1092 bool record_prologue_costs
)
1094 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1095 gimple
*stmt
= DR_STMT (dr
);
1096 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1098 switch (alignment_support_scheme
)
1102 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1103 stmt_info
, 0, vect_body
);
1105 if (dump_enabled_p ())
1106 dump_printf_loc (MSG_NOTE
, vect_location
,
1107 "vect_model_load_cost: aligned.\n");
1111 case dr_unaligned_supported
:
1113 /* Here, we assign an additional cost for the unaligned load. */
1114 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1115 unaligned_load
, stmt_info
,
1116 DR_MISALIGNMENT (dr
), vect_body
);
1118 if (dump_enabled_p ())
1119 dump_printf_loc (MSG_NOTE
, vect_location
,
1120 "vect_model_load_cost: unaligned supported by "
1125 case dr_explicit_realign
:
1127 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1128 vector_load
, stmt_info
, 0, vect_body
);
1129 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1130 vec_perm
, stmt_info
, 0, vect_body
);
1132 /* FIXME: If the misalignment remains fixed across the iterations of
1133 the containing loop, the following cost should be added to the
1135 if (targetm
.vectorize
.builtin_mask_for_load
)
1136 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1137 stmt_info
, 0, vect_body
);
1139 if (dump_enabled_p ())
1140 dump_printf_loc (MSG_NOTE
, vect_location
,
1141 "vect_model_load_cost: explicit realign\n");
1145 case dr_explicit_realign_optimized
:
1147 if (dump_enabled_p ())
1148 dump_printf_loc (MSG_NOTE
, vect_location
,
1149 "vect_model_load_cost: unaligned software "
1152 /* Unaligned software pipeline has a load of an address, an initial
1153 load, and possibly a mask operation to "prime" the loop. However,
1154 if this is an access in a group of loads, which provide grouped
1155 access, then the above cost should only be considered for one
1156 access in the group. Inside the loop, there is a load op
1157 and a realignment op. */
1159 if (add_realign_cost
&& record_prologue_costs
)
1161 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1162 vector_stmt
, stmt_info
,
1164 if (targetm
.vectorize
.builtin_mask_for_load
)
1165 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1166 vector_stmt
, stmt_info
,
1170 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1171 stmt_info
, 0, vect_body
);
1172 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1173 stmt_info
, 0, vect_body
);
1175 if (dump_enabled_p ())
1176 dump_printf_loc (MSG_NOTE
, vect_location
,
1177 "vect_model_load_cost: explicit realign optimized"
1183 case dr_unaligned_unsupported
:
1185 *inside_cost
= VECT_MAX_COST
;
1187 if (dump_enabled_p ())
1188 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1189 "vect_model_load_cost: unsupported access.\n");
1198 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1199 the loop preheader for the vectorized stmt STMT. */
1202 vect_init_vector_1 (gimple
*stmt
, gimple
*new_stmt
, gimple_stmt_iterator
*gsi
)
1205 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1208 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1209 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1213 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1217 if (nested_in_vect_loop_p (loop
, stmt
))
1220 pe
= loop_preheader_edge (loop
);
1221 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1222 gcc_assert (!new_bb
);
1226 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1228 gimple_stmt_iterator gsi_bb_start
;
1230 gcc_assert (bb_vinfo
);
1231 bb
= BB_VINFO_BB (bb_vinfo
);
1232 gsi_bb_start
= gsi_after_labels (bb
);
1233 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1237 if (dump_enabled_p ())
1239 dump_printf_loc (MSG_NOTE
, vect_location
,
1240 "created new init_stmt: ");
1241 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1245 /* Function vect_init_vector.
1247 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1248 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1249 vector type a vector with all elements equal to VAL is created first.
1250 Place the initialization at BSI if it is not NULL. Otherwise, place the
1251 initialization at the loop preheader.
1252 Return the DEF of INIT_STMT.
1253 It will be used in the vectorization of STMT. */
1256 vect_init_vector (gimple
*stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1261 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1262 if (! useless_type_conversion_p (type
, TREE_TYPE (val
)))
1264 gcc_assert (TREE_CODE (type
) == VECTOR_TYPE
);
1265 if (! types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1267 /* Scalar boolean value should be transformed into
1268 all zeros or all ones value before building a vector. */
1269 if (VECTOR_BOOLEAN_TYPE_P (type
))
1271 tree true_val
= build_all_ones_cst (TREE_TYPE (type
));
1272 tree false_val
= build_zero_cst (TREE_TYPE (type
));
1274 if (CONSTANT_CLASS_P (val
))
1275 val
= integer_zerop (val
) ? false_val
: true_val
;
1278 new_temp
= make_ssa_name (TREE_TYPE (type
));
1279 init_stmt
= gimple_build_assign (new_temp
, COND_EXPR
,
1280 val
, true_val
, false_val
);
1281 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1285 else if (CONSTANT_CLASS_P (val
))
1286 val
= fold_convert (TREE_TYPE (type
), val
);
1289 new_temp
= make_ssa_name (TREE_TYPE (type
));
1290 if (! INTEGRAL_TYPE_P (TREE_TYPE (val
)))
1291 init_stmt
= gimple_build_assign (new_temp
,
1292 fold_build1 (VIEW_CONVERT_EXPR
,
1296 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1297 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1301 val
= build_vector_from_val (type
, val
);
1304 new_temp
= vect_get_new_ssa_name (type
, vect_simple_var
, "cst_");
1305 init_stmt
= gimple_build_assign (new_temp
, val
);
1306 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1310 /* Function vect_get_vec_def_for_operand_1.
1312 For a defining stmt DEF_STMT of a scalar stmt, return a vector def with type
1313 DT that will be used in the vectorized stmt. */
1316 vect_get_vec_def_for_operand_1 (gimple
*def_stmt
, enum vect_def_type dt
)
1320 stmt_vec_info def_stmt_info
= NULL
;
1324 /* operand is a constant or a loop invariant. */
1325 case vect_constant_def
:
1326 case vect_external_def
:
1327 /* Code should use vect_get_vec_def_for_operand. */
1330 /* operand is defined inside the loop. */
1331 case vect_internal_def
:
1333 /* Get the def from the vectorized stmt. */
1334 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1336 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1337 /* Get vectorized pattern statement. */
1339 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1340 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1341 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1342 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1343 gcc_assert (vec_stmt
);
1344 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1345 vec_oprnd
= PHI_RESULT (vec_stmt
);
1346 else if (is_gimple_call (vec_stmt
))
1347 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1349 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1353 /* operand is defined by a loop header phi - reduction */
1354 case vect_reduction_def
:
1355 case vect_double_reduction_def
:
1356 case vect_nested_cycle
:
1357 /* Code should use get_initial_def_for_reduction. */
1360 /* operand is defined by loop-header phi - induction. */
1361 case vect_induction_def
:
1363 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1365 /* Get the def from the vectorized stmt. */
1366 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1367 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1368 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1369 vec_oprnd
= PHI_RESULT (vec_stmt
);
1371 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1381 /* Function vect_get_vec_def_for_operand.
1383 OP is an operand in STMT. This function returns a (vector) def that will be
1384 used in the vectorized stmt for STMT.
1386 In the case that OP is an SSA_NAME which is defined in the loop, then
1387 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1389 In case OP is an invariant or constant, a new stmt that creates a vector def
1390 needs to be introduced. VECTYPE may be used to specify a required type for
1391 vector invariant. */
1394 vect_get_vec_def_for_operand (tree op
, gimple
*stmt
, tree vectype
)
1397 enum vect_def_type dt
;
1399 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1400 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1402 if (dump_enabled_p ())
1404 dump_printf_loc (MSG_NOTE
, vect_location
,
1405 "vect_get_vec_def_for_operand: ");
1406 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1407 dump_printf (MSG_NOTE
, "\n");
1410 is_simple_use
= vect_is_simple_use (op
, loop_vinfo
, &def_stmt
, &dt
);
1411 gcc_assert (is_simple_use
);
1412 if (def_stmt
&& dump_enabled_p ())
1414 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1415 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1418 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
1420 tree stmt_vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
1424 vector_type
= vectype
;
1425 else if (TREE_CODE (TREE_TYPE (op
)) == BOOLEAN_TYPE
1426 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype
))
1427 vector_type
= build_same_sized_truth_vector_type (stmt_vectype
);
1429 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1431 gcc_assert (vector_type
);
1432 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1435 return vect_get_vec_def_for_operand_1 (def_stmt
, dt
);
1439 /* Function vect_get_vec_def_for_stmt_copy
1441 Return a vector-def for an operand. This function is used when the
1442 vectorized stmt to be created (by the caller to this function) is a "copy"
1443 created in case the vectorized result cannot fit in one vector, and several
1444 copies of the vector-stmt are required. In this case the vector-def is
1445 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1446 of the stmt that defines VEC_OPRND.
1447 DT is the type of the vector def VEC_OPRND.
1450 In case the vectorization factor (VF) is bigger than the number
1451 of elements that can fit in a vectype (nunits), we have to generate
1452 more than one vector stmt to vectorize the scalar stmt. This situation
1453 arises when there are multiple data-types operated upon in the loop; the
1454 smallest data-type determines the VF, and as a result, when vectorizing
1455 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1456 vector stmt (each computing a vector of 'nunits' results, and together
1457 computing 'VF' results in each iteration). This function is called when
1458 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1459 which VF=16 and nunits=4, so the number of copies required is 4):
1461 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1463 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1464 VS1.1: vx.1 = memref1 VS1.2
1465 VS1.2: vx.2 = memref2 VS1.3
1466 VS1.3: vx.3 = memref3
1468 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1469 VSnew.1: vz1 = vx.1 + ... VSnew.2
1470 VSnew.2: vz2 = vx.2 + ... VSnew.3
1471 VSnew.3: vz3 = vx.3 + ...
1473 The vectorization of S1 is explained in vectorizable_load.
1474 The vectorization of S2:
1475 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1476 the function 'vect_get_vec_def_for_operand' is called to
1477 get the relevant vector-def for each operand of S2. For operand x it
1478 returns the vector-def 'vx.0'.
1480 To create the remaining copies of the vector-stmt (VSnew.j), this
1481 function is called to get the relevant vector-def for each operand. It is
1482 obtained from the respective VS1.j stmt, which is recorded in the
1483 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1485 For example, to obtain the vector-def 'vx.1' in order to create the
1486 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1487 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1488 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1489 and return its def ('vx.1').
1490 Overall, to create the above sequence this function will be called 3 times:
1491 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1492 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1493 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1496 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1498 gimple
*vec_stmt_for_operand
;
1499 stmt_vec_info def_stmt_info
;
1501 /* Do nothing; can reuse same def. */
1502 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1505 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1506 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1507 gcc_assert (def_stmt_info
);
1508 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1509 gcc_assert (vec_stmt_for_operand
);
1510 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1511 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1513 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1518 /* Get vectorized definitions for the operands to create a copy of an original
1519 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1522 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1523 vec
<tree
> *vec_oprnds0
,
1524 vec
<tree
> *vec_oprnds1
)
1526 tree vec_oprnd
= vec_oprnds0
->pop ();
1528 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1529 vec_oprnds0
->quick_push (vec_oprnd
);
1531 if (vec_oprnds1
&& vec_oprnds1
->length ())
1533 vec_oprnd
= vec_oprnds1
->pop ();
1534 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1535 vec_oprnds1
->quick_push (vec_oprnd
);
1540 /* Get vectorized definitions for OP0 and OP1.
1541 REDUC_INDEX is the index of reduction operand in case of reduction,
1542 and -1 otherwise. */
1545 vect_get_vec_defs (tree op0
, tree op1
, gimple
*stmt
,
1546 vec
<tree
> *vec_oprnds0
,
1547 vec
<tree
> *vec_oprnds1
,
1548 slp_tree slp_node
, int reduc_index
)
1552 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1553 auto_vec
<tree
> ops (nops
);
1554 auto_vec
<vec
<tree
> > vec_defs (nops
);
1556 ops
.quick_push (op0
);
1558 ops
.quick_push (op1
);
1560 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1562 *vec_oprnds0
= vec_defs
[0];
1564 *vec_oprnds1
= vec_defs
[1];
1570 vec_oprnds0
->create (1);
1571 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
);
1572 vec_oprnds0
->quick_push (vec_oprnd
);
1576 vec_oprnds1
->create (1);
1577 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
);
1578 vec_oprnds1
->quick_push (vec_oprnd
);
1584 /* Function vect_finish_stmt_generation.
1586 Insert a new stmt. */
1589 vect_finish_stmt_generation (gimple
*stmt
, gimple
*vec_stmt
,
1590 gimple_stmt_iterator
*gsi
)
1592 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1593 vec_info
*vinfo
= stmt_info
->vinfo
;
1595 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1597 if (!gsi_end_p (*gsi
)
1598 && gimple_has_mem_ops (vec_stmt
))
1600 gimple
*at_stmt
= gsi_stmt (*gsi
);
1601 tree vuse
= gimple_vuse (at_stmt
);
1602 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1604 tree vdef
= gimple_vdef (at_stmt
);
1605 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1606 /* If we have an SSA vuse and insert a store, update virtual
1607 SSA form to avoid triggering the renamer. Do so only
1608 if we can easily see all uses - which is what almost always
1609 happens with the way vectorized stmts are inserted. */
1610 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1611 && ((is_gimple_assign (vec_stmt
)
1612 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1613 || (is_gimple_call (vec_stmt
)
1614 && !(gimple_call_flags (vec_stmt
)
1615 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1617 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1618 gimple_set_vdef (vec_stmt
, new_vdef
);
1619 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1623 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1625 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, vinfo
));
1627 if (dump_enabled_p ())
1629 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1630 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1633 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1635 /* While EH edges will generally prevent vectorization, stmt might
1636 e.g. be in a must-not-throw region. Ensure newly created stmts
1637 that could throw are part of the same region. */
1638 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1639 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1640 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1643 /* We want to vectorize a call to combined function CFN with function
1644 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1645 as the types of all inputs. Check whether this is possible using
1646 an internal function, returning its code if so or IFN_LAST if not. */
1649 vectorizable_internal_function (combined_fn cfn
, tree fndecl
,
1650 tree vectype_out
, tree vectype_in
)
1653 if (internal_fn_p (cfn
))
1654 ifn
= as_internal_fn (cfn
);
1656 ifn
= associated_internal_fn (fndecl
);
1657 if (ifn
!= IFN_LAST
&& direct_internal_fn_p (ifn
))
1659 const direct_internal_fn_info
&info
= direct_internal_fn (ifn
);
1660 if (info
.vectorizable
)
1662 tree type0
= (info
.type0
< 0 ? vectype_out
: vectype_in
);
1663 tree type1
= (info
.type1
< 0 ? vectype_out
: vectype_in
);
1664 if (direct_internal_fn_supported_p (ifn
, tree_pair (type0
, type1
),
1665 OPTIMIZE_FOR_SPEED
))
1673 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
*,
1674 gimple_stmt_iterator
*);
1677 /* Function vectorizable_mask_load_store.
1679 Check if STMT performs a conditional load or store that can be vectorized.
1680 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1681 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1682 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1685 vectorizable_mask_load_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
1686 gimple
**vec_stmt
, slp_tree slp_node
)
1688 tree vec_dest
= NULL
;
1689 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1690 stmt_vec_info prev_stmt_info
;
1691 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1692 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1693 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1694 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1695 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1696 tree rhs_vectype
= NULL_TREE
;
1701 tree dataref_ptr
= NULL_TREE
;
1703 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1707 gather_scatter_info gs_info
;
1711 enum vect_def_type dt
;
1713 if (slp_node
!= NULL
)
1716 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1717 gcc_assert (ncopies
>= 1);
1719 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1720 mask
= gimple_call_arg (stmt
, 2);
1722 if (TREE_CODE (TREE_TYPE (mask
)) != BOOLEAN_TYPE
)
1725 /* FORNOW. This restriction should be relaxed. */
1726 if (nested_in_vect_loop
&& ncopies
> 1)
1728 if (dump_enabled_p ())
1729 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1730 "multiple types in nested loop.");
1734 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1737 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
1741 if (!STMT_VINFO_DATA_REF (stmt_info
))
1744 elem_type
= TREE_TYPE (vectype
);
1746 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1749 if (STMT_VINFO_STRIDED_P (stmt_info
))
1752 if (TREE_CODE (mask
) != SSA_NAME
)
1755 if (!vect_is_simple_use (mask
, loop_vinfo
, &def_stmt
, &dt
, &mask_vectype
))
1759 mask_vectype
= get_mask_type_for_scalar_type (TREE_TYPE (vectype
));
1761 if (!mask_vectype
|| !VECTOR_BOOLEAN_TYPE_P (mask_vectype
)
1762 || TYPE_VECTOR_SUBPARTS (mask_vectype
) != TYPE_VECTOR_SUBPARTS (vectype
))
1767 tree rhs
= gimple_call_arg (stmt
, 3);
1768 if (!vect_is_simple_use (rhs
, loop_vinfo
, &def_stmt
, &dt
, &rhs_vectype
))
1772 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
1775 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, &gs_info
))
1777 if (!vect_is_simple_use (gs_info
.offset
, loop_vinfo
, &def_stmt
,
1778 &gs_info
.offset_dt
, &gs_info
.offset_vectype
))
1780 if (dump_enabled_p ())
1781 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1782 "gather index use not simple.");
1786 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
1788 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1789 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1791 if (dump_enabled_p ())
1792 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1793 "masked gather with integer mask not supported.");
1797 else if (tree_int_cst_compare (nested_in_vect_loop
1798 ? STMT_VINFO_DR_STEP (stmt_info
)
1799 : DR_STEP (dr
), size_zero_node
) <= 0)
1801 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1802 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
),
1803 TYPE_MODE (mask_vectype
),
1806 && !useless_type_conversion_p (vectype
, rhs_vectype
)))
1809 if (!vec_stmt
) /* transformation not required. */
1811 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1813 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1816 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1822 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
1824 tree vec_oprnd0
= NULL_TREE
, op
;
1825 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
1826 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1827 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1828 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1829 tree mask_perm_mask
= NULL_TREE
;
1830 edge pe
= loop_preheader_edge (loop
);
1833 enum { NARROW
, NONE
, WIDEN
} modifier
;
1834 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
1836 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
1837 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1838 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1839 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1840 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1841 scaletype
= TREE_VALUE (arglist
);
1842 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1843 && types_compatible_p (srctype
, masktype
));
1845 if (nunits
== gather_off_nunits
)
1847 else if (nunits
== gather_off_nunits
/ 2)
1849 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1852 for (i
= 0; i
< gather_off_nunits
; ++i
)
1853 sel
[i
] = i
| nunits
;
1855 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
1857 else if (nunits
== gather_off_nunits
* 2)
1859 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1862 for (i
= 0; i
< nunits
; ++i
)
1863 sel
[i
] = i
< gather_off_nunits
1864 ? i
: i
+ nunits
- gather_off_nunits
;
1866 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
1868 for (i
= 0; i
< nunits
; ++i
)
1869 sel
[i
] = i
| gather_off_nunits
;
1870 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
1875 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1877 ptr
= fold_convert (ptrtype
, gs_info
.base
);
1878 if (!is_gimple_min_invariant (ptr
))
1880 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1881 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1882 gcc_assert (!new_bb
);
1885 scale
= build_int_cst (scaletype
, gs_info
.scale
);
1887 prev_stmt_info
= NULL
;
1888 for (j
= 0; j
< ncopies
; ++j
)
1890 if (modifier
== WIDEN
&& (j
& 1))
1891 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1892 perm_mask
, stmt
, gsi
);
1895 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
1898 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
, vec_oprnd0
);
1900 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1902 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1903 == TYPE_VECTOR_SUBPARTS (idxtype
));
1904 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
1905 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1907 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1908 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1912 if (mask_perm_mask
&& (j
& 1))
1913 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1914 mask_perm_mask
, stmt
, gsi
);
1918 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
1921 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
1922 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
1926 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
1928 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
1929 == TYPE_VECTOR_SUBPARTS (masktype
));
1930 var
= vect_get_new_ssa_name (masktype
, vect_simple_var
);
1931 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
1933 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
1934 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1940 = gimple_build_call (gs_info
.decl
, 5, mask_op
, ptr
, op
, mask_op
,
1943 if (!useless_type_conversion_p (vectype
, rettype
))
1945 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
1946 == TYPE_VECTOR_SUBPARTS (rettype
));
1947 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
1948 gimple_call_set_lhs (new_stmt
, op
);
1949 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1950 var
= make_ssa_name (vec_dest
);
1951 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
1952 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1956 var
= make_ssa_name (vec_dest
, new_stmt
);
1957 gimple_call_set_lhs (new_stmt
, var
);
1960 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1962 if (modifier
== NARROW
)
1969 var
= permute_vec_elements (prev_res
, var
,
1970 perm_mask
, stmt
, gsi
);
1971 new_stmt
= SSA_NAME_DEF_STMT (var
);
1974 if (prev_stmt_info
== NULL
)
1975 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
1977 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1978 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1981 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
1983 if (STMT_VINFO_RELATED_STMT (stmt_info
))
1985 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
1986 stmt_info
= vinfo_for_stmt (stmt
);
1988 tree lhs
= gimple_call_lhs (stmt
);
1989 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
1990 set_vinfo_for_stmt (new_stmt
, stmt_info
);
1991 set_vinfo_for_stmt (stmt
, NULL
);
1992 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
1993 gsi_replace (gsi
, new_stmt
, true);
1998 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
1999 prev_stmt_info
= NULL
;
2000 LOOP_VINFO_HAS_MASK_STORE (loop_vinfo
) = true;
2001 for (i
= 0; i
< ncopies
; i
++)
2003 unsigned align
, misalign
;
2007 tree rhs
= gimple_call_arg (stmt
, 3);
2008 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
);
2009 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
2010 /* We should have catched mismatched types earlier. */
2011 gcc_assert (useless_type_conversion_p (vectype
,
2012 TREE_TYPE (vec_rhs
)));
2013 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2014 NULL_TREE
, &dummy
, gsi
,
2015 &ptr_incr
, false, &inv_p
);
2016 gcc_assert (!inv_p
);
2020 vect_is_simple_use (vec_rhs
, loop_vinfo
, &def_stmt
, &dt
);
2021 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2022 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2023 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2024 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2025 TYPE_SIZE_UNIT (vectype
));
2028 align
= TYPE_ALIGN_UNIT (vectype
);
2029 if (aligned_access_p (dr
))
2031 else if (DR_MISALIGNMENT (dr
) == -1)
2033 align
= TYPE_ALIGN_UNIT (elem_type
);
2037 misalign
= DR_MISALIGNMENT (dr
);
2038 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2040 tree ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)),
2041 misalign
? misalign
& -misalign
: align
);
2043 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2044 ptr
, vec_mask
, vec_rhs
);
2045 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2047 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2049 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2050 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2055 tree vec_mask
= NULL_TREE
;
2056 prev_stmt_info
= NULL
;
2057 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2058 for (i
= 0; i
< ncopies
; i
++)
2060 unsigned align
, misalign
;
2064 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
2065 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2066 NULL_TREE
, &dummy
, gsi
,
2067 &ptr_incr
, false, &inv_p
);
2068 gcc_assert (!inv_p
);
2072 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2073 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2074 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2075 TYPE_SIZE_UNIT (vectype
));
2078 align
= TYPE_ALIGN_UNIT (vectype
);
2079 if (aligned_access_p (dr
))
2081 else if (DR_MISALIGNMENT (dr
) == -1)
2083 align
= TYPE_ALIGN_UNIT (elem_type
);
2087 misalign
= DR_MISALIGNMENT (dr
);
2088 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2090 tree ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)),
2091 misalign
? misalign
& -misalign
: align
);
2093 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2095 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
));
2096 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2098 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2100 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2101 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2107 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2109 if (STMT_VINFO_RELATED_STMT (stmt_info
))
2111 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
2112 stmt_info
= vinfo_for_stmt (stmt
);
2114 tree lhs
= gimple_call_lhs (stmt
);
2115 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2116 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2117 set_vinfo_for_stmt (stmt
, NULL
);
2118 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2119 gsi_replace (gsi
, new_stmt
, true);
2125 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
2126 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
2127 in a single step. On success, store the binary pack code in
2131 simple_integer_narrowing (tree vectype_out
, tree vectype_in
,
2132 tree_code
*convert_code
)
2134 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out
))
2135 || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in
)))
2139 int multi_step_cvt
= 0;
2140 auto_vec
<tree
, 8> interm_types
;
2141 if (!supportable_narrowing_operation (NOP_EXPR
, vectype_out
, vectype_in
,
2142 &code
, &multi_step_cvt
,
2147 *convert_code
= code
;
2151 /* Function vectorizable_call.
2153 Check if GS performs a function call that can be vectorized.
2154 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2155 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2156 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2159 vectorizable_call (gimple
*gs
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
2166 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2167 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2168 tree vectype_out
, vectype_in
;
2171 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2172 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2173 vec_info
*vinfo
= stmt_info
->vinfo
;
2174 tree fndecl
, new_temp
, rhs_type
;
2176 enum vect_def_type dt
[3]
2177 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2178 gimple
*new_stmt
= NULL
;
2180 vec
<tree
> vargs
= vNULL
;
2181 enum { NARROW
, NONE
, WIDEN
} modifier
;
2185 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2188 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
2192 /* Is GS a vectorizable call? */
2193 stmt
= dyn_cast
<gcall
*> (gs
);
2197 if (gimple_call_internal_p (stmt
)
2198 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2199 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2200 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2203 if (gimple_call_lhs (stmt
) == NULL_TREE
2204 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2207 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2209 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2211 /* Process function arguments. */
2212 rhs_type
= NULL_TREE
;
2213 vectype_in
= NULL_TREE
;
2214 nargs
= gimple_call_num_args (stmt
);
2216 /* Bail out if the function has more than three arguments, we do not have
2217 interesting builtin functions to vectorize with more than two arguments
2218 except for fma. No arguments is also not good. */
2219 if (nargs
== 0 || nargs
> 3)
2222 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2223 if (gimple_call_internal_p (stmt
)
2224 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2227 rhs_type
= unsigned_type_node
;
2230 for (i
= 0; i
< nargs
; i
++)
2234 op
= gimple_call_arg (stmt
, i
);
2236 /* We can only handle calls with arguments of the same type. */
2238 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2240 if (dump_enabled_p ())
2241 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2242 "argument types differ.\n");
2246 rhs_type
= TREE_TYPE (op
);
2248 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[i
], &opvectype
))
2250 if (dump_enabled_p ())
2251 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2252 "use not simple.\n");
2257 vectype_in
= opvectype
;
2259 && opvectype
!= vectype_in
)
2261 if (dump_enabled_p ())
2262 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2263 "argument vector types differ.\n");
2267 /* If all arguments are external or constant defs use a vector type with
2268 the same size as the output vector type. */
2270 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2272 gcc_assert (vectype_in
);
2275 if (dump_enabled_p ())
2277 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2278 "no vectype for scalar type ");
2279 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2280 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2287 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2288 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2289 if (nunits_in
== nunits_out
/ 2)
2291 else if (nunits_out
== nunits_in
)
2293 else if (nunits_out
== nunits_in
/ 2)
2298 /* We only handle functions that do not read or clobber memory. */
2299 if (gimple_vuse (stmt
))
2301 if (dump_enabled_p ())
2302 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2303 "function reads from or writes to memory.\n");
2307 /* For now, we only vectorize functions if a target specific builtin
2308 is available. TODO -- in some cases, it might be profitable to
2309 insert the calls for pieces of the vector, in order to be able
2310 to vectorize other operations in the loop. */
2312 internal_fn ifn
= IFN_LAST
;
2313 combined_fn cfn
= gimple_call_combined_fn (stmt
);
2314 tree callee
= gimple_call_fndecl (stmt
);
2316 /* First try using an internal function. */
2317 tree_code convert_code
= ERROR_MARK
;
2319 && (modifier
== NONE
2320 || (modifier
== NARROW
2321 && simple_integer_narrowing (vectype_out
, vectype_in
,
2323 ifn
= vectorizable_internal_function (cfn
, callee
, vectype_out
,
2326 /* If that fails, try asking for a target-specific built-in function. */
2327 if (ifn
== IFN_LAST
)
2329 if (cfn
!= CFN_LAST
)
2330 fndecl
= targetm
.vectorize
.builtin_vectorized_function
2331 (cfn
, vectype_out
, vectype_in
);
2333 fndecl
= targetm
.vectorize
.builtin_md_vectorized_function
2334 (callee
, vectype_out
, vectype_in
);
2337 if (ifn
== IFN_LAST
&& !fndecl
)
2339 if (cfn
== CFN_GOMP_SIMD_LANE
2342 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2343 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2344 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2345 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2347 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2348 { 0, 1, 2, ... vf - 1 } vector. */
2349 gcc_assert (nargs
== 0);
2353 if (dump_enabled_p ())
2354 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2355 "function is not vectorizable.\n");
2362 else if (modifier
== NARROW
&& ifn
== IFN_LAST
)
2363 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2365 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2367 /* Sanity check: make sure that at least one copy of the vectorized stmt
2368 needs to be generated. */
2369 gcc_assert (ncopies
>= 1);
2371 if (!vec_stmt
) /* transformation not required. */
2373 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2374 if (dump_enabled_p ())
2375 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2377 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2378 if (ifn
!= IFN_LAST
&& modifier
== NARROW
&& !slp_node
)
2379 add_stmt_cost (stmt_info
->vinfo
->target_cost_data
, ncopies
/ 2,
2380 vec_promote_demote
, stmt_info
, 0, vect_body
);
2387 if (dump_enabled_p ())
2388 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2391 scalar_dest
= gimple_call_lhs (stmt
);
2392 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2394 prev_stmt_info
= NULL
;
2395 if (modifier
== NONE
|| ifn
!= IFN_LAST
)
2397 tree prev_res
= NULL_TREE
;
2398 for (j
= 0; j
< ncopies
; ++j
)
2400 /* Build argument list for the vectorized call. */
2402 vargs
.create (nargs
);
2408 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2409 vec
<tree
> vec_oprnds0
;
2411 for (i
= 0; i
< nargs
; i
++)
2412 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2413 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2414 vec_oprnds0
= vec_defs
[0];
2416 /* Arguments are ready. Create the new vector stmt. */
2417 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2420 for (k
= 0; k
< nargs
; k
++)
2422 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2423 vargs
[k
] = vec_oprndsk
[i
];
2425 if (modifier
== NARROW
)
2427 tree half_res
= make_ssa_name (vectype_in
);
2428 new_stmt
= gimple_build_call_internal_vec (ifn
, vargs
);
2429 gimple_call_set_lhs (new_stmt
, half_res
);
2430 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2433 prev_res
= half_res
;
2436 new_temp
= make_ssa_name (vec_dest
);
2437 new_stmt
= gimple_build_assign (new_temp
, convert_code
,
2438 prev_res
, half_res
);
2442 if (ifn
!= IFN_LAST
)
2443 new_stmt
= gimple_build_call_internal_vec (ifn
, vargs
);
2445 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2446 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2447 gimple_call_set_lhs (new_stmt
, new_temp
);
2449 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2450 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2453 for (i
= 0; i
< nargs
; i
++)
2455 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2456 vec_oprndsi
.release ();
2461 for (i
= 0; i
< nargs
; i
++)
2463 op
= gimple_call_arg (stmt
, i
);
2466 = vect_get_vec_def_for_operand (op
, stmt
);
2469 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2471 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2474 vargs
.quick_push (vec_oprnd0
);
2477 if (gimple_call_internal_p (stmt
)
2478 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2480 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2482 for (k
= 0; k
< nunits_out
; ++k
)
2483 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2484 tree cst
= build_vector (vectype_out
, v
);
2486 = vect_get_new_ssa_name (vectype_out
, vect_simple_var
, "cst_");
2487 gimple
*init_stmt
= gimple_build_assign (new_var
, cst
);
2488 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2489 new_temp
= make_ssa_name (vec_dest
);
2490 new_stmt
= gimple_build_assign (new_temp
, new_var
);
2492 else if (modifier
== NARROW
)
2494 tree half_res
= make_ssa_name (vectype_in
);
2495 new_stmt
= gimple_build_call_internal_vec (ifn
, vargs
);
2496 gimple_call_set_lhs (new_stmt
, half_res
);
2497 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2500 prev_res
= half_res
;
2503 new_temp
= make_ssa_name (vec_dest
);
2504 new_stmt
= gimple_build_assign (new_temp
, convert_code
,
2505 prev_res
, half_res
);
2509 if (ifn
!= IFN_LAST
)
2510 new_stmt
= gimple_build_call_internal_vec (ifn
, vargs
);
2512 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2513 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2514 gimple_call_set_lhs (new_stmt
, new_temp
);
2516 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2518 if (j
== (modifier
== NARROW
? 1 : 0))
2519 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2521 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2523 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2526 else if (modifier
== NARROW
)
2528 for (j
= 0; j
< ncopies
; ++j
)
2530 /* Build argument list for the vectorized call. */
2532 vargs
.create (nargs
* 2);
2538 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2539 vec
<tree
> vec_oprnds0
;
2541 for (i
= 0; i
< nargs
; i
++)
2542 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2543 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2544 vec_oprnds0
= vec_defs
[0];
2546 /* Arguments are ready. Create the new vector stmt. */
2547 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2551 for (k
= 0; k
< nargs
; k
++)
2553 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2554 vargs
.quick_push (vec_oprndsk
[i
]);
2555 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2557 if (ifn
!= IFN_LAST
)
2558 new_stmt
= gimple_build_call_internal_vec (ifn
, vargs
);
2560 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2561 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2562 gimple_call_set_lhs (new_stmt
, new_temp
);
2563 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2564 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2567 for (i
= 0; i
< nargs
; i
++)
2569 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2570 vec_oprndsi
.release ();
2575 for (i
= 0; i
< nargs
; i
++)
2577 op
= gimple_call_arg (stmt
, i
);
2581 = vect_get_vec_def_for_operand (op
, stmt
);
2583 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2587 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2589 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2591 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2594 vargs
.quick_push (vec_oprnd0
);
2595 vargs
.quick_push (vec_oprnd1
);
2598 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2599 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2600 gimple_call_set_lhs (new_stmt
, new_temp
);
2601 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2604 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2606 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2608 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2611 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2614 /* No current target implements this case. */
2619 /* The call in STMT might prevent it from being removed in dce.
2620 We however cannot remove it here, due to the way the ssa name
2621 it defines is mapped to the new definition. So just replace
2622 rhs of the statement with something harmless. */
2627 type
= TREE_TYPE (scalar_dest
);
2628 if (is_pattern_stmt_p (stmt_info
))
2629 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2631 lhs
= gimple_call_lhs (stmt
);
2633 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2634 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2635 set_vinfo_for_stmt (stmt
, NULL
);
2636 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2637 gsi_replace (gsi
, new_stmt
, false);
2643 struct simd_call_arg_info
2647 enum vect_def_type dt
;
2648 HOST_WIDE_INT linear_step
;
2650 bool simd_lane_linear
;
2653 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
2654 is linear within simd lane (but not within whole loop), note it in
2658 vect_simd_lane_linear (tree op
, struct loop
*loop
,
2659 struct simd_call_arg_info
*arginfo
)
2661 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
2663 if (!is_gimple_assign (def_stmt
)
2664 || gimple_assign_rhs_code (def_stmt
) != POINTER_PLUS_EXPR
2665 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt
)))
2668 tree base
= gimple_assign_rhs1 (def_stmt
);
2669 HOST_WIDE_INT linear_step
= 0;
2670 tree v
= gimple_assign_rhs2 (def_stmt
);
2671 while (TREE_CODE (v
) == SSA_NAME
)
2674 def_stmt
= SSA_NAME_DEF_STMT (v
);
2675 if (is_gimple_assign (def_stmt
))
2676 switch (gimple_assign_rhs_code (def_stmt
))
2679 t
= gimple_assign_rhs2 (def_stmt
);
2680 if (linear_step
|| TREE_CODE (t
) != INTEGER_CST
)
2682 base
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (base
), base
, t
);
2683 v
= gimple_assign_rhs1 (def_stmt
);
2686 t
= gimple_assign_rhs2 (def_stmt
);
2687 if (linear_step
|| !tree_fits_shwi_p (t
) || integer_zerop (t
))
2689 linear_step
= tree_to_shwi (t
);
2690 v
= gimple_assign_rhs1 (def_stmt
);
2693 t
= gimple_assign_rhs1 (def_stmt
);
2694 if (TREE_CODE (TREE_TYPE (t
)) != INTEGER_TYPE
2695 || (TYPE_PRECISION (TREE_TYPE (v
))
2696 < TYPE_PRECISION (TREE_TYPE (t
))))
2705 else if (is_gimple_call (def_stmt
)
2706 && gimple_call_internal_p (def_stmt
)
2707 && gimple_call_internal_fn (def_stmt
) == IFN_GOMP_SIMD_LANE
2709 && TREE_CODE (gimple_call_arg (def_stmt
, 0)) == SSA_NAME
2710 && (SSA_NAME_VAR (gimple_call_arg (def_stmt
, 0))
2715 arginfo
->linear_step
= linear_step
;
2717 arginfo
->simd_lane_linear
= true;
2723 /* Function vectorizable_simd_clone_call.
2725 Check if STMT performs a function call that can be vectorized
2726 by calling a simd clone of the function.
2727 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2728 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2729 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2732 vectorizable_simd_clone_call (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
2733 gimple
**vec_stmt
, slp_tree slp_node
)
2738 tree vec_oprnd0
= NULL_TREE
;
2739 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2741 unsigned int nunits
;
2742 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2743 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2744 vec_info
*vinfo
= stmt_info
->vinfo
;
2745 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2746 tree fndecl
, new_temp
;
2748 gimple
*new_stmt
= NULL
;
2750 auto_vec
<simd_call_arg_info
> arginfo
;
2751 vec
<tree
> vargs
= vNULL
;
2753 tree lhs
, rtype
, ratype
;
2754 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2756 /* Is STMT a vectorizable call? */
2757 if (!is_gimple_call (stmt
))
2760 fndecl
= gimple_call_fndecl (stmt
);
2761 if (fndecl
== NULL_TREE
)
2764 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2765 if (node
== NULL
|| node
->simd_clones
== NULL
)
2768 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2771 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
2775 if (gimple_call_lhs (stmt
)
2776 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2779 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2781 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2783 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2790 /* Process function arguments. */
2791 nargs
= gimple_call_num_args (stmt
);
2793 /* Bail out if the function has zero arguments. */
2797 arginfo
.reserve (nargs
, true);
2799 for (i
= 0; i
< nargs
; i
++)
2801 simd_call_arg_info thisarginfo
;
2804 thisarginfo
.linear_step
= 0;
2805 thisarginfo
.align
= 0;
2806 thisarginfo
.op
= NULL_TREE
;
2807 thisarginfo
.simd_lane_linear
= false;
2809 op
= gimple_call_arg (stmt
, i
);
2810 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &thisarginfo
.dt
,
2811 &thisarginfo
.vectype
)
2812 || thisarginfo
.dt
== vect_uninitialized_def
)
2814 if (dump_enabled_p ())
2815 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2816 "use not simple.\n");
2820 if (thisarginfo
.dt
== vect_constant_def
2821 || thisarginfo
.dt
== vect_external_def
)
2822 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2824 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2826 /* For linear arguments, the analyze phase should have saved
2827 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2828 if (i
* 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
2829 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2])
2831 gcc_assert (vec_stmt
);
2832 thisarginfo
.linear_step
2833 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2]);
2835 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 1];
2836 thisarginfo
.simd_lane_linear
2837 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 3]
2838 == boolean_true_node
);
2839 /* If loop has been peeled for alignment, we need to adjust it. */
2840 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
2841 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
2842 if (n1
!= n2
&& !thisarginfo
.simd_lane_linear
)
2844 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
2845 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2];
2846 tree opt
= TREE_TYPE (thisarginfo
.op
);
2847 bias
= fold_convert (TREE_TYPE (step
), bias
);
2848 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
2850 = fold_build2 (POINTER_TYPE_P (opt
)
2851 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
2852 thisarginfo
.op
, bias
);
2856 && thisarginfo
.dt
!= vect_constant_def
2857 && thisarginfo
.dt
!= vect_external_def
2859 && TREE_CODE (op
) == SSA_NAME
2860 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
2862 && tree_fits_shwi_p (iv
.step
))
2864 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2865 thisarginfo
.op
= iv
.base
;
2867 else if ((thisarginfo
.dt
== vect_constant_def
2868 || thisarginfo
.dt
== vect_external_def
)
2869 && POINTER_TYPE_P (TREE_TYPE (op
)))
2870 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2871 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
2873 if (POINTER_TYPE_P (TREE_TYPE (op
))
2874 && !thisarginfo
.linear_step
2876 && thisarginfo
.dt
!= vect_constant_def
2877 && thisarginfo
.dt
!= vect_external_def
2880 && TREE_CODE (op
) == SSA_NAME
)
2881 vect_simd_lane_linear (op
, loop
, &thisarginfo
);
2883 arginfo
.quick_push (thisarginfo
);
2886 unsigned int badness
= 0;
2887 struct cgraph_node
*bestn
= NULL
;
2888 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
2889 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
2891 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2892 n
= n
->simdclone
->next_clone
)
2894 unsigned int this_badness
= 0;
2895 if (n
->simdclone
->simdlen
2896 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2897 || n
->simdclone
->nargs
!= nargs
)
2899 if (n
->simdclone
->simdlen
2900 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2901 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2902 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2903 if (n
->simdclone
->inbranch
)
2904 this_badness
+= 2048;
2905 int target_badness
= targetm
.simd_clone
.usable (n
);
2906 if (target_badness
< 0)
2908 this_badness
+= target_badness
* 512;
2909 /* FORNOW: Have to add code to add the mask argument. */
2910 if (n
->simdclone
->inbranch
)
2912 for (i
= 0; i
< nargs
; i
++)
2914 switch (n
->simdclone
->args
[i
].arg_type
)
2916 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2917 if (!useless_type_conversion_p
2918 (n
->simdclone
->args
[i
].orig_type
,
2919 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2921 else if (arginfo
[i
].dt
== vect_constant_def
2922 || arginfo
[i
].dt
== vect_external_def
2923 || arginfo
[i
].linear_step
)
2926 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2927 if (arginfo
[i
].dt
!= vect_constant_def
2928 && arginfo
[i
].dt
!= vect_external_def
)
2931 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2932 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
2933 if (arginfo
[i
].dt
== vect_constant_def
2934 || arginfo
[i
].dt
== vect_external_def
2935 || (arginfo
[i
].linear_step
2936 != n
->simdclone
->args
[i
].linear_step
))
2939 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2940 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
2941 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
2942 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
2943 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
2944 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
2948 case SIMD_CLONE_ARG_TYPE_MASK
:
2951 if (i
== (size_t) -1)
2953 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2958 if (arginfo
[i
].align
)
2959 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2960 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2962 if (i
== (size_t) -1)
2964 if (bestn
== NULL
|| this_badness
< badness
)
2967 badness
= this_badness
;
2974 for (i
= 0; i
< nargs
; i
++)
2975 if ((arginfo
[i
].dt
== vect_constant_def
2976 || arginfo
[i
].dt
== vect_external_def
)
2977 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2980 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2982 if (arginfo
[i
].vectype
== NULL
2983 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2984 > bestn
->simdclone
->simdlen
))
2988 fndecl
= bestn
->decl
;
2989 nunits
= bestn
->simdclone
->simdlen
;
2990 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2992 /* If the function isn't const, only allow it in simd loops where user
2993 has asserted that at least nunits consecutive iterations can be
2994 performed using SIMD instructions. */
2995 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2996 && gimple_vuse (stmt
))
2999 /* Sanity check: make sure that at least one copy of the vectorized stmt
3000 needs to be generated. */
3001 gcc_assert (ncopies
>= 1);
3003 if (!vec_stmt
) /* transformation not required. */
3005 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
3006 for (i
= 0; i
< nargs
; i
++)
3007 if ((bestn
->simdclone
->args
[i
].arg_type
3008 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
3009 || (bestn
->simdclone
->args
[i
].arg_type
3010 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
))
3012 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 3
3014 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
3015 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
3016 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
3017 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
3018 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
3019 tree sll
= arginfo
[i
].simd_lane_linear
3020 ? boolean_true_node
: boolean_false_node
;
3021 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (sll
);
3023 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
3024 if (dump_enabled_p ())
3025 dump_printf_loc (MSG_NOTE
, vect_location
,
3026 "=== vectorizable_simd_clone_call ===\n");
3027 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
3033 if (dump_enabled_p ())
3034 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
3037 scalar_dest
= gimple_call_lhs (stmt
);
3038 vec_dest
= NULL_TREE
;
3043 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3044 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
3045 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
3048 rtype
= TREE_TYPE (ratype
);
3052 prev_stmt_info
= NULL
;
3053 for (j
= 0; j
< ncopies
; ++j
)
3055 /* Build argument list for the vectorized call. */
3057 vargs
.create (nargs
);
3061 for (i
= 0; i
< nargs
; i
++)
3063 unsigned int k
, l
, m
, o
;
3065 op
= gimple_call_arg (stmt
, i
);
3066 switch (bestn
->simdclone
->args
[i
].arg_type
)
3068 case SIMD_CLONE_ARG_TYPE_VECTOR
:
3069 atype
= bestn
->simdclone
->args
[i
].vector_type
;
3070 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
3071 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
3073 if (TYPE_VECTOR_SUBPARTS (atype
)
3074 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
3076 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
3077 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
3078 / TYPE_VECTOR_SUBPARTS (atype
));
3079 gcc_assert ((k
& (k
- 1)) == 0);
3082 = vect_get_vec_def_for_operand (op
, stmt
);
3085 vec_oprnd0
= arginfo
[i
].op
;
3086 if ((m
& (k
- 1)) == 0)
3088 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3091 arginfo
[i
].op
= vec_oprnd0
;
3093 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
3095 bitsize_int ((m
& (k
- 1)) * prec
));
3097 = gimple_build_assign (make_ssa_name (atype
),
3099 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3100 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3104 k
= (TYPE_VECTOR_SUBPARTS (atype
)
3105 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
3106 gcc_assert ((k
& (k
- 1)) == 0);
3107 vec
<constructor_elt
, va_gc
> *ctor_elts
;
3109 vec_alloc (ctor_elts
, k
);
3112 for (l
= 0; l
< k
; l
++)
3114 if (m
== 0 && l
== 0)
3116 = vect_get_vec_def_for_operand (op
, stmt
);
3119 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3121 arginfo
[i
].op
= vec_oprnd0
;
3124 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3128 vargs
.safe_push (vec_oprnd0
);
3131 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3133 = gimple_build_assign (make_ssa_name (atype
),
3135 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3136 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3141 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3142 vargs
.safe_push (op
);
3144 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3145 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
3150 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3155 edge pe
= loop_preheader_edge (loop
);
3156 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3157 gcc_assert (!new_bb
);
3159 if (arginfo
[i
].simd_lane_linear
)
3161 vargs
.safe_push (arginfo
[i
].op
);
3164 tree phi_res
= copy_ssa_name (op
);
3165 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3166 set_vinfo_for_stmt (new_phi
,
3167 new_stmt_vec_info (new_phi
, loop_vinfo
));
3168 add_phi_arg (new_phi
, arginfo
[i
].op
,
3169 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3171 = POINTER_TYPE_P (TREE_TYPE (op
))
3172 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3173 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3174 ? sizetype
: TREE_TYPE (op
);
3176 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3178 tree tcst
= wide_int_to_tree (type
, cst
);
3179 tree phi_arg
= copy_ssa_name (op
);
3181 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3182 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3183 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3184 set_vinfo_for_stmt (new_stmt
,
3185 new_stmt_vec_info (new_stmt
, loop_vinfo
));
3186 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3188 arginfo
[i
].op
= phi_res
;
3189 vargs
.safe_push (phi_res
);
3194 = POINTER_TYPE_P (TREE_TYPE (op
))
3195 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3196 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3197 ? sizetype
: TREE_TYPE (op
);
3199 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3201 tree tcst
= wide_int_to_tree (type
, cst
);
3202 new_temp
= make_ssa_name (TREE_TYPE (op
));
3203 new_stmt
= gimple_build_assign (new_temp
, code
,
3204 arginfo
[i
].op
, tcst
);
3205 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3206 vargs
.safe_push (new_temp
);
3209 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
3210 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
3211 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3212 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
3213 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
3214 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
3220 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3223 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3225 new_temp
= create_tmp_var (ratype
);
3226 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3227 == TYPE_VECTOR_SUBPARTS (rtype
))
3228 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3230 new_temp
= make_ssa_name (rtype
, new_stmt
);
3231 gimple_call_set_lhs (new_stmt
, new_temp
);
3233 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3237 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3240 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3241 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3242 gcc_assert ((k
& (k
- 1)) == 0);
3243 for (l
= 0; l
< k
; l
++)
3248 t
= build_fold_addr_expr (new_temp
);
3249 t
= build2 (MEM_REF
, vectype
, t
,
3250 build_int_cst (TREE_TYPE (t
),
3251 l
* prec
/ BITS_PER_UNIT
));
3254 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3255 size_int (prec
), bitsize_int (l
* prec
));
3257 = gimple_build_assign (make_ssa_name (vectype
), t
);
3258 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3259 if (j
== 0 && l
== 0)
3260 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3262 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3264 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3269 tree clobber
= build_constructor (ratype
, NULL
);
3270 TREE_THIS_VOLATILE (clobber
) = 1;
3271 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3272 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3276 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3278 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3279 / TYPE_VECTOR_SUBPARTS (rtype
));
3280 gcc_assert ((k
& (k
- 1)) == 0);
3281 if ((j
& (k
- 1)) == 0)
3282 vec_alloc (ret_ctor_elts
, k
);
3285 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3286 for (m
= 0; m
< o
; m
++)
3288 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3289 size_int (m
), NULL_TREE
, NULL_TREE
);
3291 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3292 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3293 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3294 gimple_assign_lhs (new_stmt
));
3296 tree clobber
= build_constructor (ratype
, NULL
);
3297 TREE_THIS_VOLATILE (clobber
) = 1;
3298 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3299 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3302 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3303 if ((j
& (k
- 1)) != k
- 1)
3305 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3307 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3308 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3310 if ((unsigned) j
== k
- 1)
3311 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3313 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3315 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3320 tree t
= build_fold_addr_expr (new_temp
);
3321 t
= build2 (MEM_REF
, vectype
, t
,
3322 build_int_cst (TREE_TYPE (t
), 0));
3324 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3325 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3326 tree clobber
= build_constructor (ratype
, NULL
);
3327 TREE_THIS_VOLATILE (clobber
) = 1;
3328 vect_finish_stmt_generation (stmt
,
3329 gimple_build_assign (new_temp
,
3335 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3337 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3339 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3344 /* The call in STMT might prevent it from being removed in dce.
3345 We however cannot remove it here, due to the way the ssa name
3346 it defines is mapped to the new definition. So just replace
3347 rhs of the statement with something harmless. */
3354 type
= TREE_TYPE (scalar_dest
);
3355 if (is_pattern_stmt_p (stmt_info
))
3356 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3358 lhs
= gimple_call_lhs (stmt
);
3359 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3362 new_stmt
= gimple_build_nop ();
3363 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3364 set_vinfo_for_stmt (stmt
, NULL
);
3365 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3366 gsi_replace (gsi
, new_stmt
, true);
3367 unlink_stmt_vdef (stmt
);
3373 /* Function vect_gen_widened_results_half
3375 Create a vector stmt whose code, type, number of arguments, and result
3376 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3377 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3378 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3379 needs to be created (DECL is a function-decl of a target-builtin).
3380 STMT is the original scalar stmt that we are vectorizing. */
3383 vect_gen_widened_results_half (enum tree_code code
,
3385 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3386 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3392 /* Generate half of the widened result: */
3393 if (code
== CALL_EXPR
)
3395 /* Target specific support */
3396 if (op_type
== binary_op
)
3397 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3399 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3400 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3401 gimple_call_set_lhs (new_stmt
, new_temp
);
3405 /* Generic support */
3406 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3407 if (op_type
!= binary_op
)
3409 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3410 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3411 gimple_assign_set_lhs (new_stmt
, new_temp
);
3413 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3419 /* Get vectorized definitions for loop-based vectorization. For the first
3420 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3421 scalar operand), and for the rest we get a copy with
3422 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3423 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3424 The vectors are collected into VEC_OPRNDS. */
3427 vect_get_loop_based_defs (tree
*oprnd
, gimple
*stmt
, enum vect_def_type dt
,
3428 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3432 /* Get first vector operand. */
3433 /* All the vector operands except the very first one (that is scalar oprnd)
3435 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3436 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
);
3438 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3440 vec_oprnds
->quick_push (vec_oprnd
);
3442 /* Get second vector operand. */
3443 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3444 vec_oprnds
->quick_push (vec_oprnd
);
3448 /* For conversion in multiple steps, continue to get operands
3451 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3455 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3456 For multi-step conversions store the resulting vectors and call the function
3460 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3461 int multi_step_cvt
, gimple
*stmt
,
3463 gimple_stmt_iterator
*gsi
,
3464 slp_tree slp_node
, enum tree_code code
,
3465 stmt_vec_info
*prev_stmt_info
)
3468 tree vop0
, vop1
, new_tmp
, vec_dest
;
3470 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3472 vec_dest
= vec_dsts
.pop ();
3474 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3476 /* Create demotion operation. */
3477 vop0
= (*vec_oprnds
)[i
];
3478 vop1
= (*vec_oprnds
)[i
+ 1];
3479 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3480 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3481 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3482 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3485 /* Store the resulting vector for next recursive call. */
3486 (*vec_oprnds
)[i
/2] = new_tmp
;
3489 /* This is the last step of the conversion sequence. Store the
3490 vectors in SLP_NODE or in vector info of the scalar statement
3491 (or in STMT_VINFO_RELATED_STMT chain). */
3493 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3496 if (!*prev_stmt_info
)
3497 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3499 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3501 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3506 /* For multi-step demotion operations we first generate demotion operations
3507 from the source type to the intermediate types, and then combine the
3508 results (stored in VEC_OPRNDS) in demotion operation to the destination
3512 /* At each level of recursion we have half of the operands we had at the
3514 vec_oprnds
->truncate ((i
+1)/2);
3515 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3516 stmt
, vec_dsts
, gsi
, slp_node
,
3517 VEC_PACK_TRUNC_EXPR
,
3521 vec_dsts
.quick_push (vec_dest
);
3525 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3526 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3527 the resulting vectors and call the function recursively. */
3530 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3531 vec
<tree
> *vec_oprnds1
,
3532 gimple
*stmt
, tree vec_dest
,
3533 gimple_stmt_iterator
*gsi
,
3534 enum tree_code code1
,
3535 enum tree_code code2
, tree decl1
,
3536 tree decl2
, int op_type
)
3539 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3540 gimple
*new_stmt1
, *new_stmt2
;
3541 vec
<tree
> vec_tmp
= vNULL
;
3543 vec_tmp
.create (vec_oprnds0
->length () * 2);
3544 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3546 if (op_type
== binary_op
)
3547 vop1
= (*vec_oprnds1
)[i
];
3551 /* Generate the two halves of promotion operation. */
3552 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3553 op_type
, vec_dest
, gsi
, stmt
);
3554 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3555 op_type
, vec_dest
, gsi
, stmt
);
3556 if (is_gimple_call (new_stmt1
))
3558 new_tmp1
= gimple_call_lhs (new_stmt1
);
3559 new_tmp2
= gimple_call_lhs (new_stmt2
);
3563 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3564 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3567 /* Store the results for the next step. */
3568 vec_tmp
.quick_push (new_tmp1
);
3569 vec_tmp
.quick_push (new_tmp2
);
3572 vec_oprnds0
->release ();
3573 *vec_oprnds0
= vec_tmp
;
3577 /* Check if STMT performs a conversion operation, that can be vectorized.
3578 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3579 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3580 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3583 vectorizable_conversion (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
3584 gimple
**vec_stmt
, slp_tree slp_node
)
3588 tree op0
, op1
= NULL_TREE
;
3589 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3590 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3591 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3592 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3593 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3594 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3597 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3598 gimple
*new_stmt
= NULL
;
3599 stmt_vec_info prev_stmt_info
;
3602 tree vectype_out
, vectype_in
;
3604 tree lhs_type
, rhs_type
;
3605 enum { NARROW
, NONE
, WIDEN
} modifier
;
3606 vec
<tree
> vec_oprnds0
= vNULL
;
3607 vec
<tree
> vec_oprnds1
= vNULL
;
3609 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3610 vec_info
*vinfo
= stmt_info
->vinfo
;
3611 int multi_step_cvt
= 0;
3612 vec
<tree
> vec_dsts
= vNULL
;
3613 vec
<tree
> interm_types
= vNULL
;
3614 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3616 machine_mode rhs_mode
;
3617 unsigned short fltsz
;
3619 /* Is STMT a vectorizable conversion? */
3621 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3624 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
3628 if (!is_gimple_assign (stmt
))
3631 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3634 code
= gimple_assign_rhs_code (stmt
);
3635 if (!CONVERT_EXPR_CODE_P (code
)
3636 && code
!= FIX_TRUNC_EXPR
3637 && code
!= FLOAT_EXPR
3638 && code
!= WIDEN_MULT_EXPR
3639 && code
!= WIDEN_LSHIFT_EXPR
)
3642 op_type
= TREE_CODE_LENGTH (code
);
3644 /* Check types of lhs and rhs. */
3645 scalar_dest
= gimple_assign_lhs (stmt
);
3646 lhs_type
= TREE_TYPE (scalar_dest
);
3647 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3649 op0
= gimple_assign_rhs1 (stmt
);
3650 rhs_type
= TREE_TYPE (op0
);
3652 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3653 && !((INTEGRAL_TYPE_P (lhs_type
)
3654 && INTEGRAL_TYPE_P (rhs_type
))
3655 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3656 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3659 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
3660 && ((INTEGRAL_TYPE_P (lhs_type
)
3661 && (TYPE_PRECISION (lhs_type
)
3662 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3663 || (INTEGRAL_TYPE_P (rhs_type
)
3664 && (TYPE_PRECISION (rhs_type
)
3665 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
))))))
3667 if (dump_enabled_p ())
3668 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3669 "type conversion to/from bit-precision unsupported."
3674 /* Check the operands of the operation. */
3675 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
3677 if (dump_enabled_p ())
3678 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3679 "use not simple.\n");
3682 if (op_type
== binary_op
)
3686 op1
= gimple_assign_rhs2 (stmt
);
3687 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3688 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3690 if (CONSTANT_CLASS_P (op0
))
3691 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &vectype_in
);
3693 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]);
3697 if (dump_enabled_p ())
3698 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3699 "use not simple.\n");
3704 /* If op0 is an external or constant defs use a vector type of
3705 the same size as the output vector type. */
3707 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3709 gcc_assert (vectype_in
);
3712 if (dump_enabled_p ())
3714 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3715 "no vectype for scalar type ");
3716 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3717 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3723 if (VECTOR_BOOLEAN_TYPE_P (vectype_out
)
3724 && !VECTOR_BOOLEAN_TYPE_P (vectype_in
))
3726 if (dump_enabled_p ())
3728 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3729 "can't convert between boolean and non "
3731 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3732 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3738 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3739 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3740 if (nunits_in
< nunits_out
)
3742 else if (nunits_out
== nunits_in
)
3747 /* Multiple types in SLP are handled by creating the appropriate number of
3748 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3752 else if (modifier
== NARROW
)
3753 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3755 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3757 /* Sanity check: make sure that at least one copy of the vectorized stmt
3758 needs to be generated. */
3759 gcc_assert (ncopies
>= 1);
3761 /* Supportable by target? */
3765 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3767 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3772 if (dump_enabled_p ())
3773 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3774 "conversion not supported by target.\n");
3778 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3779 &code1
, &code2
, &multi_step_cvt
,
3782 /* Binary widening operation can only be supported directly by the
3784 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3788 if (code
!= FLOAT_EXPR
3789 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3790 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3793 rhs_mode
= TYPE_MODE (rhs_type
);
3794 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3795 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3796 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3797 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3800 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3801 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3802 if (cvt_type
== NULL_TREE
)
3805 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3807 if (!supportable_convert_operation (code
, vectype_out
,
3808 cvt_type
, &decl1
, &codecvt1
))
3811 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3812 cvt_type
, &codecvt1
,
3813 &codecvt2
, &multi_step_cvt
,
3817 gcc_assert (multi_step_cvt
== 0);
3819 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3820 vectype_in
, &code1
, &code2
,
3821 &multi_step_cvt
, &interm_types
))
3825 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3828 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3829 codecvt2
= ERROR_MARK
;
3833 interm_types
.safe_push (cvt_type
);
3834 cvt_type
= NULL_TREE
;
3839 gcc_assert (op_type
== unary_op
);
3840 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3841 &code1
, &multi_step_cvt
,
3845 if (code
!= FIX_TRUNC_EXPR
3846 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3847 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3850 rhs_mode
= TYPE_MODE (rhs_type
);
3852 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3853 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3854 if (cvt_type
== NULL_TREE
)
3856 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3859 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3860 &code1
, &multi_step_cvt
,
3869 if (!vec_stmt
) /* transformation not required. */
3871 if (dump_enabled_p ())
3872 dump_printf_loc (MSG_NOTE
, vect_location
,
3873 "=== vectorizable_conversion ===\n");
3874 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3876 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3877 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3879 else if (modifier
== NARROW
)
3881 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3882 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3886 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3887 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3889 interm_types
.release ();
3894 if (dump_enabled_p ())
3895 dump_printf_loc (MSG_NOTE
, vect_location
,
3896 "transform conversion. ncopies = %d.\n", ncopies
);
3898 if (op_type
== binary_op
)
3900 if (CONSTANT_CLASS_P (op0
))
3901 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3902 else if (CONSTANT_CLASS_P (op1
))
3903 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3906 /* In case of multi-step conversion, we first generate conversion operations
3907 to the intermediate types, and then from that types to the final one.
3908 We create vector destinations for the intermediate type (TYPES) received
3909 from supportable_*_operation, and store them in the correct order
3910 for future use in vect_create_vectorized_*_stmts (). */
3911 vec_dsts
.create (multi_step_cvt
+ 1);
3912 vec_dest
= vect_create_destination_var (scalar_dest
,
3913 (cvt_type
&& modifier
== WIDEN
)
3914 ? cvt_type
: vectype_out
);
3915 vec_dsts
.quick_push (vec_dest
);
3919 for (i
= interm_types
.length () - 1;
3920 interm_types
.iterate (i
, &intermediate_type
); i
--)
3922 vec_dest
= vect_create_destination_var (scalar_dest
,
3924 vec_dsts
.quick_push (vec_dest
);
3929 vec_dest
= vect_create_destination_var (scalar_dest
,
3931 ? vectype_out
: cvt_type
);
3935 if (modifier
== WIDEN
)
3937 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3938 if (op_type
== binary_op
)
3939 vec_oprnds1
.create (1);
3941 else if (modifier
== NARROW
)
3942 vec_oprnds0
.create (
3943 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3945 else if (code
== WIDEN_LSHIFT_EXPR
)
3946 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3949 prev_stmt_info
= NULL
;
3953 for (j
= 0; j
< ncopies
; j
++)
3956 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3959 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3961 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3963 /* Arguments are ready, create the new vector stmt. */
3964 if (code1
== CALL_EXPR
)
3966 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3967 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3968 gimple_call_set_lhs (new_stmt
, new_temp
);
3972 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3973 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
3974 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3975 gimple_assign_set_lhs (new_stmt
, new_temp
);
3978 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3980 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3983 if (!prev_stmt_info
)
3984 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3986 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3987 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3994 /* In case the vectorization factor (VF) is bigger than the number
3995 of elements that we can fit in a vectype (nunits), we have to
3996 generate more than one vector stmt - i.e - we need to "unroll"
3997 the vector stmt by a factor VF/nunits. */
3998 for (j
= 0; j
< ncopies
; j
++)
4005 if (code
== WIDEN_LSHIFT_EXPR
)
4010 /* Store vec_oprnd1 for every vector stmt to be created
4011 for SLP_NODE. We check during the analysis that all
4012 the shift arguments are the same. */
4013 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4014 vec_oprnds1
.quick_push (vec_oprnd1
);
4016 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4020 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
4021 &vec_oprnds1
, slp_node
, -1);
4025 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
);
4026 vec_oprnds0
.quick_push (vec_oprnd0
);
4027 if (op_type
== binary_op
)
4029 if (code
== WIDEN_LSHIFT_EXPR
)
4032 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
);
4033 vec_oprnds1
.quick_push (vec_oprnd1
);
4039 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
4040 vec_oprnds0
.truncate (0);
4041 vec_oprnds0
.quick_push (vec_oprnd0
);
4042 if (op_type
== binary_op
)
4044 if (code
== WIDEN_LSHIFT_EXPR
)
4047 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
4049 vec_oprnds1
.truncate (0);
4050 vec_oprnds1
.quick_push (vec_oprnd1
);
4054 /* Arguments are ready. Create the new vector stmts. */
4055 for (i
= multi_step_cvt
; i
>= 0; i
--)
4057 tree this_dest
= vec_dsts
[i
];
4058 enum tree_code c1
= code1
, c2
= code2
;
4059 if (i
== 0 && codecvt2
!= ERROR_MARK
)
4064 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
4066 stmt
, this_dest
, gsi
,
4067 c1
, c2
, decl1
, decl2
,
4071 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4075 if (codecvt1
== CALL_EXPR
)
4077 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4078 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4079 gimple_call_set_lhs (new_stmt
, new_temp
);
4083 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4084 new_temp
= make_ssa_name (vec_dest
);
4085 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4089 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4092 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
4095 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4098 if (!prev_stmt_info
)
4099 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
4101 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4102 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4107 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4111 /* In case the vectorization factor (VF) is bigger than the number
4112 of elements that we can fit in a vectype (nunits), we have to
4113 generate more than one vector stmt - i.e - we need to "unroll"
4114 the vector stmt by a factor VF/nunits. */
4115 for (j
= 0; j
< ncopies
; j
++)
4119 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4123 vec_oprnds0
.truncate (0);
4124 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
4125 vect_pow2 (multi_step_cvt
) - 1);
4128 /* Arguments are ready. Create the new vector stmts. */
4130 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4132 if (codecvt1
== CALL_EXPR
)
4134 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4135 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4136 gimple_call_set_lhs (new_stmt
, new_temp
);
4140 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4141 new_temp
= make_ssa_name (vec_dest
);
4142 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4146 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4147 vec_oprnds0
[i
] = new_temp
;
4150 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4151 stmt
, vec_dsts
, gsi
,
4156 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4160 vec_oprnds0
.release ();
4161 vec_oprnds1
.release ();
4162 vec_dsts
.release ();
4163 interm_types
.release ();
4169 /* Function vectorizable_assignment.
4171 Check if STMT performs an assignment (copy) that can be vectorized.
4172 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4173 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4174 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4177 vectorizable_assignment (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4178 gimple
**vec_stmt
, slp_tree slp_node
)
4183 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4184 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4187 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4190 vec
<tree
> vec_oprnds
= vNULL
;
4192 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4193 vec_info
*vinfo
= stmt_info
->vinfo
;
4194 gimple
*new_stmt
= NULL
;
4195 stmt_vec_info prev_stmt_info
= NULL
;
4196 enum tree_code code
;
4199 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4202 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4206 /* Is vectorizable assignment? */
4207 if (!is_gimple_assign (stmt
))
4210 scalar_dest
= gimple_assign_lhs (stmt
);
4211 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4214 code
= gimple_assign_rhs_code (stmt
);
4215 if (gimple_assign_single_p (stmt
)
4216 || code
== PAREN_EXPR
4217 || CONVERT_EXPR_CODE_P (code
))
4218 op
= gimple_assign_rhs1 (stmt
);
4222 if (code
== VIEW_CONVERT_EXPR
)
4223 op
= TREE_OPERAND (op
, 0);
4225 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4226 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4228 /* Multiple types in SLP are handled by creating the appropriate number of
4229 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4234 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4236 gcc_assert (ncopies
>= 1);
4238 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
4240 if (dump_enabled_p ())
4241 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4242 "use not simple.\n");
4246 /* We can handle NOP_EXPR conversions that do not change the number
4247 of elements or the vector size. */
4248 if ((CONVERT_EXPR_CODE_P (code
)
4249 || code
== VIEW_CONVERT_EXPR
)
4251 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4252 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4253 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4256 /* We do not handle bit-precision changes. */
4257 if ((CONVERT_EXPR_CODE_P (code
)
4258 || code
== VIEW_CONVERT_EXPR
)
4259 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4260 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4261 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4262 || ((TYPE_PRECISION (TREE_TYPE (op
))
4263 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4264 /* But a conversion that does not change the bit-pattern is ok. */
4265 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4266 > TYPE_PRECISION (TREE_TYPE (op
)))
4267 && TYPE_UNSIGNED (TREE_TYPE (op
)))
4268 /* Conversion between boolean types of different sizes is
4269 a simple assignment in case their vectypes are same
4271 && (!VECTOR_BOOLEAN_TYPE_P (vectype
)
4272 || !VECTOR_BOOLEAN_TYPE_P (vectype_in
)))
4274 if (dump_enabled_p ())
4275 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4276 "type conversion to/from bit-precision "
4281 if (!vec_stmt
) /* transformation not required. */
4283 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4284 if (dump_enabled_p ())
4285 dump_printf_loc (MSG_NOTE
, vect_location
,
4286 "=== vectorizable_assignment ===\n");
4287 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4292 if (dump_enabled_p ())
4293 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4296 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4299 for (j
= 0; j
< ncopies
; j
++)
4303 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4305 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4307 /* Arguments are ready. create the new vector stmt. */
4308 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4310 if (CONVERT_EXPR_CODE_P (code
)
4311 || code
== VIEW_CONVERT_EXPR
)
4312 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4313 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4314 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4315 gimple_assign_set_lhs (new_stmt
, new_temp
);
4316 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4318 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4325 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4327 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4329 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4332 vec_oprnds
.release ();
4337 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4338 either as shift by a scalar or by a vector. */
4341 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4344 machine_mode vec_mode
;
4349 vectype
= get_vectype_for_scalar_type (scalar_type
);
4353 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4355 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4357 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4359 || (optab_handler (optab
, TYPE_MODE (vectype
))
4360 == CODE_FOR_nothing
))
4364 vec_mode
= TYPE_MODE (vectype
);
4365 icode
= (int) optab_handler (optab
, vec_mode
);
4366 if (icode
== CODE_FOR_nothing
)
4373 /* Function vectorizable_shift.
4375 Check if STMT performs a shift operation that can be vectorized.
4376 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4377 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4378 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4381 vectorizable_shift (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4382 gimple
**vec_stmt
, slp_tree slp_node
)
4386 tree op0
, op1
= NULL
;
4387 tree vec_oprnd1
= NULL_TREE
;
4388 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4390 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4391 enum tree_code code
;
4392 machine_mode vec_mode
;
4396 machine_mode optab_op2_mode
;
4398 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4399 gimple
*new_stmt
= NULL
;
4400 stmt_vec_info prev_stmt_info
;
4407 vec
<tree
> vec_oprnds0
= vNULL
;
4408 vec
<tree
> vec_oprnds1
= vNULL
;
4411 bool scalar_shift_arg
= true;
4412 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4413 vec_info
*vinfo
= stmt_info
->vinfo
;
4416 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4419 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4423 /* Is STMT a vectorizable binary/unary operation? */
4424 if (!is_gimple_assign (stmt
))
4427 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4430 code
= gimple_assign_rhs_code (stmt
);
4432 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4433 || code
== RROTATE_EXPR
))
4436 scalar_dest
= gimple_assign_lhs (stmt
);
4437 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4438 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4439 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4441 if (dump_enabled_p ())
4442 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4443 "bit-precision shifts not supported.\n");
4447 op0
= gimple_assign_rhs1 (stmt
);
4448 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
4450 if (dump_enabled_p ())
4451 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4452 "use not simple.\n");
4455 /* If op0 is an external or constant def use a vector type with
4456 the same size as the output vector type. */
4458 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4460 gcc_assert (vectype
);
4463 if (dump_enabled_p ())
4464 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4465 "no vectype for scalar type\n");
4469 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4470 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4471 if (nunits_out
!= nunits_in
)
4474 op1
= gimple_assign_rhs2 (stmt
);
4475 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &op1_vectype
))
4477 if (dump_enabled_p ())
4478 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4479 "use not simple.\n");
4484 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4488 /* Multiple types in SLP are handled by creating the appropriate number of
4489 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4494 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4496 gcc_assert (ncopies
>= 1);
4498 /* Determine whether the shift amount is a vector, or scalar. If the
4499 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4501 if ((dt
[1] == vect_internal_def
4502 || dt
[1] == vect_induction_def
)
4504 scalar_shift_arg
= false;
4505 else if (dt
[1] == vect_constant_def
4506 || dt
[1] == vect_external_def
4507 || dt
[1] == vect_internal_def
)
4509 /* In SLP, need to check whether the shift count is the same,
4510 in loops if it is a constant or invariant, it is always
4514 vec
<gimple
*> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4517 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4518 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4519 scalar_shift_arg
= false;
4522 /* If the shift amount is computed by a pattern stmt we cannot
4523 use the scalar amount directly thus give up and use a vector
4525 if (dt
[1] == vect_internal_def
)
4527 gimple
*def
= SSA_NAME_DEF_STMT (op1
);
4528 if (is_pattern_stmt_p (vinfo_for_stmt (def
)))
4529 scalar_shift_arg
= false;
4534 if (dump_enabled_p ())
4535 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4536 "operand mode requires invariant argument.\n");
4540 /* Vector shifted by vector. */
4541 if (!scalar_shift_arg
)
4543 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4544 if (dump_enabled_p ())
4545 dump_printf_loc (MSG_NOTE
, vect_location
,
4546 "vector/vector shift/rotate found.\n");
4549 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4550 if (op1_vectype
== NULL_TREE
4551 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4553 if (dump_enabled_p ())
4554 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4555 "unusable type for last operand in"
4556 " vector/vector shift/rotate.\n");
4560 /* See if the machine has a vector shifted by scalar insn and if not
4561 then see if it has a vector shifted by vector insn. */
4564 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4566 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4568 if (dump_enabled_p ())
4569 dump_printf_loc (MSG_NOTE
, vect_location
,
4570 "vector/scalar shift/rotate found.\n");
4574 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4576 && (optab_handler (optab
, TYPE_MODE (vectype
))
4577 != CODE_FOR_nothing
))
4579 scalar_shift_arg
= false;
4581 if (dump_enabled_p ())
4582 dump_printf_loc (MSG_NOTE
, vect_location
,
4583 "vector/vector shift/rotate found.\n");
4585 /* Unlike the other binary operators, shifts/rotates have
4586 the rhs being int, instead of the same type as the lhs,
4587 so make sure the scalar is the right type if we are
4588 dealing with vectors of long long/long/short/char. */
4589 if (dt
[1] == vect_constant_def
)
4590 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4591 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4595 && TYPE_MODE (TREE_TYPE (vectype
))
4596 != TYPE_MODE (TREE_TYPE (op1
)))
4598 if (dump_enabled_p ())
4599 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4600 "unusable type for last operand in"
4601 " vector/vector shift/rotate.\n");
4604 if (vec_stmt
&& !slp_node
)
4606 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4607 op1
= vect_init_vector (stmt
, op1
,
4608 TREE_TYPE (vectype
), NULL
);
4615 /* Supportable by target? */
4618 if (dump_enabled_p ())
4619 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4623 vec_mode
= TYPE_MODE (vectype
);
4624 icode
= (int) optab_handler (optab
, vec_mode
);
4625 if (icode
== CODE_FOR_nothing
)
4627 if (dump_enabled_p ())
4628 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4629 "op not supported by target.\n");
4630 /* Check only during analysis. */
4631 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4632 || (vf
< vect_min_worthwhile_factor (code
)
4635 if (dump_enabled_p ())
4636 dump_printf_loc (MSG_NOTE
, vect_location
,
4637 "proceeding using word mode.\n");
4640 /* Worthwhile without SIMD support? Check only during analysis. */
4641 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4642 && vf
< vect_min_worthwhile_factor (code
)
4645 if (dump_enabled_p ())
4646 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4647 "not worthwhile without SIMD support.\n");
4651 if (!vec_stmt
) /* transformation not required. */
4653 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4654 if (dump_enabled_p ())
4655 dump_printf_loc (MSG_NOTE
, vect_location
,
4656 "=== vectorizable_shift ===\n");
4657 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4663 if (dump_enabled_p ())
4664 dump_printf_loc (MSG_NOTE
, vect_location
,
4665 "transform binary/unary operation.\n");
4668 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4670 prev_stmt_info
= NULL
;
4671 for (j
= 0; j
< ncopies
; j
++)
4676 if (scalar_shift_arg
)
4678 /* Vector shl and shr insn patterns can be defined with scalar
4679 operand 2 (shift operand). In this case, use constant or loop
4680 invariant op1 directly, without extending it to vector mode
4682 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4683 if (!VECTOR_MODE_P (optab_op2_mode
))
4685 if (dump_enabled_p ())
4686 dump_printf_loc (MSG_NOTE
, vect_location
,
4687 "operand 1 using scalar mode.\n");
4689 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4690 vec_oprnds1
.quick_push (vec_oprnd1
);
4693 /* Store vec_oprnd1 for every vector stmt to be created
4694 for SLP_NODE. We check during the analysis that all
4695 the shift arguments are the same.
4696 TODO: Allow different constants for different vector
4697 stmts generated for an SLP instance. */
4698 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4699 vec_oprnds1
.quick_push (vec_oprnd1
);
4704 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4705 (a special case for certain kind of vector shifts); otherwise,
4706 operand 1 should be of a vector type (the usual case). */
4708 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4711 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4715 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4717 /* Arguments are ready. Create the new vector stmt. */
4718 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4720 vop1
= vec_oprnds1
[i
];
4721 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4722 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4723 gimple_assign_set_lhs (new_stmt
, new_temp
);
4724 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4726 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4733 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4735 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4736 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4739 vec_oprnds0
.release ();
4740 vec_oprnds1
.release ();
4746 /* Function vectorizable_operation.
4748 Check if STMT performs a binary, unary or ternary operation that can
4750 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4751 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4752 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4755 vectorizable_operation (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4756 gimple
**vec_stmt
, slp_tree slp_node
)
4760 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4761 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4763 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4764 enum tree_code code
;
4765 machine_mode vec_mode
;
4769 bool target_support_p
;
4771 enum vect_def_type dt
[3]
4772 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4773 gimple
*new_stmt
= NULL
;
4774 stmt_vec_info prev_stmt_info
;
4780 vec
<tree
> vec_oprnds0
= vNULL
;
4781 vec
<tree
> vec_oprnds1
= vNULL
;
4782 vec
<tree
> vec_oprnds2
= vNULL
;
4783 tree vop0
, vop1
, vop2
;
4784 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4785 vec_info
*vinfo
= stmt_info
->vinfo
;
4788 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4791 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4795 /* Is STMT a vectorizable binary/unary operation? */
4796 if (!is_gimple_assign (stmt
))
4799 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4802 code
= gimple_assign_rhs_code (stmt
);
4804 /* For pointer addition, we should use the normal plus for
4805 the vector addition. */
4806 if (code
== POINTER_PLUS_EXPR
)
4809 /* Support only unary or binary operations. */
4810 op_type
= TREE_CODE_LENGTH (code
);
4811 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4813 if (dump_enabled_p ())
4814 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4815 "num. args = %d (not unary/binary/ternary op).\n",
4820 scalar_dest
= gimple_assign_lhs (stmt
);
4821 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4823 /* Most operations cannot handle bit-precision types without extra
4825 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4826 && (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4827 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4828 /* Exception are bitwise binary operations. */
4829 && code
!= BIT_IOR_EXPR
4830 && code
!= BIT_XOR_EXPR
4831 && code
!= BIT_AND_EXPR
)
4833 if (dump_enabled_p ())
4834 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4835 "bit-precision arithmetic not supported.\n");
4839 op0
= gimple_assign_rhs1 (stmt
);
4840 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
4842 if (dump_enabled_p ())
4843 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4844 "use not simple.\n");
4847 /* If op0 is an external or constant def use a vector type with
4848 the same size as the output vector type. */
4851 /* For boolean type we cannot determine vectype by
4852 invariant value (don't know whether it is a vector
4853 of booleans or vector of integers). We use output
4854 vectype because operations on boolean don't change
4856 if (TREE_CODE (TREE_TYPE (op0
)) == BOOLEAN_TYPE
)
4858 if (TREE_CODE (TREE_TYPE (scalar_dest
)) != BOOLEAN_TYPE
)
4860 if (dump_enabled_p ())
4861 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4862 "not supported operation on bool value.\n");
4865 vectype
= vectype_out
;
4868 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4871 gcc_assert (vectype
);
4874 if (dump_enabled_p ())
4876 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4877 "no vectype for scalar type ");
4878 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4880 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4886 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4887 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4888 if (nunits_out
!= nunits_in
)
4891 if (op_type
== binary_op
|| op_type
== ternary_op
)
4893 op1
= gimple_assign_rhs2 (stmt
);
4894 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]))
4896 if (dump_enabled_p ())
4897 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4898 "use not simple.\n");
4902 if (op_type
== ternary_op
)
4904 op2
= gimple_assign_rhs3 (stmt
);
4905 if (!vect_is_simple_use (op2
, vinfo
, &def_stmt
, &dt
[2]))
4907 if (dump_enabled_p ())
4908 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4909 "use not simple.\n");
4915 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4919 /* Multiple types in SLP are handled by creating the appropriate number of
4920 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4925 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4927 gcc_assert (ncopies
>= 1);
4929 /* Shifts are handled in vectorizable_shift (). */
4930 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4931 || code
== RROTATE_EXPR
)
4934 /* Supportable by target? */
4936 vec_mode
= TYPE_MODE (vectype
);
4937 if (code
== MULT_HIGHPART_EXPR
)
4938 target_support_p
= can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
));
4941 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4944 if (dump_enabled_p ())
4945 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4949 target_support_p
= (optab_handler (optab
, vec_mode
)
4950 != CODE_FOR_nothing
);
4953 if (!target_support_p
)
4955 if (dump_enabled_p ())
4956 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4957 "op not supported by target.\n");
4958 /* Check only during analysis. */
4959 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4960 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4962 if (dump_enabled_p ())
4963 dump_printf_loc (MSG_NOTE
, vect_location
,
4964 "proceeding using word mode.\n");
4967 /* Worthwhile without SIMD support? Check only during analysis. */
4968 if (!VECTOR_MODE_P (vec_mode
)
4970 && vf
< vect_min_worthwhile_factor (code
))
4972 if (dump_enabled_p ())
4973 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4974 "not worthwhile without SIMD support.\n");
4978 if (!vec_stmt
) /* transformation not required. */
4980 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4981 if (dump_enabled_p ())
4982 dump_printf_loc (MSG_NOTE
, vect_location
,
4983 "=== vectorizable_operation ===\n");
4984 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4990 if (dump_enabled_p ())
4991 dump_printf_loc (MSG_NOTE
, vect_location
,
4992 "transform binary/unary operation.\n");
4995 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4997 /* In case the vectorization factor (VF) is bigger than the number
4998 of elements that we can fit in a vectype (nunits), we have to generate
4999 more than one vector stmt - i.e - we need to "unroll" the
5000 vector stmt by a factor VF/nunits. In doing so, we record a pointer
5001 from one copy of the vector stmt to the next, in the field
5002 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
5003 stages to find the correct vector defs to be used when vectorizing
5004 stmts that use the defs of the current stmt. The example below
5005 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
5006 we need to create 4 vectorized stmts):
5008 before vectorization:
5009 RELATED_STMT VEC_STMT
5013 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
5015 RELATED_STMT VEC_STMT
5016 VS1_0: vx0 = memref0 VS1_1 -
5017 VS1_1: vx1 = memref1 VS1_2 -
5018 VS1_2: vx2 = memref2 VS1_3 -
5019 VS1_3: vx3 = memref3 - -
5020 S1: x = load - VS1_0
5023 step2: vectorize stmt S2 (done here):
5024 To vectorize stmt S2 we first need to find the relevant vector
5025 def for the first operand 'x'. This is, as usual, obtained from
5026 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
5027 that defines 'x' (S1). This way we find the stmt VS1_0, and the
5028 relevant vector def 'vx0'. Having found 'vx0' we can generate
5029 the vector stmt VS2_0, and as usual, record it in the
5030 STMT_VINFO_VEC_STMT of stmt S2.
5031 When creating the second copy (VS2_1), we obtain the relevant vector
5032 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
5033 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
5034 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
5035 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
5036 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
5037 chain of stmts and pointers:
5038 RELATED_STMT VEC_STMT
5039 VS1_0: vx0 = memref0 VS1_1 -
5040 VS1_1: vx1 = memref1 VS1_2 -
5041 VS1_2: vx2 = memref2 VS1_3 -
5042 VS1_3: vx3 = memref3 - -
5043 S1: x = load - VS1_0
5044 VS2_0: vz0 = vx0 + v1 VS2_1 -
5045 VS2_1: vz1 = vx1 + v1 VS2_2 -
5046 VS2_2: vz2 = vx2 + v1 VS2_3 -
5047 VS2_3: vz3 = vx3 + v1 - -
5048 S2: z = x + 1 - VS2_0 */
5050 prev_stmt_info
= NULL
;
5051 for (j
= 0; j
< ncopies
; j
++)
5056 if (op_type
== binary_op
|| op_type
== ternary_op
)
5057 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
5060 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
5062 if (op_type
== ternary_op
)
5064 vec_oprnds2
.create (1);
5065 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
5071 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
5072 if (op_type
== ternary_op
)
5074 tree vec_oprnd
= vec_oprnds2
.pop ();
5075 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
5080 /* Arguments are ready. Create the new vector stmt. */
5081 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5083 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
5084 ? vec_oprnds1
[i
] : NULL_TREE
);
5085 vop2
= ((op_type
== ternary_op
)
5086 ? vec_oprnds2
[i
] : NULL_TREE
);
5087 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
5088 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5089 gimple_assign_set_lhs (new_stmt
, new_temp
);
5090 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5092 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
5099 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5101 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5102 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5105 vec_oprnds0
.release ();
5106 vec_oprnds1
.release ();
5107 vec_oprnds2
.release ();
5112 /* A helper function to ensure data reference DR's base alignment
5116 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
5121 if (DR_VECT_AUX (dr
)->base_misaligned
)
5123 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5124 tree base_decl
= DR_VECT_AUX (dr
)->base_decl
;
5126 if (decl_in_symtab_p (base_decl
))
5127 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
5130 SET_DECL_ALIGN (base_decl
, TYPE_ALIGN (vectype
));
5131 DECL_USER_ALIGN (base_decl
) = 1;
5133 DR_VECT_AUX (dr
)->base_misaligned
= false;
5138 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
5139 reversal of the vector elements. If that is impossible to do,
5143 perm_mask_for_reverse (tree vectype
)
5148 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5149 sel
= XALLOCAVEC (unsigned char, nunits
);
5151 for (i
= 0; i
< nunits
; ++i
)
5152 sel
[i
] = nunits
- 1 - i
;
5154 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
5156 return vect_gen_perm_mask_checked (vectype
, sel
);
5159 /* Function vectorizable_store.
5161 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
5163 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5164 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5165 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5168 vectorizable_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
5174 tree vec_oprnd
= NULL_TREE
;
5175 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5176 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5178 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5179 struct loop
*loop
= NULL
;
5180 machine_mode vec_mode
;
5182 enum dr_alignment_support alignment_support_scheme
;
5184 enum vect_def_type dt
;
5185 stmt_vec_info prev_stmt_info
= NULL
;
5186 tree dataref_ptr
= NULL_TREE
;
5187 tree dataref_offset
= NULL_TREE
;
5188 gimple
*ptr_incr
= NULL
;
5191 gimple
*next_stmt
, *first_stmt
= NULL
;
5192 bool grouped_store
= false;
5193 bool store_lanes_p
= false;
5194 unsigned int group_size
, i
;
5195 vec
<tree
> dr_chain
= vNULL
;
5196 vec
<tree
> oprnds
= vNULL
;
5197 vec
<tree
> result_chain
= vNULL
;
5199 bool negative
= false;
5200 tree offset
= NULL_TREE
;
5201 vec
<tree
> vec_oprnds
= vNULL
;
5202 bool slp
= (slp_node
!= NULL
);
5203 unsigned int vec_num
;
5204 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5205 vec_info
*vinfo
= stmt_info
->vinfo
;
5207 gather_scatter_info gs_info
;
5208 enum vect_def_type scatter_src_dt
= vect_unknown_def_type
;
5212 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5215 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5219 /* Is vectorizable store? */
5221 if (!is_gimple_assign (stmt
))
5224 scalar_dest
= gimple_assign_lhs (stmt
);
5225 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5226 && is_pattern_stmt_p (stmt_info
))
5227 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5228 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5229 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5230 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5231 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5232 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5233 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5234 && TREE_CODE (scalar_dest
) != MEM_REF
)
5237 /* Cannot have hybrid store SLP -- that would mean storing to the
5238 same location twice. */
5239 gcc_assert (slp
== PURE_SLP_STMT (stmt_info
));
5241 gcc_assert (gimple_assign_single_p (stmt
));
5243 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
), rhs_vectype
= NULL_TREE
;
5244 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5248 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5249 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5254 /* Multiple types in SLP are handled by creating the appropriate number of
5255 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5260 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5262 gcc_assert (ncopies
>= 1);
5264 /* FORNOW. This restriction should be relaxed. */
5265 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5267 if (dump_enabled_p ())
5268 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5269 "multiple types in nested loop.\n");
5273 op
= gimple_assign_rhs1 (stmt
);
5275 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
, &rhs_vectype
))
5277 if (dump_enabled_p ())
5278 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5279 "use not simple.\n");
5283 if (rhs_vectype
&& !useless_type_conversion_p (vectype
, rhs_vectype
))
5286 elem_type
= TREE_TYPE (vectype
);
5287 vec_mode
= TYPE_MODE (vectype
);
5289 /* FORNOW. In some cases can vectorize even if data-type not supported
5290 (e.g. - array initialization with 0). */
5291 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5294 if (!STMT_VINFO_DATA_REF (stmt_info
))
5297 if (!STMT_VINFO_STRIDED_P (stmt_info
))
5300 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5301 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5302 size_zero_node
) < 0;
5303 if (negative
&& ncopies
> 1)
5305 if (dump_enabled_p ())
5306 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5307 "multiple types with negative step.\n");
5312 gcc_assert (!grouped_store
);
5313 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5314 if (alignment_support_scheme
!= dr_aligned
5315 && alignment_support_scheme
!= dr_unaligned_supported
)
5317 if (dump_enabled_p ())
5318 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5319 "negative step but alignment required.\n");
5322 if (dt
!= vect_constant_def
5323 && dt
!= vect_external_def
5324 && !perm_mask_for_reverse (vectype
))
5326 if (dump_enabled_p ())
5327 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5328 "negative step and reversing not supported.\n");
5334 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5336 grouped_store
= true;
5337 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5338 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5339 if (!slp
&& !STMT_VINFO_STRIDED_P (stmt_info
))
5341 if (vect_store_lanes_supported (vectype
, group_size
))
5342 store_lanes_p
= true;
5343 else if (!vect_grouped_store_supported (vectype
, group_size
))
5347 if (first_stmt
== stmt
)
5349 /* STMT is the leader of the group. Check the operands of all the
5350 stmts of the group. */
5351 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5354 gcc_assert (gimple_assign_single_p (next_stmt
));
5355 op
= gimple_assign_rhs1 (next_stmt
);
5356 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
))
5358 if (dump_enabled_p ())
5359 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5360 "use not simple.\n");
5363 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5368 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
5371 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, &gs_info
))
5373 if (!vect_is_simple_use (gs_info
.offset
, vinfo
, &def_stmt
,
5374 &gs_info
.offset_dt
, &gs_info
.offset_vectype
))
5376 if (dump_enabled_p ())
5377 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5378 "scatter index use not simple.");
5383 if (!vec_stmt
) /* transformation not required. */
5385 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5386 /* The SLP costs are calculated during SLP analysis. */
5387 if (!PURE_SLP_STMT (stmt_info
))
5388 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5395 ensure_base_align (stmt_info
, dr
);
5397 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
5399 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
, op
, src
;
5400 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
5401 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
5402 tree ptr
, mask
, var
, scale
, perm_mask
= NULL_TREE
;
5403 edge pe
= loop_preheader_edge (loop
);
5406 enum { NARROW
, NONE
, WIDEN
} modifier
;
5407 int scatter_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
5409 if (nunits
== (unsigned int) scatter_off_nunits
)
5411 else if (nunits
== (unsigned int) scatter_off_nunits
/ 2)
5413 unsigned char *sel
= XALLOCAVEC (unsigned char, scatter_off_nunits
);
5416 for (i
= 0; i
< (unsigned int) scatter_off_nunits
; ++i
)
5417 sel
[i
] = i
| nunits
;
5419 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
5420 gcc_assert (perm_mask
!= NULL_TREE
);
5422 else if (nunits
== (unsigned int) scatter_off_nunits
* 2)
5424 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
5427 for (i
= 0; i
< (unsigned int) nunits
; ++i
)
5428 sel
[i
] = i
| scatter_off_nunits
;
5430 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
5431 gcc_assert (perm_mask
!= NULL_TREE
);
5437 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
5438 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5439 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5440 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5441 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5442 scaletype
= TREE_VALUE (arglist
);
5444 gcc_checking_assert (TREE_CODE (masktype
) == INTEGER_TYPE
5445 && TREE_CODE (rettype
) == VOID_TYPE
);
5447 ptr
= fold_convert (ptrtype
, gs_info
.base
);
5448 if (!is_gimple_min_invariant (ptr
))
5450 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
5451 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
5452 gcc_assert (!new_bb
);
5455 /* Currently we support only unconditional scatter stores,
5456 so mask should be all ones. */
5457 mask
= build_int_cst (masktype
, -1);
5458 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
5460 scale
= build_int_cst (scaletype
, gs_info
.scale
);
5462 prev_stmt_info
= NULL
;
5463 for (j
= 0; j
< ncopies
; ++j
)
5468 = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt
), stmt
);
5470 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
5472 else if (modifier
!= NONE
&& (j
& 1))
5474 if (modifier
== WIDEN
)
5477 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5478 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
, perm_mask
,
5481 else if (modifier
== NARROW
)
5483 src
= permute_vec_elements (vec_oprnd1
, vec_oprnd1
, perm_mask
,
5486 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
,
5495 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5497 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
,
5501 if (!useless_type_conversion_p (srctype
, TREE_TYPE (src
)))
5503 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src
))
5504 == TYPE_VECTOR_SUBPARTS (srctype
));
5505 var
= vect_get_new_ssa_name (srctype
, vect_simple_var
);
5506 src
= build1 (VIEW_CONVERT_EXPR
, srctype
, src
);
5507 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, src
);
5508 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5512 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
5514 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
5515 == TYPE_VECTOR_SUBPARTS (idxtype
));
5516 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
5517 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
5518 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
5519 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5524 = gimple_build_call (gs_info
.decl
, 5, ptr
, mask
, op
, src
, scale
);
5526 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5528 if (prev_stmt_info
== NULL
)
5529 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5531 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5532 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5539 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5540 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5542 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5545 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5547 /* We vectorize all the stmts of the interleaving group when we
5548 reach the last stmt in the group. */
5549 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5550 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5559 grouped_store
= false;
5560 /* VEC_NUM is the number of vect stmts to be created for this
5562 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5563 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5564 gcc_assert (GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt
)) == first_stmt
);
5565 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5566 op
= gimple_assign_rhs1 (first_stmt
);
5569 /* VEC_NUM is the number of vect stmts to be created for this
5571 vec_num
= group_size
;
5577 group_size
= vec_num
= 1;
5580 if (dump_enabled_p ())
5581 dump_printf_loc (MSG_NOTE
, vect_location
,
5582 "transform store. ncopies = %d\n", ncopies
);
5584 if (STMT_VINFO_STRIDED_P (stmt_info
))
5586 gimple_stmt_iterator incr_gsi
;
5592 gimple_seq stmts
= NULL
;
5593 tree stride_base
, stride_step
, alias_off
;
5597 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5600 = fold_build_pointer_plus
5601 (unshare_expr (DR_BASE_ADDRESS (first_dr
)),
5602 size_binop (PLUS_EXPR
,
5603 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr
))),
5604 convert_to_ptrofftype (DR_INIT(first_dr
))));
5605 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (first_dr
)));
5607 /* For a store with loop-invariant (but other than power-of-2)
5608 stride (i.e. not a grouped access) like so:
5610 for (i = 0; i < n; i += stride)
5613 we generate a new induction variable and new stores from
5614 the components of the (vectorized) rhs:
5616 for (j = 0; ; j += VF*stride)
5621 array[j + stride] = tmp2;
5625 unsigned nstores
= nunits
;
5627 tree ltype
= elem_type
;
5630 if (group_size
< nunits
5631 && nunits
% group_size
== 0)
5633 nstores
= nunits
/ group_size
;
5635 ltype
= build_vector_type (elem_type
, group_size
);
5637 else if (group_size
>= nunits
5638 && group_size
% nunits
== 0)
5644 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
5645 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5648 ivstep
= stride_step
;
5649 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
5650 build_int_cst (TREE_TYPE (ivstep
), vf
));
5652 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
5654 create_iv (stride_base
, ivstep
, NULL
,
5655 loop
, &incr_gsi
, insert_after
,
5657 incr
= gsi_stmt (incr_gsi
);
5658 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
5660 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
5662 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
5664 prev_stmt_info
= NULL
;
5665 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
5666 next_stmt
= first_stmt
;
5667 for (g
= 0; g
< group_size
; g
++)
5669 running_off
= offvar
;
5672 tree size
= TYPE_SIZE_UNIT (ltype
);
5673 tree pos
= fold_build2 (MULT_EXPR
, sizetype
, size_int (g
),
5675 tree newoff
= copy_ssa_name (running_off
, NULL
);
5676 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5678 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5679 running_off
= newoff
;
5681 unsigned int group_el
= 0;
5682 unsigned HOST_WIDE_INT
5683 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
5684 for (j
= 0; j
< ncopies
; j
++)
5686 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5687 and first_stmt == stmt. */
5692 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
, NULL
,
5694 vec_oprnd
= vec_oprnds
[0];
5698 gcc_assert (gimple_assign_single_p (next_stmt
));
5699 op
= gimple_assign_rhs1 (next_stmt
);
5700 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
5706 vec_oprnd
= vec_oprnds
[j
];
5709 vect_is_simple_use (vec_oprnd
, vinfo
, &def_stmt
, &dt
);
5710 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
5714 for (i
= 0; i
< nstores
; i
++)
5716 tree newref
, newoff
;
5717 gimple
*incr
, *assign
;
5718 tree size
= TYPE_SIZE (ltype
);
5719 /* Extract the i'th component. */
5720 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
,
5721 bitsize_int (i
), size
);
5722 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
5725 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
5729 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
5731 newref
= build2 (MEM_REF
, ltype
,
5732 running_off
, this_off
);
5734 /* And store it to *running_off. */
5735 assign
= gimple_build_assign (newref
, elem
);
5736 vect_finish_stmt_generation (stmt
, assign
, gsi
);
5740 || group_el
== group_size
)
5742 newoff
= copy_ssa_name (running_off
, NULL
);
5743 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5744 running_off
, stride_step
);
5745 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5747 running_off
= newoff
;
5750 if (g
== group_size
- 1
5753 if (j
== 0 && i
== 0)
5754 STMT_VINFO_VEC_STMT (stmt_info
)
5755 = *vec_stmt
= assign
;
5757 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
5758 prev_stmt_info
= vinfo_for_stmt (assign
);
5762 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5769 dr_chain
.create (group_size
);
5770 oprnds
.create (group_size
);
5772 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5773 gcc_assert (alignment_support_scheme
);
5774 /* Targets with store-lane instructions must not require explicit
5776 gcc_assert (!store_lanes_p
5777 || alignment_support_scheme
== dr_aligned
5778 || alignment_support_scheme
== dr_unaligned_supported
);
5781 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5784 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5786 aggr_type
= vectype
;
5788 /* In case the vectorization factor (VF) is bigger than the number
5789 of elements that we can fit in a vectype (nunits), we have to generate
5790 more than one vector stmt - i.e - we need to "unroll" the
5791 vector stmt by a factor VF/nunits. For more details see documentation in
5792 vect_get_vec_def_for_copy_stmt. */
5794 /* In case of interleaving (non-unit grouped access):
5801 We create vectorized stores starting from base address (the access of the
5802 first stmt in the chain (S2 in the above example), when the last store stmt
5803 of the chain (S4) is reached:
5806 VS2: &base + vec_size*1 = vx0
5807 VS3: &base + vec_size*2 = vx1
5808 VS4: &base + vec_size*3 = vx3
5810 Then permutation statements are generated:
5812 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5813 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5816 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5817 (the order of the data-refs in the output of vect_permute_store_chain
5818 corresponds to the order of scalar stmts in the interleaving chain - see
5819 the documentation of vect_permute_store_chain()).
5821 In case of both multiple types and interleaving, above vector stores and
5822 permutation stmts are created for every copy. The result vector stmts are
5823 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5824 STMT_VINFO_RELATED_STMT for the next copies.
5827 prev_stmt_info
= NULL
;
5828 for (j
= 0; j
< ncopies
; j
++)
5835 /* Get vectorized arguments for SLP_NODE. */
5836 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5837 NULL
, slp_node
, -1);
5839 vec_oprnd
= vec_oprnds
[0];
5843 /* For interleaved stores we collect vectorized defs for all the
5844 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5845 used as an input to vect_permute_store_chain(), and OPRNDS as
5846 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5848 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5849 OPRNDS are of size 1. */
5850 next_stmt
= first_stmt
;
5851 for (i
= 0; i
< group_size
; i
++)
5853 /* Since gaps are not supported for interleaved stores,
5854 GROUP_SIZE is the exact number of stmts in the chain.
5855 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5856 there is no interleaving, GROUP_SIZE is 1, and only one
5857 iteration of the loop will be executed. */
5858 gcc_assert (next_stmt
5859 && gimple_assign_single_p (next_stmt
));
5860 op
= gimple_assign_rhs1 (next_stmt
);
5862 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
5863 dr_chain
.quick_push (vec_oprnd
);
5864 oprnds
.quick_push (vec_oprnd
);
5865 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5869 /* We should have catched mismatched types earlier. */
5870 gcc_assert (useless_type_conversion_p (vectype
,
5871 TREE_TYPE (vec_oprnd
)));
5872 bool simd_lane_access_p
5873 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5874 if (simd_lane_access_p
5875 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5876 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5877 && integer_zerop (DR_OFFSET (first_dr
))
5878 && integer_zerop (DR_INIT (first_dr
))
5879 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5880 get_alias_set (DR_REF (first_dr
))))
5882 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5883 dataref_offset
= build_int_cst (reference_alias_ptr_type
5884 (DR_REF (first_dr
)), 0);
5889 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5890 simd_lane_access_p
? loop
: NULL
,
5891 offset
, &dummy
, gsi
, &ptr_incr
,
5892 simd_lane_access_p
, &inv_p
);
5893 gcc_assert (bb_vinfo
|| !inv_p
);
5897 /* For interleaved stores we created vectorized defs for all the
5898 defs stored in OPRNDS in the previous iteration (previous copy).
5899 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5900 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5902 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5903 OPRNDS are of size 1. */
5904 for (i
= 0; i
< group_size
; i
++)
5907 vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
);
5908 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5909 dr_chain
[i
] = vec_oprnd
;
5910 oprnds
[i
] = vec_oprnd
;
5914 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5915 TYPE_SIZE_UNIT (aggr_type
));
5917 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5918 TYPE_SIZE_UNIT (aggr_type
));
5925 /* Combine all the vectors into an array. */
5926 vec_array
= create_vector_array (vectype
, vec_num
);
5927 for (i
= 0; i
< vec_num
; i
++)
5929 vec_oprnd
= dr_chain
[i
];
5930 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5934 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5935 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5936 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5937 gimple_call_set_lhs (new_stmt
, data_ref
);
5938 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5946 result_chain
.create (group_size
);
5948 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5952 next_stmt
= first_stmt
;
5953 for (i
= 0; i
< vec_num
; i
++)
5955 unsigned align
, misalign
;
5958 /* Bump the vector pointer. */
5959 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5963 vec_oprnd
= vec_oprnds
[i
];
5964 else if (grouped_store
)
5965 /* For grouped stores vectorized defs are interleaved in
5966 vect_permute_store_chain(). */
5967 vec_oprnd
= result_chain
[i
];
5969 data_ref
= fold_build2 (MEM_REF
, TREE_TYPE (vec_oprnd
),
5973 : build_int_cst (reference_alias_ptr_type
5974 (DR_REF (first_dr
)), 0));
5975 align
= TYPE_ALIGN_UNIT (vectype
);
5976 if (aligned_access_p (first_dr
))
5978 else if (DR_MISALIGNMENT (first_dr
) == -1)
5980 if (DR_VECT_AUX (first_dr
)->base_element_aligned
)
5981 align
= TYPE_ALIGN_UNIT (elem_type
);
5983 align
= get_object_alignment (DR_REF (first_dr
))
5986 TREE_TYPE (data_ref
)
5987 = build_aligned_type (TREE_TYPE (data_ref
),
5988 align
* BITS_PER_UNIT
);
5992 TREE_TYPE (data_ref
)
5993 = build_aligned_type (TREE_TYPE (data_ref
),
5994 TYPE_ALIGN (elem_type
));
5995 misalign
= DR_MISALIGNMENT (first_dr
);
5997 if (dataref_offset
== NULL_TREE
5998 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
5999 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
6003 && dt
!= vect_constant_def
6004 && dt
!= vect_external_def
)
6006 tree perm_mask
= perm_mask_for_reverse (vectype
);
6008 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
6010 tree new_temp
= make_ssa_name (perm_dest
);
6012 /* Generate the permute statement. */
6014 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
6015 vec_oprnd
, perm_mask
);
6016 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
6018 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6019 vec_oprnd
= new_temp
;
6022 /* Arguments are ready. Create the new vector stmt. */
6023 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
6024 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6029 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
6037 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6039 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6040 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6044 dr_chain
.release ();
6046 result_chain
.release ();
6047 vec_oprnds
.release ();
6052 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
6053 VECTOR_CST mask. No checks are made that the target platform supports the
6054 mask, so callers may wish to test can_vec_perm_p separately, or use
6055 vect_gen_perm_mask_checked. */
6058 vect_gen_perm_mask_any (tree vectype
, const unsigned char *sel
)
6060 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
6063 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6065 mask_elt_type
= lang_hooks
.types
.type_for_mode
6066 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
6067 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
6069 mask_elts
= XALLOCAVEC (tree
, nunits
);
6070 for (i
= nunits
- 1; i
>= 0; i
--)
6071 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
6072 mask_vec
= build_vector (mask_type
, mask_elts
);
6077 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
6078 i.e. that the target supports the pattern _for arbitrary input vectors_. */
6081 vect_gen_perm_mask_checked (tree vectype
, const unsigned char *sel
)
6083 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, sel
));
6084 return vect_gen_perm_mask_any (vectype
, sel
);
6087 /* Given a vector variable X and Y, that was generated for the scalar
6088 STMT, generate instructions to permute the vector elements of X and Y
6089 using permutation mask MASK_VEC, insert them at *GSI and return the
6090 permuted vector variable. */
6093 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple
*stmt
,
6094 gimple_stmt_iterator
*gsi
)
6096 tree vectype
= TREE_TYPE (x
);
6097 tree perm_dest
, data_ref
;
6100 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
6101 data_ref
= make_ssa_name (perm_dest
);
6103 /* Generate the permute statement. */
6104 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
6105 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
6110 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
6111 inserting them on the loops preheader edge. Returns true if we
6112 were successful in doing so (and thus STMT can be moved then),
6113 otherwise returns false. */
6116 hoist_defs_of_uses (gimple
*stmt
, struct loop
*loop
)
6122 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6124 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6125 if (!gimple_nop_p (def_stmt
)
6126 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6128 /* Make sure we don't need to recurse. While we could do
6129 so in simple cases when there are more complex use webs
6130 we don't have an easy way to preserve stmt order to fulfil
6131 dependencies within them. */
6134 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
6136 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
6138 gimple
*def_stmt2
= SSA_NAME_DEF_STMT (op2
);
6139 if (!gimple_nop_p (def_stmt2
)
6140 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
6150 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6152 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6153 if (!gimple_nop_p (def_stmt
)
6154 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6156 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
6157 gsi_remove (&gsi
, false);
6158 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
6165 /* vectorizable_load.
6167 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
6169 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6170 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
6171 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6174 vectorizable_load (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
6175 slp_tree slp_node
, slp_instance slp_node_instance
)
6178 tree vec_dest
= NULL
;
6179 tree data_ref
= NULL
;
6180 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6181 stmt_vec_info prev_stmt_info
;
6182 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6183 struct loop
*loop
= NULL
;
6184 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
6185 bool nested_in_vect_loop
= false;
6186 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
6190 gimple
*new_stmt
= NULL
;
6192 enum dr_alignment_support alignment_support_scheme
;
6193 tree dataref_ptr
= NULL_TREE
;
6194 tree dataref_offset
= NULL_TREE
;
6195 gimple
*ptr_incr
= NULL
;
6197 int i
, j
, group_size
= -1, group_gap_adj
;
6198 tree msq
= NULL_TREE
, lsq
;
6199 tree offset
= NULL_TREE
;
6200 tree byte_offset
= NULL_TREE
;
6201 tree realignment_token
= NULL_TREE
;
6203 vec
<tree
> dr_chain
= vNULL
;
6204 bool grouped_load
= false;
6205 bool load_lanes_p
= false;
6207 gimple
*first_stmt_for_drptr
= NULL
;
6209 bool negative
= false;
6210 bool compute_in_loop
= false;
6211 struct loop
*at_loop
;
6213 bool slp
= (slp_node
!= NULL
);
6214 bool slp_perm
= false;
6215 enum tree_code code
;
6216 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6219 gather_scatter_info gs_info
;
6220 vec_info
*vinfo
= stmt_info
->vinfo
;
6222 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6225 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6229 /* Is vectorizable load? */
6230 if (!is_gimple_assign (stmt
))
6233 scalar_dest
= gimple_assign_lhs (stmt
);
6234 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
6237 code
= gimple_assign_rhs_code (stmt
);
6238 if (code
!= ARRAY_REF
6239 && code
!= BIT_FIELD_REF
6240 && code
!= INDIRECT_REF
6241 && code
!= COMPONENT_REF
6242 && code
!= IMAGPART_EXPR
6243 && code
!= REALPART_EXPR
6245 && TREE_CODE_CLASS (code
) != tcc_declaration
)
6248 if (!STMT_VINFO_DATA_REF (stmt_info
))
6251 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6252 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6256 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6257 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
6258 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
6263 /* Multiple types in SLP are handled by creating the appropriate number of
6264 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6269 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
6271 gcc_assert (ncopies
>= 1);
6273 /* FORNOW. This restriction should be relaxed. */
6274 if (nested_in_vect_loop
&& ncopies
> 1)
6276 if (dump_enabled_p ())
6277 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6278 "multiple types in nested loop.\n");
6282 /* Invalidate assumptions made by dependence analysis when vectorization
6283 on the unrolled body effectively re-orders stmts. */
6285 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6286 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6287 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6289 if (dump_enabled_p ())
6290 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6291 "cannot perform implicit CSE when unrolling "
6292 "with negative dependence distance\n");
6296 elem_type
= TREE_TYPE (vectype
);
6297 mode
= TYPE_MODE (vectype
);
6299 /* FORNOW. In some cases can vectorize even if data-type not supported
6300 (e.g. - data copies). */
6301 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
6303 if (dump_enabled_p ())
6304 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6305 "Aligned load, but unsupported type.\n");
6309 /* Check if the load is a part of an interleaving chain. */
6310 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
6312 grouped_load
= true;
6314 gcc_assert (!nested_in_vect_loop
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info
));
6316 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6317 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6319 if (!slp
&& !STMT_VINFO_STRIDED_P (stmt_info
))
6321 if (vect_load_lanes_supported (vectype
, group_size
))
6322 load_lanes_p
= true;
6323 else if (!vect_grouped_load_supported (vectype
, group_size
))
6327 /* If this is single-element interleaving with an element distance
6328 that leaves unused vector loads around punt - we at least create
6329 very sub-optimal code in that case (and blow up memory,
6331 if (first_stmt
== stmt
6332 && !GROUP_NEXT_ELEMENT (stmt_info
))
6334 if (GROUP_SIZE (stmt_info
) > TYPE_VECTOR_SUBPARTS (vectype
))
6336 if (dump_enabled_p ())
6337 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6338 "single-element interleaving not supported "
6339 "for not adjacent vector loads\n");
6343 /* Single-element interleaving requires peeling for gaps. */
6344 gcc_assert (GROUP_GAP (stmt_info
));
6347 /* If there is a gap in the end of the group then we access excess
6348 elements in the last iteration and thus need to peel that off. */
6350 && ! STMT_VINFO_STRIDED_P (stmt_info
)
6351 && GROUP_GAP (vinfo_for_stmt (first_stmt
)) != 0)
6353 if (dump_enabled_p ())
6354 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6355 "Data access with gaps requires scalar "
6359 if (dump_enabled_p ())
6360 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6361 "Peeling for outer loop is not supported\n");
6365 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) = true;
6368 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6371 /* ??? The following is overly pessimistic (as well as the loop
6372 case above) in the case we can statically determine the excess
6373 elements loaded are within the bounds of a decl that is accessed.
6374 Likewise for BB vectorizations using masked loads is a possibility. */
6375 if (bb_vinfo
&& slp_perm
&& group_size
% nunits
!= 0)
6377 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6378 "BB vectorization with gaps at the end of a load "
6379 "is not supported\n");
6383 /* Invalidate assumptions made by dependence analysis when vectorization
6384 on the unrolled body effectively re-orders stmts. */
6385 if (!PURE_SLP_STMT (stmt_info
)
6386 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6387 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6388 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6390 if (dump_enabled_p ())
6391 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6392 "cannot perform implicit CSE when performing "
6393 "group loads with negative dependence distance\n");
6397 /* Similarly when the stmt is a load that is both part of a SLP
6398 instance and a loop vectorized stmt via the same-dr mechanism
6399 we have to give up. */
6400 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
6401 && (STMT_SLP_TYPE (stmt_info
)
6402 != STMT_SLP_TYPE (vinfo_for_stmt
6403 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
6405 if (dump_enabled_p ())
6406 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6407 "conflicting SLP types for CSEd load\n");
6413 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
6416 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, &gs_info
))
6418 if (!vect_is_simple_use (gs_info
.offset
, vinfo
, &def_stmt
,
6419 &gs_info
.offset_dt
, &gs_info
.offset_vectype
))
6421 if (dump_enabled_p ())
6422 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6423 "gather index use not simple.\n");
6427 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6431 negative
= tree_int_cst_compare (nested_in_vect_loop
6432 ? STMT_VINFO_DR_STEP (stmt_info
)
6434 size_zero_node
) < 0;
6435 if (negative
&& ncopies
> 1)
6437 if (dump_enabled_p ())
6438 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6439 "multiple types with negative step.\n");
6447 if (dump_enabled_p ())
6448 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6449 "negative step for group load not supported"
6453 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
6454 if (alignment_support_scheme
!= dr_aligned
6455 && alignment_support_scheme
!= dr_unaligned_supported
)
6457 if (dump_enabled_p ())
6458 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6459 "negative step but alignment required.\n");
6462 if (!perm_mask_for_reverse (vectype
))
6464 if (dump_enabled_p ())
6465 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6466 "negative step and reversing not supported."
6473 if (!vec_stmt
) /* transformation not required. */
6475 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6476 /* The SLP costs are calculated during SLP analysis. */
6477 if (!PURE_SLP_STMT (stmt_info
))
6478 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
,
6483 if (dump_enabled_p ())
6484 dump_printf_loc (MSG_NOTE
, vect_location
,
6485 "transform load. ncopies = %d\n", ncopies
);
6489 ensure_base_align (stmt_info
, dr
);
6491 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
6493 tree vec_oprnd0
= NULL_TREE
, op
;
6494 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
6495 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6496 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6497 edge pe
= loop_preheader_edge (loop
);
6500 enum { NARROW
, NONE
, WIDEN
} modifier
;
6501 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
6503 if (nunits
== gather_off_nunits
)
6505 else if (nunits
== gather_off_nunits
/ 2)
6507 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
6510 for (i
= 0; i
< gather_off_nunits
; ++i
)
6511 sel
[i
] = i
| nunits
;
6513 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
6515 else if (nunits
== gather_off_nunits
* 2)
6517 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
6520 for (i
= 0; i
< nunits
; ++i
)
6521 sel
[i
] = i
< gather_off_nunits
6522 ? i
: i
+ nunits
- gather_off_nunits
;
6524 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6530 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
6531 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6532 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6533 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6534 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6535 scaletype
= TREE_VALUE (arglist
);
6536 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6538 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6540 ptr
= fold_convert (ptrtype
, gs_info
.base
);
6541 if (!is_gimple_min_invariant (ptr
))
6543 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6544 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6545 gcc_assert (!new_bb
);
6548 /* Currently we support only unconditional gather loads,
6549 so mask should be all ones. */
6550 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6551 mask
= build_int_cst (masktype
, -1);
6552 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6554 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6555 mask
= build_vector_from_val (masktype
, mask
);
6556 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6558 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6562 for (j
= 0; j
< 6; ++j
)
6564 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6565 mask
= build_real (TREE_TYPE (masktype
), r
);
6566 mask
= build_vector_from_val (masktype
, mask
);
6567 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6572 scale
= build_int_cst (scaletype
, gs_info
.scale
);
6574 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6575 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6576 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6580 for (j
= 0; j
< 6; ++j
)
6582 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6583 merge
= build_real (TREE_TYPE (rettype
), r
);
6587 merge
= build_vector_from_val (rettype
, merge
);
6588 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6590 prev_stmt_info
= NULL
;
6591 for (j
= 0; j
< ncopies
; ++j
)
6593 if (modifier
== WIDEN
&& (j
& 1))
6594 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6595 perm_mask
, stmt
, gsi
);
6598 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
6601 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
, vec_oprnd0
);
6603 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6605 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6606 == TYPE_VECTOR_SUBPARTS (idxtype
));
6607 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
6608 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6610 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6611 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6616 = gimple_build_call (gs_info
.decl
, 5, merge
, ptr
, op
, mask
, scale
);
6618 if (!useless_type_conversion_p (vectype
, rettype
))
6620 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6621 == TYPE_VECTOR_SUBPARTS (rettype
));
6622 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
6623 gimple_call_set_lhs (new_stmt
, op
);
6624 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6625 var
= make_ssa_name (vec_dest
);
6626 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6628 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6632 var
= make_ssa_name (vec_dest
, new_stmt
);
6633 gimple_call_set_lhs (new_stmt
, var
);
6636 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6638 if (modifier
== NARROW
)
6645 var
= permute_vec_elements (prev_res
, var
,
6646 perm_mask
, stmt
, gsi
);
6647 new_stmt
= SSA_NAME_DEF_STMT (var
);
6650 if (prev_stmt_info
== NULL
)
6651 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6653 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6654 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6658 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6660 gimple_stmt_iterator incr_gsi
;
6666 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6667 gimple_seq stmts
= NULL
;
6668 tree stride_base
, stride_step
, alias_off
;
6670 gcc_assert (!nested_in_vect_loop
);
6672 if (slp
&& grouped_load
)
6673 first_dr
= STMT_VINFO_DATA_REF
6674 (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info
)));
6679 = fold_build_pointer_plus
6680 (DR_BASE_ADDRESS (first_dr
),
6681 size_binop (PLUS_EXPR
,
6682 convert_to_ptrofftype (DR_OFFSET (first_dr
)),
6683 convert_to_ptrofftype (DR_INIT (first_dr
))));
6684 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr
));
6686 /* For a load with loop-invariant (but other than power-of-2)
6687 stride (i.e. not a grouped access) like so:
6689 for (i = 0; i < n; i += stride)
6692 we generate a new induction variable and new accesses to
6693 form a new vector (or vectors, depending on ncopies):
6695 for (j = 0; ; j += VF*stride)
6697 tmp2 = array[j + stride];
6699 vectemp = {tmp1, tmp2, ...}
6702 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
6703 build_int_cst (TREE_TYPE (stride_step
), vf
));
6705 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6707 create_iv (unshare_expr (stride_base
), unshare_expr (ivstep
), NULL
,
6708 loop
, &incr_gsi
, insert_after
,
6710 incr
= gsi_stmt (incr_gsi
);
6711 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
6713 stride_step
= force_gimple_operand (unshare_expr (stride_step
),
6714 &stmts
, true, NULL_TREE
);
6716 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6718 prev_stmt_info
= NULL
;
6719 running_off
= offvar
;
6720 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
6721 int nloads
= nunits
;
6723 tree ltype
= TREE_TYPE (vectype
);
6724 auto_vec
<tree
> dr_chain
;
6727 if (group_size
< nunits
6728 && nunits
% group_size
== 0)
6730 nloads
= nunits
/ group_size
;
6732 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
6733 ltype
= build_aligned_type (ltype
,
6734 TYPE_ALIGN (TREE_TYPE (vectype
)));
6736 else if (group_size
>= nunits
6737 && group_size
% nunits
== 0)
6742 ltype
= build_aligned_type (ltype
,
6743 TYPE_ALIGN (TREE_TYPE (vectype
)));
6745 /* For SLP permutation support we need to load the whole group,
6746 not only the number of vector stmts the permutation result
6750 ncopies
= (group_size
* vf
+ nunits
- 1) / nunits
;
6751 dr_chain
.create (ncopies
);
6754 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6757 unsigned HOST_WIDE_INT
6758 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
6759 for (j
= 0; j
< ncopies
; j
++)
6762 vec_alloc (v
, nloads
);
6763 for (i
= 0; i
< nloads
; i
++)
6765 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
6767 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
6768 build2 (MEM_REF
, ltype
,
6769 running_off
, this_off
));
6770 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6772 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
,
6773 gimple_assign_lhs (new_stmt
));
6777 || group_el
== group_size
)
6779 tree newoff
= copy_ssa_name (running_off
);
6780 gimple
*incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6781 running_off
, stride_step
);
6782 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6784 running_off
= newoff
;
6790 tree vec_inv
= build_constructor (vectype
, v
);
6791 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6792 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6798 dr_chain
.quick_push (gimple_assign_lhs (new_stmt
));
6800 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6805 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6807 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6808 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6812 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6813 slp_node_instance
, false);
6819 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6820 /* For SLP vectorization we directly vectorize a subchain
6821 without permutation. */
6822 if (slp
&& ! SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6823 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6824 /* For BB vectorization always use the first stmt to base
6825 the data ref pointer on. */
6827 first_stmt_for_drptr
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6829 /* Check if the chain of loads is already vectorized. */
6830 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6831 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6832 ??? But we can only do so if there is exactly one
6833 as we have no way to get at the rest. Leave the CSE
6835 ??? With the group load eventually participating
6836 in multiple different permutations (having multiple
6837 slp nodes which refer to the same group) the CSE
6838 is even wrong code. See PR56270. */
6841 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6844 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6845 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6848 /* VEC_NUM is the number of vect stmts to be created for this group. */
6851 grouped_load
= false;
6852 /* For SLP permutation support we need to load the whole group,
6853 not only the number of vector stmts the permutation result
6856 vec_num
= (group_size
* vf
+ nunits
- 1) / nunits
;
6858 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6859 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
6862 vec_num
= group_size
;
6868 group_size
= vec_num
= 1;
6872 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6873 gcc_assert (alignment_support_scheme
);
6874 /* Targets with load-lane instructions must not require explicit
6876 gcc_assert (!load_lanes_p
6877 || alignment_support_scheme
== dr_aligned
6878 || alignment_support_scheme
== dr_unaligned_supported
);
6880 /* In case the vectorization factor (VF) is bigger than the number
6881 of elements that we can fit in a vectype (nunits), we have to generate
6882 more than one vector stmt - i.e - we need to "unroll" the
6883 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6884 from one copy of the vector stmt to the next, in the field
6885 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6886 stages to find the correct vector defs to be used when vectorizing
6887 stmts that use the defs of the current stmt. The example below
6888 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6889 need to create 4 vectorized stmts):
6891 before vectorization:
6892 RELATED_STMT VEC_STMT
6896 step 1: vectorize stmt S1:
6897 We first create the vector stmt VS1_0, and, as usual, record a
6898 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6899 Next, we create the vector stmt VS1_1, and record a pointer to
6900 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6901 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6903 RELATED_STMT VEC_STMT
6904 VS1_0: vx0 = memref0 VS1_1 -
6905 VS1_1: vx1 = memref1 VS1_2 -
6906 VS1_2: vx2 = memref2 VS1_3 -
6907 VS1_3: vx3 = memref3 - -
6908 S1: x = load - VS1_0
6911 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6912 information we recorded in RELATED_STMT field is used to vectorize
6915 /* In case of interleaving (non-unit grouped access):
6922 Vectorized loads are created in the order of memory accesses
6923 starting from the access of the first stmt of the chain:
6926 VS2: vx1 = &base + vec_size*1
6927 VS3: vx3 = &base + vec_size*2
6928 VS4: vx4 = &base + vec_size*3
6930 Then permutation statements are generated:
6932 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6933 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6936 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6937 (the order of the data-refs in the output of vect_permute_load_chain
6938 corresponds to the order of scalar stmts in the interleaving chain - see
6939 the documentation of vect_permute_load_chain()).
6940 The generation of permutation stmts and recording them in
6941 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6943 In case of both multiple types and interleaving, the vector loads and
6944 permutation stmts above are created for every copy. The result vector
6945 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6946 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6948 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6949 on a target that supports unaligned accesses (dr_unaligned_supported)
6950 we generate the following code:
6954 p = p + indx * vectype_size;
6959 Otherwise, the data reference is potentially unaligned on a target that
6960 does not support unaligned accesses (dr_explicit_realign_optimized) -
6961 then generate the following code, in which the data in each iteration is
6962 obtained by two vector loads, one from the previous iteration, and one
6963 from the current iteration:
6965 msq_init = *(floor(p1))
6966 p2 = initial_addr + VS - 1;
6967 realignment_token = call target_builtin;
6970 p2 = p2 + indx * vectype_size
6972 vec_dest = realign_load (msq, lsq, realignment_token)
6977 /* If the misalignment remains the same throughout the execution of the
6978 loop, we can create the init_addr and permutation mask at the loop
6979 preheader. Otherwise, it needs to be created inside the loop.
6980 This can only occur when vectorizing memory accesses in the inner-loop
6981 nested within an outer-loop that is being vectorized. */
6983 if (nested_in_vect_loop
6984 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6985 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6987 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6988 compute_in_loop
= true;
6991 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6992 || alignment_support_scheme
== dr_explicit_realign
)
6993 && !compute_in_loop
)
6995 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6996 alignment_support_scheme
, NULL_TREE
,
6998 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
7000 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
7001 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
7009 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
7012 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
7014 aggr_type
= vectype
;
7016 prev_stmt_info
= NULL
;
7017 for (j
= 0; j
< ncopies
; j
++)
7019 /* 1. Create the vector or array pointer update chain. */
7022 bool simd_lane_access_p
7023 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
7024 if (simd_lane_access_p
7025 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
7026 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
7027 && integer_zerop (DR_OFFSET (first_dr
))
7028 && integer_zerop (DR_INIT (first_dr
))
7029 && alias_sets_conflict_p (get_alias_set (aggr_type
),
7030 get_alias_set (DR_REF (first_dr
)))
7031 && (alignment_support_scheme
== dr_aligned
7032 || alignment_support_scheme
== dr_unaligned_supported
))
7034 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
7035 dataref_offset
= build_int_cst (reference_alias_ptr_type
7036 (DR_REF (first_dr
)), 0);
7039 else if (first_stmt_for_drptr
7040 && first_stmt
!= first_stmt_for_drptr
)
7043 = vect_create_data_ref_ptr (first_stmt_for_drptr
, aggr_type
,
7044 at_loop
, offset
, &dummy
, gsi
,
7045 &ptr_incr
, simd_lane_access_p
,
7046 &inv_p
, byte_offset
);
7047 /* Adjust the pointer by the difference to first_stmt. */
7048 data_reference_p ptrdr
7049 = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr
));
7050 tree diff
= fold_convert (sizetype
,
7051 size_binop (MINUS_EXPR
,
7054 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7059 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
7060 offset
, &dummy
, gsi
, &ptr_incr
,
7061 simd_lane_access_p
, &inv_p
,
7064 else if (dataref_offset
)
7065 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
7066 TYPE_SIZE_UNIT (aggr_type
));
7068 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
7069 TYPE_SIZE_UNIT (aggr_type
));
7071 if (grouped_load
|| slp_perm
)
7072 dr_chain
.create (vec_num
);
7078 vec_array
= create_vector_array (vectype
, vec_num
);
7081 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
7082 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
7083 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
7084 gimple_call_set_lhs (new_stmt
, vec_array
);
7085 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7087 /* Extract each vector into an SSA_NAME. */
7088 for (i
= 0; i
< vec_num
; i
++)
7090 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
7092 dr_chain
.quick_push (new_temp
);
7095 /* Record the mapping between SSA_NAMEs and statements. */
7096 vect_record_grouped_load_vectors (stmt
, dr_chain
);
7100 for (i
= 0; i
< vec_num
; i
++)
7103 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7106 /* 2. Create the vector-load in the loop. */
7107 switch (alignment_support_scheme
)
7110 case dr_unaligned_supported
:
7112 unsigned int align
, misalign
;
7115 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
7118 : build_int_cst (reference_alias_ptr_type
7119 (DR_REF (first_dr
)), 0));
7120 align
= TYPE_ALIGN_UNIT (vectype
);
7121 if (alignment_support_scheme
== dr_aligned
)
7123 gcc_assert (aligned_access_p (first_dr
));
7126 else if (DR_MISALIGNMENT (first_dr
) == -1)
7128 if (DR_VECT_AUX (first_dr
)->base_element_aligned
)
7129 align
= TYPE_ALIGN_UNIT (elem_type
);
7131 align
= (get_object_alignment (DR_REF (first_dr
))
7134 TREE_TYPE (data_ref
)
7135 = build_aligned_type (TREE_TYPE (data_ref
),
7136 align
* BITS_PER_UNIT
);
7140 TREE_TYPE (data_ref
)
7141 = build_aligned_type (TREE_TYPE (data_ref
),
7142 TYPE_ALIGN (elem_type
));
7143 misalign
= DR_MISALIGNMENT (first_dr
);
7145 if (dataref_offset
== NULL_TREE
7146 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
7147 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
7151 case dr_explicit_realign
:
7155 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
7157 if (compute_in_loop
)
7158 msq
= vect_setup_realignment (first_stmt
, gsi
,
7160 dr_explicit_realign
,
7163 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
7164 ptr
= copy_ssa_name (dataref_ptr
);
7166 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
7167 new_stmt
= gimple_build_assign
7168 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
7170 (TREE_TYPE (dataref_ptr
),
7171 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7172 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7174 = build2 (MEM_REF
, vectype
, ptr
,
7175 build_int_cst (reference_alias_ptr_type
7176 (DR_REF (first_dr
)), 0));
7177 vec_dest
= vect_create_destination_var (scalar_dest
,
7179 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
7180 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7181 gimple_assign_set_lhs (new_stmt
, new_temp
);
7182 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
7183 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
7184 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7187 bump
= size_binop (MULT_EXPR
, vs
,
7188 TYPE_SIZE_UNIT (elem_type
));
7189 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
7190 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
7191 new_stmt
= gimple_build_assign
7192 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
7195 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7196 ptr
= copy_ssa_name (ptr
, new_stmt
);
7197 gimple_assign_set_lhs (new_stmt
, ptr
);
7198 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7200 = build2 (MEM_REF
, vectype
, ptr
,
7201 build_int_cst (reference_alias_ptr_type
7202 (DR_REF (first_dr
)), 0));
7205 case dr_explicit_realign_optimized
:
7206 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
7207 new_temp
= copy_ssa_name (dataref_ptr
);
7209 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
7210 new_stmt
= gimple_build_assign
7211 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
7213 (TREE_TYPE (dataref_ptr
),
7214 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7215 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7217 = build2 (MEM_REF
, vectype
, new_temp
,
7218 build_int_cst (reference_alias_ptr_type
7219 (DR_REF (first_dr
)), 0));
7224 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7225 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
7226 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7227 gimple_assign_set_lhs (new_stmt
, new_temp
);
7228 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7230 /* 3. Handle explicit realignment if necessary/supported.
7232 vec_dest = realign_load (msq, lsq, realignment_token) */
7233 if (alignment_support_scheme
== dr_explicit_realign_optimized
7234 || alignment_support_scheme
== dr_explicit_realign
)
7236 lsq
= gimple_assign_lhs (new_stmt
);
7237 if (!realignment_token
)
7238 realignment_token
= dataref_ptr
;
7239 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7240 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
7241 msq
, lsq
, realignment_token
);
7242 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7243 gimple_assign_set_lhs (new_stmt
, new_temp
);
7244 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7246 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
7249 if (i
== vec_num
- 1 && j
== ncopies
- 1)
7250 add_phi_arg (phi
, lsq
,
7251 loop_latch_edge (containing_loop
),
7257 /* 4. Handle invariant-load. */
7258 if (inv_p
&& !bb_vinfo
)
7260 gcc_assert (!grouped_load
);
7261 /* If we have versioned for aliasing or the loop doesn't
7262 have any data dependencies that would preclude this,
7263 then we are sure this is a loop invariant load and
7264 thus we can insert it on the preheader edge. */
7265 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
7266 && !nested_in_vect_loop
7267 && hoist_defs_of_uses (stmt
, loop
))
7269 if (dump_enabled_p ())
7271 dump_printf_loc (MSG_NOTE
, vect_location
,
7272 "hoisting out of the vectorized "
7274 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7276 tree tem
= copy_ssa_name (scalar_dest
);
7277 gsi_insert_on_edge_immediate
7278 (loop_preheader_edge (loop
),
7279 gimple_build_assign (tem
,
7281 (gimple_assign_rhs1 (stmt
))));
7282 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
7283 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7284 set_vinfo_for_stmt (new_stmt
,
7285 new_stmt_vec_info (new_stmt
, vinfo
));
7289 gimple_stmt_iterator gsi2
= *gsi
;
7291 new_temp
= vect_init_vector (stmt
, scalar_dest
,
7293 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7299 tree perm_mask
= perm_mask_for_reverse (vectype
);
7300 new_temp
= permute_vec_elements (new_temp
, new_temp
,
7301 perm_mask
, stmt
, gsi
);
7302 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7305 /* Collect vector loads and later create their permutation in
7306 vect_transform_grouped_load (). */
7307 if (grouped_load
|| slp_perm
)
7308 dr_chain
.quick_push (new_temp
);
7310 /* Store vector loads in the corresponding SLP_NODE. */
7311 if (slp
&& !slp_perm
)
7312 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7314 /* Bump the vector pointer to account for a gap or for excess
7315 elements loaded for a permuted SLP load. */
7316 if (group_gap_adj
!= 0)
7320 = wide_int_to_tree (sizetype
,
7321 wi::smul (TYPE_SIZE_UNIT (elem_type
),
7322 group_gap_adj
, &ovf
));
7323 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7328 if (slp
&& !slp_perm
)
7333 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
7334 slp_node_instance
, false))
7336 dr_chain
.release ();
7345 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
7346 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7351 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7353 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7354 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7357 dr_chain
.release ();
7363 /* Function vect_is_simple_cond.
7366 LOOP - the loop that is being vectorized.
7367 COND - Condition that is checked for simple use.
7370 *COMP_VECTYPE - the vector type for the comparison.
7372 Returns whether a COND can be vectorized. Checks whether
7373 condition operands are supportable using vec_is_simple_use. */
7376 vect_is_simple_cond (tree cond
, vec_info
*vinfo
, tree
*comp_vectype
)
7379 enum vect_def_type dt
;
7380 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7383 if (TREE_CODE (cond
) == SSA_NAME
7384 && TREE_CODE (TREE_TYPE (cond
)) == BOOLEAN_TYPE
)
7386 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (cond
);
7387 if (!vect_is_simple_use (cond
, vinfo
, &lhs_def_stmt
,
7390 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype
))
7395 if (!COMPARISON_CLASS_P (cond
))
7398 lhs
= TREE_OPERAND (cond
, 0);
7399 rhs
= TREE_OPERAND (cond
, 1);
7401 if (TREE_CODE (lhs
) == SSA_NAME
)
7403 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
7404 if (!vect_is_simple_use (lhs
, vinfo
, &lhs_def_stmt
, &dt
, &vectype1
))
7407 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
7408 && TREE_CODE (lhs
) != FIXED_CST
)
7411 if (TREE_CODE (rhs
) == SSA_NAME
)
7413 gimple
*rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
7414 if (!vect_is_simple_use (rhs
, vinfo
, &rhs_def_stmt
, &dt
, &vectype2
))
7417 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
7418 && TREE_CODE (rhs
) != FIXED_CST
)
7421 if (vectype1
&& vectype2
7422 && TYPE_VECTOR_SUBPARTS (vectype1
) != TYPE_VECTOR_SUBPARTS (vectype2
))
7425 *comp_vectype
= vectype1
? vectype1
: vectype2
;
7429 /* vectorizable_condition.
7431 Check if STMT is conditional modify expression that can be vectorized.
7432 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7433 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
7436 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
7437 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
7438 else clause if it is 2).
7440 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7443 vectorizable_condition (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7444 gimple
**vec_stmt
, tree reduc_def
, int reduc_index
,
7447 tree scalar_dest
= NULL_TREE
;
7448 tree vec_dest
= NULL_TREE
;
7449 tree cond_expr
, then_clause
, else_clause
;
7450 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7451 tree comp_vectype
= NULL_TREE
;
7452 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
7453 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
7456 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7457 enum vect_def_type dt
, dts
[4];
7459 enum tree_code code
;
7460 stmt_vec_info prev_stmt_info
= NULL
;
7462 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7463 vec
<tree
> vec_oprnds0
= vNULL
;
7464 vec
<tree
> vec_oprnds1
= vNULL
;
7465 vec
<tree
> vec_oprnds2
= vNULL
;
7466 vec
<tree
> vec_oprnds3
= vNULL
;
7468 bool masked
= false;
7470 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
7473 if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info
) == TREE_CODE_REDUCTION
)
7475 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7478 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7479 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7483 /* FORNOW: not yet supported. */
7484 if (STMT_VINFO_LIVE_P (stmt_info
))
7486 if (dump_enabled_p ())
7487 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7488 "value used after loop.\n");
7493 /* Is vectorizable conditional operation? */
7494 if (!is_gimple_assign (stmt
))
7497 code
= gimple_assign_rhs_code (stmt
);
7499 if (code
!= COND_EXPR
)
7502 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7503 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7504 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7509 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7511 gcc_assert (ncopies
>= 1);
7512 if (reduc_index
&& ncopies
> 1)
7513 return false; /* FORNOW */
7515 cond_expr
= gimple_assign_rhs1 (stmt
);
7516 then_clause
= gimple_assign_rhs2 (stmt
);
7517 else_clause
= gimple_assign_rhs3 (stmt
);
7519 if (!vect_is_simple_cond (cond_expr
, stmt_info
->vinfo
, &comp_vectype
)
7524 if (!vect_is_simple_use (then_clause
, stmt_info
->vinfo
, &def_stmt
, &dt
,
7527 if (!vect_is_simple_use (else_clause
, stmt_info
->vinfo
, &def_stmt
, &dt
,
7531 if (vectype1
&& !useless_type_conversion_p (vectype
, vectype1
))
7534 if (vectype2
&& !useless_type_conversion_p (vectype
, vectype2
))
7537 masked
= !COMPARISON_CLASS_P (cond_expr
);
7538 vec_cmp_type
= build_same_sized_truth_vector_type (comp_vectype
);
7540 if (vec_cmp_type
== NULL_TREE
)
7545 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
7546 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
7553 vec_oprnds0
.create (1);
7554 vec_oprnds1
.create (1);
7555 vec_oprnds2
.create (1);
7556 vec_oprnds3
.create (1);
7560 scalar_dest
= gimple_assign_lhs (stmt
);
7561 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7563 /* Handle cond expr. */
7564 for (j
= 0; j
< ncopies
; j
++)
7566 gassign
*new_stmt
= NULL
;
7571 auto_vec
<tree
, 4> ops
;
7572 auto_vec
<vec
<tree
>, 4> vec_defs
;
7575 ops
.safe_push (cond_expr
);
7578 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
7579 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
7581 ops
.safe_push (then_clause
);
7582 ops
.safe_push (else_clause
);
7583 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7584 vec_oprnds3
= vec_defs
.pop ();
7585 vec_oprnds2
= vec_defs
.pop ();
7587 vec_oprnds1
= vec_defs
.pop ();
7588 vec_oprnds0
= vec_defs
.pop ();
7591 vec_defs
.release ();
7599 = vect_get_vec_def_for_operand (cond_expr
, stmt
,
7601 vect_is_simple_use (cond_expr
, stmt_info
->vinfo
,
7607 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
7608 stmt
, comp_vectype
);
7609 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0),
7610 loop_vinfo
, >emp
, &dts
[0]);
7613 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
7614 stmt
, comp_vectype
);
7615 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1),
7616 loop_vinfo
, >emp
, &dts
[1]);
7618 if (reduc_index
== 1)
7619 vec_then_clause
= reduc_def
;
7622 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
7624 vect_is_simple_use (then_clause
, loop_vinfo
,
7627 if (reduc_index
== 2)
7628 vec_else_clause
= reduc_def
;
7631 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
7633 vect_is_simple_use (else_clause
, loop_vinfo
, >emp
, &dts
[3]);
7640 = vect_get_vec_def_for_stmt_copy (dts
[0],
7641 vec_oprnds0
.pop ());
7644 = vect_get_vec_def_for_stmt_copy (dts
[1],
7645 vec_oprnds1
.pop ());
7647 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
7648 vec_oprnds2
.pop ());
7649 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
7650 vec_oprnds3
.pop ());
7655 vec_oprnds0
.quick_push (vec_cond_lhs
);
7657 vec_oprnds1
.quick_push (vec_cond_rhs
);
7658 vec_oprnds2
.quick_push (vec_then_clause
);
7659 vec_oprnds3
.quick_push (vec_else_clause
);
7662 /* Arguments are ready. Create the new vector stmt. */
7663 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
7665 vec_then_clause
= vec_oprnds2
[i
];
7666 vec_else_clause
= vec_oprnds3
[i
];
7669 vec_compare
= vec_cond_lhs
;
7672 vec_cond_rhs
= vec_oprnds1
[i
];
7673 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
7674 vec_cond_lhs
, vec_cond_rhs
);
7676 new_temp
= make_ssa_name (vec_dest
);
7677 new_stmt
= gimple_build_assign (new_temp
, VEC_COND_EXPR
,
7678 vec_compare
, vec_then_clause
,
7680 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7682 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7689 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7691 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7693 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7696 vec_oprnds0
.release ();
7697 vec_oprnds1
.release ();
7698 vec_oprnds2
.release ();
7699 vec_oprnds3
.release ();
7704 /* vectorizable_comparison.
7706 Check if STMT is comparison expression that can be vectorized.
7707 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7708 comparison, put it in VEC_STMT, and insert it at GSI.
7710 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7713 vectorizable_comparison (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7714 gimple
**vec_stmt
, tree reduc_def
,
7717 tree lhs
, rhs1
, rhs2
;
7718 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7719 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7720 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7721 tree vec_rhs1
= NULL_TREE
, vec_rhs2
= NULL_TREE
;
7723 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7724 enum vect_def_type dts
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
7727 enum tree_code code
, bitop1
= NOP_EXPR
, bitop2
= NOP_EXPR
;
7728 stmt_vec_info prev_stmt_info
= NULL
;
7730 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7731 vec
<tree
> vec_oprnds0
= vNULL
;
7732 vec
<tree
> vec_oprnds1
= vNULL
;
7737 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7740 if (!vectype
|| !VECTOR_BOOLEAN_TYPE_P (vectype
))
7743 mask_type
= vectype
;
7744 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7749 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7751 gcc_assert (ncopies
>= 1);
7752 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7753 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7757 if (STMT_VINFO_LIVE_P (stmt_info
))
7759 if (dump_enabled_p ())
7760 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7761 "value used after loop.\n");
7765 if (!is_gimple_assign (stmt
))
7768 code
= gimple_assign_rhs_code (stmt
);
7770 if (TREE_CODE_CLASS (code
) != tcc_comparison
)
7773 rhs1
= gimple_assign_rhs1 (stmt
);
7774 rhs2
= gimple_assign_rhs2 (stmt
);
7776 if (!vect_is_simple_use (rhs1
, stmt_info
->vinfo
, &def_stmt
,
7777 &dts
[0], &vectype1
))
7780 if (!vect_is_simple_use (rhs2
, stmt_info
->vinfo
, &def_stmt
,
7781 &dts
[1], &vectype2
))
7784 if (vectype1
&& vectype2
7785 && TYPE_VECTOR_SUBPARTS (vectype1
) != TYPE_VECTOR_SUBPARTS (vectype2
))
7788 vectype
= vectype1
? vectype1
: vectype2
;
7790 /* Invariant comparison. */
7793 vectype
= get_vectype_for_scalar_type (TREE_TYPE (rhs1
));
7794 if (TYPE_VECTOR_SUBPARTS (vectype
) != nunits
)
7797 else if (nunits
!= TYPE_VECTOR_SUBPARTS (vectype
))
7800 /* Can't compare mask and non-mask types. */
7801 if (vectype1
&& vectype2
7802 && (VECTOR_BOOLEAN_TYPE_P (vectype1
) ^ VECTOR_BOOLEAN_TYPE_P (vectype2
)))
7805 /* Boolean values may have another representation in vectors
7806 and therefore we prefer bit operations over comparison for
7807 them (which also works for scalar masks). We store opcodes
7808 to use in bitop1 and bitop2. Statement is vectorized as
7809 BITOP2 (rhs1 BITOP1 rhs2) or
7810 rhs1 BITOP2 (BITOP1 rhs2)
7811 depending on bitop1 and bitop2 arity. */
7812 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
7814 if (code
== GT_EXPR
)
7816 bitop1
= BIT_NOT_EXPR
;
7817 bitop2
= BIT_AND_EXPR
;
7819 else if (code
== GE_EXPR
)
7821 bitop1
= BIT_NOT_EXPR
;
7822 bitop2
= BIT_IOR_EXPR
;
7824 else if (code
== LT_EXPR
)
7826 bitop1
= BIT_NOT_EXPR
;
7827 bitop2
= BIT_AND_EXPR
;
7828 std::swap (rhs1
, rhs2
);
7829 std::swap (dts
[0], dts
[1]);
7831 else if (code
== LE_EXPR
)
7833 bitop1
= BIT_NOT_EXPR
;
7834 bitop2
= BIT_IOR_EXPR
;
7835 std::swap (rhs1
, rhs2
);
7836 std::swap (dts
[0], dts
[1]);
7840 bitop1
= BIT_XOR_EXPR
;
7841 if (code
== EQ_EXPR
)
7842 bitop2
= BIT_NOT_EXPR
;
7848 STMT_VINFO_TYPE (stmt_info
) = comparison_vec_info_type
;
7849 vect_model_simple_cost (stmt_info
, ncopies
* (1 + (bitop2
!= NOP_EXPR
)),
7851 if (bitop1
== NOP_EXPR
)
7852 return expand_vec_cmp_expr_p (vectype
, mask_type
);
7855 machine_mode mode
= TYPE_MODE (vectype
);
7858 optab
= optab_for_tree_code (bitop1
, vectype
, optab_default
);
7859 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
7862 if (bitop2
!= NOP_EXPR
)
7864 optab
= optab_for_tree_code (bitop2
, vectype
, optab_default
);
7865 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
7875 vec_oprnds0
.create (1);
7876 vec_oprnds1
.create (1);
7880 lhs
= gimple_assign_lhs (stmt
);
7881 mask
= vect_create_destination_var (lhs
, mask_type
);
7883 /* Handle cmp expr. */
7884 for (j
= 0; j
< ncopies
; j
++)
7886 gassign
*new_stmt
= NULL
;
7891 auto_vec
<tree
, 2> ops
;
7892 auto_vec
<vec
<tree
>, 2> vec_defs
;
7894 ops
.safe_push (rhs1
);
7895 ops
.safe_push (rhs2
);
7896 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7897 vec_oprnds1
= vec_defs
.pop ();
7898 vec_oprnds0
= vec_defs
.pop ();
7902 vec_rhs1
= vect_get_vec_def_for_operand (rhs1
, stmt
, vectype
);
7903 vec_rhs2
= vect_get_vec_def_for_operand (rhs2
, stmt
, vectype
);
7908 vec_rhs1
= vect_get_vec_def_for_stmt_copy (dts
[0],
7909 vec_oprnds0
.pop ());
7910 vec_rhs2
= vect_get_vec_def_for_stmt_copy (dts
[1],
7911 vec_oprnds1
.pop ());
7916 vec_oprnds0
.quick_push (vec_rhs1
);
7917 vec_oprnds1
.quick_push (vec_rhs2
);
7920 /* Arguments are ready. Create the new vector stmt. */
7921 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_rhs1
)
7923 vec_rhs2
= vec_oprnds1
[i
];
7925 new_temp
= make_ssa_name (mask
);
7926 if (bitop1
== NOP_EXPR
)
7928 new_stmt
= gimple_build_assign (new_temp
, code
,
7929 vec_rhs1
, vec_rhs2
);
7930 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7934 if (bitop1
== BIT_NOT_EXPR
)
7935 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs2
);
7937 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs1
,
7939 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7940 if (bitop2
!= NOP_EXPR
)
7942 tree res
= make_ssa_name (mask
);
7943 if (bitop2
== BIT_NOT_EXPR
)
7944 new_stmt
= gimple_build_assign (res
, bitop2
, new_temp
);
7946 new_stmt
= gimple_build_assign (res
, bitop2
, vec_rhs1
,
7948 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7952 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7959 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7961 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7963 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7966 vec_oprnds0
.release ();
7967 vec_oprnds1
.release ();
7972 /* Make sure the statement is vectorizable. */
7975 vect_analyze_stmt (gimple
*stmt
, bool *need_to_vectorize
, slp_tree node
)
7977 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7978 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7979 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
7981 tree scalar_type
, vectype
;
7982 gimple
*pattern_stmt
;
7983 gimple_seq pattern_def_seq
;
7985 if (dump_enabled_p ())
7987 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
7988 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7991 if (gimple_has_volatile_ops (stmt
))
7993 if (dump_enabled_p ())
7994 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7995 "not vectorized: stmt has volatile operands\n");
8000 /* Skip stmts that do not need to be vectorized. In loops this is expected
8002 - the COND_EXPR which is the loop exit condition
8003 - any LABEL_EXPRs in the loop
8004 - computations that are used only for array indexing or loop control.
8005 In basic blocks we only analyze statements that are a part of some SLP
8006 instance, therefore, all the statements are relevant.
8008 Pattern statement needs to be analyzed instead of the original statement
8009 if the original statement is not relevant. Otherwise, we analyze both
8010 statements. In basic blocks we are called from some SLP instance
8011 traversal, don't analyze pattern stmts instead, the pattern stmts
8012 already will be part of SLP instance. */
8014 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
8015 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
8016 && !STMT_VINFO_LIVE_P (stmt_info
))
8018 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8020 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
8021 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
8023 /* Analyze PATTERN_STMT instead of the original stmt. */
8024 stmt
= pattern_stmt
;
8025 stmt_info
= vinfo_for_stmt (pattern_stmt
);
8026 if (dump_enabled_p ())
8028 dump_printf_loc (MSG_NOTE
, vect_location
,
8029 "==> examining pattern statement: ");
8030 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
8035 if (dump_enabled_p ())
8036 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
8041 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8044 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
8045 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
8047 /* Analyze PATTERN_STMT too. */
8048 if (dump_enabled_p ())
8050 dump_printf_loc (MSG_NOTE
, vect_location
,
8051 "==> examining pattern statement: ");
8052 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
8055 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
8059 if (is_pattern_stmt_p (stmt_info
)
8061 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
8063 gimple_stmt_iterator si
;
8065 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
8067 gimple
*pattern_def_stmt
= gsi_stmt (si
);
8068 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
8069 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
8071 /* Analyze def stmt of STMT if it's a pattern stmt. */
8072 if (dump_enabled_p ())
8074 dump_printf_loc (MSG_NOTE
, vect_location
,
8075 "==> examining pattern def statement: ");
8076 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
8079 if (!vect_analyze_stmt (pattern_def_stmt
,
8080 need_to_vectorize
, node
))
8086 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
8088 case vect_internal_def
:
8091 case vect_reduction_def
:
8092 case vect_nested_cycle
:
8093 gcc_assert (!bb_vinfo
8094 && (relevance
== vect_used_in_outer
8095 || relevance
== vect_used_in_outer_by_reduction
8096 || relevance
== vect_used_by_reduction
8097 || relevance
== vect_unused_in_scope
8098 || relevance
== vect_used_only_live
));
8101 case vect_induction_def
:
8102 case vect_constant_def
:
8103 case vect_external_def
:
8104 case vect_unknown_def_type
:
8111 gcc_assert (PURE_SLP_STMT (stmt_info
));
8113 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
8114 if (dump_enabled_p ())
8116 dump_printf_loc (MSG_NOTE
, vect_location
,
8117 "get vectype for scalar type: ");
8118 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
8119 dump_printf (MSG_NOTE
, "\n");
8122 vectype
= get_vectype_for_scalar_type (scalar_type
);
8125 if (dump_enabled_p ())
8127 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8128 "not SLPed: unsupported data-type ");
8129 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
8131 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
8136 if (dump_enabled_p ())
8138 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
8139 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
8140 dump_printf (MSG_NOTE
, "\n");
8143 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
8146 if (STMT_VINFO_RELEVANT_P (stmt_info
))
8148 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
8149 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
8150 || (is_gimple_call (stmt
)
8151 && gimple_call_lhs (stmt
) == NULL_TREE
));
8152 *need_to_vectorize
= true;
8155 if (PURE_SLP_STMT (stmt_info
) && !node
)
8157 dump_printf_loc (MSG_NOTE
, vect_location
,
8158 "handled only by SLP analysis\n");
8164 && (STMT_VINFO_RELEVANT_P (stmt_info
)
8165 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
8166 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
8167 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
8168 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
8169 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
8170 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
8171 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
8172 || vectorizable_call (stmt
, NULL
, NULL
, node
)
8173 || vectorizable_store (stmt
, NULL
, NULL
, node
)
8174 || vectorizable_reduction (stmt
, NULL
, NULL
, node
)
8175 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
8176 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
8180 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
8181 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
8182 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
8183 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
8184 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
8185 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
8186 || vectorizable_call (stmt
, NULL
, NULL
, node
)
8187 || vectorizable_store (stmt
, NULL
, NULL
, node
)
8188 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
8189 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
8194 if (dump_enabled_p ())
8196 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8197 "not vectorized: relevant stmt not ");
8198 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
8199 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
8208 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
8209 need extra handling, except for vectorizable reductions. */
8210 if (STMT_VINFO_LIVE_P (stmt_info
)
8211 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
8212 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
, -1, NULL
);
8216 if (dump_enabled_p ())
8218 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8219 "not vectorized: live stmt not ");
8220 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
8221 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
8231 /* Function vect_transform_stmt.
8233 Create a vectorized stmt to replace STMT, and insert it at BSI. */
8236 vect_transform_stmt (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
8237 bool *grouped_store
, slp_tree slp_node
,
8238 slp_instance slp_node_instance
)
8240 bool is_store
= false;
8241 gimple
*vec_stmt
= NULL
;
8242 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8245 gcc_assert (slp_node
|| !PURE_SLP_STMT (stmt_info
));
8246 gimple
*old_vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
8248 switch (STMT_VINFO_TYPE (stmt_info
))
8250 case type_demotion_vec_info_type
:
8251 case type_promotion_vec_info_type
:
8252 case type_conversion_vec_info_type
:
8253 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
8257 case induc_vec_info_type
:
8258 gcc_assert (!slp_node
);
8259 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
8263 case shift_vec_info_type
:
8264 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
8268 case op_vec_info_type
:
8269 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
8273 case assignment_vec_info_type
:
8274 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
8278 case load_vec_info_type
:
8279 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
8284 case store_vec_info_type
:
8285 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
8287 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
8289 /* In case of interleaving, the whole chain is vectorized when the
8290 last store in the chain is reached. Store stmts before the last
8291 one are skipped, and there vec_stmt_info shouldn't be freed
8293 *grouped_store
= true;
8294 if (STMT_VINFO_VEC_STMT (stmt_info
))
8301 case condition_vec_info_type
:
8302 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
8306 case comparison_vec_info_type
:
8307 done
= vectorizable_comparison (stmt
, gsi
, &vec_stmt
, NULL
, slp_node
);
8311 case call_vec_info_type
:
8312 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8313 stmt
= gsi_stmt (*gsi
);
8314 if (is_gimple_call (stmt
)
8315 && gimple_call_internal_p (stmt
)
8316 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
8320 case call_simd_clone_vec_info_type
:
8321 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8322 stmt
= gsi_stmt (*gsi
);
8325 case reduc_vec_info_type
:
8326 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
8331 if (!STMT_VINFO_LIVE_P (stmt_info
))
8333 if (dump_enabled_p ())
8334 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8335 "stmt not supported.\n");
8340 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
8341 This would break hybrid SLP vectorization. */
8343 gcc_assert (!vec_stmt
8344 && STMT_VINFO_VEC_STMT (stmt_info
) == old_vec_stmt
);
8346 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
8347 is being vectorized, but outside the immediately enclosing loop. */
8349 && STMT_VINFO_LOOP_VINFO (stmt_info
)
8350 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
8351 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
8352 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
8353 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
8354 || STMT_VINFO_RELEVANT (stmt_info
) ==
8355 vect_used_in_outer_by_reduction
))
8357 struct loop
*innerloop
= LOOP_VINFO_LOOP (
8358 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
8359 imm_use_iterator imm_iter
;
8360 use_operand_p use_p
;
8364 if (dump_enabled_p ())
8365 dump_printf_loc (MSG_NOTE
, vect_location
,
8366 "Record the vdef for outer-loop vectorization.\n");
8368 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
8369 (to be used when vectorizing outer-loop stmts that use the DEF of
8371 if (gimple_code (stmt
) == GIMPLE_PHI
)
8372 scalar_dest
= PHI_RESULT (stmt
);
8374 scalar_dest
= gimple_assign_lhs (stmt
);
8376 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
8378 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
8380 exit_phi
= USE_STMT (use_p
);
8381 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
8386 /* Handle stmts whose DEF is used outside the loop-nest that is
8387 being vectorized. */
8392 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node
), i
, slp_stmt
)
8394 stmt_vec_info slp_stmt_info
= vinfo_for_stmt (slp_stmt
);
8395 if (STMT_VINFO_LIVE_P (slp_stmt_info
)
8396 && STMT_VINFO_TYPE (slp_stmt_info
) != reduc_vec_info_type
)
8398 done
= vectorizable_live_operation (slp_stmt
, gsi
, slp_node
, i
,
8404 else if (STMT_VINFO_LIVE_P (stmt_info
)
8405 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
8407 done
= vectorizable_live_operation (stmt
, gsi
, slp_node
, -1, &vec_stmt
);
8412 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
8418 /* Remove a group of stores (for SLP or interleaving), free their
8422 vect_remove_stores (gimple
*first_stmt
)
8424 gimple
*next
= first_stmt
;
8426 gimple_stmt_iterator next_si
;
8430 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
8432 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
8433 if (is_pattern_stmt_p (stmt_info
))
8434 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
8435 /* Free the attached stmt_vec_info and remove the stmt. */
8436 next_si
= gsi_for_stmt (next
);
8437 unlink_stmt_vdef (next
);
8438 gsi_remove (&next_si
, true);
8439 release_defs (next
);
8440 free_stmt_vec_info (next
);
8446 /* Function new_stmt_vec_info.
8448 Create and initialize a new stmt_vec_info struct for STMT. */
8451 new_stmt_vec_info (gimple
*stmt
, vec_info
*vinfo
)
8454 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
8456 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
8457 STMT_VINFO_STMT (res
) = stmt
;
8459 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
8460 STMT_VINFO_LIVE_P (res
) = false;
8461 STMT_VINFO_VECTYPE (res
) = NULL
;
8462 STMT_VINFO_VEC_STMT (res
) = NULL
;
8463 STMT_VINFO_VECTORIZABLE (res
) = true;
8464 STMT_VINFO_IN_PATTERN_P (res
) = false;
8465 STMT_VINFO_RELATED_STMT (res
) = NULL
;
8466 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
8467 STMT_VINFO_DATA_REF (res
) = NULL
;
8468 STMT_VINFO_VEC_REDUCTION_TYPE (res
) = TREE_CODE_REDUCTION
;
8470 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
8471 STMT_VINFO_DR_OFFSET (res
) = NULL
;
8472 STMT_VINFO_DR_INIT (res
) = NULL
;
8473 STMT_VINFO_DR_STEP (res
) = NULL
;
8474 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
8476 if (gimple_code (stmt
) == GIMPLE_PHI
8477 && is_loop_header_bb_p (gimple_bb (stmt
)))
8478 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
8480 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
8482 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
8483 STMT_SLP_TYPE (res
) = loop_vect
;
8484 STMT_VINFO_NUM_SLP_USES (res
) = 0;
8486 GROUP_FIRST_ELEMENT (res
) = NULL
;
8487 GROUP_NEXT_ELEMENT (res
) = NULL
;
8488 GROUP_SIZE (res
) = 0;
8489 GROUP_STORE_COUNT (res
) = 0;
8490 GROUP_GAP (res
) = 0;
8491 GROUP_SAME_DR_STMT (res
) = NULL
;
8497 /* Create a hash table for stmt_vec_info. */
8500 init_stmt_vec_info_vec (void)
8502 gcc_assert (!stmt_vec_info_vec
.exists ());
8503 stmt_vec_info_vec
.create (50);
8507 /* Free hash table for stmt_vec_info. */
8510 free_stmt_vec_info_vec (void)
8514 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
8516 free_stmt_vec_info (STMT_VINFO_STMT (info
));
8517 gcc_assert (stmt_vec_info_vec
.exists ());
8518 stmt_vec_info_vec
.release ();
8522 /* Free stmt vectorization related info. */
8525 free_stmt_vec_info (gimple
*stmt
)
8527 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8532 /* Check if this statement has a related "pattern stmt"
8533 (introduced by the vectorizer during the pattern recognition
8534 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
8536 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
8538 stmt_vec_info patt_info
8539 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8542 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
8543 gimple
*patt_stmt
= STMT_VINFO_STMT (patt_info
);
8544 gimple_set_bb (patt_stmt
, NULL
);
8545 tree lhs
= gimple_get_lhs (patt_stmt
);
8546 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
8547 release_ssa_name (lhs
);
8550 gimple_stmt_iterator si
;
8551 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
8553 gimple
*seq_stmt
= gsi_stmt (si
);
8554 gimple_set_bb (seq_stmt
, NULL
);
8555 lhs
= gimple_get_lhs (seq_stmt
);
8556 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
8557 release_ssa_name (lhs
);
8558 free_stmt_vec_info (seq_stmt
);
8561 free_stmt_vec_info (patt_stmt
);
8565 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
8566 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
8567 set_vinfo_for_stmt (stmt
, NULL
);
8572 /* Function get_vectype_for_scalar_type_and_size.
8574 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
8578 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
8580 machine_mode inner_mode
= TYPE_MODE (scalar_type
);
8581 machine_mode simd_mode
;
8582 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
8589 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
8590 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
8593 /* For vector types of elements whose mode precision doesn't
8594 match their types precision we use a element type of mode
8595 precision. The vectorization routines will have to make sure
8596 they support the proper result truncation/extension.
8597 We also make sure to build vector types with INTEGER_TYPE
8598 component type only. */
8599 if (INTEGRAL_TYPE_P (scalar_type
)
8600 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
8601 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
8602 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
8603 TYPE_UNSIGNED (scalar_type
));
8605 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
8606 When the component mode passes the above test simply use a type
8607 corresponding to that mode. The theory is that any use that
8608 would cause problems with this will disable vectorization anyway. */
8609 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
8610 && !INTEGRAL_TYPE_P (scalar_type
))
8611 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
8613 /* We can't build a vector type of elements with alignment bigger than
8615 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
8616 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
8617 TYPE_UNSIGNED (scalar_type
));
8619 /* If we felt back to using the mode fail if there was
8620 no scalar type for it. */
8621 if (scalar_type
== NULL_TREE
)
8624 /* If no size was supplied use the mode the target prefers. Otherwise
8625 lookup a vector mode of the specified size. */
8627 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
8629 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
8630 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
8634 vectype
= build_vector_type (scalar_type
, nunits
);
8636 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
8637 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
8643 unsigned int current_vector_size
;
8645 /* Function get_vectype_for_scalar_type.
8647 Returns the vector type corresponding to SCALAR_TYPE as supported
8651 get_vectype_for_scalar_type (tree scalar_type
)
8654 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
8655 current_vector_size
);
8657 && current_vector_size
== 0)
8658 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
8662 /* Function get_mask_type_for_scalar_type.
8664 Returns the mask type corresponding to a result of comparison
8665 of vectors of specified SCALAR_TYPE as supported by target. */
8668 get_mask_type_for_scalar_type (tree scalar_type
)
8670 tree vectype
= get_vectype_for_scalar_type (scalar_type
);
8675 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype
),
8676 current_vector_size
);
8679 /* Function get_same_sized_vectype
8681 Returns a vector type corresponding to SCALAR_TYPE of size
8682 VECTOR_TYPE if supported by the target. */
8685 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
8687 if (TREE_CODE (scalar_type
) == BOOLEAN_TYPE
)
8688 return build_same_sized_truth_vector_type (vector_type
);
8690 return get_vectype_for_scalar_type_and_size
8691 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
8694 /* Function vect_is_simple_use.
8697 VINFO - the vect info of the loop or basic block that is being vectorized.
8698 OPERAND - operand in the loop or bb.
8700 DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
8701 DT - the type of definition
8703 Returns whether a stmt with OPERAND can be vectorized.
8704 For loops, supportable operands are constants, loop invariants, and operands
8705 that are defined by the current iteration of the loop. Unsupportable
8706 operands are those that are defined by a previous iteration of the loop (as
8707 is the case in reduction/induction computations).
8708 For basic blocks, supportable operands are constants and bb invariants.
8709 For now, operands defined outside the basic block are not supported. */
8712 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
8713 gimple
**def_stmt
, enum vect_def_type
*dt
)
8716 *dt
= vect_unknown_def_type
;
8718 if (dump_enabled_p ())
8720 dump_printf_loc (MSG_NOTE
, vect_location
,
8721 "vect_is_simple_use: operand ");
8722 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
8723 dump_printf (MSG_NOTE
, "\n");
8726 if (CONSTANT_CLASS_P (operand
))
8728 *dt
= vect_constant_def
;
8732 if (is_gimple_min_invariant (operand
))
8734 *dt
= vect_external_def
;
8738 if (TREE_CODE (operand
) != SSA_NAME
)
8740 if (dump_enabled_p ())
8741 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8746 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
8748 *dt
= vect_external_def
;
8752 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
8753 if (dump_enabled_p ())
8755 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
8756 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
8759 if (! vect_stmt_in_region_p (vinfo
, *def_stmt
))
8760 *dt
= vect_external_def
;
8763 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
8764 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
8767 if (dump_enabled_p ())
8769 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
8772 case vect_uninitialized_def
:
8773 dump_printf (MSG_NOTE
, "uninitialized\n");
8775 case vect_constant_def
:
8776 dump_printf (MSG_NOTE
, "constant\n");
8778 case vect_external_def
:
8779 dump_printf (MSG_NOTE
, "external\n");
8781 case vect_internal_def
:
8782 dump_printf (MSG_NOTE
, "internal\n");
8784 case vect_induction_def
:
8785 dump_printf (MSG_NOTE
, "induction\n");
8787 case vect_reduction_def
:
8788 dump_printf (MSG_NOTE
, "reduction\n");
8790 case vect_double_reduction_def
:
8791 dump_printf (MSG_NOTE
, "double reduction\n");
8793 case vect_nested_cycle
:
8794 dump_printf (MSG_NOTE
, "nested cycle\n");
8796 case vect_unknown_def_type
:
8797 dump_printf (MSG_NOTE
, "unknown\n");
8802 if (*dt
== vect_unknown_def_type
)
8804 if (dump_enabled_p ())
8805 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8806 "Unsupported pattern.\n");
8810 switch (gimple_code (*def_stmt
))
8817 if (dump_enabled_p ())
8818 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8819 "unsupported defining stmt:\n");
8826 /* Function vect_is_simple_use.
8828 Same as vect_is_simple_use but also determines the vector operand
8829 type of OPERAND and stores it to *VECTYPE. If the definition of
8830 OPERAND is vect_uninitialized_def, vect_constant_def or
8831 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8832 is responsible to compute the best suited vector type for the
8836 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
8837 gimple
**def_stmt
, enum vect_def_type
*dt
, tree
*vectype
)
8839 if (!vect_is_simple_use (operand
, vinfo
, def_stmt
, dt
))
8842 /* Now get a vector type if the def is internal, otherwise supply
8843 NULL_TREE and leave it up to the caller to figure out a proper
8844 type for the use stmt. */
8845 if (*dt
== vect_internal_def
8846 || *dt
== vect_induction_def
8847 || *dt
== vect_reduction_def
8848 || *dt
== vect_double_reduction_def
8849 || *dt
== vect_nested_cycle
)
8851 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
8853 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8854 && !STMT_VINFO_RELEVANT (stmt_info
)
8855 && !STMT_VINFO_LIVE_P (stmt_info
))
8856 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8858 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8859 gcc_assert (*vectype
!= NULL_TREE
);
8861 else if (*dt
== vect_uninitialized_def
8862 || *dt
== vect_constant_def
8863 || *dt
== vect_external_def
)
8864 *vectype
= NULL_TREE
;
8872 /* Function supportable_widening_operation
8874 Check whether an operation represented by the code CODE is a
8875 widening operation that is supported by the target platform in
8876 vector form (i.e., when operating on arguments of type VECTYPE_IN
8877 producing a result of type VECTYPE_OUT).
8879 Widening operations we currently support are NOP (CONVERT), FLOAT
8880 and WIDEN_MULT. This function checks if these operations are supported
8881 by the target platform either directly (via vector tree-codes), or via
8885 - CODE1 and CODE2 are codes of vector operations to be used when
8886 vectorizing the operation, if available.
8887 - MULTI_STEP_CVT determines the number of required intermediate steps in
8888 case of multi-step conversion (like char->short->int - in that case
8889 MULTI_STEP_CVT will be 1).
8890 - INTERM_TYPES contains the intermediate type required to perform the
8891 widening operation (short in the above example). */
8894 supportable_widening_operation (enum tree_code code
, gimple
*stmt
,
8895 tree vectype_out
, tree vectype_in
,
8896 enum tree_code
*code1
, enum tree_code
*code2
,
8897 int *multi_step_cvt
,
8898 vec
<tree
> *interm_types
)
8900 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8901 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8902 struct loop
*vect_loop
= NULL
;
8903 machine_mode vec_mode
;
8904 enum insn_code icode1
, icode2
;
8905 optab optab1
, optab2
;
8906 tree vectype
= vectype_in
;
8907 tree wide_vectype
= vectype_out
;
8908 enum tree_code c1
, c2
;
8910 tree prev_type
, intermediate_type
;
8911 machine_mode intermediate_mode
, prev_mode
;
8912 optab optab3
, optab4
;
8914 *multi_step_cvt
= 0;
8916 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
8920 case WIDEN_MULT_EXPR
:
8921 /* The result of a vectorized widening operation usually requires
8922 two vectors (because the widened results do not fit into one vector).
8923 The generated vector results would normally be expected to be
8924 generated in the same order as in the original scalar computation,
8925 i.e. if 8 results are generated in each vector iteration, they are
8926 to be organized as follows:
8927 vect1: [res1,res2,res3,res4],
8928 vect2: [res5,res6,res7,res8].
8930 However, in the special case that the result of the widening
8931 operation is used in a reduction computation only, the order doesn't
8932 matter (because when vectorizing a reduction we change the order of
8933 the computation). Some targets can take advantage of this and
8934 generate more efficient code. For example, targets like Altivec,
8935 that support widen_mult using a sequence of {mult_even,mult_odd}
8936 generate the following vectors:
8937 vect1: [res1,res3,res5,res7],
8938 vect2: [res2,res4,res6,res8].
8940 When vectorizing outer-loops, we execute the inner-loop sequentially
8941 (each vectorized inner-loop iteration contributes to VF outer-loop
8942 iterations in parallel). We therefore don't allow to change the
8943 order of the computation in the inner-loop during outer-loop
8945 /* TODO: Another case in which order doesn't *really* matter is when we
8946 widen and then contract again, e.g. (short)((int)x * y >> 8).
8947 Normally, pack_trunc performs an even/odd permute, whereas the
8948 repack from an even/odd expansion would be an interleave, which
8949 would be significantly simpler for e.g. AVX2. */
8950 /* In any case, in order to avoid duplicating the code below, recurse
8951 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8952 are properly set up for the caller. If we fail, we'll continue with
8953 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8955 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
8956 && !nested_in_vect_loop_p (vect_loop
, stmt
)
8957 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
8958 stmt
, vectype_out
, vectype_in
,
8959 code1
, code2
, multi_step_cvt
,
8962 /* Elements in a vector with vect_used_by_reduction property cannot
8963 be reordered if the use chain with this property does not have the
8964 same operation. One such an example is s += a * b, where elements
8965 in a and b cannot be reordered. Here we check if the vector defined
8966 by STMT is only directly used in the reduction statement. */
8967 tree lhs
= gimple_assign_lhs (stmt
);
8968 use_operand_p dummy
;
8970 stmt_vec_info use_stmt_info
= NULL
;
8971 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
8972 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
8973 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
8976 c1
= VEC_WIDEN_MULT_LO_EXPR
;
8977 c2
= VEC_WIDEN_MULT_HI_EXPR
;
8990 case VEC_WIDEN_MULT_EVEN_EXPR
:
8991 /* Support the recursion induced just above. */
8992 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
8993 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
8996 case WIDEN_LSHIFT_EXPR
:
8997 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
8998 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
9002 c1
= VEC_UNPACK_LO_EXPR
;
9003 c2
= VEC_UNPACK_HI_EXPR
;
9007 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
9008 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
9011 case FIX_TRUNC_EXPR
:
9012 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
9013 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
9014 computing the operation. */
9021 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
9024 if (code
== FIX_TRUNC_EXPR
)
9026 /* The signedness is determined from output operand. */
9027 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
9028 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
9032 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
9033 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
9036 if (!optab1
|| !optab2
)
9039 vec_mode
= TYPE_MODE (vectype
);
9040 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
9041 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
9047 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
9048 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
9049 /* For scalar masks we may have different boolean
9050 vector types having the same QImode. Thus we
9051 add additional check for elements number. */
9052 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9053 || (TYPE_VECTOR_SUBPARTS (vectype
) / 2
9054 == TYPE_VECTOR_SUBPARTS (wide_vectype
)));
9056 /* Check if it's a multi-step conversion that can be done using intermediate
9059 prev_type
= vectype
;
9060 prev_mode
= vec_mode
;
9062 if (!CONVERT_EXPR_CODE_P (code
))
9065 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
9066 intermediate steps in promotion sequence. We try
9067 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
9069 interm_types
->create (MAX_INTERM_CVT_STEPS
);
9070 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
9072 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
9073 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
9076 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type
) / 2,
9077 current_vector_size
);
9078 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
9083 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
9084 TYPE_UNSIGNED (prev_type
));
9086 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
9087 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
9089 if (!optab3
|| !optab4
9090 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
9091 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
9092 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
9093 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
9094 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
9095 == CODE_FOR_nothing
)
9096 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
9097 == CODE_FOR_nothing
))
9100 interm_types
->quick_push (intermediate_type
);
9101 (*multi_step_cvt
)++;
9103 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
9104 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
9105 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9106 || (TYPE_VECTOR_SUBPARTS (intermediate_type
) / 2
9107 == TYPE_VECTOR_SUBPARTS (wide_vectype
)));
9109 prev_type
= intermediate_type
;
9110 prev_mode
= intermediate_mode
;
9113 interm_types
->release ();
9118 /* Function supportable_narrowing_operation
9120 Check whether an operation represented by the code CODE is a
9121 narrowing operation that is supported by the target platform in
9122 vector form (i.e., when operating on arguments of type VECTYPE_IN
9123 and producing a result of type VECTYPE_OUT).
9125 Narrowing operations we currently support are NOP (CONVERT) and
9126 FIX_TRUNC. This function checks if these operations are supported by
9127 the target platform directly via vector tree-codes.
9130 - CODE1 is the code of a vector operation to be used when
9131 vectorizing the operation, if available.
9132 - MULTI_STEP_CVT determines the number of required intermediate steps in
9133 case of multi-step conversion (like int->short->char - in that case
9134 MULTI_STEP_CVT will be 1).
9135 - INTERM_TYPES contains the intermediate type required to perform the
9136 narrowing operation (short in the above example). */
9139 supportable_narrowing_operation (enum tree_code code
,
9140 tree vectype_out
, tree vectype_in
,
9141 enum tree_code
*code1
, int *multi_step_cvt
,
9142 vec
<tree
> *interm_types
)
9144 machine_mode vec_mode
;
9145 enum insn_code icode1
;
9146 optab optab1
, interm_optab
;
9147 tree vectype
= vectype_in
;
9148 tree narrow_vectype
= vectype_out
;
9150 tree intermediate_type
, prev_type
;
9151 machine_mode intermediate_mode
, prev_mode
;
9155 *multi_step_cvt
= 0;
9159 c1
= VEC_PACK_TRUNC_EXPR
;
9162 case FIX_TRUNC_EXPR
:
9163 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
9167 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
9168 tree code and optabs used for computing the operation. */
9175 if (code
== FIX_TRUNC_EXPR
)
9176 /* The signedness is determined from output operand. */
9177 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
9179 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
9184 vec_mode
= TYPE_MODE (vectype
);
9185 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
9190 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
9191 /* For scalar masks we may have different boolean
9192 vector types having the same QImode. Thus we
9193 add additional check for elements number. */
9194 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9195 || (TYPE_VECTOR_SUBPARTS (vectype
) * 2
9196 == TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
9198 /* Check if it's a multi-step conversion that can be done using intermediate
9200 prev_mode
= vec_mode
;
9201 prev_type
= vectype
;
9202 if (code
== FIX_TRUNC_EXPR
)
9203 uns
= TYPE_UNSIGNED (vectype_out
);
9205 uns
= TYPE_UNSIGNED (vectype
);
9207 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
9208 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
9209 costly than signed. */
9210 if (code
== FIX_TRUNC_EXPR
&& uns
)
9212 enum insn_code icode2
;
9215 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
9217 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
9218 if (interm_optab
!= unknown_optab
9219 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
9220 && insn_data
[icode1
].operand
[0].mode
9221 == insn_data
[icode2
].operand
[0].mode
)
9224 optab1
= interm_optab
;
9229 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
9230 intermediate steps in promotion sequence. We try
9231 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
9232 interm_types
->create (MAX_INTERM_CVT_STEPS
);
9233 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
9235 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
9236 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
9239 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type
) * 2,
9240 current_vector_size
);
9241 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
9246 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
9248 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
9251 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
9252 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
9253 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
9254 == CODE_FOR_nothing
))
9257 interm_types
->quick_push (intermediate_type
);
9258 (*multi_step_cvt
)++;
9260 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
9261 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9262 || (TYPE_VECTOR_SUBPARTS (intermediate_type
) * 2
9263 == TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
9265 prev_mode
= intermediate_mode
;
9266 prev_type
= intermediate_type
;
9267 optab1
= interm_optab
;
9270 interm_types
->release ();