1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2017 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
37 #include "fold-const.h"
38 #include "stor-layout.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
44 #include "tree-ssa-loop-manip.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
50 #include "internal-fn.h"
52 /* For lang_hooks.types.type_for_mode. */
53 #include "langhooks.h"
55 /* Says whether a statement is a load, a store of a vectorized statement
56 result, or a store of an invariant value. */
57 enum vec_load_store_type
{
63 /* Return the vectorized type for the given statement. */
66 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
68 return STMT_VINFO_VECTYPE (stmt_info
);
71 /* Return TRUE iff the given statement is in an inner loop relative to
72 the loop being vectorized. */
74 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
76 gimple
*stmt
= STMT_VINFO_STMT (stmt_info
);
77 basic_block bb
= gimple_bb (stmt
);
78 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
84 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
86 return (bb
->loop_father
== loop
->inner
);
89 /* Record the cost of a statement, either by directly informing the
90 target model or by saving it in a vector for later processing.
91 Return a preliminary estimate of the statement's cost. */
94 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
95 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
96 int misalign
, enum vect_cost_model_location where
)
100 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
101 stmt_info_for_cost si
= { count
, kind
,
102 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
104 body_cost_vec
->safe_push (si
);
106 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
109 return add_stmt_cost (stmt_info
->vinfo
->target_cost_data
,
110 count
, kind
, stmt_info
, misalign
, where
);
113 /* Return a variable of type ELEM_TYPE[NELEMS]. */
116 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
118 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
122 /* ARRAY is an array of vectors created by create_vector_array.
123 Return an SSA_NAME for the vector in index N. The reference
124 is part of the vectorization of STMT and the vector is associated
125 with scalar destination SCALAR_DEST. */
128 read_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
129 tree array
, unsigned HOST_WIDE_INT n
)
131 tree vect_type
, vect
, vect_name
, array_ref
;
134 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
135 vect_type
= TREE_TYPE (TREE_TYPE (array
));
136 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
137 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
138 build_int_cst (size_type_node
, n
),
139 NULL_TREE
, NULL_TREE
);
141 new_stmt
= gimple_build_assign (vect
, array_ref
);
142 vect_name
= make_ssa_name (vect
, new_stmt
);
143 gimple_assign_set_lhs (new_stmt
, vect_name
);
144 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
149 /* ARRAY is an array of vectors created by create_vector_array.
150 Emit code to store SSA_NAME VECT in index N of the array.
151 The store is part of the vectorization of STMT. */
154 write_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
155 tree array
, unsigned HOST_WIDE_INT n
)
160 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
161 build_int_cst (size_type_node
, n
),
162 NULL_TREE
, NULL_TREE
);
164 new_stmt
= gimple_build_assign (array_ref
, vect
);
165 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
168 /* PTR is a pointer to an array of type TYPE. Return a representation
169 of *PTR. The memory reference replaces those in FIRST_DR
173 create_array_ref (tree type
, tree ptr
, tree alias_ptr_type
)
177 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
178 /* Arrays have the same alignment as their type. */
179 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
183 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
185 /* Function vect_mark_relevant.
187 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
190 vect_mark_relevant (vec
<gimple
*> *worklist
, gimple
*stmt
,
191 enum vect_relevant relevant
, bool live_p
)
193 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
194 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
195 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
196 gimple
*pattern_stmt
;
198 if (dump_enabled_p ())
200 dump_printf_loc (MSG_NOTE
, vect_location
,
201 "mark relevant %d, live %d: ", relevant
, live_p
);
202 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
205 /* If this stmt is an original stmt in a pattern, we might need to mark its
206 related pattern stmt instead of the original stmt. However, such stmts
207 may have their own uses that are not in any pattern, in such cases the
208 stmt itself should be marked. */
209 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
211 /* This is the last stmt in a sequence that was detected as a
212 pattern that can potentially be vectorized. Don't mark the stmt
213 as relevant/live because it's not going to be vectorized.
214 Instead mark the pattern-stmt that replaces it. */
216 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
218 if (dump_enabled_p ())
219 dump_printf_loc (MSG_NOTE
, vect_location
,
220 "last stmt in pattern. don't mark"
221 " relevant/live.\n");
222 stmt_info
= vinfo_for_stmt (pattern_stmt
);
223 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
224 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
225 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
229 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
230 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
231 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
233 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
234 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
236 if (dump_enabled_p ())
237 dump_printf_loc (MSG_NOTE
, vect_location
,
238 "already marked relevant/live.\n");
242 worklist
->safe_push (stmt
);
246 /* Function is_simple_and_all_uses_invariant
248 Return true if STMT is simple and all uses of it are invariant. */
251 is_simple_and_all_uses_invariant (gimple
*stmt
, loop_vec_info loop_vinfo
)
257 if (!is_gimple_assign (stmt
))
260 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
262 enum vect_def_type dt
= vect_uninitialized_def
;
264 if (!vect_is_simple_use (op
, loop_vinfo
, &def_stmt
, &dt
))
266 if (dump_enabled_p ())
267 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
268 "use not simple.\n");
272 if (dt
!= vect_external_def
&& dt
!= vect_constant_def
)
278 /* Function vect_stmt_relevant_p.
280 Return true if STMT in loop that is represented by LOOP_VINFO is
281 "relevant for vectorization".
283 A stmt is considered "relevant for vectorization" if:
284 - it has uses outside the loop.
285 - it has vdefs (it alters memory).
286 - control stmts in the loop (except for the exit condition).
288 CHECKME: what other side effects would the vectorizer allow? */
291 vect_stmt_relevant_p (gimple
*stmt
, loop_vec_info loop_vinfo
,
292 enum vect_relevant
*relevant
, bool *live_p
)
294 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
296 imm_use_iterator imm_iter
;
300 *relevant
= vect_unused_in_scope
;
303 /* cond stmt other than loop exit cond. */
304 if (is_ctrl_stmt (stmt
)
305 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
306 != loop_exit_ctrl_vec_info_type
)
307 *relevant
= vect_used_in_scope
;
309 /* changing memory. */
310 if (gimple_code (stmt
) != GIMPLE_PHI
)
311 if (gimple_vdef (stmt
)
312 && !gimple_clobber_p (stmt
))
314 if (dump_enabled_p ())
315 dump_printf_loc (MSG_NOTE
, vect_location
,
316 "vec_stmt_relevant_p: stmt has vdefs.\n");
317 *relevant
= vect_used_in_scope
;
320 /* uses outside the loop. */
321 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
323 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
325 basic_block bb
= gimple_bb (USE_STMT (use_p
));
326 if (!flow_bb_inside_loop_p (loop
, bb
))
328 if (dump_enabled_p ())
329 dump_printf_loc (MSG_NOTE
, vect_location
,
330 "vec_stmt_relevant_p: used out of loop.\n");
332 if (is_gimple_debug (USE_STMT (use_p
)))
335 /* We expect all such uses to be in the loop exit phis
336 (because of loop closed form) */
337 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
338 gcc_assert (bb
== single_exit (loop
)->dest
);
345 if (*live_p
&& *relevant
== vect_unused_in_scope
346 && !is_simple_and_all_uses_invariant (stmt
, loop_vinfo
))
348 if (dump_enabled_p ())
349 dump_printf_loc (MSG_NOTE
, vect_location
,
350 "vec_stmt_relevant_p: stmt live but not relevant.\n");
351 *relevant
= vect_used_only_live
;
354 return (*live_p
|| *relevant
);
358 /* Function exist_non_indexing_operands_for_use_p
360 USE is one of the uses attached to STMT. Check if USE is
361 used in STMT for anything other than indexing an array. */
364 exist_non_indexing_operands_for_use_p (tree use
, gimple
*stmt
)
367 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
369 /* USE corresponds to some operand in STMT. If there is no data
370 reference in STMT, then any operand that corresponds to USE
371 is not indexing an array. */
372 if (!STMT_VINFO_DATA_REF (stmt_info
))
375 /* STMT has a data_ref. FORNOW this means that its of one of
379 (This should have been verified in analyze_data_refs).
381 'var' in the second case corresponds to a def, not a use,
382 so USE cannot correspond to any operands that are not used
385 Therefore, all we need to check is if STMT falls into the
386 first case, and whether var corresponds to USE. */
388 if (!gimple_assign_copy_p (stmt
))
390 if (is_gimple_call (stmt
)
391 && gimple_call_internal_p (stmt
))
392 switch (gimple_call_internal_fn (stmt
))
395 operand
= gimple_call_arg (stmt
, 3);
400 operand
= gimple_call_arg (stmt
, 2);
410 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
412 operand
= gimple_assign_rhs1 (stmt
);
413 if (TREE_CODE (operand
) != SSA_NAME
)
424 Function process_use.
427 - a USE in STMT in a loop represented by LOOP_VINFO
428 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
429 that defined USE. This is done by calling mark_relevant and passing it
430 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
431 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
435 Generally, LIVE_P and RELEVANT are used to define the liveness and
436 relevance info of the DEF_STMT of this USE:
437 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
438 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
440 - case 1: If USE is used only for address computations (e.g. array indexing),
441 which does not need to be directly vectorized, then the liveness/relevance
442 of the respective DEF_STMT is left unchanged.
443 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
444 skip DEF_STMT cause it had already been processed.
445 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
446 be modified accordingly.
448 Return true if everything is as expected. Return false otherwise. */
451 process_use (gimple
*stmt
, tree use
, loop_vec_info loop_vinfo
,
452 enum vect_relevant relevant
, vec
<gimple
*> *worklist
,
455 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
456 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
457 stmt_vec_info dstmt_vinfo
;
458 basic_block bb
, def_bb
;
460 enum vect_def_type dt
;
462 /* case 1: we are only interested in uses that need to be vectorized. Uses
463 that are used for address computation are not considered relevant. */
464 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
467 if (!vect_is_simple_use (use
, loop_vinfo
, &def_stmt
, &dt
))
469 if (dump_enabled_p ())
470 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
471 "not vectorized: unsupported use in stmt.\n");
475 if (!def_stmt
|| gimple_nop_p (def_stmt
))
478 def_bb
= gimple_bb (def_stmt
);
479 if (!flow_bb_inside_loop_p (loop
, def_bb
))
481 if (dump_enabled_p ())
482 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
486 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
487 DEF_STMT must have already been processed, because this should be the
488 only way that STMT, which is a reduction-phi, was put in the worklist,
489 as there should be no other uses for DEF_STMT in the loop. So we just
490 check that everything is as expected, and we are done. */
491 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
492 bb
= gimple_bb (stmt
);
493 if (gimple_code (stmt
) == GIMPLE_PHI
494 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
495 && gimple_code (def_stmt
) != GIMPLE_PHI
496 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
497 && bb
->loop_father
== def_bb
->loop_father
)
499 if (dump_enabled_p ())
500 dump_printf_loc (MSG_NOTE
, vect_location
,
501 "reduc-stmt defining reduc-phi in the same nest.\n");
502 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
503 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
504 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
505 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
506 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
510 /* case 3a: outer-loop stmt defining an inner-loop stmt:
511 outer-loop-header-bb:
517 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
519 if (dump_enabled_p ())
520 dump_printf_loc (MSG_NOTE
, vect_location
,
521 "outer-loop def-stmt defining inner-loop stmt.\n");
525 case vect_unused_in_scope
:
526 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
527 vect_used_in_scope
: vect_unused_in_scope
;
530 case vect_used_in_outer_by_reduction
:
531 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
532 relevant
= vect_used_by_reduction
;
535 case vect_used_in_outer
:
536 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
537 relevant
= vect_used_in_scope
;
540 case vect_used_in_scope
:
548 /* case 3b: inner-loop stmt defining an outer-loop stmt:
549 outer-loop-header-bb:
553 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
555 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
557 if (dump_enabled_p ())
558 dump_printf_loc (MSG_NOTE
, vect_location
,
559 "inner-loop def-stmt defining outer-loop stmt.\n");
563 case vect_unused_in_scope
:
564 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
565 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
566 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
569 case vect_used_by_reduction
:
570 case vect_used_only_live
:
571 relevant
= vect_used_in_outer_by_reduction
;
574 case vect_used_in_scope
:
575 relevant
= vect_used_in_outer
;
582 /* We are also not interested in uses on loop PHI backedges that are
583 inductions. Otherwise we'll needlessly vectorize the IV increment
584 and cause hybrid SLP for SLP inductions. Unless the PHI is live
586 else if (gimple_code (stmt
) == GIMPLE_PHI
587 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_induction_def
588 && ! STMT_VINFO_LIVE_P (stmt_vinfo
)
589 && (PHI_ARG_DEF_FROM_EDGE (stmt
, loop_latch_edge (bb
->loop_father
))
592 if (dump_enabled_p ())
593 dump_printf_loc (MSG_NOTE
, vect_location
,
594 "induction value on backedge.\n");
599 vect_mark_relevant (worklist
, def_stmt
, relevant
, false);
604 /* Function vect_mark_stmts_to_be_vectorized.
606 Not all stmts in the loop need to be vectorized. For example:
615 Stmt 1 and 3 do not need to be vectorized, because loop control and
616 addressing of vectorized data-refs are handled differently.
618 This pass detects such stmts. */
621 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
623 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
624 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
625 unsigned int nbbs
= loop
->num_nodes
;
626 gimple_stmt_iterator si
;
629 stmt_vec_info stmt_vinfo
;
633 enum vect_relevant relevant
;
635 if (dump_enabled_p ())
636 dump_printf_loc (MSG_NOTE
, vect_location
,
637 "=== vect_mark_stmts_to_be_vectorized ===\n");
639 auto_vec
<gimple
*, 64> worklist
;
641 /* 1. Init worklist. */
642 for (i
= 0; i
< nbbs
; i
++)
645 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
648 if (dump_enabled_p ())
650 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
651 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
654 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
655 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
);
657 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
659 stmt
= gsi_stmt (si
);
660 if (dump_enabled_p ())
662 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
663 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
666 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
667 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
);
671 /* 2. Process_worklist */
672 while (worklist
.length () > 0)
677 stmt
= worklist
.pop ();
678 if (dump_enabled_p ())
680 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
681 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
684 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
685 (DEF_STMT) as relevant/irrelevant according to the relevance property
687 stmt_vinfo
= vinfo_for_stmt (stmt
);
688 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
690 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
691 propagated as is to the DEF_STMTs of its USEs.
693 One exception is when STMT has been identified as defining a reduction
694 variable; in this case we set the relevance to vect_used_by_reduction.
695 This is because we distinguish between two kinds of relevant stmts -
696 those that are used by a reduction computation, and those that are
697 (also) used by a regular computation. This allows us later on to
698 identify stmts that are used solely by a reduction, and therefore the
699 order of the results that they produce does not have to be kept. */
701 switch (STMT_VINFO_DEF_TYPE (stmt_vinfo
))
703 case vect_reduction_def
:
704 gcc_assert (relevant
!= vect_unused_in_scope
);
705 if (relevant
!= vect_unused_in_scope
706 && relevant
!= vect_used_in_scope
707 && relevant
!= vect_used_by_reduction
708 && relevant
!= vect_used_only_live
)
710 if (dump_enabled_p ())
711 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
712 "unsupported use of reduction.\n");
717 case vect_nested_cycle
:
718 if (relevant
!= vect_unused_in_scope
719 && relevant
!= vect_used_in_outer_by_reduction
720 && relevant
!= vect_used_in_outer
)
722 if (dump_enabled_p ())
723 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
724 "unsupported use of nested cycle.\n");
730 case vect_double_reduction_def
:
731 if (relevant
!= vect_unused_in_scope
732 && relevant
!= vect_used_by_reduction
733 && relevant
!= vect_used_only_live
)
735 if (dump_enabled_p ())
736 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
737 "unsupported use of double reduction.\n");
747 if (is_pattern_stmt_p (stmt_vinfo
))
749 /* Pattern statements are not inserted into the code, so
750 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
751 have to scan the RHS or function arguments instead. */
752 if (is_gimple_assign (stmt
))
754 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
755 tree op
= gimple_assign_rhs1 (stmt
);
758 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
760 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
761 relevant
, &worklist
, false)
762 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
763 relevant
, &worklist
, false))
767 for (; i
< gimple_num_ops (stmt
); i
++)
769 op
= gimple_op (stmt
, i
);
770 if (TREE_CODE (op
) == SSA_NAME
771 && !process_use (stmt
, op
, loop_vinfo
, relevant
,
776 else if (is_gimple_call (stmt
))
778 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
780 tree arg
= gimple_call_arg (stmt
, i
);
781 if (!process_use (stmt
, arg
, loop_vinfo
, relevant
,
788 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
790 tree op
= USE_FROM_PTR (use_p
);
791 if (!process_use (stmt
, op
, loop_vinfo
, relevant
,
796 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo
))
798 gather_scatter_info gs_info
;
799 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, &gs_info
))
801 if (!process_use (stmt
, gs_info
.offset
, loop_vinfo
, relevant
,
805 } /* while worklist */
811 /* Function vect_model_simple_cost.
813 Models cost for simple operations, i.e. those that only emit ncopies of a
814 single op. Right now, this does not account for multiple insns that could
815 be generated for the single vector op. We will handle that shortly. */
818 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
819 enum vect_def_type
*dt
,
821 stmt_vector_for_cost
*prologue_cost_vec
,
822 stmt_vector_for_cost
*body_cost_vec
)
825 int inside_cost
= 0, prologue_cost
= 0;
827 /* The SLP costs were already calculated during SLP tree build. */
828 if (PURE_SLP_STMT (stmt_info
))
831 /* Cost the "broadcast" of a scalar operand in to a vector operand.
832 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
834 for (i
= 0; i
< ndts
; i
++)
835 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
836 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
837 stmt_info
, 0, vect_prologue
);
839 /* Pass the inside-of-loop statements to the target-specific cost model. */
840 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
841 stmt_info
, 0, vect_body
);
843 if (dump_enabled_p ())
844 dump_printf_loc (MSG_NOTE
, vect_location
,
845 "vect_model_simple_cost: inside_cost = %d, "
846 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
850 /* Model cost for type demotion and promotion operations. PWR is normally
851 zero for single-step promotions and demotions. It will be one if
852 two-step promotion/demotion is required, and so on. Each additional
853 step doubles the number of instructions required. */
856 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
857 enum vect_def_type
*dt
, int pwr
)
860 int inside_cost
= 0, prologue_cost
= 0;
861 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
862 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
863 void *target_cost_data
;
865 /* The SLP costs were already calculated during SLP tree build. */
866 if (PURE_SLP_STMT (stmt_info
))
870 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
872 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
874 for (i
= 0; i
< pwr
+ 1; i
++)
876 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
878 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
879 vec_promote_demote
, stmt_info
, 0,
883 /* FORNOW: Assuming maximum 2 args per stmts. */
884 for (i
= 0; i
< 2; i
++)
885 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
886 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
887 stmt_info
, 0, vect_prologue
);
889 if (dump_enabled_p ())
890 dump_printf_loc (MSG_NOTE
, vect_location
,
891 "vect_model_promotion_demotion_cost: inside_cost = %d, "
892 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
895 /* Function vect_model_store_cost
897 Models cost for stores. In the case of grouped accesses, one access
898 has the overhead of the grouped access attributed to it. */
901 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
902 vect_memory_access_type memory_access_type
,
903 enum vect_def_type dt
, slp_tree slp_node
,
904 stmt_vector_for_cost
*prologue_cost_vec
,
905 stmt_vector_for_cost
*body_cost_vec
)
907 unsigned int inside_cost
= 0, prologue_cost
= 0;
908 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
909 gimple
*first_stmt
= STMT_VINFO_STMT (stmt_info
);
910 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
912 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
913 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
914 stmt_info
, 0, vect_prologue
);
916 /* Grouped stores update all elements in the group at once,
917 so we want the DR for the first statement. */
918 if (!slp_node
&& grouped_access_p
)
920 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
921 dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
924 /* True if we should include any once-per-group costs as well as
925 the cost of the statement itself. For SLP we only get called
926 once per group anyhow. */
927 bool first_stmt_p
= (first_stmt
== STMT_VINFO_STMT (stmt_info
));
929 /* We assume that the cost of a single store-lanes instruction is
930 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
931 access is instead being provided by a permute-and-store operation,
932 include the cost of the permutes. */
934 && memory_access_type
== VMAT_CONTIGUOUS_PERMUTE
)
936 /* Uses a high and low interleave or shuffle operations for each
938 int group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
939 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
940 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
941 stmt_info
, 0, vect_body
);
943 if (dump_enabled_p ())
944 dump_printf_loc (MSG_NOTE
, vect_location
,
945 "vect_model_store_cost: strided group_size = %d .\n",
949 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
950 /* Costs of the stores. */
951 if (memory_access_type
== VMAT_ELEMENTWISE
952 || memory_access_type
== VMAT_GATHER_SCATTER
)
953 /* N scalar stores plus extracting the elements. */
954 inside_cost
+= record_stmt_cost (body_cost_vec
,
955 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
956 scalar_store
, stmt_info
, 0, vect_body
);
958 vect_get_store_cost (dr
, ncopies
, &inside_cost
, body_cost_vec
);
960 if (memory_access_type
== VMAT_ELEMENTWISE
961 || memory_access_type
== VMAT_STRIDED_SLP
)
962 inside_cost
+= record_stmt_cost (body_cost_vec
,
963 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
964 vec_to_scalar
, stmt_info
, 0, vect_body
);
966 if (dump_enabled_p ())
967 dump_printf_loc (MSG_NOTE
, vect_location
,
968 "vect_model_store_cost: inside_cost = %d, "
969 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
973 /* Calculate cost of DR's memory access. */
975 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
976 unsigned int *inside_cost
,
977 stmt_vector_for_cost
*body_cost_vec
)
979 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
980 gimple
*stmt
= DR_STMT (dr
);
981 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
983 switch (alignment_support_scheme
)
987 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
988 vector_store
, stmt_info
, 0,
991 if (dump_enabled_p ())
992 dump_printf_loc (MSG_NOTE
, vect_location
,
993 "vect_model_store_cost: aligned.\n");
997 case dr_unaligned_supported
:
999 /* Here, we assign an additional cost for the unaligned store. */
1000 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1001 unaligned_store
, stmt_info
,
1002 DR_MISALIGNMENT (dr
), vect_body
);
1003 if (dump_enabled_p ())
1004 dump_printf_loc (MSG_NOTE
, vect_location
,
1005 "vect_model_store_cost: unaligned supported by "
1010 case dr_unaligned_unsupported
:
1012 *inside_cost
= VECT_MAX_COST
;
1014 if (dump_enabled_p ())
1015 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1016 "vect_model_store_cost: unsupported access.\n");
1026 /* Function vect_model_load_cost
1028 Models cost for loads. In the case of grouped accesses, one access has
1029 the overhead of the grouped access attributed to it. Since unaligned
1030 accesses are supported for loads, we also account for the costs of the
1031 access scheme chosen. */
1034 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1035 vect_memory_access_type memory_access_type
,
1037 stmt_vector_for_cost
*prologue_cost_vec
,
1038 stmt_vector_for_cost
*body_cost_vec
)
1040 gimple
*first_stmt
= STMT_VINFO_STMT (stmt_info
);
1041 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1042 unsigned int inside_cost
= 0, prologue_cost
= 0;
1043 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
1045 /* Grouped loads read all elements in the group at once,
1046 so we want the DR for the first statement. */
1047 if (!slp_node
&& grouped_access_p
)
1049 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1050 dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1053 /* True if we should include any once-per-group costs as well as
1054 the cost of the statement itself. For SLP we only get called
1055 once per group anyhow. */
1056 bool first_stmt_p
= (first_stmt
== STMT_VINFO_STMT (stmt_info
));
1058 /* We assume that the cost of a single load-lanes instruction is
1059 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1060 access is instead being provided by a load-and-permute operation,
1061 include the cost of the permutes. */
1063 && memory_access_type
== VMAT_CONTIGUOUS_PERMUTE
)
1065 /* Uses an even and odd extract operations or shuffle operations
1066 for each needed permute. */
1067 int group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
1068 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1069 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1070 stmt_info
, 0, vect_body
);
1072 if (dump_enabled_p ())
1073 dump_printf_loc (MSG_NOTE
, vect_location
,
1074 "vect_model_load_cost: strided group_size = %d .\n",
1078 /* The loads themselves. */
1079 if (memory_access_type
== VMAT_ELEMENTWISE
1080 || memory_access_type
== VMAT_GATHER_SCATTER
)
1082 /* N scalar loads plus gathering them into a vector. */
1083 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1084 inside_cost
+= record_stmt_cost (body_cost_vec
,
1085 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1086 scalar_load
, stmt_info
, 0, vect_body
);
1089 vect_get_load_cost (dr
, ncopies
, first_stmt_p
,
1090 &inside_cost
, &prologue_cost
,
1091 prologue_cost_vec
, body_cost_vec
, true);
1092 if (memory_access_type
== VMAT_ELEMENTWISE
1093 || memory_access_type
== VMAT_STRIDED_SLP
)
1094 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1095 stmt_info
, 0, vect_body
);
1097 if (dump_enabled_p ())
1098 dump_printf_loc (MSG_NOTE
, vect_location
,
1099 "vect_model_load_cost: inside_cost = %d, "
1100 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1104 /* Calculate cost of DR's memory access. */
1106 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1107 bool add_realign_cost
, unsigned int *inside_cost
,
1108 unsigned int *prologue_cost
,
1109 stmt_vector_for_cost
*prologue_cost_vec
,
1110 stmt_vector_for_cost
*body_cost_vec
,
1111 bool record_prologue_costs
)
1113 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1114 gimple
*stmt
= DR_STMT (dr
);
1115 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1117 switch (alignment_support_scheme
)
1121 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1122 stmt_info
, 0, vect_body
);
1124 if (dump_enabled_p ())
1125 dump_printf_loc (MSG_NOTE
, vect_location
,
1126 "vect_model_load_cost: aligned.\n");
1130 case dr_unaligned_supported
:
1132 /* Here, we assign an additional cost for the unaligned load. */
1133 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1134 unaligned_load
, stmt_info
,
1135 DR_MISALIGNMENT (dr
), vect_body
);
1137 if (dump_enabled_p ())
1138 dump_printf_loc (MSG_NOTE
, vect_location
,
1139 "vect_model_load_cost: unaligned supported by "
1144 case dr_explicit_realign
:
1146 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1147 vector_load
, stmt_info
, 0, vect_body
);
1148 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1149 vec_perm
, stmt_info
, 0, vect_body
);
1151 /* FIXME: If the misalignment remains fixed across the iterations of
1152 the containing loop, the following cost should be added to the
1154 if (targetm
.vectorize
.builtin_mask_for_load
)
1155 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1156 stmt_info
, 0, vect_body
);
1158 if (dump_enabled_p ())
1159 dump_printf_loc (MSG_NOTE
, vect_location
,
1160 "vect_model_load_cost: explicit realign\n");
1164 case dr_explicit_realign_optimized
:
1166 if (dump_enabled_p ())
1167 dump_printf_loc (MSG_NOTE
, vect_location
,
1168 "vect_model_load_cost: unaligned software "
1171 /* Unaligned software pipeline has a load of an address, an initial
1172 load, and possibly a mask operation to "prime" the loop. However,
1173 if this is an access in a group of loads, which provide grouped
1174 access, then the above cost should only be considered for one
1175 access in the group. Inside the loop, there is a load op
1176 and a realignment op. */
1178 if (add_realign_cost
&& record_prologue_costs
)
1180 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1181 vector_stmt
, stmt_info
,
1183 if (targetm
.vectorize
.builtin_mask_for_load
)
1184 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1185 vector_stmt
, stmt_info
,
1189 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1190 stmt_info
, 0, vect_body
);
1191 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1192 stmt_info
, 0, vect_body
);
1194 if (dump_enabled_p ())
1195 dump_printf_loc (MSG_NOTE
, vect_location
,
1196 "vect_model_load_cost: explicit realign optimized"
1202 case dr_unaligned_unsupported
:
1204 *inside_cost
= VECT_MAX_COST
;
1206 if (dump_enabled_p ())
1207 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1208 "vect_model_load_cost: unsupported access.\n");
1217 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1218 the loop preheader for the vectorized stmt STMT. */
1221 vect_init_vector_1 (gimple
*stmt
, gimple
*new_stmt
, gimple_stmt_iterator
*gsi
)
1224 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1227 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1228 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1232 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1236 if (nested_in_vect_loop_p (loop
, stmt
))
1239 pe
= loop_preheader_edge (loop
);
1240 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1241 gcc_assert (!new_bb
);
1245 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1247 gimple_stmt_iterator gsi_bb_start
;
1249 gcc_assert (bb_vinfo
);
1250 bb
= BB_VINFO_BB (bb_vinfo
);
1251 gsi_bb_start
= gsi_after_labels (bb
);
1252 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1256 if (dump_enabled_p ())
1258 dump_printf_loc (MSG_NOTE
, vect_location
,
1259 "created new init_stmt: ");
1260 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1264 /* Function vect_init_vector.
1266 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1267 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1268 vector type a vector with all elements equal to VAL is created first.
1269 Place the initialization at BSI if it is not NULL. Otherwise, place the
1270 initialization at the loop preheader.
1271 Return the DEF of INIT_STMT.
1272 It will be used in the vectorization of STMT. */
1275 vect_init_vector (gimple
*stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1280 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1281 if (! useless_type_conversion_p (type
, TREE_TYPE (val
)))
1283 gcc_assert (TREE_CODE (type
) == VECTOR_TYPE
);
1284 if (! types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1286 /* Scalar boolean value should be transformed into
1287 all zeros or all ones value before building a vector. */
1288 if (VECTOR_BOOLEAN_TYPE_P (type
))
1290 tree true_val
= build_all_ones_cst (TREE_TYPE (type
));
1291 tree false_val
= build_zero_cst (TREE_TYPE (type
));
1293 if (CONSTANT_CLASS_P (val
))
1294 val
= integer_zerop (val
) ? false_val
: true_val
;
1297 new_temp
= make_ssa_name (TREE_TYPE (type
));
1298 init_stmt
= gimple_build_assign (new_temp
, COND_EXPR
,
1299 val
, true_val
, false_val
);
1300 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1304 else if (CONSTANT_CLASS_P (val
))
1305 val
= fold_convert (TREE_TYPE (type
), val
);
1308 new_temp
= make_ssa_name (TREE_TYPE (type
));
1309 if (! INTEGRAL_TYPE_P (TREE_TYPE (val
)))
1310 init_stmt
= gimple_build_assign (new_temp
,
1311 fold_build1 (VIEW_CONVERT_EXPR
,
1315 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1316 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1320 val
= build_vector_from_val (type
, val
);
1323 new_temp
= vect_get_new_ssa_name (type
, vect_simple_var
, "cst_");
1324 init_stmt
= gimple_build_assign (new_temp
, val
);
1325 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1329 /* Function vect_get_vec_def_for_operand_1.
1331 For a defining stmt DEF_STMT of a scalar stmt, return a vector def with type
1332 DT that will be used in the vectorized stmt. */
1335 vect_get_vec_def_for_operand_1 (gimple
*def_stmt
, enum vect_def_type dt
)
1339 stmt_vec_info def_stmt_info
= NULL
;
1343 /* operand is a constant or a loop invariant. */
1344 case vect_constant_def
:
1345 case vect_external_def
:
1346 /* Code should use vect_get_vec_def_for_operand. */
1349 /* operand is defined inside the loop. */
1350 case vect_internal_def
:
1352 /* Get the def from the vectorized stmt. */
1353 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1355 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1356 /* Get vectorized pattern statement. */
1358 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1359 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1360 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1361 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1362 gcc_assert (vec_stmt
);
1363 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1364 vec_oprnd
= PHI_RESULT (vec_stmt
);
1365 else if (is_gimple_call (vec_stmt
))
1366 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1368 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1372 /* operand is defined by a loop header phi. */
1373 case vect_reduction_def
:
1374 case vect_double_reduction_def
:
1375 case vect_nested_cycle
:
1376 case vect_induction_def
:
1378 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1380 /* Get the def from the vectorized stmt. */
1381 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1382 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1383 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1384 vec_oprnd
= PHI_RESULT (vec_stmt
);
1386 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1396 /* Function vect_get_vec_def_for_operand.
1398 OP is an operand in STMT. This function returns a (vector) def that will be
1399 used in the vectorized stmt for STMT.
1401 In the case that OP is an SSA_NAME which is defined in the loop, then
1402 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1404 In case OP is an invariant or constant, a new stmt that creates a vector def
1405 needs to be introduced. VECTYPE may be used to specify a required type for
1406 vector invariant. */
1409 vect_get_vec_def_for_operand (tree op
, gimple
*stmt
, tree vectype
)
1412 enum vect_def_type dt
;
1414 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1415 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1417 if (dump_enabled_p ())
1419 dump_printf_loc (MSG_NOTE
, vect_location
,
1420 "vect_get_vec_def_for_operand: ");
1421 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1422 dump_printf (MSG_NOTE
, "\n");
1425 is_simple_use
= vect_is_simple_use (op
, loop_vinfo
, &def_stmt
, &dt
);
1426 gcc_assert (is_simple_use
);
1427 if (def_stmt
&& dump_enabled_p ())
1429 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1430 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1433 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
1435 tree stmt_vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
1439 vector_type
= vectype
;
1440 else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op
))
1441 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype
))
1442 vector_type
= build_same_sized_truth_vector_type (stmt_vectype
);
1444 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1446 gcc_assert (vector_type
);
1447 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1450 return vect_get_vec_def_for_operand_1 (def_stmt
, dt
);
1454 /* Function vect_get_vec_def_for_stmt_copy
1456 Return a vector-def for an operand. This function is used when the
1457 vectorized stmt to be created (by the caller to this function) is a "copy"
1458 created in case the vectorized result cannot fit in one vector, and several
1459 copies of the vector-stmt are required. In this case the vector-def is
1460 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1461 of the stmt that defines VEC_OPRND.
1462 DT is the type of the vector def VEC_OPRND.
1465 In case the vectorization factor (VF) is bigger than the number
1466 of elements that can fit in a vectype (nunits), we have to generate
1467 more than one vector stmt to vectorize the scalar stmt. This situation
1468 arises when there are multiple data-types operated upon in the loop; the
1469 smallest data-type determines the VF, and as a result, when vectorizing
1470 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1471 vector stmt (each computing a vector of 'nunits' results, and together
1472 computing 'VF' results in each iteration). This function is called when
1473 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1474 which VF=16 and nunits=4, so the number of copies required is 4):
1476 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1478 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1479 VS1.1: vx.1 = memref1 VS1.2
1480 VS1.2: vx.2 = memref2 VS1.3
1481 VS1.3: vx.3 = memref3
1483 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1484 VSnew.1: vz1 = vx.1 + ... VSnew.2
1485 VSnew.2: vz2 = vx.2 + ... VSnew.3
1486 VSnew.3: vz3 = vx.3 + ...
1488 The vectorization of S1 is explained in vectorizable_load.
1489 The vectorization of S2:
1490 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1491 the function 'vect_get_vec_def_for_operand' is called to
1492 get the relevant vector-def for each operand of S2. For operand x it
1493 returns the vector-def 'vx.0'.
1495 To create the remaining copies of the vector-stmt (VSnew.j), this
1496 function is called to get the relevant vector-def for each operand. It is
1497 obtained from the respective VS1.j stmt, which is recorded in the
1498 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1500 For example, to obtain the vector-def 'vx.1' in order to create the
1501 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1502 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1503 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1504 and return its def ('vx.1').
1505 Overall, to create the above sequence this function will be called 3 times:
1506 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1507 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1508 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1511 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1513 gimple
*vec_stmt_for_operand
;
1514 stmt_vec_info def_stmt_info
;
1516 /* Do nothing; can reuse same def. */
1517 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1520 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1521 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1522 gcc_assert (def_stmt_info
);
1523 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1524 gcc_assert (vec_stmt_for_operand
);
1525 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1526 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1528 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1533 /* Get vectorized definitions for the operands to create a copy of an original
1534 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1537 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1538 vec
<tree
> *vec_oprnds0
,
1539 vec
<tree
> *vec_oprnds1
)
1541 tree vec_oprnd
= vec_oprnds0
->pop ();
1543 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1544 vec_oprnds0
->quick_push (vec_oprnd
);
1546 if (vec_oprnds1
&& vec_oprnds1
->length ())
1548 vec_oprnd
= vec_oprnds1
->pop ();
1549 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1550 vec_oprnds1
->quick_push (vec_oprnd
);
1555 /* Get vectorized definitions for OP0 and OP1. */
1558 vect_get_vec_defs (tree op0
, tree op1
, gimple
*stmt
,
1559 vec
<tree
> *vec_oprnds0
,
1560 vec
<tree
> *vec_oprnds1
,
1565 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1566 auto_vec
<tree
> ops (nops
);
1567 auto_vec
<vec
<tree
> > vec_defs (nops
);
1569 ops
.quick_push (op0
);
1571 ops
.quick_push (op1
);
1573 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
1575 *vec_oprnds0
= vec_defs
[0];
1577 *vec_oprnds1
= vec_defs
[1];
1583 vec_oprnds0
->create (1);
1584 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
);
1585 vec_oprnds0
->quick_push (vec_oprnd
);
1589 vec_oprnds1
->create (1);
1590 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
);
1591 vec_oprnds1
->quick_push (vec_oprnd
);
1597 /* Function vect_finish_stmt_generation.
1599 Insert a new stmt. */
1602 vect_finish_stmt_generation (gimple
*stmt
, gimple
*vec_stmt
,
1603 gimple_stmt_iterator
*gsi
)
1605 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1606 vec_info
*vinfo
= stmt_info
->vinfo
;
1608 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1610 if (!gsi_end_p (*gsi
)
1611 && gimple_has_mem_ops (vec_stmt
))
1613 gimple
*at_stmt
= gsi_stmt (*gsi
);
1614 tree vuse
= gimple_vuse (at_stmt
);
1615 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1617 tree vdef
= gimple_vdef (at_stmt
);
1618 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1619 /* If we have an SSA vuse and insert a store, update virtual
1620 SSA form to avoid triggering the renamer. Do so only
1621 if we can easily see all uses - which is what almost always
1622 happens with the way vectorized stmts are inserted. */
1623 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1624 && ((is_gimple_assign (vec_stmt
)
1625 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1626 || (is_gimple_call (vec_stmt
)
1627 && !(gimple_call_flags (vec_stmt
)
1628 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1630 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1631 gimple_set_vdef (vec_stmt
, new_vdef
);
1632 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1636 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1638 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, vinfo
));
1640 if (dump_enabled_p ())
1642 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1643 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1646 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1648 /* While EH edges will generally prevent vectorization, stmt might
1649 e.g. be in a must-not-throw region. Ensure newly created stmts
1650 that could throw are part of the same region. */
1651 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1652 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1653 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1656 /* We want to vectorize a call to combined function CFN with function
1657 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1658 as the types of all inputs. Check whether this is possible using
1659 an internal function, returning its code if so or IFN_LAST if not. */
1662 vectorizable_internal_function (combined_fn cfn
, tree fndecl
,
1663 tree vectype_out
, tree vectype_in
)
1666 if (internal_fn_p (cfn
))
1667 ifn
= as_internal_fn (cfn
);
1669 ifn
= associated_internal_fn (fndecl
);
1670 if (ifn
!= IFN_LAST
&& direct_internal_fn_p (ifn
))
1672 const direct_internal_fn_info
&info
= direct_internal_fn (ifn
);
1673 if (info
.vectorizable
)
1675 tree type0
= (info
.type0
< 0 ? vectype_out
: vectype_in
);
1676 tree type1
= (info
.type1
< 0 ? vectype_out
: vectype_in
);
1677 if (direct_internal_fn_supported_p (ifn
, tree_pair (type0
, type1
),
1678 OPTIMIZE_FOR_SPEED
))
1686 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
*,
1687 gimple_stmt_iterator
*);
1689 /* STMT is a non-strided load or store, meaning that it accesses
1690 elements with a known constant step. Return -1 if that step
1691 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1694 compare_step_with_zero (gimple
*stmt
)
1696 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1697 data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1698 return tree_int_cst_compare (vect_dr_behavior (dr
)->step
,
1702 /* If the target supports a permute mask that reverses the elements in
1703 a vector of type VECTYPE, return that mask, otherwise return null. */
1706 perm_mask_for_reverse (tree vectype
)
1710 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1712 auto_vec_perm_indices
sel (nunits
);
1713 for (i
= 0; i
< nunits
; ++i
)
1714 sel
.quick_push (nunits
- 1 - i
);
1716 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, &sel
))
1718 return vect_gen_perm_mask_checked (vectype
, sel
);
1721 /* A subroutine of get_load_store_type, with a subset of the same
1722 arguments. Handle the case where STMT is part of a grouped load
1725 For stores, the statements in the group are all consecutive
1726 and there is no gap at the end. For loads, the statements in the
1727 group might not be consecutive; there can be gaps between statements
1728 as well as at the end. */
1731 get_group_load_store_type (gimple
*stmt
, tree vectype
, bool slp
,
1732 vec_load_store_type vls_type
,
1733 vect_memory_access_type
*memory_access_type
)
1735 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1736 vec_info
*vinfo
= stmt_info
->vinfo
;
1737 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1738 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
1739 gimple
*first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1740 unsigned int group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
1741 bool single_element_p
= (stmt
== first_stmt
1742 && !GROUP_NEXT_ELEMENT (stmt_info
));
1743 unsigned HOST_WIDE_INT gap
= GROUP_GAP (vinfo_for_stmt (first_stmt
));
1744 unsigned nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1746 /* True if the vectorized statements would access beyond the last
1747 statement in the group. */
1748 bool overrun_p
= false;
1750 /* True if we can cope with such overrun by peeling for gaps, so that
1751 there is at least one final scalar iteration after the vector loop. */
1752 bool can_overrun_p
= (vls_type
== VLS_LOAD
&& loop_vinfo
&& !loop
->inner
);
1754 /* There can only be a gap at the end of the group if the stride is
1755 known at compile time. */
1756 gcc_assert (!STMT_VINFO_STRIDED_P (stmt_info
) || gap
== 0);
1758 /* Stores can't yet have gaps. */
1759 gcc_assert (slp
|| vls_type
== VLS_LOAD
|| gap
== 0);
1763 if (STMT_VINFO_STRIDED_P (stmt_info
))
1765 /* Try to use consecutive accesses of GROUP_SIZE elements,
1766 separated by the stride, until we have a complete vector.
1767 Fall back to scalar accesses if that isn't possible. */
1768 if (nunits
% group_size
== 0)
1769 *memory_access_type
= VMAT_STRIDED_SLP
;
1771 *memory_access_type
= VMAT_ELEMENTWISE
;
1775 overrun_p
= loop_vinfo
&& gap
!= 0;
1776 if (overrun_p
&& vls_type
!= VLS_LOAD
)
1778 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1779 "Grouped store with gaps requires"
1780 " non-consecutive accesses\n");
1783 /* If the access is aligned an overrun is fine. */
1786 (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
))))
1788 if (overrun_p
&& !can_overrun_p
)
1790 if (dump_enabled_p ())
1791 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1792 "Peeling for outer loop is not supported\n");
1795 *memory_access_type
= VMAT_CONTIGUOUS
;
1800 /* We can always handle this case using elementwise accesses,
1801 but see if something more efficient is available. */
1802 *memory_access_type
= VMAT_ELEMENTWISE
;
1804 /* If there is a gap at the end of the group then these optimizations
1805 would access excess elements in the last iteration. */
1806 bool would_overrun_p
= (gap
!= 0);
1807 /* If the access is aligned an overrun is fine, but only if the
1808 overrun is not inside an unused vector (if the gap is as large
1809 or larger than a vector). */
1813 (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
))))
1814 would_overrun_p
= false;
1815 if (!STMT_VINFO_STRIDED_P (stmt_info
)
1816 && (can_overrun_p
|| !would_overrun_p
)
1817 && compare_step_with_zero (stmt
) > 0)
1819 /* First try using LOAD/STORE_LANES. */
1820 if (vls_type
== VLS_LOAD
1821 ? vect_load_lanes_supported (vectype
, group_size
)
1822 : vect_store_lanes_supported (vectype
, group_size
))
1824 *memory_access_type
= VMAT_LOAD_STORE_LANES
;
1825 overrun_p
= would_overrun_p
;
1828 /* If that fails, try using permuting loads. */
1829 if (*memory_access_type
== VMAT_ELEMENTWISE
1830 && (vls_type
== VLS_LOAD
1831 ? vect_grouped_load_supported (vectype
, single_element_p
,
1833 : vect_grouped_store_supported (vectype
, group_size
)))
1835 *memory_access_type
= VMAT_CONTIGUOUS_PERMUTE
;
1836 overrun_p
= would_overrun_p
;
1841 if (vls_type
!= VLS_LOAD
&& first_stmt
== stmt
)
1843 /* STMT is the leader of the group. Check the operands of all the
1844 stmts of the group. */
1845 gimple
*next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
1848 gcc_assert (gimple_assign_single_p (next_stmt
));
1849 tree op
= gimple_assign_rhs1 (next_stmt
);
1851 enum vect_def_type dt
;
1852 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
))
1854 if (dump_enabled_p ())
1855 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1856 "use not simple.\n");
1859 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
1865 gcc_assert (can_overrun_p
);
1866 if (dump_enabled_p ())
1867 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1868 "Data access with gaps requires scalar "
1870 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) = true;
1876 /* A subroutine of get_load_store_type, with a subset of the same
1877 arguments. Handle the case where STMT is a load or store that
1878 accesses consecutive elements with a negative step. */
1880 static vect_memory_access_type
1881 get_negative_load_store_type (gimple
*stmt
, tree vectype
,
1882 vec_load_store_type vls_type
,
1883 unsigned int ncopies
)
1885 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1886 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1887 dr_alignment_support alignment_support_scheme
;
1891 if (dump_enabled_p ())
1892 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1893 "multiple types with negative step.\n");
1894 return VMAT_ELEMENTWISE
;
1897 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1898 if (alignment_support_scheme
!= dr_aligned
1899 && alignment_support_scheme
!= dr_unaligned_supported
)
1901 if (dump_enabled_p ())
1902 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1903 "negative step but alignment required.\n");
1904 return VMAT_ELEMENTWISE
;
1907 if (vls_type
== VLS_STORE_INVARIANT
)
1909 if (dump_enabled_p ())
1910 dump_printf_loc (MSG_NOTE
, vect_location
,
1911 "negative step with invariant source;"
1912 " no permute needed.\n");
1913 return VMAT_CONTIGUOUS_DOWN
;
1916 if (!perm_mask_for_reverse (vectype
))
1918 if (dump_enabled_p ())
1919 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1920 "negative step and reversing not supported.\n");
1921 return VMAT_ELEMENTWISE
;
1924 return VMAT_CONTIGUOUS_REVERSE
;
1927 /* Analyze load or store statement STMT of type VLS_TYPE. Return true
1928 if there is a memory access type that the vectorized form can use,
1929 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
1930 or scatters, fill in GS_INFO accordingly.
1932 SLP says whether we're performing SLP rather than loop vectorization.
1933 VECTYPE is the vector type that the vectorized statements will use.
1934 NCOPIES is the number of vector statements that will be needed. */
1937 get_load_store_type (gimple
*stmt
, tree vectype
, bool slp
,
1938 vec_load_store_type vls_type
, unsigned int ncopies
,
1939 vect_memory_access_type
*memory_access_type
,
1940 gather_scatter_info
*gs_info
)
1942 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1943 vec_info
*vinfo
= stmt_info
->vinfo
;
1944 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1945 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
1947 *memory_access_type
= VMAT_GATHER_SCATTER
;
1949 if (!vect_check_gather_scatter (stmt
, loop_vinfo
, gs_info
))
1951 else if (!vect_is_simple_use (gs_info
->offset
, vinfo
, &def_stmt
,
1952 &gs_info
->offset_dt
,
1953 &gs_info
->offset_vectype
))
1955 if (dump_enabled_p ())
1956 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1957 "%s index use not simple.\n",
1958 vls_type
== VLS_LOAD
? "gather" : "scatter");
1962 else if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1964 if (!get_group_load_store_type (stmt
, vectype
, slp
, vls_type
,
1965 memory_access_type
))
1968 else if (STMT_VINFO_STRIDED_P (stmt_info
))
1971 *memory_access_type
= VMAT_ELEMENTWISE
;
1975 int cmp
= compare_step_with_zero (stmt
);
1977 *memory_access_type
= get_negative_load_store_type
1978 (stmt
, vectype
, vls_type
, ncopies
);
1981 gcc_assert (vls_type
== VLS_LOAD
);
1982 *memory_access_type
= VMAT_INVARIANT
;
1985 *memory_access_type
= VMAT_CONTIGUOUS
;
1988 /* FIXME: At the moment the cost model seems to underestimate the
1989 cost of using elementwise accesses. This check preserves the
1990 traditional behavior until that can be fixed. */
1991 if (*memory_access_type
== VMAT_ELEMENTWISE
1992 && !STMT_VINFO_STRIDED_P (stmt_info
))
1994 if (dump_enabled_p ())
1995 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1996 "not falling back to elementwise accesses\n");
2002 /* Function vectorizable_mask_load_store.
2004 Check if STMT performs a conditional load or store that can be vectorized.
2005 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2006 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
2007 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2010 vectorizable_mask_load_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
2011 gimple
**vec_stmt
, slp_tree slp_node
)
2013 tree vec_dest
= NULL
;
2014 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2015 stmt_vec_info prev_stmt_info
;
2016 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2017 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2018 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
2019 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
2020 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2021 tree rhs_vectype
= NULL_TREE
;
2026 tree dataref_ptr
= NULL_TREE
;
2028 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2032 gather_scatter_info gs_info
;
2033 vec_load_store_type vls_type
;
2036 enum vect_def_type dt
;
2038 if (slp_node
!= NULL
)
2041 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
2042 gcc_assert (ncopies
>= 1);
2044 mask
= gimple_call_arg (stmt
, 2);
2046 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask
)))
2049 /* FORNOW. This restriction should be relaxed. */
2050 if (nested_in_vect_loop
&& ncopies
> 1)
2052 if (dump_enabled_p ())
2053 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2054 "multiple types in nested loop.");
2058 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
2061 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
2065 if (!STMT_VINFO_DATA_REF (stmt_info
))
2068 elem_type
= TREE_TYPE (vectype
);
2070 if (TREE_CODE (mask
) != SSA_NAME
)
2073 if (!vect_is_simple_use (mask
, loop_vinfo
, &def_stmt
, &dt
, &mask_vectype
))
2077 mask_vectype
= get_mask_type_for_scalar_type (TREE_TYPE (vectype
));
2079 if (!mask_vectype
|| !VECTOR_BOOLEAN_TYPE_P (mask_vectype
)
2080 || TYPE_VECTOR_SUBPARTS (mask_vectype
) != TYPE_VECTOR_SUBPARTS (vectype
))
2083 if (gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
2085 tree rhs
= gimple_call_arg (stmt
, 3);
2086 if (!vect_is_simple_use (rhs
, loop_vinfo
, &def_stmt
, &dt
, &rhs_vectype
))
2088 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
2089 vls_type
= VLS_STORE_INVARIANT
;
2091 vls_type
= VLS_STORE
;
2094 vls_type
= VLS_LOAD
;
2096 vect_memory_access_type memory_access_type
;
2097 if (!get_load_store_type (stmt
, vectype
, false, vls_type
, ncopies
,
2098 &memory_access_type
, &gs_info
))
2101 if (memory_access_type
== VMAT_GATHER_SCATTER
)
2103 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
2105 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
2106 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
2108 if (dump_enabled_p ())
2109 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2110 "masked gather with integer mask not supported.");
2114 else if (memory_access_type
!= VMAT_CONTIGUOUS
)
2116 if (dump_enabled_p ())
2117 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2118 "unsupported access type for masked %s.\n",
2119 vls_type
== VLS_LOAD
? "load" : "store");
2122 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
2123 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
),
2124 TYPE_MODE (mask_vectype
),
2125 vls_type
== VLS_LOAD
)
2127 && !useless_type_conversion_p (vectype
, rhs_vectype
)))
2130 if (!vec_stmt
) /* transformation not required. */
2132 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
) = memory_access_type
;
2133 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2134 if (vls_type
== VLS_LOAD
)
2135 vect_model_load_cost (stmt_info
, ncopies
, memory_access_type
,
2138 vect_model_store_cost (stmt_info
, ncopies
, memory_access_type
,
2139 dt
, NULL
, NULL
, NULL
);
2142 gcc_assert (memory_access_type
== STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
));
2146 if (memory_access_type
== VMAT_GATHER_SCATTER
)
2148 tree vec_oprnd0
= NULL_TREE
, op
;
2149 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
2150 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
2151 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
2152 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
2153 tree mask_perm_mask
= NULL_TREE
;
2154 edge pe
= loop_preheader_edge (loop
);
2157 enum { NARROW
, NONE
, WIDEN
} modifier
;
2158 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
2160 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
2161 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2162 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2163 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2164 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2165 scaletype
= TREE_VALUE (arglist
);
2166 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
2167 && types_compatible_p (srctype
, masktype
));
2169 if (nunits
== gather_off_nunits
)
2171 else if (nunits
== gather_off_nunits
/ 2)
2175 auto_vec_perm_indices
sel (gather_off_nunits
);
2176 for (i
= 0; i
< gather_off_nunits
; ++i
)
2177 sel
.quick_push (i
| nunits
);
2179 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
2181 else if (nunits
== gather_off_nunits
* 2)
2185 auto_vec_perm_indices
sel (nunits
);
2186 sel
.quick_grow (nunits
);
2187 for (i
= 0; i
< nunits
; ++i
)
2188 sel
[i
] = i
< gather_off_nunits
2189 ? i
: i
+ nunits
- gather_off_nunits
;
2191 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
2193 for (i
= 0; i
< nunits
; ++i
)
2194 sel
[i
] = i
| gather_off_nunits
;
2195 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
2200 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2202 ptr
= fold_convert (ptrtype
, gs_info
.base
);
2203 if (!is_gimple_min_invariant (ptr
))
2205 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
2206 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
2207 gcc_assert (!new_bb
);
2210 scale
= build_int_cst (scaletype
, gs_info
.scale
);
2212 prev_stmt_info
= NULL
;
2213 for (j
= 0; j
< ncopies
; ++j
)
2215 if (modifier
== WIDEN
&& (j
& 1))
2216 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
2217 perm_mask
, stmt
, gsi
);
2220 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
2223 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
, vec_oprnd0
);
2225 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
2227 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
2228 == TYPE_VECTOR_SUBPARTS (idxtype
));
2229 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
2230 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
2232 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2233 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2237 if (mask_perm_mask
&& (j
& 1))
2238 mask_op
= permute_vec_elements (mask_op
, mask_op
,
2239 mask_perm_mask
, stmt
, gsi
);
2243 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
2246 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2247 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2251 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
2253 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
2254 == TYPE_VECTOR_SUBPARTS (masktype
));
2255 var
= vect_get_new_ssa_name (masktype
, vect_simple_var
);
2256 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
2258 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
2259 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2265 = gimple_build_call (gs_info
.decl
, 5, mask_op
, ptr
, op
, mask_op
,
2268 if (!useless_type_conversion_p (vectype
, rettype
))
2270 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
2271 == TYPE_VECTOR_SUBPARTS (rettype
));
2272 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
2273 gimple_call_set_lhs (new_stmt
, op
);
2274 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2275 var
= make_ssa_name (vec_dest
);
2276 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2277 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2281 var
= make_ssa_name (vec_dest
, new_stmt
);
2282 gimple_call_set_lhs (new_stmt
, var
);
2285 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2287 if (modifier
== NARROW
)
2294 var
= permute_vec_elements (prev_res
, var
,
2295 perm_mask
, stmt
, gsi
);
2296 new_stmt
= SSA_NAME_DEF_STMT (var
);
2299 if (prev_stmt_info
== NULL
)
2300 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2302 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2303 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2306 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2308 if (STMT_VINFO_RELATED_STMT (stmt_info
))
2310 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
2311 stmt_info
= vinfo_for_stmt (stmt
);
2313 tree lhs
= gimple_call_lhs (stmt
);
2314 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2315 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2316 set_vinfo_for_stmt (stmt
, NULL
);
2317 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2318 gsi_replace (gsi
, new_stmt
, true);
2321 else if (vls_type
!= VLS_LOAD
)
2323 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
2324 prev_stmt_info
= NULL
;
2325 LOOP_VINFO_HAS_MASK_STORE (loop_vinfo
) = true;
2326 for (i
= 0; i
< ncopies
; i
++)
2328 unsigned align
, misalign
;
2332 tree rhs
= gimple_call_arg (stmt
, 3);
2333 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
);
2334 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
,
2336 /* We should have catched mismatched types earlier. */
2337 gcc_assert (useless_type_conversion_p (vectype
,
2338 TREE_TYPE (vec_rhs
)));
2339 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2340 NULL_TREE
, &dummy
, gsi
,
2341 &ptr_incr
, false, &inv_p
);
2342 gcc_assert (!inv_p
);
2346 vect_is_simple_use (vec_rhs
, loop_vinfo
, &def_stmt
, &dt
);
2347 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2348 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2349 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2350 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2351 TYPE_SIZE_UNIT (vectype
));
2354 align
= TYPE_ALIGN_UNIT (vectype
);
2355 if (aligned_access_p (dr
))
2357 else if (DR_MISALIGNMENT (dr
) == -1)
2359 align
= TYPE_ALIGN_UNIT (elem_type
);
2363 misalign
= DR_MISALIGNMENT (dr
);
2364 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2366 tree ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)),
2367 misalign
? least_bit_hwi (misalign
) : align
);
2369 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2370 ptr
, vec_mask
, vec_rhs
);
2371 gimple_call_set_nothrow (call
, true);
2373 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2375 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2377 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2378 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2383 tree vec_mask
= NULL_TREE
;
2384 prev_stmt_info
= NULL
;
2385 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2386 for (i
= 0; i
< ncopies
; i
++)
2388 unsigned align
, misalign
;
2392 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
,
2394 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2395 NULL_TREE
, &dummy
, gsi
,
2396 &ptr_incr
, false, &inv_p
);
2397 gcc_assert (!inv_p
);
2401 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2402 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2403 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2404 TYPE_SIZE_UNIT (vectype
));
2407 align
= TYPE_ALIGN_UNIT (vectype
);
2408 if (aligned_access_p (dr
))
2410 else if (DR_MISALIGNMENT (dr
) == -1)
2412 align
= TYPE_ALIGN_UNIT (elem_type
);
2416 misalign
= DR_MISALIGNMENT (dr
);
2417 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2419 tree ptr
= build_int_cst (TREE_TYPE (gimple_call_arg (stmt
, 1)),
2420 misalign
? least_bit_hwi (misalign
) : align
);
2422 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2424 gimple_call_set_lhs (call
, make_ssa_name (vec_dest
));
2425 gimple_call_set_nothrow (call
, true);
2426 vect_finish_stmt_generation (stmt
, call
, gsi
);
2428 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= call
;
2430 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = call
;
2431 prev_stmt_info
= vinfo_for_stmt (call
);
2435 if (vls_type
== VLS_LOAD
)
2437 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2439 if (STMT_VINFO_RELATED_STMT (stmt_info
))
2441 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
2442 stmt_info
= vinfo_for_stmt (stmt
);
2444 tree lhs
= gimple_call_lhs (stmt
);
2445 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2446 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2447 set_vinfo_for_stmt (stmt
, NULL
);
2448 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2449 gsi_replace (gsi
, new_stmt
, true);
2455 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */
2458 vectorizable_bswap (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
2459 gimple
**vec_stmt
, slp_tree slp_node
,
2460 tree vectype_in
, enum vect_def_type
*dt
)
2463 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2464 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2465 unsigned ncopies
, nunits
;
2467 op
= gimple_call_arg (stmt
, 0);
2468 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2469 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2471 /* Multiple types in SLP are handled by creating the appropriate number of
2472 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2477 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
2479 gcc_assert (ncopies
>= 1);
2481 tree char_vectype
= get_same_sized_vectype (char_type_node
, vectype_in
);
2485 unsigned int num_bytes
= TYPE_VECTOR_SUBPARTS (char_vectype
);
2486 unsigned word_bytes
= num_bytes
/ nunits
;
2488 auto_vec_perm_indices
elts (num_bytes
);
2489 for (unsigned i
= 0; i
< nunits
; ++i
)
2490 for (unsigned j
= 0; j
< word_bytes
; ++j
)
2491 elts
.quick_push ((i
+ 1) * word_bytes
- j
- 1);
2493 if (! can_vec_perm_p (TYPE_MODE (char_vectype
), false, &elts
))
2498 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2499 if (dump_enabled_p ())
2500 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_bswap ==="
2502 if (! PURE_SLP_STMT (stmt_info
))
2504 add_stmt_cost (stmt_info
->vinfo
->target_cost_data
,
2505 1, vector_stmt
, stmt_info
, 0, vect_prologue
);
2506 add_stmt_cost (stmt_info
->vinfo
->target_cost_data
,
2507 ncopies
, vec_perm
, stmt_info
, 0, vect_body
);
2512 auto_vec
<tree
, 32> telts (num_bytes
);
2513 for (unsigned i
= 0; i
< num_bytes
; ++i
)
2514 telts
.quick_push (build_int_cst (char_type_node
, elts
[i
]));
2515 tree bswap_vconst
= build_vector (char_vectype
, telts
);
2518 vec
<tree
> vec_oprnds
= vNULL
;
2519 gimple
*new_stmt
= NULL
;
2520 stmt_vec_info prev_stmt_info
= NULL
;
2521 for (unsigned j
= 0; j
< ncopies
; j
++)
2525 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
);
2527 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
2529 /* Arguments are ready. create the new vector stmt. */
2532 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
2534 tree tem
= make_ssa_name (char_vectype
);
2535 new_stmt
= gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
2536 char_vectype
, vop
));
2537 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2538 tree tem2
= make_ssa_name (char_vectype
);
2539 new_stmt
= gimple_build_assign (tem2
, VEC_PERM_EXPR
,
2540 tem
, tem
, bswap_vconst
);
2541 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2542 tem
= make_ssa_name (vectype
);
2543 new_stmt
= gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
2545 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2547 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2554 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2556 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2558 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2561 vec_oprnds
.release ();
2565 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
2566 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
2567 in a single step. On success, store the binary pack code in
2571 simple_integer_narrowing (tree vectype_out
, tree vectype_in
,
2572 tree_code
*convert_code
)
2574 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out
))
2575 || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in
)))
2579 int multi_step_cvt
= 0;
2580 auto_vec
<tree
, 8> interm_types
;
2581 if (!supportable_narrowing_operation (NOP_EXPR
, vectype_out
, vectype_in
,
2582 &code
, &multi_step_cvt
,
2587 *convert_code
= code
;
2591 /* Function vectorizable_call.
2593 Check if GS performs a function call that can be vectorized.
2594 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2595 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2596 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2599 vectorizable_call (gimple
*gs
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
2606 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2607 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2608 tree vectype_out
, vectype_in
;
2611 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2612 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2613 vec_info
*vinfo
= stmt_info
->vinfo
;
2614 tree fndecl
, new_temp
, rhs_type
;
2616 enum vect_def_type dt
[3]
2617 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2619 gimple
*new_stmt
= NULL
;
2621 vec
<tree
> vargs
= vNULL
;
2622 enum { NARROW
, NONE
, WIDEN
} modifier
;
2626 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2629 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
2633 /* Is GS a vectorizable call? */
2634 stmt
= dyn_cast
<gcall
*> (gs
);
2638 if (gimple_call_internal_p (stmt
)
2639 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2640 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2641 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2644 if (gimple_call_lhs (stmt
) == NULL_TREE
2645 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2648 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2650 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2652 /* Process function arguments. */
2653 rhs_type
= NULL_TREE
;
2654 vectype_in
= NULL_TREE
;
2655 nargs
= gimple_call_num_args (stmt
);
2657 /* Bail out if the function has more than three arguments, we do not have
2658 interesting builtin functions to vectorize with more than two arguments
2659 except for fma. No arguments is also not good. */
2660 if (nargs
== 0 || nargs
> 3)
2663 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2664 if (gimple_call_internal_p (stmt
)
2665 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2668 rhs_type
= unsigned_type_node
;
2671 for (i
= 0; i
< nargs
; i
++)
2675 op
= gimple_call_arg (stmt
, i
);
2677 /* We can only handle calls with arguments of the same type. */
2679 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2681 if (dump_enabled_p ())
2682 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2683 "argument types differ.\n");
2687 rhs_type
= TREE_TYPE (op
);
2689 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[i
], &opvectype
))
2691 if (dump_enabled_p ())
2692 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2693 "use not simple.\n");
2698 vectype_in
= opvectype
;
2700 && opvectype
!= vectype_in
)
2702 if (dump_enabled_p ())
2703 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2704 "argument vector types differ.\n");
2708 /* If all arguments are external or constant defs use a vector type with
2709 the same size as the output vector type. */
2711 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2713 gcc_assert (vectype_in
);
2716 if (dump_enabled_p ())
2718 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2719 "no vectype for scalar type ");
2720 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2721 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2728 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2729 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2730 if (nunits_in
== nunits_out
/ 2)
2732 else if (nunits_out
== nunits_in
)
2734 else if (nunits_out
== nunits_in
/ 2)
2739 /* We only handle functions that do not read or clobber memory. */
2740 if (gimple_vuse (stmt
))
2742 if (dump_enabled_p ())
2743 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2744 "function reads from or writes to memory.\n");
2748 /* For now, we only vectorize functions if a target specific builtin
2749 is available. TODO -- in some cases, it might be profitable to
2750 insert the calls for pieces of the vector, in order to be able
2751 to vectorize other operations in the loop. */
2753 internal_fn ifn
= IFN_LAST
;
2754 combined_fn cfn
= gimple_call_combined_fn (stmt
);
2755 tree callee
= gimple_call_fndecl (stmt
);
2757 /* First try using an internal function. */
2758 tree_code convert_code
= ERROR_MARK
;
2760 && (modifier
== NONE
2761 || (modifier
== NARROW
2762 && simple_integer_narrowing (vectype_out
, vectype_in
,
2764 ifn
= vectorizable_internal_function (cfn
, callee
, vectype_out
,
2767 /* If that fails, try asking for a target-specific built-in function. */
2768 if (ifn
== IFN_LAST
)
2770 if (cfn
!= CFN_LAST
)
2771 fndecl
= targetm
.vectorize
.builtin_vectorized_function
2772 (cfn
, vectype_out
, vectype_in
);
2774 fndecl
= targetm
.vectorize
.builtin_md_vectorized_function
2775 (callee
, vectype_out
, vectype_in
);
2778 if (ifn
== IFN_LAST
&& !fndecl
)
2780 if (cfn
== CFN_GOMP_SIMD_LANE
2783 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2784 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2785 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2786 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2788 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2789 { 0, 1, 2, ... vf - 1 } vector. */
2790 gcc_assert (nargs
== 0);
2792 else if (modifier
== NONE
2793 && (gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP16
)
2794 || gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP32
)
2795 || gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP64
)))
2796 return vectorizable_bswap (stmt
, gsi
, vec_stmt
, slp_node
,
2800 if (dump_enabled_p ())
2801 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2802 "function is not vectorizable.\n");
2809 else if (modifier
== NARROW
&& ifn
== IFN_LAST
)
2810 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_out
);
2812 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
2814 /* Sanity check: make sure that at least one copy of the vectorized stmt
2815 needs to be generated. */
2816 gcc_assert (ncopies
>= 1);
2818 if (!vec_stmt
) /* transformation not required. */
2820 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2821 if (dump_enabled_p ())
2822 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2824 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, NULL
, NULL
);
2825 if (ifn
!= IFN_LAST
&& modifier
== NARROW
&& !slp_node
)
2826 add_stmt_cost (stmt_info
->vinfo
->target_cost_data
, ncopies
/ 2,
2827 vec_promote_demote
, stmt_info
, 0, vect_body
);
2834 if (dump_enabled_p ())
2835 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2838 scalar_dest
= gimple_call_lhs (stmt
);
2839 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2841 prev_stmt_info
= NULL
;
2842 if (modifier
== NONE
|| ifn
!= IFN_LAST
)
2844 tree prev_res
= NULL_TREE
;
2845 for (j
= 0; j
< ncopies
; ++j
)
2847 /* Build argument list for the vectorized call. */
2849 vargs
.create (nargs
);
2855 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2856 vec
<tree
> vec_oprnds0
;
2858 for (i
= 0; i
< nargs
; i
++)
2859 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2860 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
);
2861 vec_oprnds0
= vec_defs
[0];
2863 /* Arguments are ready. Create the new vector stmt. */
2864 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2867 for (k
= 0; k
< nargs
; k
++)
2869 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2870 vargs
[k
] = vec_oprndsk
[i
];
2872 if (modifier
== NARROW
)
2874 tree half_res
= make_ssa_name (vectype_in
);
2876 = gimple_build_call_internal_vec (ifn
, vargs
);
2877 gimple_call_set_lhs (call
, half_res
);
2878 gimple_call_set_nothrow (call
, true);
2880 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2883 prev_res
= half_res
;
2886 new_temp
= make_ssa_name (vec_dest
);
2887 new_stmt
= gimple_build_assign (new_temp
, convert_code
,
2888 prev_res
, half_res
);
2893 if (ifn
!= IFN_LAST
)
2894 call
= gimple_build_call_internal_vec (ifn
, vargs
);
2896 call
= gimple_build_call_vec (fndecl
, vargs
);
2897 new_temp
= make_ssa_name (vec_dest
, call
);
2898 gimple_call_set_lhs (call
, new_temp
);
2899 gimple_call_set_nothrow (call
, true);
2902 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2903 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2906 for (i
= 0; i
< nargs
; i
++)
2908 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2909 vec_oprndsi
.release ();
2914 for (i
= 0; i
< nargs
; i
++)
2916 op
= gimple_call_arg (stmt
, i
);
2919 = vect_get_vec_def_for_operand (op
, stmt
);
2922 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2924 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2927 vargs
.quick_push (vec_oprnd0
);
2930 if (gimple_call_internal_p (stmt
)
2931 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2933 auto_vec
<tree
, 32> v (nunits_out
);
2934 for (int k
= 0; k
< nunits_out
; ++k
)
2935 v
.quick_push (build_int_cst (unsigned_type_node
,
2936 j
* nunits_out
+ k
));
2937 tree cst
= build_vector (vectype_out
, v
);
2939 = vect_get_new_ssa_name (vectype_out
, vect_simple_var
, "cst_");
2940 gimple
*init_stmt
= gimple_build_assign (new_var
, cst
);
2941 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2942 new_temp
= make_ssa_name (vec_dest
);
2943 new_stmt
= gimple_build_assign (new_temp
, new_var
);
2945 else if (modifier
== NARROW
)
2947 tree half_res
= make_ssa_name (vectype_in
);
2948 gcall
*call
= gimple_build_call_internal_vec (ifn
, vargs
);
2949 gimple_call_set_lhs (call
, half_res
);
2950 gimple_call_set_nothrow (call
, true);
2952 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2955 prev_res
= half_res
;
2958 new_temp
= make_ssa_name (vec_dest
);
2959 new_stmt
= gimple_build_assign (new_temp
, convert_code
,
2960 prev_res
, half_res
);
2965 if (ifn
!= IFN_LAST
)
2966 call
= gimple_build_call_internal_vec (ifn
, vargs
);
2968 call
= gimple_build_call_vec (fndecl
, vargs
);
2969 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2970 gimple_call_set_lhs (call
, new_temp
);
2971 gimple_call_set_nothrow (call
, true);
2974 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2976 if (j
== (modifier
== NARROW
? 1 : 0))
2977 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2979 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2981 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2984 else if (modifier
== NARROW
)
2986 for (j
= 0; j
< ncopies
; ++j
)
2988 /* Build argument list for the vectorized call. */
2990 vargs
.create (nargs
* 2);
2996 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2997 vec
<tree
> vec_oprnds0
;
2999 for (i
= 0; i
< nargs
; i
++)
3000 vargs
.quick_push (gimple_call_arg (stmt
, i
));
3001 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
);
3002 vec_oprnds0
= vec_defs
[0];
3004 /* Arguments are ready. Create the new vector stmt. */
3005 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
3009 for (k
= 0; k
< nargs
; k
++)
3011 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
3012 vargs
.quick_push (vec_oprndsk
[i
]);
3013 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
3016 if (ifn
!= IFN_LAST
)
3017 call
= gimple_build_call_internal_vec (ifn
, vargs
);
3019 call
= gimple_build_call_vec (fndecl
, vargs
);
3020 new_temp
= make_ssa_name (vec_dest
, call
);
3021 gimple_call_set_lhs (call
, new_temp
);
3022 gimple_call_set_nothrow (call
, true);
3024 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3025 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3028 for (i
= 0; i
< nargs
; i
++)
3030 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
3031 vec_oprndsi
.release ();
3036 for (i
= 0; i
< nargs
; i
++)
3038 op
= gimple_call_arg (stmt
, i
);
3042 = vect_get_vec_def_for_operand (op
, stmt
);
3044 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
3048 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
3050 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
3052 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
3055 vargs
.quick_push (vec_oprnd0
);
3056 vargs
.quick_push (vec_oprnd1
);
3059 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3060 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3061 gimple_call_set_lhs (new_stmt
, new_temp
);
3062 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3065 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3067 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3069 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3072 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3075 /* No current target implements this case. */
3080 /* The call in STMT might prevent it from being removed in dce.
3081 We however cannot remove it here, due to the way the ssa name
3082 it defines is mapped to the new definition. So just replace
3083 rhs of the statement with something harmless. */
3088 type
= TREE_TYPE (scalar_dest
);
3089 if (is_pattern_stmt_p (stmt_info
))
3090 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3092 lhs
= gimple_call_lhs (stmt
);
3094 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3095 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3096 set_vinfo_for_stmt (stmt
, NULL
);
3097 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3098 gsi_replace (gsi
, new_stmt
, false);
3104 struct simd_call_arg_info
3108 HOST_WIDE_INT linear_step
;
3109 enum vect_def_type dt
;
3111 bool simd_lane_linear
;
3114 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3115 is linear within simd lane (but not within whole loop), note it in
3119 vect_simd_lane_linear (tree op
, struct loop
*loop
,
3120 struct simd_call_arg_info
*arginfo
)
3122 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
3124 if (!is_gimple_assign (def_stmt
)
3125 || gimple_assign_rhs_code (def_stmt
) != POINTER_PLUS_EXPR
3126 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt
)))
3129 tree base
= gimple_assign_rhs1 (def_stmt
);
3130 HOST_WIDE_INT linear_step
= 0;
3131 tree v
= gimple_assign_rhs2 (def_stmt
);
3132 while (TREE_CODE (v
) == SSA_NAME
)
3135 def_stmt
= SSA_NAME_DEF_STMT (v
);
3136 if (is_gimple_assign (def_stmt
))
3137 switch (gimple_assign_rhs_code (def_stmt
))
3140 t
= gimple_assign_rhs2 (def_stmt
);
3141 if (linear_step
|| TREE_CODE (t
) != INTEGER_CST
)
3143 base
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (base
), base
, t
);
3144 v
= gimple_assign_rhs1 (def_stmt
);
3147 t
= gimple_assign_rhs2 (def_stmt
);
3148 if (linear_step
|| !tree_fits_shwi_p (t
) || integer_zerop (t
))
3150 linear_step
= tree_to_shwi (t
);
3151 v
= gimple_assign_rhs1 (def_stmt
);
3154 t
= gimple_assign_rhs1 (def_stmt
);
3155 if (TREE_CODE (TREE_TYPE (t
)) != INTEGER_TYPE
3156 || (TYPE_PRECISION (TREE_TYPE (v
))
3157 < TYPE_PRECISION (TREE_TYPE (t
))))
3166 else if (gimple_call_internal_p (def_stmt
, IFN_GOMP_SIMD_LANE
)
3168 && TREE_CODE (gimple_call_arg (def_stmt
, 0)) == SSA_NAME
3169 && (SSA_NAME_VAR (gimple_call_arg (def_stmt
, 0))
3174 arginfo
->linear_step
= linear_step
;
3176 arginfo
->simd_lane_linear
= true;
3182 /* Function vectorizable_simd_clone_call.
3184 Check if STMT performs a function call that can be vectorized
3185 by calling a simd clone of the function.
3186 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3187 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3188 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3191 vectorizable_simd_clone_call (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
3192 gimple
**vec_stmt
, slp_tree slp_node
)
3197 tree vec_oprnd0
= NULL_TREE
;
3198 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
3200 unsigned int nunits
;
3201 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3202 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3203 vec_info
*vinfo
= stmt_info
->vinfo
;
3204 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
3205 tree fndecl
, new_temp
;
3207 gimple
*new_stmt
= NULL
;
3209 auto_vec
<simd_call_arg_info
> arginfo
;
3210 vec
<tree
> vargs
= vNULL
;
3212 tree lhs
, rtype
, ratype
;
3213 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
3215 /* Is STMT a vectorizable call? */
3216 if (!is_gimple_call (stmt
))
3219 fndecl
= gimple_call_fndecl (stmt
);
3220 if (fndecl
== NULL_TREE
)
3223 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
3224 if (node
== NULL
|| node
->simd_clones
== NULL
)
3227 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3230 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
3234 if (gimple_call_lhs (stmt
)
3235 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
3238 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
3240 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3242 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
3249 /* Process function arguments. */
3250 nargs
= gimple_call_num_args (stmt
);
3252 /* Bail out if the function has zero arguments. */
3256 arginfo
.reserve (nargs
, true);
3258 for (i
= 0; i
< nargs
; i
++)
3260 simd_call_arg_info thisarginfo
;
3263 thisarginfo
.linear_step
= 0;
3264 thisarginfo
.align
= 0;
3265 thisarginfo
.op
= NULL_TREE
;
3266 thisarginfo
.simd_lane_linear
= false;
3268 op
= gimple_call_arg (stmt
, i
);
3269 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &thisarginfo
.dt
,
3270 &thisarginfo
.vectype
)
3271 || thisarginfo
.dt
== vect_uninitialized_def
)
3273 if (dump_enabled_p ())
3274 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3275 "use not simple.\n");
3279 if (thisarginfo
.dt
== vect_constant_def
3280 || thisarginfo
.dt
== vect_external_def
)
3281 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
3283 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
3285 /* For linear arguments, the analyze phase should have saved
3286 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
3287 if (i
* 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
3288 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2])
3290 gcc_assert (vec_stmt
);
3291 thisarginfo
.linear_step
3292 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2]);
3294 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 1];
3295 thisarginfo
.simd_lane_linear
3296 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 3]
3297 == boolean_true_node
);
3298 /* If loop has been peeled for alignment, we need to adjust it. */
3299 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
3300 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
3301 if (n1
!= n2
&& !thisarginfo
.simd_lane_linear
)
3303 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
3304 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2];
3305 tree opt
= TREE_TYPE (thisarginfo
.op
);
3306 bias
= fold_convert (TREE_TYPE (step
), bias
);
3307 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
3309 = fold_build2 (POINTER_TYPE_P (opt
)
3310 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
3311 thisarginfo
.op
, bias
);
3315 && thisarginfo
.dt
!= vect_constant_def
3316 && thisarginfo
.dt
!= vect_external_def
3318 && TREE_CODE (op
) == SSA_NAME
3319 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
3321 && tree_fits_shwi_p (iv
.step
))
3323 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
3324 thisarginfo
.op
= iv
.base
;
3326 else if ((thisarginfo
.dt
== vect_constant_def
3327 || thisarginfo
.dt
== vect_external_def
)
3328 && POINTER_TYPE_P (TREE_TYPE (op
)))
3329 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
3330 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
3332 if (POINTER_TYPE_P (TREE_TYPE (op
))
3333 && !thisarginfo
.linear_step
3335 && thisarginfo
.dt
!= vect_constant_def
3336 && thisarginfo
.dt
!= vect_external_def
3339 && TREE_CODE (op
) == SSA_NAME
)
3340 vect_simd_lane_linear (op
, loop
, &thisarginfo
);
3342 arginfo
.quick_push (thisarginfo
);
3345 unsigned int badness
= 0;
3346 struct cgraph_node
*bestn
= NULL
;
3347 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
3348 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
3350 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
3351 n
= n
->simdclone
->next_clone
)
3353 unsigned int this_badness
= 0;
3354 if (n
->simdclone
->simdlen
3355 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
3356 || n
->simdclone
->nargs
!= nargs
)
3358 if (n
->simdclone
->simdlen
3359 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
3360 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
3361 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
3362 if (n
->simdclone
->inbranch
)
3363 this_badness
+= 2048;
3364 int target_badness
= targetm
.simd_clone
.usable (n
);
3365 if (target_badness
< 0)
3367 this_badness
+= target_badness
* 512;
3368 /* FORNOW: Have to add code to add the mask argument. */
3369 if (n
->simdclone
->inbranch
)
3371 for (i
= 0; i
< nargs
; i
++)
3373 switch (n
->simdclone
->args
[i
].arg_type
)
3375 case SIMD_CLONE_ARG_TYPE_VECTOR
:
3376 if (!useless_type_conversion_p
3377 (n
->simdclone
->args
[i
].orig_type
,
3378 TREE_TYPE (gimple_call_arg (stmt
, i
))))
3380 else if (arginfo
[i
].dt
== vect_constant_def
3381 || arginfo
[i
].dt
== vect_external_def
3382 || arginfo
[i
].linear_step
)
3385 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3386 if (arginfo
[i
].dt
!= vect_constant_def
3387 && arginfo
[i
].dt
!= vect_external_def
)
3390 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3391 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
3392 if (arginfo
[i
].dt
== vect_constant_def
3393 || arginfo
[i
].dt
== vect_external_def
3394 || (arginfo
[i
].linear_step
3395 != n
->simdclone
->args
[i
].linear_step
))
3398 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3399 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
3400 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
3401 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
3402 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
3403 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
3407 case SIMD_CLONE_ARG_TYPE_MASK
:
3410 if (i
== (size_t) -1)
3412 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
3417 if (arginfo
[i
].align
)
3418 this_badness
+= (exact_log2 (arginfo
[i
].align
)
3419 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
3421 if (i
== (size_t) -1)
3423 if (bestn
== NULL
|| this_badness
< badness
)
3426 badness
= this_badness
;
3433 for (i
= 0; i
< nargs
; i
++)
3434 if ((arginfo
[i
].dt
== vect_constant_def
3435 || arginfo
[i
].dt
== vect_external_def
)
3436 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
3439 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
3441 if (arginfo
[i
].vectype
== NULL
3442 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
3443 > bestn
->simdclone
->simdlen
))
3447 fndecl
= bestn
->decl
;
3448 nunits
= bestn
->simdclone
->simdlen
;
3449 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
3451 /* If the function isn't const, only allow it in simd loops where user
3452 has asserted that at least nunits consecutive iterations can be
3453 performed using SIMD instructions. */
3454 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
3455 && gimple_vuse (stmt
))
3458 /* Sanity check: make sure that at least one copy of the vectorized stmt
3459 needs to be generated. */
3460 gcc_assert (ncopies
>= 1);
3462 if (!vec_stmt
) /* transformation not required. */
3464 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
3465 for (i
= 0; i
< nargs
; i
++)
3466 if ((bestn
->simdclone
->args
[i
].arg_type
3467 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
3468 || (bestn
->simdclone
->args
[i
].arg_type
3469 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
))
3471 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 3
3473 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
3474 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
3475 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
3476 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
3477 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
3478 tree sll
= arginfo
[i
].simd_lane_linear
3479 ? boolean_true_node
: boolean_false_node
;
3480 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (sll
);
3482 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
3483 if (dump_enabled_p ())
3484 dump_printf_loc (MSG_NOTE
, vect_location
,
3485 "=== vectorizable_simd_clone_call ===\n");
3486 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
3492 if (dump_enabled_p ())
3493 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
3496 scalar_dest
= gimple_call_lhs (stmt
);
3497 vec_dest
= NULL_TREE
;
3502 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3503 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
3504 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
3507 rtype
= TREE_TYPE (ratype
);
3511 prev_stmt_info
= NULL
;
3512 for (j
= 0; j
< ncopies
; ++j
)
3514 /* Build argument list for the vectorized call. */
3516 vargs
.create (nargs
);
3520 for (i
= 0; i
< nargs
; i
++)
3522 unsigned int k
, l
, m
, o
;
3524 op
= gimple_call_arg (stmt
, i
);
3525 switch (bestn
->simdclone
->args
[i
].arg_type
)
3527 case SIMD_CLONE_ARG_TYPE_VECTOR
:
3528 atype
= bestn
->simdclone
->args
[i
].vector_type
;
3529 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
3530 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
3532 if (TYPE_VECTOR_SUBPARTS (atype
)
3533 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
3535 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
3536 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
3537 / TYPE_VECTOR_SUBPARTS (atype
));
3538 gcc_assert ((k
& (k
- 1)) == 0);
3541 = vect_get_vec_def_for_operand (op
, stmt
);
3544 vec_oprnd0
= arginfo
[i
].op
;
3545 if ((m
& (k
- 1)) == 0)
3547 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3550 arginfo
[i
].op
= vec_oprnd0
;
3552 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
3554 bitsize_int ((m
& (k
- 1)) * prec
));
3556 = gimple_build_assign (make_ssa_name (atype
),
3558 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3559 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3563 k
= (TYPE_VECTOR_SUBPARTS (atype
)
3564 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
3565 gcc_assert ((k
& (k
- 1)) == 0);
3566 vec
<constructor_elt
, va_gc
> *ctor_elts
;
3568 vec_alloc (ctor_elts
, k
);
3571 for (l
= 0; l
< k
; l
++)
3573 if (m
== 0 && l
== 0)
3575 = vect_get_vec_def_for_operand (op
, stmt
);
3578 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3580 arginfo
[i
].op
= vec_oprnd0
;
3583 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3587 vargs
.safe_push (vec_oprnd0
);
3590 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3592 = gimple_build_assign (make_ssa_name (atype
),
3594 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3595 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3600 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3601 vargs
.safe_push (op
);
3603 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3604 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
3609 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3614 edge pe
= loop_preheader_edge (loop
);
3615 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3616 gcc_assert (!new_bb
);
3618 if (arginfo
[i
].simd_lane_linear
)
3620 vargs
.safe_push (arginfo
[i
].op
);
3623 tree phi_res
= copy_ssa_name (op
);
3624 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3625 set_vinfo_for_stmt (new_phi
,
3626 new_stmt_vec_info (new_phi
, loop_vinfo
));
3627 add_phi_arg (new_phi
, arginfo
[i
].op
,
3628 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3630 = POINTER_TYPE_P (TREE_TYPE (op
))
3631 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3632 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3633 ? sizetype
: TREE_TYPE (op
);
3635 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3637 tree tcst
= wide_int_to_tree (type
, cst
);
3638 tree phi_arg
= copy_ssa_name (op
);
3640 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3641 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3642 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3643 set_vinfo_for_stmt (new_stmt
,
3644 new_stmt_vec_info (new_stmt
, loop_vinfo
));
3645 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3647 arginfo
[i
].op
= phi_res
;
3648 vargs
.safe_push (phi_res
);
3653 = POINTER_TYPE_P (TREE_TYPE (op
))
3654 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3655 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3656 ? sizetype
: TREE_TYPE (op
);
3658 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3660 tree tcst
= wide_int_to_tree (type
, cst
);
3661 new_temp
= make_ssa_name (TREE_TYPE (op
));
3662 new_stmt
= gimple_build_assign (new_temp
, code
,
3663 arginfo
[i
].op
, tcst
);
3664 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3665 vargs
.safe_push (new_temp
);
3668 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
3669 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
3670 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3671 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
3672 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
3673 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
3679 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3682 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3684 new_temp
= create_tmp_var (ratype
);
3685 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3686 == TYPE_VECTOR_SUBPARTS (rtype
))
3687 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3689 new_temp
= make_ssa_name (rtype
, new_stmt
);
3690 gimple_call_set_lhs (new_stmt
, new_temp
);
3692 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3696 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3699 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3700 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3701 gcc_assert ((k
& (k
- 1)) == 0);
3702 for (l
= 0; l
< k
; l
++)
3707 t
= build_fold_addr_expr (new_temp
);
3708 t
= build2 (MEM_REF
, vectype
, t
,
3709 build_int_cst (TREE_TYPE (t
),
3710 l
* prec
/ BITS_PER_UNIT
));
3713 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3714 bitsize_int (prec
), bitsize_int (l
* prec
));
3716 = gimple_build_assign (make_ssa_name (vectype
), t
);
3717 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3718 if (j
== 0 && l
== 0)
3719 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3721 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3723 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3728 tree clobber
= build_constructor (ratype
, NULL
);
3729 TREE_THIS_VOLATILE (clobber
) = 1;
3730 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3731 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3735 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3737 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3738 / TYPE_VECTOR_SUBPARTS (rtype
));
3739 gcc_assert ((k
& (k
- 1)) == 0);
3740 if ((j
& (k
- 1)) == 0)
3741 vec_alloc (ret_ctor_elts
, k
);
3744 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3745 for (m
= 0; m
< o
; m
++)
3747 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3748 size_int (m
), NULL_TREE
, NULL_TREE
);
3750 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3751 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3752 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3753 gimple_assign_lhs (new_stmt
));
3755 tree clobber
= build_constructor (ratype
, NULL
);
3756 TREE_THIS_VOLATILE (clobber
) = 1;
3757 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3758 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3761 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3762 if ((j
& (k
- 1)) != k
- 1)
3764 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3766 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3767 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3769 if ((unsigned) j
== k
- 1)
3770 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3772 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3774 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3779 tree t
= build_fold_addr_expr (new_temp
);
3780 t
= build2 (MEM_REF
, vectype
, t
,
3781 build_int_cst (TREE_TYPE (t
), 0));
3783 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3784 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3785 tree clobber
= build_constructor (ratype
, NULL
);
3786 TREE_THIS_VOLATILE (clobber
) = 1;
3787 vect_finish_stmt_generation (stmt
,
3788 gimple_build_assign (new_temp
,
3794 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3796 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3798 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3803 /* The call in STMT might prevent it from being removed in dce.
3804 We however cannot remove it here, due to the way the ssa name
3805 it defines is mapped to the new definition. So just replace
3806 rhs of the statement with something harmless. */
3813 type
= TREE_TYPE (scalar_dest
);
3814 if (is_pattern_stmt_p (stmt_info
))
3815 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3817 lhs
= gimple_call_lhs (stmt
);
3818 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3821 new_stmt
= gimple_build_nop ();
3822 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3823 set_vinfo_for_stmt (stmt
, NULL
);
3824 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3825 gsi_replace (gsi
, new_stmt
, true);
3826 unlink_stmt_vdef (stmt
);
3832 /* Function vect_gen_widened_results_half
3834 Create a vector stmt whose code, type, number of arguments, and result
3835 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3836 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3837 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3838 needs to be created (DECL is a function-decl of a target-builtin).
3839 STMT is the original scalar stmt that we are vectorizing. */
3842 vect_gen_widened_results_half (enum tree_code code
,
3844 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3845 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3851 /* Generate half of the widened result: */
3852 if (code
== CALL_EXPR
)
3854 /* Target specific support */
3855 if (op_type
== binary_op
)
3856 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3858 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3859 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3860 gimple_call_set_lhs (new_stmt
, new_temp
);
3864 /* Generic support */
3865 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3866 if (op_type
!= binary_op
)
3868 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3869 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3870 gimple_assign_set_lhs (new_stmt
, new_temp
);
3872 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3878 /* Get vectorized definitions for loop-based vectorization. For the first
3879 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3880 scalar operand), and for the rest we get a copy with
3881 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3882 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3883 The vectors are collected into VEC_OPRNDS. */
3886 vect_get_loop_based_defs (tree
*oprnd
, gimple
*stmt
, enum vect_def_type dt
,
3887 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3891 /* Get first vector operand. */
3892 /* All the vector operands except the very first one (that is scalar oprnd)
3894 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3895 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
);
3897 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3899 vec_oprnds
->quick_push (vec_oprnd
);
3901 /* Get second vector operand. */
3902 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3903 vec_oprnds
->quick_push (vec_oprnd
);
3907 /* For conversion in multiple steps, continue to get operands
3910 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3914 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3915 For multi-step conversions store the resulting vectors and call the function
3919 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3920 int multi_step_cvt
, gimple
*stmt
,
3922 gimple_stmt_iterator
*gsi
,
3923 slp_tree slp_node
, enum tree_code code
,
3924 stmt_vec_info
*prev_stmt_info
)
3927 tree vop0
, vop1
, new_tmp
, vec_dest
;
3929 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3931 vec_dest
= vec_dsts
.pop ();
3933 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3935 /* Create demotion operation. */
3936 vop0
= (*vec_oprnds
)[i
];
3937 vop1
= (*vec_oprnds
)[i
+ 1];
3938 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3939 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3940 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3941 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3944 /* Store the resulting vector for next recursive call. */
3945 (*vec_oprnds
)[i
/2] = new_tmp
;
3948 /* This is the last step of the conversion sequence. Store the
3949 vectors in SLP_NODE or in vector info of the scalar statement
3950 (or in STMT_VINFO_RELATED_STMT chain). */
3952 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3955 if (!*prev_stmt_info
)
3956 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3958 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3960 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3965 /* For multi-step demotion operations we first generate demotion operations
3966 from the source type to the intermediate types, and then combine the
3967 results (stored in VEC_OPRNDS) in demotion operation to the destination
3971 /* At each level of recursion we have half of the operands we had at the
3973 vec_oprnds
->truncate ((i
+1)/2);
3974 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3975 stmt
, vec_dsts
, gsi
, slp_node
,
3976 VEC_PACK_TRUNC_EXPR
,
3980 vec_dsts
.quick_push (vec_dest
);
3984 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3985 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3986 the resulting vectors and call the function recursively. */
3989 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3990 vec
<tree
> *vec_oprnds1
,
3991 gimple
*stmt
, tree vec_dest
,
3992 gimple_stmt_iterator
*gsi
,
3993 enum tree_code code1
,
3994 enum tree_code code2
, tree decl1
,
3995 tree decl2
, int op_type
)
3998 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3999 gimple
*new_stmt1
, *new_stmt2
;
4000 vec
<tree
> vec_tmp
= vNULL
;
4002 vec_tmp
.create (vec_oprnds0
->length () * 2);
4003 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
4005 if (op_type
== binary_op
)
4006 vop1
= (*vec_oprnds1
)[i
];
4010 /* Generate the two halves of promotion operation. */
4011 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
4012 op_type
, vec_dest
, gsi
, stmt
);
4013 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
4014 op_type
, vec_dest
, gsi
, stmt
);
4015 if (is_gimple_call (new_stmt1
))
4017 new_tmp1
= gimple_call_lhs (new_stmt1
);
4018 new_tmp2
= gimple_call_lhs (new_stmt2
);
4022 new_tmp1
= gimple_assign_lhs (new_stmt1
);
4023 new_tmp2
= gimple_assign_lhs (new_stmt2
);
4026 /* Store the results for the next step. */
4027 vec_tmp
.quick_push (new_tmp1
);
4028 vec_tmp
.quick_push (new_tmp2
);
4031 vec_oprnds0
->release ();
4032 *vec_oprnds0
= vec_tmp
;
4036 /* Check if STMT performs a conversion operation, that can be vectorized.
4037 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4038 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
4039 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4042 vectorizable_conversion (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4043 gimple
**vec_stmt
, slp_tree slp_node
)
4047 tree op0
, op1
= NULL_TREE
;
4048 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
4049 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4050 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4051 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
4052 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
4053 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
4056 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4058 gimple
*new_stmt
= NULL
;
4059 stmt_vec_info prev_stmt_info
;
4062 tree vectype_out
, vectype_in
;
4064 tree lhs_type
, rhs_type
;
4065 enum { NARROW
, NONE
, WIDEN
} modifier
;
4066 vec
<tree
> vec_oprnds0
= vNULL
;
4067 vec
<tree
> vec_oprnds1
= vNULL
;
4069 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4070 vec_info
*vinfo
= stmt_info
->vinfo
;
4071 int multi_step_cvt
= 0;
4072 vec
<tree
> interm_types
= vNULL
;
4073 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
4075 unsigned short fltsz
;
4077 /* Is STMT a vectorizable conversion? */
4079 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4082 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4086 if (!is_gimple_assign (stmt
))
4089 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4092 code
= gimple_assign_rhs_code (stmt
);
4093 if (!CONVERT_EXPR_CODE_P (code
)
4094 && code
!= FIX_TRUNC_EXPR
4095 && code
!= FLOAT_EXPR
4096 && code
!= WIDEN_MULT_EXPR
4097 && code
!= WIDEN_LSHIFT_EXPR
)
4100 op_type
= TREE_CODE_LENGTH (code
);
4102 /* Check types of lhs and rhs. */
4103 scalar_dest
= gimple_assign_lhs (stmt
);
4104 lhs_type
= TREE_TYPE (scalar_dest
);
4105 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4107 op0
= gimple_assign_rhs1 (stmt
);
4108 rhs_type
= TREE_TYPE (op0
);
4110 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
4111 && !((INTEGRAL_TYPE_P (lhs_type
)
4112 && INTEGRAL_TYPE_P (rhs_type
))
4113 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
4114 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
4117 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4118 && ((INTEGRAL_TYPE_P (lhs_type
)
4119 && !type_has_mode_precision_p (lhs_type
))
4120 || (INTEGRAL_TYPE_P (rhs_type
)
4121 && !type_has_mode_precision_p (rhs_type
))))
4123 if (dump_enabled_p ())
4124 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4125 "type conversion to/from bit-precision unsupported."
4130 /* Check the operands of the operation. */
4131 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
4133 if (dump_enabled_p ())
4134 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4135 "use not simple.\n");
4138 if (op_type
== binary_op
)
4142 op1
= gimple_assign_rhs2 (stmt
);
4143 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
4144 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
4146 if (CONSTANT_CLASS_P (op0
))
4147 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &vectype_in
);
4149 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]);
4153 if (dump_enabled_p ())
4154 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4155 "use not simple.\n");
4160 /* If op0 is an external or constant defs use a vector type of
4161 the same size as the output vector type. */
4163 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
4165 gcc_assert (vectype_in
);
4168 if (dump_enabled_p ())
4170 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4171 "no vectype for scalar type ");
4172 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
4173 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4179 if (VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4180 && !VECTOR_BOOLEAN_TYPE_P (vectype_in
))
4182 if (dump_enabled_p ())
4184 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4185 "can't convert between boolean and non "
4187 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
4188 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4194 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
4195 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4196 if (nunits_in
< nunits_out
)
4198 else if (nunits_out
== nunits_in
)
4203 /* Multiple types in SLP are handled by creating the appropriate number of
4204 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4208 else if (modifier
== NARROW
)
4209 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_out
);
4211 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
4213 /* Sanity check: make sure that at least one copy of the vectorized stmt
4214 needs to be generated. */
4215 gcc_assert (ncopies
>= 1);
4217 bool found_mode
= false;
4218 scalar_mode lhs_mode
= SCALAR_TYPE_MODE (lhs_type
);
4219 scalar_mode rhs_mode
= SCALAR_TYPE_MODE (rhs_type
);
4220 opt_scalar_mode rhs_mode_iter
;
4222 /* Supportable by target? */
4226 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
4228 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
4233 if (dump_enabled_p ())
4234 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4235 "conversion not supported by target.\n");
4239 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
4240 &code1
, &code2
, &multi_step_cvt
,
4243 /* Binary widening operation can only be supported directly by the
4245 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
4249 if (code
!= FLOAT_EXPR
4250 || GET_MODE_SIZE (lhs_mode
) <= GET_MODE_SIZE (rhs_mode
))
4253 fltsz
= GET_MODE_SIZE (lhs_mode
);
4254 FOR_EACH_2XWIDER_MODE (rhs_mode_iter
, rhs_mode
)
4256 rhs_mode
= rhs_mode_iter
.require ();
4257 if (GET_MODE_SIZE (rhs_mode
) > fltsz
)
4261 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
4262 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
4263 if (cvt_type
== NULL_TREE
)
4266 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
4268 if (!supportable_convert_operation (code
, vectype_out
,
4269 cvt_type
, &decl1
, &codecvt1
))
4272 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
4273 cvt_type
, &codecvt1
,
4274 &codecvt2
, &multi_step_cvt
,
4278 gcc_assert (multi_step_cvt
== 0);
4280 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
4281 vectype_in
, &code1
, &code2
,
4282 &multi_step_cvt
, &interm_types
))
4292 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
4293 codecvt2
= ERROR_MARK
;
4297 interm_types
.safe_push (cvt_type
);
4298 cvt_type
= NULL_TREE
;
4303 gcc_assert (op_type
== unary_op
);
4304 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
4305 &code1
, &multi_step_cvt
,
4309 if (code
!= FIX_TRUNC_EXPR
4310 || GET_MODE_SIZE (lhs_mode
) >= GET_MODE_SIZE (rhs_mode
))
4314 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
4315 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
4316 if (cvt_type
== NULL_TREE
)
4318 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
4321 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
4322 &code1
, &multi_step_cvt
,
4331 if (!vec_stmt
) /* transformation not required. */
4333 if (dump_enabled_p ())
4334 dump_printf_loc (MSG_NOTE
, vect_location
,
4335 "=== vectorizable_conversion ===\n");
4336 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
4338 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
4339 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, NULL
, NULL
);
4341 else if (modifier
== NARROW
)
4343 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
4344 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
4348 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
4349 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
4351 interm_types
.release ();
4356 if (dump_enabled_p ())
4357 dump_printf_loc (MSG_NOTE
, vect_location
,
4358 "transform conversion. ncopies = %d.\n", ncopies
);
4360 if (op_type
== binary_op
)
4362 if (CONSTANT_CLASS_P (op0
))
4363 op0
= fold_convert (TREE_TYPE (op1
), op0
);
4364 else if (CONSTANT_CLASS_P (op1
))
4365 op1
= fold_convert (TREE_TYPE (op0
), op1
);
4368 /* In case of multi-step conversion, we first generate conversion operations
4369 to the intermediate types, and then from that types to the final one.
4370 We create vector destinations for the intermediate type (TYPES) received
4371 from supportable_*_operation, and store them in the correct order
4372 for future use in vect_create_vectorized_*_stmts (). */
4373 auto_vec
<tree
> vec_dsts (multi_step_cvt
+ 1);
4374 vec_dest
= vect_create_destination_var (scalar_dest
,
4375 (cvt_type
&& modifier
== WIDEN
)
4376 ? cvt_type
: vectype_out
);
4377 vec_dsts
.quick_push (vec_dest
);
4381 for (i
= interm_types
.length () - 1;
4382 interm_types
.iterate (i
, &intermediate_type
); i
--)
4384 vec_dest
= vect_create_destination_var (scalar_dest
,
4386 vec_dsts
.quick_push (vec_dest
);
4391 vec_dest
= vect_create_destination_var (scalar_dest
,
4393 ? vectype_out
: cvt_type
);
4397 if (modifier
== WIDEN
)
4399 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
4400 if (op_type
== binary_op
)
4401 vec_oprnds1
.create (1);
4403 else if (modifier
== NARROW
)
4404 vec_oprnds0
.create (
4405 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
4407 else if (code
== WIDEN_LSHIFT_EXPR
)
4408 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
4411 prev_stmt_info
= NULL
;
4415 for (j
= 0; j
< ncopies
; j
++)
4418 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
);
4420 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
4422 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4424 /* Arguments are ready, create the new vector stmt. */
4425 if (code1
== CALL_EXPR
)
4427 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4428 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4429 gimple_call_set_lhs (new_stmt
, new_temp
);
4433 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
4434 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
4435 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4436 gimple_assign_set_lhs (new_stmt
, new_temp
);
4439 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4441 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4444 if (!prev_stmt_info
)
4445 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4447 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4448 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4455 /* In case the vectorization factor (VF) is bigger than the number
4456 of elements that we can fit in a vectype (nunits), we have to
4457 generate more than one vector stmt - i.e - we need to "unroll"
4458 the vector stmt by a factor VF/nunits. */
4459 for (j
= 0; j
< ncopies
; j
++)
4466 if (code
== WIDEN_LSHIFT_EXPR
)
4471 /* Store vec_oprnd1 for every vector stmt to be created
4472 for SLP_NODE. We check during the analysis that all
4473 the shift arguments are the same. */
4474 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4475 vec_oprnds1
.quick_push (vec_oprnd1
);
4477 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4481 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
4482 &vec_oprnds1
, slp_node
);
4486 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
);
4487 vec_oprnds0
.quick_push (vec_oprnd0
);
4488 if (op_type
== binary_op
)
4490 if (code
== WIDEN_LSHIFT_EXPR
)
4493 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
);
4494 vec_oprnds1
.quick_push (vec_oprnd1
);
4500 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
4501 vec_oprnds0
.truncate (0);
4502 vec_oprnds0
.quick_push (vec_oprnd0
);
4503 if (op_type
== binary_op
)
4505 if (code
== WIDEN_LSHIFT_EXPR
)
4508 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
4510 vec_oprnds1
.truncate (0);
4511 vec_oprnds1
.quick_push (vec_oprnd1
);
4515 /* Arguments are ready. Create the new vector stmts. */
4516 for (i
= multi_step_cvt
; i
>= 0; i
--)
4518 tree this_dest
= vec_dsts
[i
];
4519 enum tree_code c1
= code1
, c2
= code2
;
4520 if (i
== 0 && codecvt2
!= ERROR_MARK
)
4525 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
4527 stmt
, this_dest
, gsi
,
4528 c1
, c2
, decl1
, decl2
,
4532 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4536 if (codecvt1
== CALL_EXPR
)
4538 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4539 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4540 gimple_call_set_lhs (new_stmt
, new_temp
);
4544 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4545 new_temp
= make_ssa_name (vec_dest
);
4546 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4550 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4553 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
4556 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4559 if (!prev_stmt_info
)
4560 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
4562 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4563 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4568 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4572 /* In case the vectorization factor (VF) is bigger than the number
4573 of elements that we can fit in a vectype (nunits), we have to
4574 generate more than one vector stmt - i.e - we need to "unroll"
4575 the vector stmt by a factor VF/nunits. */
4576 for (j
= 0; j
< ncopies
; j
++)
4580 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4584 vec_oprnds0
.truncate (0);
4585 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
4586 vect_pow2 (multi_step_cvt
) - 1);
4589 /* Arguments are ready. Create the new vector stmts. */
4591 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4593 if (codecvt1
== CALL_EXPR
)
4595 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4596 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4597 gimple_call_set_lhs (new_stmt
, new_temp
);
4601 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4602 new_temp
= make_ssa_name (vec_dest
);
4603 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4607 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4608 vec_oprnds0
[i
] = new_temp
;
4611 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4612 stmt
, vec_dsts
, gsi
,
4617 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4621 vec_oprnds0
.release ();
4622 vec_oprnds1
.release ();
4623 interm_types
.release ();
4629 /* Function vectorizable_assignment.
4631 Check if STMT performs an assignment (copy) that can be vectorized.
4632 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4633 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4634 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4637 vectorizable_assignment (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4638 gimple
**vec_stmt
, slp_tree slp_node
)
4643 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4644 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4647 enum vect_def_type dt
[1] = {vect_unknown_def_type
};
4651 vec
<tree
> vec_oprnds
= vNULL
;
4653 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4654 vec_info
*vinfo
= stmt_info
->vinfo
;
4655 gimple
*new_stmt
= NULL
;
4656 stmt_vec_info prev_stmt_info
= NULL
;
4657 enum tree_code code
;
4660 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4663 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4667 /* Is vectorizable assignment? */
4668 if (!is_gimple_assign (stmt
))
4671 scalar_dest
= gimple_assign_lhs (stmt
);
4672 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4675 code
= gimple_assign_rhs_code (stmt
);
4676 if (gimple_assign_single_p (stmt
)
4677 || code
== PAREN_EXPR
4678 || CONVERT_EXPR_CODE_P (code
))
4679 op
= gimple_assign_rhs1 (stmt
);
4683 if (code
== VIEW_CONVERT_EXPR
)
4684 op
= TREE_OPERAND (op
, 0);
4686 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4687 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4689 /* Multiple types in SLP are handled by creating the appropriate number of
4690 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4695 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
4697 gcc_assert (ncopies
>= 1);
4699 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
4701 if (dump_enabled_p ())
4702 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4703 "use not simple.\n");
4707 /* We can handle NOP_EXPR conversions that do not change the number
4708 of elements or the vector size. */
4709 if ((CONVERT_EXPR_CODE_P (code
)
4710 || code
== VIEW_CONVERT_EXPR
)
4712 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4713 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4714 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4717 /* We do not handle bit-precision changes. */
4718 if ((CONVERT_EXPR_CODE_P (code
)
4719 || code
== VIEW_CONVERT_EXPR
)
4720 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4721 && (!type_has_mode_precision_p (TREE_TYPE (scalar_dest
))
4722 || !type_has_mode_precision_p (TREE_TYPE (op
)))
4723 /* But a conversion that does not change the bit-pattern is ok. */
4724 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4725 > TYPE_PRECISION (TREE_TYPE (op
)))
4726 && TYPE_UNSIGNED (TREE_TYPE (op
)))
4727 /* Conversion between boolean types of different sizes is
4728 a simple assignment in case their vectypes are same
4730 && (!VECTOR_BOOLEAN_TYPE_P (vectype
)
4731 || !VECTOR_BOOLEAN_TYPE_P (vectype_in
)))
4733 if (dump_enabled_p ())
4734 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4735 "type conversion to/from bit-precision "
4740 if (!vec_stmt
) /* transformation not required. */
4742 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4743 if (dump_enabled_p ())
4744 dump_printf_loc (MSG_NOTE
, vect_location
,
4745 "=== vectorizable_assignment ===\n");
4746 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, NULL
, NULL
);
4751 if (dump_enabled_p ())
4752 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4755 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4758 for (j
= 0; j
< ncopies
; j
++)
4762 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
);
4764 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4766 /* Arguments are ready. create the new vector stmt. */
4767 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4769 if (CONVERT_EXPR_CODE_P (code
)
4770 || code
== VIEW_CONVERT_EXPR
)
4771 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4772 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4773 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4774 gimple_assign_set_lhs (new_stmt
, new_temp
);
4775 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4777 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4784 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4786 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4788 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4791 vec_oprnds
.release ();
4796 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4797 either as shift by a scalar or by a vector. */
4800 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4803 machine_mode vec_mode
;
4808 vectype
= get_vectype_for_scalar_type (scalar_type
);
4812 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4814 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4816 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4818 || (optab_handler (optab
, TYPE_MODE (vectype
))
4819 == CODE_FOR_nothing
))
4823 vec_mode
= TYPE_MODE (vectype
);
4824 icode
= (int) optab_handler (optab
, vec_mode
);
4825 if (icode
== CODE_FOR_nothing
)
4832 /* Function vectorizable_shift.
4834 Check if STMT performs a shift operation that can be vectorized.
4835 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4836 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4837 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4840 vectorizable_shift (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4841 gimple
**vec_stmt
, slp_tree slp_node
)
4845 tree op0
, op1
= NULL
;
4846 tree vec_oprnd1
= NULL_TREE
;
4847 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4849 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4850 enum tree_code code
;
4851 machine_mode vec_mode
;
4855 machine_mode optab_op2_mode
;
4857 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4859 gimple
*new_stmt
= NULL
;
4860 stmt_vec_info prev_stmt_info
;
4867 vec
<tree
> vec_oprnds0
= vNULL
;
4868 vec
<tree
> vec_oprnds1
= vNULL
;
4871 bool scalar_shift_arg
= true;
4872 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4873 vec_info
*vinfo
= stmt_info
->vinfo
;
4875 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4878 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4882 /* Is STMT a vectorizable binary/unary operation? */
4883 if (!is_gimple_assign (stmt
))
4886 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4889 code
= gimple_assign_rhs_code (stmt
);
4891 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4892 || code
== RROTATE_EXPR
))
4895 scalar_dest
= gimple_assign_lhs (stmt
);
4896 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4897 if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest
)))
4899 if (dump_enabled_p ())
4900 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4901 "bit-precision shifts not supported.\n");
4905 op0
= gimple_assign_rhs1 (stmt
);
4906 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
4908 if (dump_enabled_p ())
4909 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4910 "use not simple.\n");
4913 /* If op0 is an external or constant def use a vector type with
4914 the same size as the output vector type. */
4916 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4918 gcc_assert (vectype
);
4921 if (dump_enabled_p ())
4922 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4923 "no vectype for scalar type\n");
4927 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4928 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4929 if (nunits_out
!= nunits_in
)
4932 op1
= gimple_assign_rhs2 (stmt
);
4933 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &op1_vectype
))
4935 if (dump_enabled_p ())
4936 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4937 "use not simple.\n");
4941 /* Multiple types in SLP are handled by creating the appropriate number of
4942 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4947 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
4949 gcc_assert (ncopies
>= 1);
4951 /* Determine whether the shift amount is a vector, or scalar. If the
4952 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4954 if ((dt
[1] == vect_internal_def
4955 || dt
[1] == vect_induction_def
)
4957 scalar_shift_arg
= false;
4958 else if (dt
[1] == vect_constant_def
4959 || dt
[1] == vect_external_def
4960 || dt
[1] == vect_internal_def
)
4962 /* In SLP, need to check whether the shift count is the same,
4963 in loops if it is a constant or invariant, it is always
4967 vec
<gimple
*> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4970 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4971 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4972 scalar_shift_arg
= false;
4975 /* If the shift amount is computed by a pattern stmt we cannot
4976 use the scalar amount directly thus give up and use a vector
4978 if (dt
[1] == vect_internal_def
)
4980 gimple
*def
= SSA_NAME_DEF_STMT (op1
);
4981 if (is_pattern_stmt_p (vinfo_for_stmt (def
)))
4982 scalar_shift_arg
= false;
4987 if (dump_enabled_p ())
4988 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4989 "operand mode requires invariant argument.\n");
4993 /* Vector shifted by vector. */
4994 if (!scalar_shift_arg
)
4996 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4997 if (dump_enabled_p ())
4998 dump_printf_loc (MSG_NOTE
, vect_location
,
4999 "vector/vector shift/rotate found.\n");
5002 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
5003 if (op1_vectype
== NULL_TREE
5004 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
5006 if (dump_enabled_p ())
5007 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5008 "unusable type for last operand in"
5009 " vector/vector shift/rotate.\n");
5013 /* See if the machine has a vector shifted by scalar insn and if not
5014 then see if it has a vector shifted by vector insn. */
5017 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
5019 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
5021 if (dump_enabled_p ())
5022 dump_printf_loc (MSG_NOTE
, vect_location
,
5023 "vector/scalar shift/rotate found.\n");
5027 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
5029 && (optab_handler (optab
, TYPE_MODE (vectype
))
5030 != CODE_FOR_nothing
))
5032 scalar_shift_arg
= false;
5034 if (dump_enabled_p ())
5035 dump_printf_loc (MSG_NOTE
, vect_location
,
5036 "vector/vector shift/rotate found.\n");
5038 /* Unlike the other binary operators, shifts/rotates have
5039 the rhs being int, instead of the same type as the lhs,
5040 so make sure the scalar is the right type if we are
5041 dealing with vectors of long long/long/short/char. */
5042 if (dt
[1] == vect_constant_def
)
5043 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
5044 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
5048 && TYPE_MODE (TREE_TYPE (vectype
))
5049 != TYPE_MODE (TREE_TYPE (op1
)))
5051 if (dump_enabled_p ())
5052 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5053 "unusable type for last operand in"
5054 " vector/vector shift/rotate.\n");
5057 if (vec_stmt
&& !slp_node
)
5059 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
5060 op1
= vect_init_vector (stmt
, op1
,
5061 TREE_TYPE (vectype
), NULL
);
5068 /* Supportable by target? */
5071 if (dump_enabled_p ())
5072 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5076 vec_mode
= TYPE_MODE (vectype
);
5077 icode
= (int) optab_handler (optab
, vec_mode
);
5078 if (icode
== CODE_FOR_nothing
)
5080 if (dump_enabled_p ())
5081 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5082 "op not supported by target.\n");
5083 /* Check only during analysis. */
5084 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
5086 && !vect_worthwhile_without_simd_p (vinfo
, code
)))
5088 if (dump_enabled_p ())
5089 dump_printf_loc (MSG_NOTE
, vect_location
,
5090 "proceeding using word mode.\n");
5093 /* Worthwhile without SIMD support? Check only during analysis. */
5095 && !VECTOR_MODE_P (TYPE_MODE (vectype
))
5096 && !vect_worthwhile_without_simd_p (vinfo
, code
))
5098 if (dump_enabled_p ())
5099 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5100 "not worthwhile without SIMD support.\n");
5104 if (!vec_stmt
) /* transformation not required. */
5106 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
5107 if (dump_enabled_p ())
5108 dump_printf_loc (MSG_NOTE
, vect_location
,
5109 "=== vectorizable_shift ===\n");
5110 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, NULL
, NULL
);
5116 if (dump_enabled_p ())
5117 dump_printf_loc (MSG_NOTE
, vect_location
,
5118 "transform binary/unary operation.\n");
5121 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
5123 prev_stmt_info
= NULL
;
5124 for (j
= 0; j
< ncopies
; j
++)
5129 if (scalar_shift_arg
)
5131 /* Vector shl and shr insn patterns can be defined with scalar
5132 operand 2 (shift operand). In this case, use constant or loop
5133 invariant op1 directly, without extending it to vector mode
5135 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
5136 if (!VECTOR_MODE_P (optab_op2_mode
))
5138 if (dump_enabled_p ())
5139 dump_printf_loc (MSG_NOTE
, vect_location
,
5140 "operand 1 using scalar mode.\n");
5142 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
5143 vec_oprnds1
.quick_push (vec_oprnd1
);
5146 /* Store vec_oprnd1 for every vector stmt to be created
5147 for SLP_NODE. We check during the analysis that all
5148 the shift arguments are the same.
5149 TODO: Allow different constants for different vector
5150 stmts generated for an SLP instance. */
5151 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
5152 vec_oprnds1
.quick_push (vec_oprnd1
);
5157 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
5158 (a special case for certain kind of vector shifts); otherwise,
5159 operand 1 should be of a vector type (the usual case). */
5161 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
5164 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
5168 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
5170 /* Arguments are ready. Create the new vector stmt. */
5171 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5173 vop1
= vec_oprnds1
[i
];
5174 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
5175 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5176 gimple_assign_set_lhs (new_stmt
, new_temp
);
5177 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5179 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
5186 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5188 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5189 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5192 vec_oprnds0
.release ();
5193 vec_oprnds1
.release ();
5199 /* Function vectorizable_operation.
5201 Check if STMT performs a binary, unary or ternary operation that can
5203 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5204 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5205 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5208 vectorizable_operation (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
5209 gimple
**vec_stmt
, slp_tree slp_node
)
5213 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
5214 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5216 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5217 enum tree_code code
;
5218 machine_mode vec_mode
;
5222 bool target_support_p
;
5224 enum vect_def_type dt
[3]
5225 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
5227 gimple
*new_stmt
= NULL
;
5228 stmt_vec_info prev_stmt_info
;
5234 vec
<tree
> vec_oprnds0
= vNULL
;
5235 vec
<tree
> vec_oprnds1
= vNULL
;
5236 vec
<tree
> vec_oprnds2
= vNULL
;
5237 tree vop0
, vop1
, vop2
;
5238 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5239 vec_info
*vinfo
= stmt_info
->vinfo
;
5241 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5244 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5248 /* Is STMT a vectorizable binary/unary operation? */
5249 if (!is_gimple_assign (stmt
))
5252 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
5255 code
= gimple_assign_rhs_code (stmt
);
5257 /* For pointer addition, we should use the normal plus for
5258 the vector addition. */
5259 if (code
== POINTER_PLUS_EXPR
)
5262 /* Support only unary or binary operations. */
5263 op_type
= TREE_CODE_LENGTH (code
);
5264 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
5266 if (dump_enabled_p ())
5267 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5268 "num. args = %d (not unary/binary/ternary op).\n",
5273 scalar_dest
= gimple_assign_lhs (stmt
);
5274 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
5276 /* Most operations cannot handle bit-precision types without extra
5278 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
5279 && !type_has_mode_precision_p (TREE_TYPE (scalar_dest
))
5280 /* Exception are bitwise binary operations. */
5281 && code
!= BIT_IOR_EXPR
5282 && code
!= BIT_XOR_EXPR
5283 && code
!= BIT_AND_EXPR
)
5285 if (dump_enabled_p ())
5286 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5287 "bit-precision arithmetic not supported.\n");
5291 op0
= gimple_assign_rhs1 (stmt
);
5292 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
5294 if (dump_enabled_p ())
5295 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5296 "use not simple.\n");
5299 /* If op0 is an external or constant def use a vector type with
5300 the same size as the output vector type. */
5303 /* For boolean type we cannot determine vectype by
5304 invariant value (don't know whether it is a vector
5305 of booleans or vector of integers). We use output
5306 vectype because operations on boolean don't change
5308 if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)))
5310 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest
)))
5312 if (dump_enabled_p ())
5313 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5314 "not supported operation on bool value.\n");
5317 vectype
= vectype_out
;
5320 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
5323 gcc_assert (vectype
);
5326 if (dump_enabled_p ())
5328 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5329 "no vectype for scalar type ");
5330 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
5332 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
5338 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
5339 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
5340 if (nunits_out
!= nunits_in
)
5343 if (op_type
== binary_op
|| op_type
== ternary_op
)
5345 op1
= gimple_assign_rhs2 (stmt
);
5346 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]))
5348 if (dump_enabled_p ())
5349 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5350 "use not simple.\n");
5354 if (op_type
== ternary_op
)
5356 op2
= gimple_assign_rhs3 (stmt
);
5357 if (!vect_is_simple_use (op2
, vinfo
, &def_stmt
, &dt
[2]))
5359 if (dump_enabled_p ())
5360 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5361 "use not simple.\n");
5366 /* Multiple types in SLP are handled by creating the appropriate number of
5367 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5372 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
5374 gcc_assert (ncopies
>= 1);
5376 /* Shifts are handled in vectorizable_shift (). */
5377 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
5378 || code
== RROTATE_EXPR
)
5381 /* Supportable by target? */
5383 vec_mode
= TYPE_MODE (vectype
);
5384 if (code
== MULT_HIGHPART_EXPR
)
5385 target_support_p
= can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
));
5388 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
5391 if (dump_enabled_p ())
5392 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5396 target_support_p
= (optab_handler (optab
, vec_mode
)
5397 != CODE_FOR_nothing
);
5400 if (!target_support_p
)
5402 if (dump_enabled_p ())
5403 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5404 "op not supported by target.\n");
5405 /* Check only during analysis. */
5406 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
5407 || (!vec_stmt
&& !vect_worthwhile_without_simd_p (vinfo
, code
)))
5409 if (dump_enabled_p ())
5410 dump_printf_loc (MSG_NOTE
, vect_location
,
5411 "proceeding using word mode.\n");
5414 /* Worthwhile without SIMD support? Check only during analysis. */
5415 if (!VECTOR_MODE_P (vec_mode
)
5417 && !vect_worthwhile_without_simd_p (vinfo
, code
))
5419 if (dump_enabled_p ())
5420 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5421 "not worthwhile without SIMD support.\n");
5425 if (!vec_stmt
) /* transformation not required. */
5427 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
5428 if (dump_enabled_p ())
5429 dump_printf_loc (MSG_NOTE
, vect_location
,
5430 "=== vectorizable_operation ===\n");
5431 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, NULL
, NULL
);
5437 if (dump_enabled_p ())
5438 dump_printf_loc (MSG_NOTE
, vect_location
,
5439 "transform binary/unary operation.\n");
5442 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
5444 /* In case the vectorization factor (VF) is bigger than the number
5445 of elements that we can fit in a vectype (nunits), we have to generate
5446 more than one vector stmt - i.e - we need to "unroll" the
5447 vector stmt by a factor VF/nunits. In doing so, we record a pointer
5448 from one copy of the vector stmt to the next, in the field
5449 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
5450 stages to find the correct vector defs to be used when vectorizing
5451 stmts that use the defs of the current stmt. The example below
5452 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
5453 we need to create 4 vectorized stmts):
5455 before vectorization:
5456 RELATED_STMT VEC_STMT
5460 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
5462 RELATED_STMT VEC_STMT
5463 VS1_0: vx0 = memref0 VS1_1 -
5464 VS1_1: vx1 = memref1 VS1_2 -
5465 VS1_2: vx2 = memref2 VS1_3 -
5466 VS1_3: vx3 = memref3 - -
5467 S1: x = load - VS1_0
5470 step2: vectorize stmt S2 (done here):
5471 To vectorize stmt S2 we first need to find the relevant vector
5472 def for the first operand 'x'. This is, as usual, obtained from
5473 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
5474 that defines 'x' (S1). This way we find the stmt VS1_0, and the
5475 relevant vector def 'vx0'. Having found 'vx0' we can generate
5476 the vector stmt VS2_0, and as usual, record it in the
5477 STMT_VINFO_VEC_STMT of stmt S2.
5478 When creating the second copy (VS2_1), we obtain the relevant vector
5479 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
5480 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
5481 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
5482 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
5483 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
5484 chain of stmts and pointers:
5485 RELATED_STMT VEC_STMT
5486 VS1_0: vx0 = memref0 VS1_1 -
5487 VS1_1: vx1 = memref1 VS1_2 -
5488 VS1_2: vx2 = memref2 VS1_3 -
5489 VS1_3: vx3 = memref3 - -
5490 S1: x = load - VS1_0
5491 VS2_0: vz0 = vx0 + v1 VS2_1 -
5492 VS2_1: vz1 = vx1 + v1 VS2_2 -
5493 VS2_2: vz2 = vx2 + v1 VS2_3 -
5494 VS2_3: vz3 = vx3 + v1 - -
5495 S2: z = x + 1 - VS2_0 */
5497 prev_stmt_info
= NULL
;
5498 for (j
= 0; j
< ncopies
; j
++)
5503 if (op_type
== binary_op
|| op_type
== ternary_op
)
5504 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
5507 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
5509 if (op_type
== ternary_op
)
5510 vect_get_vec_defs (op2
, NULL_TREE
, stmt
, &vec_oprnds2
, NULL
,
5515 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
5516 if (op_type
== ternary_op
)
5518 tree vec_oprnd
= vec_oprnds2
.pop ();
5519 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
5524 /* Arguments are ready. Create the new vector stmt. */
5525 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5527 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
5528 ? vec_oprnds1
[i
] : NULL_TREE
);
5529 vop2
= ((op_type
== ternary_op
)
5530 ? vec_oprnds2
[i
] : NULL_TREE
);
5531 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
5532 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5533 gimple_assign_set_lhs (new_stmt
, new_temp
);
5534 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5536 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
5543 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5545 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5546 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5549 vec_oprnds0
.release ();
5550 vec_oprnds1
.release ();
5551 vec_oprnds2
.release ();
5556 /* A helper function to ensure data reference DR's base alignment
5560 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
5565 if (DR_VECT_AUX (dr
)->base_misaligned
)
5567 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5568 tree base_decl
= DR_VECT_AUX (dr
)->base_decl
;
5570 if (decl_in_symtab_p (base_decl
))
5571 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
5574 SET_DECL_ALIGN (base_decl
, TYPE_ALIGN (vectype
));
5575 DECL_USER_ALIGN (base_decl
) = 1;
5577 DR_VECT_AUX (dr
)->base_misaligned
= false;
5582 /* Function get_group_alias_ptr_type.
5584 Return the alias type for the group starting at FIRST_STMT. */
5587 get_group_alias_ptr_type (gimple
*first_stmt
)
5589 struct data_reference
*first_dr
, *next_dr
;
5592 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5593 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (first_stmt
));
5596 next_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (next_stmt
));
5597 if (get_alias_set (DR_REF (first_dr
))
5598 != get_alias_set (DR_REF (next_dr
)))
5600 if (dump_enabled_p ())
5601 dump_printf_loc (MSG_NOTE
, vect_location
,
5602 "conflicting alias set types.\n");
5603 return ptr_type_node
;
5605 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5607 return reference_alias_ptr_type (DR_REF (first_dr
));
5611 /* Function vectorizable_store.
5613 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
5615 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5616 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5617 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5620 vectorizable_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
5626 tree vec_oprnd
= NULL_TREE
;
5627 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5628 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5630 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5631 struct loop
*loop
= NULL
;
5632 machine_mode vec_mode
;
5634 enum dr_alignment_support alignment_support_scheme
;
5636 enum vect_def_type dt
;
5637 stmt_vec_info prev_stmt_info
= NULL
;
5638 tree dataref_ptr
= NULL_TREE
;
5639 tree dataref_offset
= NULL_TREE
;
5640 gimple
*ptr_incr
= NULL
;
5643 gimple
*next_stmt
, *first_stmt
;
5645 unsigned int group_size
, i
;
5646 vec
<tree
> oprnds
= vNULL
;
5647 vec
<tree
> result_chain
= vNULL
;
5649 tree offset
= NULL_TREE
;
5650 vec
<tree
> vec_oprnds
= vNULL
;
5651 bool slp
= (slp_node
!= NULL
);
5652 unsigned int vec_num
;
5653 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5654 vec_info
*vinfo
= stmt_info
->vinfo
;
5656 gather_scatter_info gs_info
;
5657 enum vect_def_type scatter_src_dt
= vect_unknown_def_type
;
5660 vec_load_store_type vls_type
;
5663 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5666 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5670 /* Is vectorizable store? */
5672 if (!is_gimple_assign (stmt
))
5675 scalar_dest
= gimple_assign_lhs (stmt
);
5676 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5677 && is_pattern_stmt_p (stmt_info
))
5678 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5679 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5680 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5681 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5682 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5683 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5684 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5685 && TREE_CODE (scalar_dest
) != MEM_REF
)
5688 /* Cannot have hybrid store SLP -- that would mean storing to the
5689 same location twice. */
5690 gcc_assert (slp
== PURE_SLP_STMT (stmt_info
));
5692 gcc_assert (gimple_assign_single_p (stmt
));
5694 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
), rhs_vectype
= NULL_TREE
;
5695 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5699 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5700 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5705 /* Multiple types in SLP are handled by creating the appropriate number of
5706 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5711 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
5713 gcc_assert (ncopies
>= 1);
5715 /* FORNOW. This restriction should be relaxed. */
5716 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5718 if (dump_enabled_p ())
5719 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5720 "multiple types in nested loop.\n");
5724 op
= gimple_assign_rhs1 (stmt
);
5726 /* In the case this is a store from a STRING_CST make sure
5727 native_encode_expr can handle it. */
5728 if (TREE_CODE (op
) == STRING_CST
5729 && ! can_native_encode_string_p (op
))
5732 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
, &rhs_vectype
))
5734 if (dump_enabled_p ())
5735 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5736 "use not simple.\n");
5740 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
5741 vls_type
= VLS_STORE_INVARIANT
;
5743 vls_type
= VLS_STORE
;
5745 if (rhs_vectype
&& !useless_type_conversion_p (vectype
, rhs_vectype
))
5748 elem_type
= TREE_TYPE (vectype
);
5749 vec_mode
= TYPE_MODE (vectype
);
5751 /* FORNOW. In some cases can vectorize even if data-type not supported
5752 (e.g. - array initialization with 0). */
5753 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5756 if (!STMT_VINFO_DATA_REF (stmt_info
))
5759 vect_memory_access_type memory_access_type
;
5760 if (!get_load_store_type (stmt
, vectype
, slp
, vls_type
, ncopies
,
5761 &memory_access_type
, &gs_info
))
5764 if (!vec_stmt
) /* transformation not required. */
5766 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
) = memory_access_type
;
5767 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5768 /* The SLP costs are calculated during SLP analysis. */
5769 if (!PURE_SLP_STMT (stmt_info
))
5770 vect_model_store_cost (stmt_info
, ncopies
, memory_access_type
, dt
,
5774 gcc_assert (memory_access_type
== STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
));
5778 ensure_base_align (stmt_info
, dr
);
5780 if (memory_access_type
== VMAT_GATHER_SCATTER
)
5782 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
, op
, src
;
5783 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
5784 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
5785 tree ptr
, mask
, var
, scale
, perm_mask
= NULL_TREE
;
5786 edge pe
= loop_preheader_edge (loop
);
5789 enum { NARROW
, NONE
, WIDEN
} modifier
;
5790 int scatter_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
5792 if (nunits
== (unsigned int) scatter_off_nunits
)
5794 else if (nunits
== (unsigned int) scatter_off_nunits
/ 2)
5798 auto_vec_perm_indices
sel (scatter_off_nunits
);
5799 for (i
= 0; i
< (unsigned int) scatter_off_nunits
; ++i
)
5800 sel
.quick_push (i
| nunits
);
5802 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
5803 gcc_assert (perm_mask
!= NULL_TREE
);
5805 else if (nunits
== (unsigned int) scatter_off_nunits
* 2)
5809 auto_vec_perm_indices
sel (nunits
);
5810 for (i
= 0; i
< (unsigned int) nunits
; ++i
)
5811 sel
.quick_push (i
| scatter_off_nunits
);
5813 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
5814 gcc_assert (perm_mask
!= NULL_TREE
);
5820 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
5821 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5822 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5823 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5824 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5825 scaletype
= TREE_VALUE (arglist
);
5827 gcc_checking_assert (TREE_CODE (masktype
) == INTEGER_TYPE
5828 && TREE_CODE (rettype
) == VOID_TYPE
);
5830 ptr
= fold_convert (ptrtype
, gs_info
.base
);
5831 if (!is_gimple_min_invariant (ptr
))
5833 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
5834 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
5835 gcc_assert (!new_bb
);
5838 /* Currently we support only unconditional scatter stores,
5839 so mask should be all ones. */
5840 mask
= build_int_cst (masktype
, -1);
5841 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
5843 scale
= build_int_cst (scaletype
, gs_info
.scale
);
5845 prev_stmt_info
= NULL
;
5846 for (j
= 0; j
< ncopies
; ++j
)
5851 = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt
), stmt
);
5853 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
5855 else if (modifier
!= NONE
&& (j
& 1))
5857 if (modifier
== WIDEN
)
5860 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5861 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
, perm_mask
,
5864 else if (modifier
== NARROW
)
5866 src
= permute_vec_elements (vec_oprnd1
, vec_oprnd1
, perm_mask
,
5869 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
,
5878 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5880 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
,
5884 if (!useless_type_conversion_p (srctype
, TREE_TYPE (src
)))
5886 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src
))
5887 == TYPE_VECTOR_SUBPARTS (srctype
));
5888 var
= vect_get_new_ssa_name (srctype
, vect_simple_var
);
5889 src
= build1 (VIEW_CONVERT_EXPR
, srctype
, src
);
5890 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, src
);
5891 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5895 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
5897 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
5898 == TYPE_VECTOR_SUBPARTS (idxtype
));
5899 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
5900 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
5901 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
5902 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5907 = gimple_build_call (gs_info
.decl
, 5, ptr
, mask
, op
, src
, scale
);
5909 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5911 if (prev_stmt_info
== NULL
)
5912 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5914 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5915 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5920 grouped_store
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
5923 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5924 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5925 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5927 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5930 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5932 /* We vectorize all the stmts of the interleaving group when we
5933 reach the last stmt in the group. */
5934 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5935 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5944 grouped_store
= false;
5945 /* VEC_NUM is the number of vect stmts to be created for this
5947 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5948 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5949 gcc_assert (GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt
)) == first_stmt
);
5950 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5951 op
= gimple_assign_rhs1 (first_stmt
);
5954 /* VEC_NUM is the number of vect stmts to be created for this
5956 vec_num
= group_size
;
5958 ref_type
= get_group_alias_ptr_type (first_stmt
);
5964 group_size
= vec_num
= 1;
5965 ref_type
= reference_alias_ptr_type (DR_REF (first_dr
));
5968 if (dump_enabled_p ())
5969 dump_printf_loc (MSG_NOTE
, vect_location
,
5970 "transform store. ncopies = %d\n", ncopies
);
5972 if (memory_access_type
== VMAT_ELEMENTWISE
5973 || memory_access_type
== VMAT_STRIDED_SLP
)
5975 gimple_stmt_iterator incr_gsi
;
5981 gimple_seq stmts
= NULL
;
5982 tree stride_base
, stride_step
, alias_off
;
5986 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5989 = fold_build_pointer_plus
5990 (unshare_expr (DR_BASE_ADDRESS (first_dr
)),
5991 size_binop (PLUS_EXPR
,
5992 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr
))),
5993 convert_to_ptrofftype (DR_INIT (first_dr
))));
5994 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (first_dr
)));
5996 /* For a store with loop-invariant (but other than power-of-2)
5997 stride (i.e. not a grouped access) like so:
5999 for (i = 0; i < n; i += stride)
6002 we generate a new induction variable and new stores from
6003 the components of the (vectorized) rhs:
6005 for (j = 0; ; j += VF*stride)
6010 array[j + stride] = tmp2;
6014 unsigned nstores
= nunits
;
6016 tree ltype
= elem_type
;
6017 tree lvectype
= vectype
;
6020 if (group_size
< nunits
6021 && nunits
% group_size
== 0)
6023 nstores
= nunits
/ group_size
;
6025 ltype
= build_vector_type (elem_type
, group_size
);
6028 /* First check if vec_extract optab doesn't support extraction
6029 of vector elts directly. */
6030 scalar_mode elmode
= SCALAR_TYPE_MODE (elem_type
);
6032 if (!mode_for_vector (elmode
, group_size
).exists (&vmode
)
6033 || !VECTOR_MODE_P (vmode
)
6034 || (convert_optab_handler (vec_extract_optab
,
6035 TYPE_MODE (vectype
), vmode
)
6036 == CODE_FOR_nothing
))
6038 /* Try to avoid emitting an extract of vector elements
6039 by performing the extracts using an integer type of the
6040 same size, extracting from a vector of those and then
6041 re-interpreting it as the original vector type if
6044 = group_size
* GET_MODE_BITSIZE (elmode
);
6045 elmode
= int_mode_for_size (lsize
, 0).require ();
6046 /* If we can't construct such a vector fall back to
6047 element extracts from the original vector type and
6048 element size stores. */
6049 if (mode_for_vector (elmode
,
6050 nunits
/ group_size
).exists (&vmode
)
6051 && VECTOR_MODE_P (vmode
)
6052 && (convert_optab_handler (vec_extract_optab
,
6054 != CODE_FOR_nothing
))
6056 nstores
= nunits
/ group_size
;
6058 ltype
= build_nonstandard_integer_type (lsize
, 1);
6059 lvectype
= build_vector_type (ltype
, nstores
);
6061 /* Else fall back to vector extraction anyway.
6062 Fewer stores are more important than avoiding spilling
6063 of the vector we extract from. Compared to the
6064 construction case in vectorizable_load no store-forwarding
6065 issue exists here for reasonable archs. */
6068 else if (group_size
>= nunits
6069 && group_size
% nunits
== 0)
6076 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
6077 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6080 ivstep
= stride_step
;
6081 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
6082 build_int_cst (TREE_TYPE (ivstep
), vf
));
6084 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6086 create_iv (stride_base
, ivstep
, NULL
,
6087 loop
, &incr_gsi
, insert_after
,
6089 incr
= gsi_stmt (incr_gsi
);
6090 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
6092 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
6094 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6096 prev_stmt_info
= NULL
;
6097 alias_off
= build_int_cst (ref_type
, 0);
6098 next_stmt
= first_stmt
;
6099 for (g
= 0; g
< group_size
; g
++)
6101 running_off
= offvar
;
6104 tree size
= TYPE_SIZE_UNIT (ltype
);
6105 tree pos
= fold_build2 (MULT_EXPR
, sizetype
, size_int (g
),
6107 tree newoff
= copy_ssa_name (running_off
, NULL
);
6108 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6110 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6111 running_off
= newoff
;
6113 unsigned int group_el
= 0;
6114 unsigned HOST_WIDE_INT
6115 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
6116 for (j
= 0; j
< ncopies
; j
++)
6118 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
6119 and first_stmt == stmt. */
6124 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
, NULL
,
6126 vec_oprnd
= vec_oprnds
[0];
6130 gcc_assert (gimple_assign_single_p (next_stmt
));
6131 op
= gimple_assign_rhs1 (next_stmt
);
6132 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
6138 vec_oprnd
= vec_oprnds
[j
];
6141 vect_is_simple_use (vec_oprnd
, vinfo
, &def_stmt
, &dt
);
6142 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
6145 /* Pun the vector to extract from if necessary. */
6146 if (lvectype
!= vectype
)
6148 tree tem
= make_ssa_name (lvectype
);
6150 = gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
6151 lvectype
, vec_oprnd
));
6152 vect_finish_stmt_generation (stmt
, pun
, gsi
);
6155 for (i
= 0; i
< nstores
; i
++)
6157 tree newref
, newoff
;
6158 gimple
*incr
, *assign
;
6159 tree size
= TYPE_SIZE (ltype
);
6160 /* Extract the i'th component. */
6161 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
,
6162 bitsize_int (i
), size
);
6163 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
6166 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
6170 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
6172 newref
= build2 (MEM_REF
, ltype
,
6173 running_off
, this_off
);
6175 /* And store it to *running_off. */
6176 assign
= gimple_build_assign (newref
, elem
);
6177 vect_finish_stmt_generation (stmt
, assign
, gsi
);
6181 || group_el
== group_size
)
6183 newoff
= copy_ssa_name (running_off
, NULL
);
6184 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6185 running_off
, stride_step
);
6186 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6188 running_off
= newoff
;
6191 if (g
== group_size
- 1
6194 if (j
== 0 && i
== 0)
6195 STMT_VINFO_VEC_STMT (stmt_info
)
6196 = *vec_stmt
= assign
;
6198 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
6199 prev_stmt_info
= vinfo_for_stmt (assign
);
6203 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
6208 vec_oprnds
.release ();
6212 auto_vec
<tree
> dr_chain (group_size
);
6213 oprnds
.create (group_size
);
6215 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6216 gcc_assert (alignment_support_scheme
);
6217 /* Targets with store-lane instructions must not require explicit
6219 gcc_assert (memory_access_type
!= VMAT_LOAD_STORE_LANES
6220 || alignment_support_scheme
== dr_aligned
6221 || alignment_support_scheme
== dr_unaligned_supported
);
6223 if (memory_access_type
== VMAT_CONTIGUOUS_DOWN
6224 || memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
6225 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6227 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
6228 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6230 aggr_type
= vectype
;
6232 /* In case the vectorization factor (VF) is bigger than the number
6233 of elements that we can fit in a vectype (nunits), we have to generate
6234 more than one vector stmt - i.e - we need to "unroll" the
6235 vector stmt by a factor VF/nunits. For more details see documentation in
6236 vect_get_vec_def_for_copy_stmt. */
6238 /* In case of interleaving (non-unit grouped access):
6245 We create vectorized stores starting from base address (the access of the
6246 first stmt in the chain (S2 in the above example), when the last store stmt
6247 of the chain (S4) is reached:
6250 VS2: &base + vec_size*1 = vx0
6251 VS3: &base + vec_size*2 = vx1
6252 VS4: &base + vec_size*3 = vx3
6254 Then permutation statements are generated:
6256 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
6257 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
6260 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6261 (the order of the data-refs in the output of vect_permute_store_chain
6262 corresponds to the order of scalar stmts in the interleaving chain - see
6263 the documentation of vect_permute_store_chain()).
6265 In case of both multiple types and interleaving, above vector stores and
6266 permutation stmts are created for every copy. The result vector stmts are
6267 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
6268 STMT_VINFO_RELATED_STMT for the next copies.
6271 prev_stmt_info
= NULL
;
6272 for (j
= 0; j
< ncopies
; j
++)
6279 /* Get vectorized arguments for SLP_NODE. */
6280 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
6283 vec_oprnd
= vec_oprnds
[0];
6287 /* For interleaved stores we collect vectorized defs for all the
6288 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
6289 used as an input to vect_permute_store_chain(), and OPRNDS as
6290 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
6292 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
6293 OPRNDS are of size 1. */
6294 next_stmt
= first_stmt
;
6295 for (i
= 0; i
< group_size
; i
++)
6297 /* Since gaps are not supported for interleaved stores,
6298 GROUP_SIZE is the exact number of stmts in the chain.
6299 Therefore, NEXT_STMT can't be NULL_TREE. In case that
6300 there is no interleaving, GROUP_SIZE is 1, and only one
6301 iteration of the loop will be executed. */
6302 gcc_assert (next_stmt
6303 && gimple_assign_single_p (next_stmt
));
6304 op
= gimple_assign_rhs1 (next_stmt
);
6306 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
6307 dr_chain
.quick_push (vec_oprnd
);
6308 oprnds
.quick_push (vec_oprnd
);
6309 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
6313 /* We should have catched mismatched types earlier. */
6314 gcc_assert (useless_type_conversion_p (vectype
,
6315 TREE_TYPE (vec_oprnd
)));
6316 bool simd_lane_access_p
6317 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6318 if (simd_lane_access_p
6319 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6320 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6321 && integer_zerop (DR_OFFSET (first_dr
))
6322 && integer_zerop (DR_INIT (first_dr
))
6323 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6324 get_alias_set (TREE_TYPE (ref_type
))))
6326 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6327 dataref_offset
= build_int_cst (ref_type
, 0);
6332 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
6333 simd_lane_access_p
? loop
: NULL
,
6334 offset
, &dummy
, gsi
, &ptr_incr
,
6335 simd_lane_access_p
, &inv_p
);
6336 gcc_assert (bb_vinfo
|| !inv_p
);
6340 /* For interleaved stores we created vectorized defs for all the
6341 defs stored in OPRNDS in the previous iteration (previous copy).
6342 DR_CHAIN is then used as an input to vect_permute_store_chain(),
6343 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
6345 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
6346 OPRNDS are of size 1. */
6347 for (i
= 0; i
< group_size
; i
++)
6350 vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
);
6351 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
6352 dr_chain
[i
] = vec_oprnd
;
6353 oprnds
[i
] = vec_oprnd
;
6357 = int_const_binop (PLUS_EXPR
, dataref_offset
,
6358 TYPE_SIZE_UNIT (aggr_type
));
6360 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6361 TYPE_SIZE_UNIT (aggr_type
));
6364 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
6368 /* Combine all the vectors into an array. */
6369 vec_array
= create_vector_array (vectype
, vec_num
);
6370 for (i
= 0; i
< vec_num
; i
++)
6372 vec_oprnd
= dr_chain
[i
];
6373 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
6377 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
6378 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, ref_type
);
6379 gcall
*call
= gimple_build_call_internal (IFN_STORE_LANES
, 1,
6381 gimple_call_set_lhs (call
, data_ref
);
6382 gimple_call_set_nothrow (call
, true);
6384 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6392 result_chain
.create (group_size
);
6394 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
6398 next_stmt
= first_stmt
;
6399 for (i
= 0; i
< vec_num
; i
++)
6401 unsigned align
, misalign
;
6404 /* Bump the vector pointer. */
6405 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6409 vec_oprnd
= vec_oprnds
[i
];
6410 else if (grouped_store
)
6411 /* For grouped stores vectorized defs are interleaved in
6412 vect_permute_store_chain(). */
6413 vec_oprnd
= result_chain
[i
];
6415 data_ref
= fold_build2 (MEM_REF
, vectype
,
6419 : build_int_cst (ref_type
, 0));
6420 align
= TYPE_ALIGN_UNIT (vectype
);
6421 if (aligned_access_p (first_dr
))
6423 else if (DR_MISALIGNMENT (first_dr
) == -1)
6425 align
= dr_alignment (vect_dr_behavior (first_dr
));
6427 TREE_TYPE (data_ref
)
6428 = build_aligned_type (TREE_TYPE (data_ref
),
6429 align
* BITS_PER_UNIT
);
6433 TREE_TYPE (data_ref
)
6434 = build_aligned_type (TREE_TYPE (data_ref
),
6435 TYPE_ALIGN (elem_type
));
6436 misalign
= DR_MISALIGNMENT (first_dr
);
6438 if (dataref_offset
== NULL_TREE
6439 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
6440 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
6443 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
6445 tree perm_mask
= perm_mask_for_reverse (vectype
);
6447 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
6449 tree new_temp
= make_ssa_name (perm_dest
);
6451 /* Generate the permute statement. */
6453 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
6454 vec_oprnd
, perm_mask
);
6455 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
6457 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6458 vec_oprnd
= new_temp
;
6461 /* Arguments are ready. Create the new vector stmt. */
6462 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
6463 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6468 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
6476 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6478 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6479 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6484 result_chain
.release ();
6485 vec_oprnds
.release ();
6490 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
6491 VECTOR_CST mask. No checks are made that the target platform supports the
6492 mask, so callers may wish to test can_vec_perm_p separately, or use
6493 vect_gen_perm_mask_checked. */
6496 vect_gen_perm_mask_any (tree vectype
, vec_perm_indices sel
)
6498 tree mask_elt_type
, mask_type
, mask_vec
;
6500 unsigned int nunits
= sel
.length ();
6501 gcc_checking_assert (nunits
== TYPE_VECTOR_SUBPARTS (vectype
));
6503 mask_elt_type
= lang_hooks
.types
.type_for_mode
6504 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))).require (), 1);
6505 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
6507 auto_vec
<tree
, 32> mask_elts (nunits
);
6508 for (unsigned int i
= 0; i
< nunits
; ++i
)
6509 mask_elts
.quick_push (build_int_cst (mask_elt_type
, sel
[i
]));
6510 mask_vec
= build_vector (mask_type
, mask_elts
);
6515 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
6516 i.e. that the target supports the pattern _for arbitrary input vectors_. */
6519 vect_gen_perm_mask_checked (tree vectype
, vec_perm_indices sel
)
6521 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, &sel
));
6522 return vect_gen_perm_mask_any (vectype
, sel
);
6525 /* Given a vector variable X and Y, that was generated for the scalar
6526 STMT, generate instructions to permute the vector elements of X and Y
6527 using permutation mask MASK_VEC, insert them at *GSI and return the
6528 permuted vector variable. */
6531 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple
*stmt
,
6532 gimple_stmt_iterator
*gsi
)
6534 tree vectype
= TREE_TYPE (x
);
6535 tree perm_dest
, data_ref
;
6538 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
6539 data_ref
= make_ssa_name (perm_dest
);
6541 /* Generate the permute statement. */
6542 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
6543 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
6548 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
6549 inserting them on the loops preheader edge. Returns true if we
6550 were successful in doing so (and thus STMT can be moved then),
6551 otherwise returns false. */
6554 hoist_defs_of_uses (gimple
*stmt
, struct loop
*loop
)
6560 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6562 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6563 if (!gimple_nop_p (def_stmt
)
6564 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6566 /* Make sure we don't need to recurse. While we could do
6567 so in simple cases when there are more complex use webs
6568 we don't have an easy way to preserve stmt order to fulfil
6569 dependencies within them. */
6572 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
6574 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
6576 gimple
*def_stmt2
= SSA_NAME_DEF_STMT (op2
);
6577 if (!gimple_nop_p (def_stmt2
)
6578 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
6588 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6590 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6591 if (!gimple_nop_p (def_stmt
)
6592 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6594 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
6595 gsi_remove (&gsi
, false);
6596 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
6603 /* vectorizable_load.
6605 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
6607 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6608 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
6609 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6612 vectorizable_load (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
6613 slp_tree slp_node
, slp_instance slp_node_instance
)
6616 tree vec_dest
= NULL
;
6617 tree data_ref
= NULL
;
6618 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6619 stmt_vec_info prev_stmt_info
;
6620 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6621 struct loop
*loop
= NULL
;
6622 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
6623 bool nested_in_vect_loop
= false;
6624 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
6628 gimple
*new_stmt
= NULL
;
6630 enum dr_alignment_support alignment_support_scheme
;
6631 tree dataref_ptr
= NULL_TREE
;
6632 tree dataref_offset
= NULL_TREE
;
6633 gimple
*ptr_incr
= NULL
;
6635 int i
, j
, group_size
, group_gap_adj
;
6636 tree msq
= NULL_TREE
, lsq
;
6637 tree offset
= NULL_TREE
;
6638 tree byte_offset
= NULL_TREE
;
6639 tree realignment_token
= NULL_TREE
;
6641 vec
<tree
> dr_chain
= vNULL
;
6642 bool grouped_load
= false;
6644 gimple
*first_stmt_for_drptr
= NULL
;
6646 bool compute_in_loop
= false;
6647 struct loop
*at_loop
;
6649 bool slp
= (slp_node
!= NULL
);
6650 bool slp_perm
= false;
6651 enum tree_code code
;
6652 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6655 gather_scatter_info gs_info
;
6656 vec_info
*vinfo
= stmt_info
->vinfo
;
6659 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6662 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6666 /* Is vectorizable load? */
6667 if (!is_gimple_assign (stmt
))
6670 scalar_dest
= gimple_assign_lhs (stmt
);
6671 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
6674 code
= gimple_assign_rhs_code (stmt
);
6675 if (code
!= ARRAY_REF
6676 && code
!= BIT_FIELD_REF
6677 && code
!= INDIRECT_REF
6678 && code
!= COMPONENT_REF
6679 && code
!= IMAGPART_EXPR
6680 && code
!= REALPART_EXPR
6682 && TREE_CODE_CLASS (code
) != tcc_declaration
)
6685 if (!STMT_VINFO_DATA_REF (stmt_info
))
6688 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6689 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6693 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6694 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
6695 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
6700 /* Multiple types in SLP are handled by creating the appropriate number of
6701 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6706 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
6708 gcc_assert (ncopies
>= 1);
6710 /* FORNOW. This restriction should be relaxed. */
6711 if (nested_in_vect_loop
&& ncopies
> 1)
6713 if (dump_enabled_p ())
6714 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6715 "multiple types in nested loop.\n");
6719 /* Invalidate assumptions made by dependence analysis when vectorization
6720 on the unrolled body effectively re-orders stmts. */
6722 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6723 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6724 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6726 if (dump_enabled_p ())
6727 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6728 "cannot perform implicit CSE when unrolling "
6729 "with negative dependence distance\n");
6733 elem_type
= TREE_TYPE (vectype
);
6734 mode
= TYPE_MODE (vectype
);
6736 /* FORNOW. In some cases can vectorize even if data-type not supported
6737 (e.g. - data copies). */
6738 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
6740 if (dump_enabled_p ())
6741 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6742 "Aligned load, but unsupported type.\n");
6746 /* Check if the load is a part of an interleaving chain. */
6747 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
6749 grouped_load
= true;
6751 gcc_assert (!nested_in_vect_loop
);
6752 gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info
));
6754 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6755 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6757 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6760 /* Invalidate assumptions made by dependence analysis when vectorization
6761 on the unrolled body effectively re-orders stmts. */
6762 if (!PURE_SLP_STMT (stmt_info
)
6763 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6764 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6765 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6767 if (dump_enabled_p ())
6768 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6769 "cannot perform implicit CSE when performing "
6770 "group loads with negative dependence distance\n");
6774 /* Similarly when the stmt is a load that is both part of a SLP
6775 instance and a loop vectorized stmt via the same-dr mechanism
6776 we have to give up. */
6777 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
6778 && (STMT_SLP_TYPE (stmt_info
)
6779 != STMT_SLP_TYPE (vinfo_for_stmt
6780 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
6782 if (dump_enabled_p ())
6783 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6784 "conflicting SLP types for CSEd load\n");
6789 vect_memory_access_type memory_access_type
;
6790 if (!get_load_store_type (stmt
, vectype
, slp
, VLS_LOAD
, ncopies
,
6791 &memory_access_type
, &gs_info
))
6794 if (!vec_stmt
) /* transformation not required. */
6797 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
) = memory_access_type
;
6798 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6799 /* The SLP costs are calculated during SLP analysis. */
6800 if (!PURE_SLP_STMT (stmt_info
))
6801 vect_model_load_cost (stmt_info
, ncopies
, memory_access_type
,
6807 gcc_assert (memory_access_type
6808 == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
));
6810 if (dump_enabled_p ())
6811 dump_printf_loc (MSG_NOTE
, vect_location
,
6812 "transform load. ncopies = %d\n", ncopies
);
6816 ensure_base_align (stmt_info
, dr
);
6818 if (memory_access_type
== VMAT_GATHER_SCATTER
)
6820 tree vec_oprnd0
= NULL_TREE
, op
;
6821 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
6822 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6823 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6824 edge pe
= loop_preheader_edge (loop
);
6827 enum { NARROW
, NONE
, WIDEN
} modifier
;
6828 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
6830 if (nunits
== gather_off_nunits
)
6832 else if (nunits
== gather_off_nunits
/ 2)
6836 auto_vec_perm_indices
sel (gather_off_nunits
);
6837 for (i
= 0; i
< gather_off_nunits
; ++i
)
6838 sel
.quick_push (i
| nunits
);
6840 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
, sel
);
6842 else if (nunits
== gather_off_nunits
* 2)
6846 auto_vec_perm_indices
sel (nunits
);
6847 for (i
= 0; i
< nunits
; ++i
)
6848 sel
.quick_push (i
< gather_off_nunits
6849 ? i
: i
+ nunits
- gather_off_nunits
);
6851 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6857 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
6858 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6859 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6860 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6861 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6862 scaletype
= TREE_VALUE (arglist
);
6863 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6865 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6867 ptr
= fold_convert (ptrtype
, gs_info
.base
);
6868 if (!is_gimple_min_invariant (ptr
))
6870 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6871 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6872 gcc_assert (!new_bb
);
6875 /* Currently we support only unconditional gather loads,
6876 so mask should be all ones. */
6877 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6878 mask
= build_int_cst (masktype
, -1);
6879 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6881 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6882 mask
= build_vector_from_val (masktype
, mask
);
6883 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6885 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6889 for (j
= 0; j
< 6; ++j
)
6891 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6892 mask
= build_real (TREE_TYPE (masktype
), r
);
6893 mask
= build_vector_from_val (masktype
, mask
);
6894 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6899 scale
= build_int_cst (scaletype
, gs_info
.scale
);
6901 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6902 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6903 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6907 for (j
= 0; j
< 6; ++j
)
6909 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6910 merge
= build_real (TREE_TYPE (rettype
), r
);
6914 merge
= build_vector_from_val (rettype
, merge
);
6915 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6917 prev_stmt_info
= NULL
;
6918 for (j
= 0; j
< ncopies
; ++j
)
6920 if (modifier
== WIDEN
&& (j
& 1))
6921 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6922 perm_mask
, stmt
, gsi
);
6925 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt
);
6928 = vect_get_vec_def_for_stmt_copy (gs_info
.offset_dt
, vec_oprnd0
);
6930 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6932 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6933 == TYPE_VECTOR_SUBPARTS (idxtype
));
6934 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
6935 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6937 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6938 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6943 = gimple_build_call (gs_info
.decl
, 5, merge
, ptr
, op
, mask
, scale
);
6945 if (!useless_type_conversion_p (vectype
, rettype
))
6947 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6948 == TYPE_VECTOR_SUBPARTS (rettype
));
6949 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
6950 gimple_call_set_lhs (new_stmt
, op
);
6951 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6952 var
= make_ssa_name (vec_dest
);
6953 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6955 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6959 var
= make_ssa_name (vec_dest
, new_stmt
);
6960 gimple_call_set_lhs (new_stmt
, var
);
6963 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6965 if (modifier
== NARROW
)
6972 var
= permute_vec_elements (prev_res
, var
,
6973 perm_mask
, stmt
, gsi
);
6974 new_stmt
= SSA_NAME_DEF_STMT (var
);
6977 if (prev_stmt_info
== NULL
)
6978 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6980 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6981 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6986 if (memory_access_type
== VMAT_ELEMENTWISE
6987 || memory_access_type
== VMAT_STRIDED_SLP
)
6989 gimple_stmt_iterator incr_gsi
;
6995 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6996 gimple_seq stmts
= NULL
;
6997 tree stride_base
, stride_step
, alias_off
;
6999 gcc_assert (!nested_in_vect_loop
);
7001 if (slp
&& grouped_load
)
7003 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
7004 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
7005 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
7006 ref_type
= get_group_alias_ptr_type (first_stmt
);
7013 ref_type
= reference_alias_ptr_type (DR_REF (first_dr
));
7017 = fold_build_pointer_plus
7018 (DR_BASE_ADDRESS (first_dr
),
7019 size_binop (PLUS_EXPR
,
7020 convert_to_ptrofftype (DR_OFFSET (first_dr
)),
7021 convert_to_ptrofftype (DR_INIT (first_dr
))));
7022 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr
));
7024 /* For a load with loop-invariant (but other than power-of-2)
7025 stride (i.e. not a grouped access) like so:
7027 for (i = 0; i < n; i += stride)
7030 we generate a new induction variable and new accesses to
7031 form a new vector (or vectors, depending on ncopies):
7033 for (j = 0; ; j += VF*stride)
7035 tmp2 = array[j + stride];
7037 vectemp = {tmp1, tmp2, ...}
7040 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
7041 build_int_cst (TREE_TYPE (stride_step
), vf
));
7043 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
7045 create_iv (unshare_expr (stride_base
), unshare_expr (ivstep
), NULL
,
7046 loop
, &incr_gsi
, insert_after
,
7048 incr
= gsi_stmt (incr_gsi
);
7049 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
7051 stride_step
= force_gimple_operand (unshare_expr (stride_step
),
7052 &stmts
, true, NULL_TREE
);
7054 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
7056 prev_stmt_info
= NULL
;
7057 running_off
= offvar
;
7058 alias_off
= build_int_cst (ref_type
, 0);
7059 int nloads
= nunits
;
7061 tree ltype
= TREE_TYPE (vectype
);
7062 tree lvectype
= vectype
;
7063 auto_vec
<tree
> dr_chain
;
7064 if (memory_access_type
== VMAT_STRIDED_SLP
)
7066 if (group_size
< nunits
)
7068 /* First check if vec_init optab supports construction from
7069 vector elts directly. */
7070 scalar_mode elmode
= SCALAR_TYPE_MODE (TREE_TYPE (vectype
));
7072 if (mode_for_vector (elmode
, group_size
).exists (&vmode
)
7073 && VECTOR_MODE_P (vmode
)
7074 && (convert_optab_handler (vec_init_optab
,
7075 TYPE_MODE (vectype
), vmode
)
7076 != CODE_FOR_nothing
))
7078 nloads
= nunits
/ group_size
;
7080 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
7084 /* Otherwise avoid emitting a constructor of vector elements
7085 by performing the loads using an integer type of the same
7086 size, constructing a vector of those and then
7087 re-interpreting it as the original vector type.
7088 This avoids a huge runtime penalty due to the general
7089 inability to perform store forwarding from smaller stores
7090 to a larger load. */
7092 = group_size
* TYPE_PRECISION (TREE_TYPE (vectype
));
7093 elmode
= int_mode_for_size (lsize
, 0).require ();
7094 /* If we can't construct such a vector fall back to
7095 element loads of the original vector type. */
7096 if (mode_for_vector (elmode
,
7097 nunits
/ group_size
).exists (&vmode
)
7098 && VECTOR_MODE_P (vmode
)
7099 && (convert_optab_handler (vec_init_optab
, vmode
, elmode
)
7100 != CODE_FOR_nothing
))
7102 nloads
= nunits
/ group_size
;
7104 ltype
= build_nonstandard_integer_type (lsize
, 1);
7105 lvectype
= build_vector_type (ltype
, nloads
);
7115 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
7119 /* For SLP permutation support we need to load the whole group,
7120 not only the number of vector stmts the permutation result
7124 ncopies
= (group_size
* vf
+ nunits
- 1) / nunits
;
7125 dr_chain
.create (ncopies
);
7128 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
7131 unsigned HOST_WIDE_INT
7132 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
7133 for (j
= 0; j
< ncopies
; j
++)
7136 vec_alloc (v
, nloads
);
7137 for (i
= 0; i
< nloads
; i
++)
7139 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
7141 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
7142 build2 (MEM_REF
, ltype
,
7143 running_off
, this_off
));
7144 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7146 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
,
7147 gimple_assign_lhs (new_stmt
));
7151 || group_el
== group_size
)
7153 tree newoff
= copy_ssa_name (running_off
);
7154 gimple
*incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
7155 running_off
, stride_step
);
7156 vect_finish_stmt_generation (stmt
, incr
, gsi
);
7158 running_off
= newoff
;
7164 tree vec_inv
= build_constructor (lvectype
, v
);
7165 new_temp
= vect_init_vector (stmt
, vec_inv
, lvectype
, gsi
);
7166 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7167 if (lvectype
!= vectype
)
7169 new_stmt
= gimple_build_assign (make_ssa_name (vectype
),
7171 build1 (VIEW_CONVERT_EXPR
,
7172 vectype
, new_temp
));
7173 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7180 dr_chain
.quick_push (gimple_assign_lhs (new_stmt
));
7182 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7187 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7189 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7190 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7196 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
7197 slp_node_instance
, false, &n_perms
);
7204 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
7205 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
7206 /* For SLP vectorization we directly vectorize a subchain
7207 without permutation. */
7208 if (slp
&& ! SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
7209 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
7210 /* For BB vectorization always use the first stmt to base
7211 the data ref pointer on. */
7213 first_stmt_for_drptr
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
7215 /* Check if the chain of loads is already vectorized. */
7216 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
7217 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
7218 ??? But we can only do so if there is exactly one
7219 as we have no way to get at the rest. Leave the CSE
7221 ??? With the group load eventually participating
7222 in multiple different permutations (having multiple
7223 slp nodes which refer to the same group) the CSE
7224 is even wrong code. See PR56270. */
7227 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7230 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
7233 /* VEC_NUM is the number of vect stmts to be created for this group. */
7236 grouped_load
= false;
7237 /* For SLP permutation support we need to load the whole group,
7238 not only the number of vector stmts the permutation result
7242 vec_num
= (group_size
* vf
+ nunits
- 1) / nunits
;
7243 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
7247 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
7249 = group_size
- SLP_INSTANCE_GROUP_SIZE (slp_node_instance
);
7253 vec_num
= group_size
;
7255 ref_type
= get_group_alias_ptr_type (first_stmt
);
7261 group_size
= vec_num
= 1;
7263 ref_type
= reference_alias_ptr_type (DR_REF (first_dr
));
7266 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
7267 gcc_assert (alignment_support_scheme
);
7268 /* Targets with load-lane instructions must not require explicit
7270 gcc_assert (memory_access_type
!= VMAT_LOAD_STORE_LANES
7271 || alignment_support_scheme
== dr_aligned
7272 || alignment_support_scheme
== dr_unaligned_supported
);
7274 /* In case the vectorization factor (VF) is bigger than the number
7275 of elements that we can fit in a vectype (nunits), we have to generate
7276 more than one vector stmt - i.e - we need to "unroll" the
7277 vector stmt by a factor VF/nunits. In doing so, we record a pointer
7278 from one copy of the vector stmt to the next, in the field
7279 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
7280 stages to find the correct vector defs to be used when vectorizing
7281 stmts that use the defs of the current stmt. The example below
7282 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
7283 need to create 4 vectorized stmts):
7285 before vectorization:
7286 RELATED_STMT VEC_STMT
7290 step 1: vectorize stmt S1:
7291 We first create the vector stmt VS1_0, and, as usual, record a
7292 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
7293 Next, we create the vector stmt VS1_1, and record a pointer to
7294 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
7295 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
7297 RELATED_STMT VEC_STMT
7298 VS1_0: vx0 = memref0 VS1_1 -
7299 VS1_1: vx1 = memref1 VS1_2 -
7300 VS1_2: vx2 = memref2 VS1_3 -
7301 VS1_3: vx3 = memref3 - -
7302 S1: x = load - VS1_0
7305 See in documentation in vect_get_vec_def_for_stmt_copy for how the
7306 information we recorded in RELATED_STMT field is used to vectorize
7309 /* In case of interleaving (non-unit grouped access):
7316 Vectorized loads are created in the order of memory accesses
7317 starting from the access of the first stmt of the chain:
7320 VS2: vx1 = &base + vec_size*1
7321 VS3: vx3 = &base + vec_size*2
7322 VS4: vx4 = &base + vec_size*3
7324 Then permutation statements are generated:
7326 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
7327 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
7330 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
7331 (the order of the data-refs in the output of vect_permute_load_chain
7332 corresponds to the order of scalar stmts in the interleaving chain - see
7333 the documentation of vect_permute_load_chain()).
7334 The generation of permutation stmts and recording them in
7335 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
7337 In case of both multiple types and interleaving, the vector loads and
7338 permutation stmts above are created for every copy. The result vector
7339 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
7340 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
7342 /* If the data reference is aligned (dr_aligned) or potentially unaligned
7343 on a target that supports unaligned accesses (dr_unaligned_supported)
7344 we generate the following code:
7348 p = p + indx * vectype_size;
7353 Otherwise, the data reference is potentially unaligned on a target that
7354 does not support unaligned accesses (dr_explicit_realign_optimized) -
7355 then generate the following code, in which the data in each iteration is
7356 obtained by two vector loads, one from the previous iteration, and one
7357 from the current iteration:
7359 msq_init = *(floor(p1))
7360 p2 = initial_addr + VS - 1;
7361 realignment_token = call target_builtin;
7364 p2 = p2 + indx * vectype_size
7366 vec_dest = realign_load (msq, lsq, realignment_token)
7371 /* If the misalignment remains the same throughout the execution of the
7372 loop, we can create the init_addr and permutation mask at the loop
7373 preheader. Otherwise, it needs to be created inside the loop.
7374 This can only occur when vectorizing memory accesses in the inner-loop
7375 nested within an outer-loop that is being vectorized. */
7377 if (nested_in_vect_loop
7378 && (DR_STEP_ALIGNMENT (dr
) % GET_MODE_SIZE (TYPE_MODE (vectype
))) != 0)
7380 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
7381 compute_in_loop
= true;
7384 if ((alignment_support_scheme
== dr_explicit_realign_optimized
7385 || alignment_support_scheme
== dr_explicit_realign
)
7386 && !compute_in_loop
)
7388 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
7389 alignment_support_scheme
, NULL_TREE
,
7391 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
7393 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
7394 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
7401 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
7402 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
7404 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
7405 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
7407 aggr_type
= vectype
;
7409 prev_stmt_info
= NULL
;
7411 for (j
= 0; j
< ncopies
; j
++)
7413 /* 1. Create the vector or array pointer update chain. */
7416 bool simd_lane_access_p
7417 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
7418 if (simd_lane_access_p
7419 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
7420 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
7421 && integer_zerop (DR_OFFSET (first_dr
))
7422 && integer_zerop (DR_INIT (first_dr
))
7423 && alias_sets_conflict_p (get_alias_set (aggr_type
),
7424 get_alias_set (TREE_TYPE (ref_type
)))
7425 && (alignment_support_scheme
== dr_aligned
7426 || alignment_support_scheme
== dr_unaligned_supported
))
7428 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
7429 dataref_offset
= build_int_cst (ref_type
, 0);
7432 else if (first_stmt_for_drptr
7433 && first_stmt
!= first_stmt_for_drptr
)
7436 = vect_create_data_ref_ptr (first_stmt_for_drptr
, aggr_type
,
7437 at_loop
, offset
, &dummy
, gsi
,
7438 &ptr_incr
, simd_lane_access_p
,
7439 &inv_p
, byte_offset
);
7440 /* Adjust the pointer by the difference to first_stmt. */
7441 data_reference_p ptrdr
7442 = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr
));
7443 tree diff
= fold_convert (sizetype
,
7444 size_binop (MINUS_EXPR
,
7447 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7452 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
7453 offset
, &dummy
, gsi
, &ptr_incr
,
7454 simd_lane_access_p
, &inv_p
,
7457 else if (dataref_offset
)
7458 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
7459 TYPE_SIZE_UNIT (aggr_type
));
7461 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
7462 TYPE_SIZE_UNIT (aggr_type
));
7464 if (grouped_load
|| slp_perm
)
7465 dr_chain
.create (vec_num
);
7467 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
7471 vec_array
= create_vector_array (vectype
, vec_num
);
7474 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
7475 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, ref_type
);
7476 gcall
*call
= gimple_build_call_internal (IFN_LOAD_LANES
, 1,
7478 gimple_call_set_lhs (call
, vec_array
);
7479 gimple_call_set_nothrow (call
, true);
7481 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7483 /* Extract each vector into an SSA_NAME. */
7484 for (i
= 0; i
< vec_num
; i
++)
7486 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
7488 dr_chain
.quick_push (new_temp
);
7491 /* Record the mapping between SSA_NAMEs and statements. */
7492 vect_record_grouped_load_vectors (stmt
, dr_chain
);
7496 for (i
= 0; i
< vec_num
; i
++)
7499 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7502 /* 2. Create the vector-load in the loop. */
7503 switch (alignment_support_scheme
)
7506 case dr_unaligned_supported
:
7508 unsigned int align
, misalign
;
7511 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
7514 : build_int_cst (ref_type
, 0));
7515 align
= TYPE_ALIGN_UNIT (vectype
);
7516 if (alignment_support_scheme
== dr_aligned
)
7518 gcc_assert (aligned_access_p (first_dr
));
7521 else if (DR_MISALIGNMENT (first_dr
) == -1)
7523 align
= dr_alignment (vect_dr_behavior (first_dr
));
7525 TREE_TYPE (data_ref
)
7526 = build_aligned_type (TREE_TYPE (data_ref
),
7527 align
* BITS_PER_UNIT
);
7531 TREE_TYPE (data_ref
)
7532 = build_aligned_type (TREE_TYPE (data_ref
),
7533 TYPE_ALIGN (elem_type
));
7534 misalign
= DR_MISALIGNMENT (first_dr
);
7536 if (dataref_offset
== NULL_TREE
7537 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
7538 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
7542 case dr_explicit_realign
:
7546 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
7548 if (compute_in_loop
)
7549 msq
= vect_setup_realignment (first_stmt
, gsi
,
7551 dr_explicit_realign
,
7554 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
7555 ptr
= copy_ssa_name (dataref_ptr
);
7557 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
7558 new_stmt
= gimple_build_assign
7559 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
7561 (TREE_TYPE (dataref_ptr
),
7562 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7563 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7565 = build2 (MEM_REF
, vectype
, ptr
,
7566 build_int_cst (ref_type
, 0));
7567 vec_dest
= vect_create_destination_var (scalar_dest
,
7569 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
7570 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7571 gimple_assign_set_lhs (new_stmt
, new_temp
);
7572 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
7573 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
7574 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7577 bump
= size_binop (MULT_EXPR
, vs
,
7578 TYPE_SIZE_UNIT (elem_type
));
7579 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
7580 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
7581 new_stmt
= gimple_build_assign
7582 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
7585 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7586 ptr
= copy_ssa_name (ptr
, new_stmt
);
7587 gimple_assign_set_lhs (new_stmt
, ptr
);
7588 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7590 = build2 (MEM_REF
, vectype
, ptr
,
7591 build_int_cst (ref_type
, 0));
7594 case dr_explicit_realign_optimized
:
7595 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
7596 new_temp
= copy_ssa_name (dataref_ptr
);
7598 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
7599 new_stmt
= gimple_build_assign
7600 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
7602 (TREE_TYPE (dataref_ptr
),
7603 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7604 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7606 = build2 (MEM_REF
, vectype
, new_temp
,
7607 build_int_cst (ref_type
, 0));
7612 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7613 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
7614 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7615 gimple_assign_set_lhs (new_stmt
, new_temp
);
7616 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7618 /* 3. Handle explicit realignment if necessary/supported.
7620 vec_dest = realign_load (msq, lsq, realignment_token) */
7621 if (alignment_support_scheme
== dr_explicit_realign_optimized
7622 || alignment_support_scheme
== dr_explicit_realign
)
7624 lsq
= gimple_assign_lhs (new_stmt
);
7625 if (!realignment_token
)
7626 realignment_token
= dataref_ptr
;
7627 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7628 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
7629 msq
, lsq
, realignment_token
);
7630 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7631 gimple_assign_set_lhs (new_stmt
, new_temp
);
7632 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7634 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
7637 if (i
== vec_num
- 1 && j
== ncopies
- 1)
7638 add_phi_arg (phi
, lsq
,
7639 loop_latch_edge (containing_loop
),
7645 /* 4. Handle invariant-load. */
7646 if (inv_p
&& !bb_vinfo
)
7648 gcc_assert (!grouped_load
);
7649 /* If we have versioned for aliasing or the loop doesn't
7650 have any data dependencies that would preclude this,
7651 then we are sure this is a loop invariant load and
7652 thus we can insert it on the preheader edge. */
7653 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
7654 && !nested_in_vect_loop
7655 && hoist_defs_of_uses (stmt
, loop
))
7657 if (dump_enabled_p ())
7659 dump_printf_loc (MSG_NOTE
, vect_location
,
7660 "hoisting out of the vectorized "
7662 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7664 tree tem
= copy_ssa_name (scalar_dest
);
7665 gsi_insert_on_edge_immediate
7666 (loop_preheader_edge (loop
),
7667 gimple_build_assign (tem
,
7669 (gimple_assign_rhs1 (stmt
))));
7670 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
7671 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7672 set_vinfo_for_stmt (new_stmt
,
7673 new_stmt_vec_info (new_stmt
, vinfo
));
7677 gimple_stmt_iterator gsi2
= *gsi
;
7679 new_temp
= vect_init_vector (stmt
, scalar_dest
,
7681 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7685 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
7687 tree perm_mask
= perm_mask_for_reverse (vectype
);
7688 new_temp
= permute_vec_elements (new_temp
, new_temp
,
7689 perm_mask
, stmt
, gsi
);
7690 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7693 /* Collect vector loads and later create their permutation in
7694 vect_transform_grouped_load (). */
7695 if (grouped_load
|| slp_perm
)
7696 dr_chain
.quick_push (new_temp
);
7698 /* Store vector loads in the corresponding SLP_NODE. */
7699 if (slp
&& !slp_perm
)
7700 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7702 /* With SLP permutation we load the gaps as well, without
7703 we need to skip the gaps after we manage to fully load
7704 all elements. group_gap_adj is GROUP_SIZE here. */
7705 group_elt
+= nunits
;
7706 if (group_gap_adj
!= 0 && ! slp_perm
7707 && group_elt
== group_size
- group_gap_adj
)
7711 = wide_int_to_tree (sizetype
,
7712 wi::smul (TYPE_SIZE_UNIT (elem_type
),
7713 group_gap_adj
, &ovf
));
7714 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7719 /* Bump the vector pointer to account for a gap or for excess
7720 elements loaded for a permuted SLP load. */
7721 if (group_gap_adj
!= 0 && slp_perm
)
7725 = wide_int_to_tree (sizetype
,
7726 wi::smul (TYPE_SIZE_UNIT (elem_type
),
7727 group_gap_adj
, &ovf
));
7728 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7733 if (slp
&& !slp_perm
)
7739 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
7740 slp_node_instance
, false,
7743 dr_chain
.release ();
7751 if (memory_access_type
!= VMAT_LOAD_STORE_LANES
)
7752 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
7753 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7758 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7760 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7761 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7764 dr_chain
.release ();
7770 /* Function vect_is_simple_cond.
7773 LOOP - the loop that is being vectorized.
7774 COND - Condition that is checked for simple use.
7777 *COMP_VECTYPE - the vector type for the comparison.
7778 *DTS - The def types for the arguments of the comparison
7780 Returns whether a COND can be vectorized. Checks whether
7781 condition operands are supportable using vec_is_simple_use. */
7784 vect_is_simple_cond (tree cond
, vec_info
*vinfo
,
7785 tree
*comp_vectype
, enum vect_def_type
*dts
)
7788 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7791 if (TREE_CODE (cond
) == SSA_NAME
7792 && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond
)))
7794 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (cond
);
7795 if (!vect_is_simple_use (cond
, vinfo
, &lhs_def_stmt
,
7796 &dts
[0], comp_vectype
)
7798 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype
))
7803 if (!COMPARISON_CLASS_P (cond
))
7806 lhs
= TREE_OPERAND (cond
, 0);
7807 rhs
= TREE_OPERAND (cond
, 1);
7809 if (TREE_CODE (lhs
) == SSA_NAME
)
7811 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
7812 if (!vect_is_simple_use (lhs
, vinfo
, &lhs_def_stmt
, &dts
[0], &vectype1
))
7815 else if (TREE_CODE (lhs
) == INTEGER_CST
|| TREE_CODE (lhs
) == REAL_CST
7816 || TREE_CODE (lhs
) == FIXED_CST
)
7817 dts
[0] = vect_constant_def
;
7821 if (TREE_CODE (rhs
) == SSA_NAME
)
7823 gimple
*rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
7824 if (!vect_is_simple_use (rhs
, vinfo
, &rhs_def_stmt
, &dts
[1], &vectype2
))
7827 else if (TREE_CODE (rhs
) == INTEGER_CST
|| TREE_CODE (rhs
) == REAL_CST
7828 || TREE_CODE (rhs
) == FIXED_CST
)
7829 dts
[1] = vect_constant_def
;
7833 if (vectype1
&& vectype2
7834 && TYPE_VECTOR_SUBPARTS (vectype1
) != TYPE_VECTOR_SUBPARTS (vectype2
))
7837 *comp_vectype
= vectype1
? vectype1
: vectype2
;
7841 /* vectorizable_condition.
7843 Check if STMT is conditional modify expression that can be vectorized.
7844 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7845 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
7848 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
7849 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
7850 else clause if it is 2).
7852 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7855 vectorizable_condition (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7856 gimple
**vec_stmt
, tree reduc_def
, int reduc_index
,
7859 tree scalar_dest
= NULL_TREE
;
7860 tree vec_dest
= NULL_TREE
;
7861 tree cond_expr
, cond_expr0
= NULL_TREE
, cond_expr1
= NULL_TREE
;
7862 tree then_clause
, else_clause
;
7863 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7864 tree comp_vectype
= NULL_TREE
;
7865 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
7866 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
7869 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7870 enum vect_def_type dts
[4]
7871 = {vect_unknown_def_type
, vect_unknown_def_type
,
7872 vect_unknown_def_type
, vect_unknown_def_type
};
7875 enum tree_code code
, cond_code
, bitop1
= NOP_EXPR
, bitop2
= NOP_EXPR
;
7876 stmt_vec_info prev_stmt_info
= NULL
;
7878 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7879 vec
<tree
> vec_oprnds0
= vNULL
;
7880 vec
<tree
> vec_oprnds1
= vNULL
;
7881 vec
<tree
> vec_oprnds2
= vNULL
;
7882 vec
<tree
> vec_oprnds3
= vNULL
;
7884 bool masked
= false;
7886 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
7889 if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info
) == TREE_CODE_REDUCTION
)
7891 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7894 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7895 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7899 /* FORNOW: not yet supported. */
7900 if (STMT_VINFO_LIVE_P (stmt_info
))
7902 if (dump_enabled_p ())
7903 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7904 "value used after loop.\n");
7909 /* Is vectorizable conditional operation? */
7910 if (!is_gimple_assign (stmt
))
7913 code
= gimple_assign_rhs_code (stmt
);
7915 if (code
!= COND_EXPR
)
7918 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7919 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7924 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
7926 gcc_assert (ncopies
>= 1);
7927 if (reduc_index
&& ncopies
> 1)
7928 return false; /* FORNOW */
7930 cond_expr
= gimple_assign_rhs1 (stmt
);
7931 then_clause
= gimple_assign_rhs2 (stmt
);
7932 else_clause
= gimple_assign_rhs3 (stmt
);
7934 if (!vect_is_simple_cond (cond_expr
, stmt_info
->vinfo
,
7935 &comp_vectype
, &dts
[0])
7940 if (!vect_is_simple_use (then_clause
, stmt_info
->vinfo
, &def_stmt
, &dts
[2],
7943 if (!vect_is_simple_use (else_clause
, stmt_info
->vinfo
, &def_stmt
, &dts
[3],
7947 if (vectype1
&& !useless_type_conversion_p (vectype
, vectype1
))
7950 if (vectype2
&& !useless_type_conversion_p (vectype
, vectype2
))
7953 masked
= !COMPARISON_CLASS_P (cond_expr
);
7954 vec_cmp_type
= build_same_sized_truth_vector_type (comp_vectype
);
7956 if (vec_cmp_type
== NULL_TREE
)
7959 cond_code
= TREE_CODE (cond_expr
);
7962 cond_expr0
= TREE_OPERAND (cond_expr
, 0);
7963 cond_expr1
= TREE_OPERAND (cond_expr
, 1);
7966 if (!masked
&& VECTOR_BOOLEAN_TYPE_P (comp_vectype
))
7968 /* Boolean values may have another representation in vectors
7969 and therefore we prefer bit operations over comparison for
7970 them (which also works for scalar masks). We store opcodes
7971 to use in bitop1 and bitop2. Statement is vectorized as
7972 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
7973 depending on bitop1 and bitop2 arity. */
7977 bitop1
= BIT_NOT_EXPR
;
7978 bitop2
= BIT_AND_EXPR
;
7981 bitop1
= BIT_NOT_EXPR
;
7982 bitop2
= BIT_IOR_EXPR
;
7985 bitop1
= BIT_NOT_EXPR
;
7986 bitop2
= BIT_AND_EXPR
;
7987 std::swap (cond_expr0
, cond_expr1
);
7990 bitop1
= BIT_NOT_EXPR
;
7991 bitop2
= BIT_IOR_EXPR
;
7992 std::swap (cond_expr0
, cond_expr1
);
7995 bitop1
= BIT_XOR_EXPR
;
7998 bitop1
= BIT_XOR_EXPR
;
7999 bitop2
= BIT_NOT_EXPR
;
8004 cond_code
= SSA_NAME
;
8009 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
8010 if (bitop1
!= NOP_EXPR
)
8012 machine_mode mode
= TYPE_MODE (comp_vectype
);
8015 optab
= optab_for_tree_code (bitop1
, comp_vectype
, optab_default
);
8016 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8019 if (bitop2
!= NOP_EXPR
)
8021 optab
= optab_for_tree_code (bitop2
, comp_vectype
,
8023 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8027 if (expand_vec_cond_expr_p (vectype
, comp_vectype
,
8030 vect_model_simple_cost (stmt_info
, ncopies
, dts
, ndts
, NULL
, NULL
);
8040 vec_oprnds0
.create (1);
8041 vec_oprnds1
.create (1);
8042 vec_oprnds2
.create (1);
8043 vec_oprnds3
.create (1);
8047 scalar_dest
= gimple_assign_lhs (stmt
);
8048 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
8050 /* Handle cond expr. */
8051 for (j
= 0; j
< ncopies
; j
++)
8053 gassign
*new_stmt
= NULL
;
8058 auto_vec
<tree
, 4> ops
;
8059 auto_vec
<vec
<tree
>, 4> vec_defs
;
8062 ops
.safe_push (cond_expr
);
8065 ops
.safe_push (cond_expr0
);
8066 ops
.safe_push (cond_expr1
);
8068 ops
.safe_push (then_clause
);
8069 ops
.safe_push (else_clause
);
8070 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
8071 vec_oprnds3
= vec_defs
.pop ();
8072 vec_oprnds2
= vec_defs
.pop ();
8074 vec_oprnds1
= vec_defs
.pop ();
8075 vec_oprnds0
= vec_defs
.pop ();
8083 = vect_get_vec_def_for_operand (cond_expr
, stmt
,
8085 vect_is_simple_use (cond_expr
, stmt_info
->vinfo
,
8091 = vect_get_vec_def_for_operand (cond_expr0
,
8092 stmt
, comp_vectype
);
8093 vect_is_simple_use (cond_expr0
, loop_vinfo
, >emp
, &dts
[0]);
8096 = vect_get_vec_def_for_operand (cond_expr1
,
8097 stmt
, comp_vectype
);
8098 vect_is_simple_use (cond_expr1
, loop_vinfo
, >emp
, &dts
[1]);
8100 if (reduc_index
== 1)
8101 vec_then_clause
= reduc_def
;
8104 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
8106 vect_is_simple_use (then_clause
, loop_vinfo
,
8109 if (reduc_index
== 2)
8110 vec_else_clause
= reduc_def
;
8113 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
8115 vect_is_simple_use (else_clause
, loop_vinfo
, >emp
, &dts
[3]);
8122 = vect_get_vec_def_for_stmt_copy (dts
[0],
8123 vec_oprnds0
.pop ());
8126 = vect_get_vec_def_for_stmt_copy (dts
[1],
8127 vec_oprnds1
.pop ());
8129 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
8130 vec_oprnds2
.pop ());
8131 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
8132 vec_oprnds3
.pop ());
8137 vec_oprnds0
.quick_push (vec_cond_lhs
);
8139 vec_oprnds1
.quick_push (vec_cond_rhs
);
8140 vec_oprnds2
.quick_push (vec_then_clause
);
8141 vec_oprnds3
.quick_push (vec_else_clause
);
8144 /* Arguments are ready. Create the new vector stmt. */
8145 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
8147 vec_then_clause
= vec_oprnds2
[i
];
8148 vec_else_clause
= vec_oprnds3
[i
];
8151 vec_compare
= vec_cond_lhs
;
8154 vec_cond_rhs
= vec_oprnds1
[i
];
8155 if (bitop1
== NOP_EXPR
)
8156 vec_compare
= build2 (cond_code
, vec_cmp_type
,
8157 vec_cond_lhs
, vec_cond_rhs
);
8160 new_temp
= make_ssa_name (vec_cmp_type
);
8161 if (bitop1
== BIT_NOT_EXPR
)
8162 new_stmt
= gimple_build_assign (new_temp
, bitop1
,
8166 = gimple_build_assign (new_temp
, bitop1
, vec_cond_lhs
,
8168 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8169 if (bitop2
== NOP_EXPR
)
8170 vec_compare
= new_temp
;
8171 else if (bitop2
== BIT_NOT_EXPR
)
8173 /* Instead of doing ~x ? y : z do x ? z : y. */
8174 vec_compare
= new_temp
;
8175 std::swap (vec_then_clause
, vec_else_clause
);
8179 vec_compare
= make_ssa_name (vec_cmp_type
);
8181 = gimple_build_assign (vec_compare
, bitop2
,
8182 vec_cond_lhs
, new_temp
);
8183 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8187 new_temp
= make_ssa_name (vec_dest
);
8188 new_stmt
= gimple_build_assign (new_temp
, VEC_COND_EXPR
,
8189 vec_compare
, vec_then_clause
,
8191 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8193 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
8200 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
8202 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
8204 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
8207 vec_oprnds0
.release ();
8208 vec_oprnds1
.release ();
8209 vec_oprnds2
.release ();
8210 vec_oprnds3
.release ();
8215 /* vectorizable_comparison.
8217 Check if STMT is comparison expression that can be vectorized.
8218 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
8219 comparison, put it in VEC_STMT, and insert it at GSI.
8221 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
8224 vectorizable_comparison (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
8225 gimple
**vec_stmt
, tree reduc_def
,
8228 tree lhs
, rhs1
, rhs2
;
8229 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8230 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
8231 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8232 tree vec_rhs1
= NULL_TREE
, vec_rhs2
= NULL_TREE
;
8234 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8235 enum vect_def_type dts
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
8239 enum tree_code code
, bitop1
= NOP_EXPR
, bitop2
= NOP_EXPR
;
8240 stmt_vec_info prev_stmt_info
= NULL
;
8242 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
8243 vec
<tree
> vec_oprnds0
= vNULL
;
8244 vec
<tree
> vec_oprnds1
= vNULL
;
8249 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
8252 if (!vectype
|| !VECTOR_BOOLEAN_TYPE_P (vectype
))
8255 mask_type
= vectype
;
8256 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
8261 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
8263 gcc_assert (ncopies
>= 1);
8264 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
8265 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
8269 if (STMT_VINFO_LIVE_P (stmt_info
))
8271 if (dump_enabled_p ())
8272 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8273 "value used after loop.\n");
8277 if (!is_gimple_assign (stmt
))
8280 code
= gimple_assign_rhs_code (stmt
);
8282 if (TREE_CODE_CLASS (code
) != tcc_comparison
)
8285 rhs1
= gimple_assign_rhs1 (stmt
);
8286 rhs2
= gimple_assign_rhs2 (stmt
);
8288 if (!vect_is_simple_use (rhs1
, stmt_info
->vinfo
, &def_stmt
,
8289 &dts
[0], &vectype1
))
8292 if (!vect_is_simple_use (rhs2
, stmt_info
->vinfo
, &def_stmt
,
8293 &dts
[1], &vectype2
))
8296 if (vectype1
&& vectype2
8297 && TYPE_VECTOR_SUBPARTS (vectype1
) != TYPE_VECTOR_SUBPARTS (vectype2
))
8300 vectype
= vectype1
? vectype1
: vectype2
;
8302 /* Invariant comparison. */
8305 vectype
= get_vectype_for_scalar_type (TREE_TYPE (rhs1
));
8306 if (TYPE_VECTOR_SUBPARTS (vectype
) != nunits
)
8309 else if (nunits
!= TYPE_VECTOR_SUBPARTS (vectype
))
8312 /* Can't compare mask and non-mask types. */
8313 if (vectype1
&& vectype2
8314 && (VECTOR_BOOLEAN_TYPE_P (vectype1
) ^ VECTOR_BOOLEAN_TYPE_P (vectype2
)))
8317 /* Boolean values may have another representation in vectors
8318 and therefore we prefer bit operations over comparison for
8319 them (which also works for scalar masks). We store opcodes
8320 to use in bitop1 and bitop2. Statement is vectorized as
8321 BITOP2 (rhs1 BITOP1 rhs2) or
8322 rhs1 BITOP2 (BITOP1 rhs2)
8323 depending on bitop1 and bitop2 arity. */
8324 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
8326 if (code
== GT_EXPR
)
8328 bitop1
= BIT_NOT_EXPR
;
8329 bitop2
= BIT_AND_EXPR
;
8331 else if (code
== GE_EXPR
)
8333 bitop1
= BIT_NOT_EXPR
;
8334 bitop2
= BIT_IOR_EXPR
;
8336 else if (code
== LT_EXPR
)
8338 bitop1
= BIT_NOT_EXPR
;
8339 bitop2
= BIT_AND_EXPR
;
8340 std::swap (rhs1
, rhs2
);
8341 std::swap (dts
[0], dts
[1]);
8343 else if (code
== LE_EXPR
)
8345 bitop1
= BIT_NOT_EXPR
;
8346 bitop2
= BIT_IOR_EXPR
;
8347 std::swap (rhs1
, rhs2
);
8348 std::swap (dts
[0], dts
[1]);
8352 bitop1
= BIT_XOR_EXPR
;
8353 if (code
== EQ_EXPR
)
8354 bitop2
= BIT_NOT_EXPR
;
8360 STMT_VINFO_TYPE (stmt_info
) = comparison_vec_info_type
;
8361 vect_model_simple_cost (stmt_info
, ncopies
* (1 + (bitop2
!= NOP_EXPR
)),
8362 dts
, ndts
, NULL
, NULL
);
8363 if (bitop1
== NOP_EXPR
)
8364 return expand_vec_cmp_expr_p (vectype
, mask_type
, code
);
8367 machine_mode mode
= TYPE_MODE (vectype
);
8370 optab
= optab_for_tree_code (bitop1
, vectype
, optab_default
);
8371 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8374 if (bitop2
!= NOP_EXPR
)
8376 optab
= optab_for_tree_code (bitop2
, vectype
, optab_default
);
8377 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8387 vec_oprnds0
.create (1);
8388 vec_oprnds1
.create (1);
8392 lhs
= gimple_assign_lhs (stmt
);
8393 mask
= vect_create_destination_var (lhs
, mask_type
);
8395 /* Handle cmp expr. */
8396 for (j
= 0; j
< ncopies
; j
++)
8398 gassign
*new_stmt
= NULL
;
8403 auto_vec
<tree
, 2> ops
;
8404 auto_vec
<vec
<tree
>, 2> vec_defs
;
8406 ops
.safe_push (rhs1
);
8407 ops
.safe_push (rhs2
);
8408 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
8409 vec_oprnds1
= vec_defs
.pop ();
8410 vec_oprnds0
= vec_defs
.pop ();
8414 vec_rhs1
= vect_get_vec_def_for_operand (rhs1
, stmt
, vectype
);
8415 vec_rhs2
= vect_get_vec_def_for_operand (rhs2
, stmt
, vectype
);
8420 vec_rhs1
= vect_get_vec_def_for_stmt_copy (dts
[0],
8421 vec_oprnds0
.pop ());
8422 vec_rhs2
= vect_get_vec_def_for_stmt_copy (dts
[1],
8423 vec_oprnds1
.pop ());
8428 vec_oprnds0
.quick_push (vec_rhs1
);
8429 vec_oprnds1
.quick_push (vec_rhs2
);
8432 /* Arguments are ready. Create the new vector stmt. */
8433 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_rhs1
)
8435 vec_rhs2
= vec_oprnds1
[i
];
8437 new_temp
= make_ssa_name (mask
);
8438 if (bitop1
== NOP_EXPR
)
8440 new_stmt
= gimple_build_assign (new_temp
, code
,
8441 vec_rhs1
, vec_rhs2
);
8442 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8446 if (bitop1
== BIT_NOT_EXPR
)
8447 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs2
);
8449 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs1
,
8451 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8452 if (bitop2
!= NOP_EXPR
)
8454 tree res
= make_ssa_name (mask
);
8455 if (bitop2
== BIT_NOT_EXPR
)
8456 new_stmt
= gimple_build_assign (res
, bitop2
, new_temp
);
8458 new_stmt
= gimple_build_assign (res
, bitop2
, vec_rhs1
,
8460 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
8464 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
8471 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
8473 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
8475 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
8478 vec_oprnds0
.release ();
8479 vec_oprnds1
.release ();
8484 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
8485 can handle all live statements in the node. Otherwise return true
8486 if STMT is not live or if vectorizable_live_operation can handle it.
8487 GSI and VEC_STMT are as for vectorizable_live_operation. */
8490 can_vectorize_live_stmts (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
8491 slp_tree slp_node
, gimple
**vec_stmt
)
8497 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node
), i
, slp_stmt
)
8499 stmt_vec_info slp_stmt_info
= vinfo_for_stmt (slp_stmt
);
8500 if (STMT_VINFO_LIVE_P (slp_stmt_info
)
8501 && !vectorizable_live_operation (slp_stmt
, gsi
, slp_node
, i
,
8506 else if (STMT_VINFO_LIVE_P (vinfo_for_stmt (stmt
))
8507 && !vectorizable_live_operation (stmt
, gsi
, slp_node
, -1, vec_stmt
))
8513 /* Make sure the statement is vectorizable. */
8516 vect_analyze_stmt (gimple
*stmt
, bool *need_to_vectorize
, slp_tree node
,
8517 slp_instance node_instance
)
8519 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8520 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
8521 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
8523 gimple
*pattern_stmt
;
8524 gimple_seq pattern_def_seq
;
8526 if (dump_enabled_p ())
8528 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
8529 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
8532 if (gimple_has_volatile_ops (stmt
))
8534 if (dump_enabled_p ())
8535 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8536 "not vectorized: stmt has volatile operands\n");
8541 /* Skip stmts that do not need to be vectorized. In loops this is expected
8543 - the COND_EXPR which is the loop exit condition
8544 - any LABEL_EXPRs in the loop
8545 - computations that are used only for array indexing or loop control.
8546 In basic blocks we only analyze statements that are a part of some SLP
8547 instance, therefore, all the statements are relevant.
8549 Pattern statement needs to be analyzed instead of the original statement
8550 if the original statement is not relevant. Otherwise, we analyze both
8551 statements. In basic blocks we are called from some SLP instance
8552 traversal, don't analyze pattern stmts instead, the pattern stmts
8553 already will be part of SLP instance. */
8555 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
8556 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
8557 && !STMT_VINFO_LIVE_P (stmt_info
))
8559 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8561 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
8562 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
8564 /* Analyze PATTERN_STMT instead of the original stmt. */
8565 stmt
= pattern_stmt
;
8566 stmt_info
= vinfo_for_stmt (pattern_stmt
);
8567 if (dump_enabled_p ())
8569 dump_printf_loc (MSG_NOTE
, vect_location
,
8570 "==> examining pattern statement: ");
8571 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
8576 if (dump_enabled_p ())
8577 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
8582 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8585 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
8586 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
8588 /* Analyze PATTERN_STMT too. */
8589 if (dump_enabled_p ())
8591 dump_printf_loc (MSG_NOTE
, vect_location
,
8592 "==> examining pattern statement: ");
8593 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
8596 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
,
8601 if (is_pattern_stmt_p (stmt_info
)
8603 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
8605 gimple_stmt_iterator si
;
8607 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
8609 gimple
*pattern_def_stmt
= gsi_stmt (si
);
8610 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
8611 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
8613 /* Analyze def stmt of STMT if it's a pattern stmt. */
8614 if (dump_enabled_p ())
8616 dump_printf_loc (MSG_NOTE
, vect_location
,
8617 "==> examining pattern def statement: ");
8618 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
8621 if (!vect_analyze_stmt (pattern_def_stmt
,
8622 need_to_vectorize
, node
, node_instance
))
8628 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
8630 case vect_internal_def
:
8633 case vect_reduction_def
:
8634 case vect_nested_cycle
:
8635 gcc_assert (!bb_vinfo
8636 && (relevance
== vect_used_in_outer
8637 || relevance
== vect_used_in_outer_by_reduction
8638 || relevance
== vect_used_by_reduction
8639 || relevance
== vect_unused_in_scope
8640 || relevance
== vect_used_only_live
));
8643 case vect_induction_def
:
8644 gcc_assert (!bb_vinfo
);
8647 case vect_constant_def
:
8648 case vect_external_def
:
8649 case vect_unknown_def_type
:
8654 if (STMT_VINFO_RELEVANT_P (stmt_info
))
8656 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
8657 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
8658 || (is_gimple_call (stmt
)
8659 && gimple_call_lhs (stmt
) == NULL_TREE
));
8660 *need_to_vectorize
= true;
8663 if (PURE_SLP_STMT (stmt_info
) && !node
)
8665 dump_printf_loc (MSG_NOTE
, vect_location
,
8666 "handled only by SLP analysis\n");
8672 && (STMT_VINFO_RELEVANT_P (stmt_info
)
8673 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
8674 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
8675 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
8676 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
8677 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
8678 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
8679 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
8680 || vectorizable_call (stmt
, NULL
, NULL
, node
)
8681 || vectorizable_store (stmt
, NULL
, NULL
, node
)
8682 || vectorizable_reduction (stmt
, NULL
, NULL
, node
, node_instance
)
8683 || vectorizable_induction (stmt
, NULL
, NULL
, node
)
8684 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
8685 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
8689 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
8690 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
8691 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
8692 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
8693 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
8694 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
8695 || vectorizable_call (stmt
, NULL
, NULL
, node
)
8696 || vectorizable_store (stmt
, NULL
, NULL
, node
)
8697 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
8698 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
8703 if (dump_enabled_p ())
8705 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8706 "not vectorized: relevant stmt not ");
8707 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
8708 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
8717 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
8718 need extra handling, except for vectorizable reductions. */
8719 if (STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
8720 && !can_vectorize_live_stmts (stmt
, NULL
, node
, NULL
))
8722 if (dump_enabled_p ())
8724 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8725 "not vectorized: live stmt not supported: ");
8726 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
8736 /* Function vect_transform_stmt.
8738 Create a vectorized stmt to replace STMT, and insert it at BSI. */
8741 vect_transform_stmt (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
8742 bool *grouped_store
, slp_tree slp_node
,
8743 slp_instance slp_node_instance
)
8745 bool is_store
= false;
8746 gimple
*vec_stmt
= NULL
;
8747 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8750 gcc_assert (slp_node
|| !PURE_SLP_STMT (stmt_info
));
8751 gimple
*old_vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
8753 switch (STMT_VINFO_TYPE (stmt_info
))
8755 case type_demotion_vec_info_type
:
8756 case type_promotion_vec_info_type
:
8757 case type_conversion_vec_info_type
:
8758 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
8762 case induc_vec_info_type
:
8763 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
, slp_node
);
8767 case shift_vec_info_type
:
8768 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
8772 case op_vec_info_type
:
8773 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
8777 case assignment_vec_info_type
:
8778 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
8782 case load_vec_info_type
:
8783 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
8788 case store_vec_info_type
:
8789 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
8791 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
8793 /* In case of interleaving, the whole chain is vectorized when the
8794 last store in the chain is reached. Store stmts before the last
8795 one are skipped, and there vec_stmt_info shouldn't be freed
8797 *grouped_store
= true;
8798 if (STMT_VINFO_VEC_STMT (stmt_info
))
8805 case condition_vec_info_type
:
8806 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
8810 case comparison_vec_info_type
:
8811 done
= vectorizable_comparison (stmt
, gsi
, &vec_stmt
, NULL
, slp_node
);
8815 case call_vec_info_type
:
8816 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8817 stmt
= gsi_stmt (*gsi
);
8818 if (gimple_call_internal_p (stmt
, IFN_MASK_STORE
))
8822 case call_simd_clone_vec_info_type
:
8823 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8824 stmt
= gsi_stmt (*gsi
);
8827 case reduc_vec_info_type
:
8828 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
,
8834 if (!STMT_VINFO_LIVE_P (stmt_info
))
8836 if (dump_enabled_p ())
8837 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8838 "stmt not supported.\n");
8843 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
8844 This would break hybrid SLP vectorization. */
8846 gcc_assert (!vec_stmt
8847 && STMT_VINFO_VEC_STMT (stmt_info
) == old_vec_stmt
);
8849 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
8850 is being vectorized, but outside the immediately enclosing loop. */
8852 && STMT_VINFO_LOOP_VINFO (stmt_info
)
8853 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
8854 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
8855 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
8856 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
8857 || STMT_VINFO_RELEVANT (stmt_info
) ==
8858 vect_used_in_outer_by_reduction
))
8860 struct loop
*innerloop
= LOOP_VINFO_LOOP (
8861 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
8862 imm_use_iterator imm_iter
;
8863 use_operand_p use_p
;
8867 if (dump_enabled_p ())
8868 dump_printf_loc (MSG_NOTE
, vect_location
,
8869 "Record the vdef for outer-loop vectorization.\n");
8871 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
8872 (to be used when vectorizing outer-loop stmts that use the DEF of
8874 if (gimple_code (stmt
) == GIMPLE_PHI
)
8875 scalar_dest
= PHI_RESULT (stmt
);
8877 scalar_dest
= gimple_assign_lhs (stmt
);
8879 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
8881 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
8883 exit_phi
= USE_STMT (use_p
);
8884 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
8889 /* Handle stmts whose DEF is used outside the loop-nest that is
8890 being vectorized. */
8891 if (STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
8893 done
= can_vectorize_live_stmts (stmt
, gsi
, slp_node
, &vec_stmt
);
8898 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
8904 /* Remove a group of stores (for SLP or interleaving), free their
8908 vect_remove_stores (gimple
*first_stmt
)
8910 gimple
*next
= first_stmt
;
8912 gimple_stmt_iterator next_si
;
8916 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
8918 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
8919 if (is_pattern_stmt_p (stmt_info
))
8920 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
8921 /* Free the attached stmt_vec_info and remove the stmt. */
8922 next_si
= gsi_for_stmt (next
);
8923 unlink_stmt_vdef (next
);
8924 gsi_remove (&next_si
, true);
8925 release_defs (next
);
8926 free_stmt_vec_info (next
);
8932 /* Function new_stmt_vec_info.
8934 Create and initialize a new stmt_vec_info struct for STMT. */
8937 new_stmt_vec_info (gimple
*stmt
, vec_info
*vinfo
)
8940 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
8942 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
8943 STMT_VINFO_STMT (res
) = stmt
;
8945 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
8946 STMT_VINFO_LIVE_P (res
) = false;
8947 STMT_VINFO_VECTYPE (res
) = NULL
;
8948 STMT_VINFO_VEC_STMT (res
) = NULL
;
8949 STMT_VINFO_VECTORIZABLE (res
) = true;
8950 STMT_VINFO_IN_PATTERN_P (res
) = false;
8951 STMT_VINFO_RELATED_STMT (res
) = NULL
;
8952 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
8953 STMT_VINFO_DATA_REF (res
) = NULL
;
8954 STMT_VINFO_VEC_REDUCTION_TYPE (res
) = TREE_CODE_REDUCTION
;
8955 STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res
) = ERROR_MARK
;
8957 if (gimple_code (stmt
) == GIMPLE_PHI
8958 && is_loop_header_bb_p (gimple_bb (stmt
)))
8959 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
8961 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
8963 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
8964 STMT_SLP_TYPE (res
) = loop_vect
;
8965 STMT_VINFO_NUM_SLP_USES (res
) = 0;
8967 GROUP_FIRST_ELEMENT (res
) = NULL
;
8968 GROUP_NEXT_ELEMENT (res
) = NULL
;
8969 GROUP_SIZE (res
) = 0;
8970 GROUP_STORE_COUNT (res
) = 0;
8971 GROUP_GAP (res
) = 0;
8972 GROUP_SAME_DR_STMT (res
) = NULL
;
8978 /* Create a hash table for stmt_vec_info. */
8981 init_stmt_vec_info_vec (void)
8983 gcc_assert (!stmt_vec_info_vec
.exists ());
8984 stmt_vec_info_vec
.create (50);
8988 /* Free hash table for stmt_vec_info. */
8991 free_stmt_vec_info_vec (void)
8995 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
8997 free_stmt_vec_info (STMT_VINFO_STMT (info
));
8998 gcc_assert (stmt_vec_info_vec
.exists ());
8999 stmt_vec_info_vec
.release ();
9003 /* Free stmt vectorization related info. */
9006 free_stmt_vec_info (gimple
*stmt
)
9008 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
9013 /* Check if this statement has a related "pattern stmt"
9014 (introduced by the vectorizer during the pattern recognition
9015 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
9017 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
9019 stmt_vec_info patt_info
9020 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
9023 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
9024 gimple
*patt_stmt
= STMT_VINFO_STMT (patt_info
);
9025 gimple_set_bb (patt_stmt
, NULL
);
9026 tree lhs
= gimple_get_lhs (patt_stmt
);
9027 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
9028 release_ssa_name (lhs
);
9031 gimple_stmt_iterator si
;
9032 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
9034 gimple
*seq_stmt
= gsi_stmt (si
);
9035 gimple_set_bb (seq_stmt
, NULL
);
9036 lhs
= gimple_get_lhs (seq_stmt
);
9037 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
9038 release_ssa_name (lhs
);
9039 free_stmt_vec_info (seq_stmt
);
9042 free_stmt_vec_info (patt_stmt
);
9046 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
9047 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
9048 set_vinfo_for_stmt (stmt
, NULL
);
9053 /* Function get_vectype_for_scalar_type_and_size.
9055 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
9059 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
9061 tree orig_scalar_type
= scalar_type
;
9062 scalar_mode inner_mode
;
9063 machine_mode simd_mode
;
9067 if (!is_int_mode (TYPE_MODE (scalar_type
), &inner_mode
)
9068 && !is_float_mode (TYPE_MODE (scalar_type
), &inner_mode
))
9071 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
9073 /* For vector types of elements whose mode precision doesn't
9074 match their types precision we use a element type of mode
9075 precision. The vectorization routines will have to make sure
9076 they support the proper result truncation/extension.
9077 We also make sure to build vector types with INTEGER_TYPE
9078 component type only. */
9079 if (INTEGRAL_TYPE_P (scalar_type
)
9080 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
9081 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
9082 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
9083 TYPE_UNSIGNED (scalar_type
));
9085 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
9086 When the component mode passes the above test simply use a type
9087 corresponding to that mode. The theory is that any use that
9088 would cause problems with this will disable vectorization anyway. */
9089 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
9090 && !INTEGRAL_TYPE_P (scalar_type
))
9091 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
9093 /* We can't build a vector type of elements with alignment bigger than
9095 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
9096 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
9097 TYPE_UNSIGNED (scalar_type
));
9099 /* If we felt back to using the mode fail if there was
9100 no scalar type for it. */
9101 if (scalar_type
== NULL_TREE
)
9104 /* If no size was supplied use the mode the target prefers. Otherwise
9105 lookup a vector mode of the specified size. */
9107 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
9108 else if (!mode_for_vector (inner_mode
, size
/ nbytes
).exists (&simd_mode
))
9110 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
9111 /* NOTE: nunits == 1 is allowed to support single element vector types. */
9115 vectype
= build_vector_type (scalar_type
, nunits
);
9117 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
9118 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
9121 /* Re-attach the address-space qualifier if we canonicalized the scalar
9123 if (TYPE_ADDR_SPACE (orig_scalar_type
) != TYPE_ADDR_SPACE (vectype
))
9124 return build_qualified_type
9125 (vectype
, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type
)));
9130 unsigned int current_vector_size
;
9132 /* Function get_vectype_for_scalar_type.
9134 Returns the vector type corresponding to SCALAR_TYPE as supported
9138 get_vectype_for_scalar_type (tree scalar_type
)
9141 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
9142 current_vector_size
);
9144 && current_vector_size
== 0)
9145 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
9149 /* Function get_mask_type_for_scalar_type.
9151 Returns the mask type corresponding to a result of comparison
9152 of vectors of specified SCALAR_TYPE as supported by target. */
9155 get_mask_type_for_scalar_type (tree scalar_type
)
9157 tree vectype
= get_vectype_for_scalar_type (scalar_type
);
9162 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype
),
9163 current_vector_size
);
9166 /* Function get_same_sized_vectype
9168 Returns a vector type corresponding to SCALAR_TYPE of size
9169 VECTOR_TYPE if supported by the target. */
9172 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
9174 if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type
))
9175 return build_same_sized_truth_vector_type (vector_type
);
9177 return get_vectype_for_scalar_type_and_size
9178 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
9181 /* Function vect_is_simple_use.
9184 VINFO - the vect info of the loop or basic block that is being vectorized.
9185 OPERAND - operand in the loop or bb.
9187 DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
9188 DT - the type of definition
9190 Returns whether a stmt with OPERAND can be vectorized.
9191 For loops, supportable operands are constants, loop invariants, and operands
9192 that are defined by the current iteration of the loop. Unsupportable
9193 operands are those that are defined by a previous iteration of the loop (as
9194 is the case in reduction/induction computations).
9195 For basic blocks, supportable operands are constants and bb invariants.
9196 For now, operands defined outside the basic block are not supported. */
9199 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
9200 gimple
**def_stmt
, enum vect_def_type
*dt
)
9203 *dt
= vect_unknown_def_type
;
9205 if (dump_enabled_p ())
9207 dump_printf_loc (MSG_NOTE
, vect_location
,
9208 "vect_is_simple_use: operand ");
9209 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
9210 dump_printf (MSG_NOTE
, "\n");
9213 if (CONSTANT_CLASS_P (operand
))
9215 *dt
= vect_constant_def
;
9219 if (is_gimple_min_invariant (operand
))
9221 *dt
= vect_external_def
;
9225 if (TREE_CODE (operand
) != SSA_NAME
)
9227 if (dump_enabled_p ())
9228 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9233 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
9235 *dt
= vect_external_def
;
9239 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
9240 if (dump_enabled_p ())
9242 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
9243 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
9246 if (! vect_stmt_in_region_p (vinfo
, *def_stmt
))
9247 *dt
= vect_external_def
;
9250 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
9251 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
9254 if (dump_enabled_p ())
9256 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
9259 case vect_uninitialized_def
:
9260 dump_printf (MSG_NOTE
, "uninitialized\n");
9262 case vect_constant_def
:
9263 dump_printf (MSG_NOTE
, "constant\n");
9265 case vect_external_def
:
9266 dump_printf (MSG_NOTE
, "external\n");
9268 case vect_internal_def
:
9269 dump_printf (MSG_NOTE
, "internal\n");
9271 case vect_induction_def
:
9272 dump_printf (MSG_NOTE
, "induction\n");
9274 case vect_reduction_def
:
9275 dump_printf (MSG_NOTE
, "reduction\n");
9277 case vect_double_reduction_def
:
9278 dump_printf (MSG_NOTE
, "double reduction\n");
9280 case vect_nested_cycle
:
9281 dump_printf (MSG_NOTE
, "nested cycle\n");
9283 case vect_unknown_def_type
:
9284 dump_printf (MSG_NOTE
, "unknown\n");
9289 if (*dt
== vect_unknown_def_type
)
9291 if (dump_enabled_p ())
9292 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9293 "Unsupported pattern.\n");
9297 switch (gimple_code (*def_stmt
))
9304 if (dump_enabled_p ())
9305 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9306 "unsupported defining stmt:\n");
9313 /* Function vect_is_simple_use.
9315 Same as vect_is_simple_use but also determines the vector operand
9316 type of OPERAND and stores it to *VECTYPE. If the definition of
9317 OPERAND is vect_uninitialized_def, vect_constant_def or
9318 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
9319 is responsible to compute the best suited vector type for the
9323 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
9324 gimple
**def_stmt
, enum vect_def_type
*dt
, tree
*vectype
)
9326 if (!vect_is_simple_use (operand
, vinfo
, def_stmt
, dt
))
9329 /* Now get a vector type if the def is internal, otherwise supply
9330 NULL_TREE and leave it up to the caller to figure out a proper
9331 type for the use stmt. */
9332 if (*dt
== vect_internal_def
9333 || *dt
== vect_induction_def
9334 || *dt
== vect_reduction_def
9335 || *dt
== vect_double_reduction_def
9336 || *dt
== vect_nested_cycle
)
9338 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
9340 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
9341 && !STMT_VINFO_RELEVANT (stmt_info
)
9342 && !STMT_VINFO_LIVE_P (stmt_info
))
9343 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
9345 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
9346 gcc_assert (*vectype
!= NULL_TREE
);
9348 else if (*dt
== vect_uninitialized_def
9349 || *dt
== vect_constant_def
9350 || *dt
== vect_external_def
)
9351 *vectype
= NULL_TREE
;
9359 /* Function supportable_widening_operation
9361 Check whether an operation represented by the code CODE is a
9362 widening operation that is supported by the target platform in
9363 vector form (i.e., when operating on arguments of type VECTYPE_IN
9364 producing a result of type VECTYPE_OUT).
9366 Widening operations we currently support are NOP (CONVERT), FLOAT
9367 and WIDEN_MULT. This function checks if these operations are supported
9368 by the target platform either directly (via vector tree-codes), or via
9372 - CODE1 and CODE2 are codes of vector operations to be used when
9373 vectorizing the operation, if available.
9374 - MULTI_STEP_CVT determines the number of required intermediate steps in
9375 case of multi-step conversion (like char->short->int - in that case
9376 MULTI_STEP_CVT will be 1).
9377 - INTERM_TYPES contains the intermediate type required to perform the
9378 widening operation (short in the above example). */
9381 supportable_widening_operation (enum tree_code code
, gimple
*stmt
,
9382 tree vectype_out
, tree vectype_in
,
9383 enum tree_code
*code1
, enum tree_code
*code2
,
9384 int *multi_step_cvt
,
9385 vec
<tree
> *interm_types
)
9387 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
9388 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
9389 struct loop
*vect_loop
= NULL
;
9390 machine_mode vec_mode
;
9391 enum insn_code icode1
, icode2
;
9392 optab optab1
, optab2
;
9393 tree vectype
= vectype_in
;
9394 tree wide_vectype
= vectype_out
;
9395 enum tree_code c1
, c2
;
9397 tree prev_type
, intermediate_type
;
9398 machine_mode intermediate_mode
, prev_mode
;
9399 optab optab3
, optab4
;
9401 *multi_step_cvt
= 0;
9403 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
9407 case WIDEN_MULT_EXPR
:
9408 /* The result of a vectorized widening operation usually requires
9409 two vectors (because the widened results do not fit into one vector).
9410 The generated vector results would normally be expected to be
9411 generated in the same order as in the original scalar computation,
9412 i.e. if 8 results are generated in each vector iteration, they are
9413 to be organized as follows:
9414 vect1: [res1,res2,res3,res4],
9415 vect2: [res5,res6,res7,res8].
9417 However, in the special case that the result of the widening
9418 operation is used in a reduction computation only, the order doesn't
9419 matter (because when vectorizing a reduction we change the order of
9420 the computation). Some targets can take advantage of this and
9421 generate more efficient code. For example, targets like Altivec,
9422 that support widen_mult using a sequence of {mult_even,mult_odd}
9423 generate the following vectors:
9424 vect1: [res1,res3,res5,res7],
9425 vect2: [res2,res4,res6,res8].
9427 When vectorizing outer-loops, we execute the inner-loop sequentially
9428 (each vectorized inner-loop iteration contributes to VF outer-loop
9429 iterations in parallel). We therefore don't allow to change the
9430 order of the computation in the inner-loop during outer-loop
9432 /* TODO: Another case in which order doesn't *really* matter is when we
9433 widen and then contract again, e.g. (short)((int)x * y >> 8).
9434 Normally, pack_trunc performs an even/odd permute, whereas the
9435 repack from an even/odd expansion would be an interleave, which
9436 would be significantly simpler for e.g. AVX2. */
9437 /* In any case, in order to avoid duplicating the code below, recurse
9438 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
9439 are properly set up for the caller. If we fail, we'll continue with
9440 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
9442 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
9443 && !nested_in_vect_loop_p (vect_loop
, stmt
)
9444 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
9445 stmt
, vectype_out
, vectype_in
,
9446 code1
, code2
, multi_step_cvt
,
9449 /* Elements in a vector with vect_used_by_reduction property cannot
9450 be reordered if the use chain with this property does not have the
9451 same operation. One such an example is s += a * b, where elements
9452 in a and b cannot be reordered. Here we check if the vector defined
9453 by STMT is only directly used in the reduction statement. */
9454 tree lhs
= gimple_assign_lhs (stmt
);
9455 use_operand_p dummy
;
9457 stmt_vec_info use_stmt_info
= NULL
;
9458 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
9459 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
9460 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
9463 c1
= VEC_WIDEN_MULT_LO_EXPR
;
9464 c2
= VEC_WIDEN_MULT_HI_EXPR
;
9477 case VEC_WIDEN_MULT_EVEN_EXPR
:
9478 /* Support the recursion induced just above. */
9479 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
9480 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
9483 case WIDEN_LSHIFT_EXPR
:
9484 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
9485 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
9489 c1
= VEC_UNPACK_LO_EXPR
;
9490 c2
= VEC_UNPACK_HI_EXPR
;
9494 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
9495 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
9498 case FIX_TRUNC_EXPR
:
9499 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
9500 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
9501 computing the operation. */
9508 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
9511 if (code
== FIX_TRUNC_EXPR
)
9513 /* The signedness is determined from output operand. */
9514 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
9515 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
9519 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
9520 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
9523 if (!optab1
|| !optab2
)
9526 vec_mode
= TYPE_MODE (vectype
);
9527 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
9528 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
9534 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
9535 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
9536 /* For scalar masks we may have different boolean
9537 vector types having the same QImode. Thus we
9538 add additional check for elements number. */
9539 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9540 || (TYPE_VECTOR_SUBPARTS (vectype
) / 2
9541 == TYPE_VECTOR_SUBPARTS (wide_vectype
)));
9543 /* Check if it's a multi-step conversion that can be done using intermediate
9546 prev_type
= vectype
;
9547 prev_mode
= vec_mode
;
9549 if (!CONVERT_EXPR_CODE_P (code
))
9552 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
9553 intermediate steps in promotion sequence. We try
9554 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
9556 interm_types
->create (MAX_INTERM_CVT_STEPS
);
9557 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
9559 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
9560 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
9563 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type
) / 2,
9564 current_vector_size
);
9565 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
9570 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
9571 TYPE_UNSIGNED (prev_type
));
9573 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
9574 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
9576 if (!optab3
|| !optab4
9577 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
9578 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
9579 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
9580 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
9581 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
9582 == CODE_FOR_nothing
)
9583 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
9584 == CODE_FOR_nothing
))
9587 interm_types
->quick_push (intermediate_type
);
9588 (*multi_step_cvt
)++;
9590 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
9591 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
9592 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9593 || (TYPE_VECTOR_SUBPARTS (intermediate_type
) / 2
9594 == TYPE_VECTOR_SUBPARTS (wide_vectype
)));
9596 prev_type
= intermediate_type
;
9597 prev_mode
= intermediate_mode
;
9600 interm_types
->release ();
9605 /* Function supportable_narrowing_operation
9607 Check whether an operation represented by the code CODE is a
9608 narrowing operation that is supported by the target platform in
9609 vector form (i.e., when operating on arguments of type VECTYPE_IN
9610 and producing a result of type VECTYPE_OUT).
9612 Narrowing operations we currently support are NOP (CONVERT) and
9613 FIX_TRUNC. This function checks if these operations are supported by
9614 the target platform directly via vector tree-codes.
9617 - CODE1 is the code of a vector operation to be used when
9618 vectorizing the operation, if available.
9619 - MULTI_STEP_CVT determines the number of required intermediate steps in
9620 case of multi-step conversion (like int->short->char - in that case
9621 MULTI_STEP_CVT will be 1).
9622 - INTERM_TYPES contains the intermediate type required to perform the
9623 narrowing operation (short in the above example). */
9626 supportable_narrowing_operation (enum tree_code code
,
9627 tree vectype_out
, tree vectype_in
,
9628 enum tree_code
*code1
, int *multi_step_cvt
,
9629 vec
<tree
> *interm_types
)
9631 machine_mode vec_mode
;
9632 enum insn_code icode1
;
9633 optab optab1
, interm_optab
;
9634 tree vectype
= vectype_in
;
9635 tree narrow_vectype
= vectype_out
;
9637 tree intermediate_type
, prev_type
;
9638 machine_mode intermediate_mode
, prev_mode
;
9642 *multi_step_cvt
= 0;
9646 c1
= VEC_PACK_TRUNC_EXPR
;
9649 case FIX_TRUNC_EXPR
:
9650 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
9654 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
9655 tree code and optabs used for computing the operation. */
9662 if (code
== FIX_TRUNC_EXPR
)
9663 /* The signedness is determined from output operand. */
9664 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
9666 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
9671 vec_mode
= TYPE_MODE (vectype
);
9672 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
9677 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
9678 /* For scalar masks we may have different boolean
9679 vector types having the same QImode. Thus we
9680 add additional check for elements number. */
9681 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9682 || (TYPE_VECTOR_SUBPARTS (vectype
) * 2
9683 == TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
9685 /* Check if it's a multi-step conversion that can be done using intermediate
9687 prev_mode
= vec_mode
;
9688 prev_type
= vectype
;
9689 if (code
== FIX_TRUNC_EXPR
)
9690 uns
= TYPE_UNSIGNED (vectype_out
);
9692 uns
= TYPE_UNSIGNED (vectype
);
9694 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
9695 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
9696 costly than signed. */
9697 if (code
== FIX_TRUNC_EXPR
&& uns
)
9699 enum insn_code icode2
;
9702 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
9704 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
9705 if (interm_optab
!= unknown_optab
9706 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
9707 && insn_data
[icode1
].operand
[0].mode
9708 == insn_data
[icode2
].operand
[0].mode
)
9711 optab1
= interm_optab
;
9716 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
9717 intermediate steps in promotion sequence. We try
9718 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
9719 interm_types
->create (MAX_INTERM_CVT_STEPS
);
9720 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
9722 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
9723 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
9726 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type
) * 2,
9727 current_vector_size
);
9728 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
9733 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
9735 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
9738 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
9739 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
9740 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
9741 == CODE_FOR_nothing
))
9744 interm_types
->quick_push (intermediate_type
);
9745 (*multi_step_cvt
)++;
9747 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
9748 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
9749 || (TYPE_VECTOR_SUBPARTS (intermediate_type
) * 2
9750 == TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
9752 prev_mode
= intermediate_mode
;
9753 prev_type
= intermediate_type
;
9754 optab1
= interm_optab
;
9757 interm_types
->release ();