1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2014 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
28 #include "stor-layout.h"
30 #include "basic-block.h"
31 #include "gimple-pretty-print.h"
32 #include "tree-ssa-alias.h"
33 #include "internal-fn.h"
35 #include "gimple-expr.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-ssa.h"
43 #include "tree-phinodes.h"
44 #include "ssa-iterators.h"
45 #include "stringpool.h"
46 #include "tree-ssanames.h"
47 #include "tree-ssa-loop-manip.h"
49 #include "tree-ssa-loop.h"
50 #include "tree-scalar-evolution.h"
52 #include "recog.h" /* FIXME: for insn_data */
54 #include "diagnostic-core.h"
55 #include "tree-vectorizer.h"
60 /* For lang_hooks.types.type_for_mode. */
61 #include "langhooks.h"
63 /* Return the vectorized type for the given statement. */
66 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
68 return STMT_VINFO_VECTYPE (stmt_info
);
71 /* Return TRUE iff the given statement is in an inner loop relative to
72 the loop being vectorized. */
74 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
76 gimple stmt
= STMT_VINFO_STMT (stmt_info
);
77 basic_block bb
= gimple_bb (stmt
);
78 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
84 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
86 return (bb
->loop_father
== loop
->inner
);
89 /* Record the cost of a statement, either by directly informing the
90 target model or by saving it in a vector for later processing.
91 Return a preliminary estimate of the statement's cost. */
94 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
95 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
96 int misalign
, enum vect_cost_model_location where
)
100 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
101 add_stmt_info_to_vec (body_cost_vec
, count
, kind
,
102 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
105 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
110 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
111 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
112 void *target_cost_data
;
115 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
117 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
119 return add_stmt_cost (target_cost_data
, count
, kind
, stmt_info
,
124 /* Return a variable of type ELEM_TYPE[NELEMS]. */
127 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
129 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
133 /* ARRAY is an array of vectors created by create_vector_array.
134 Return an SSA_NAME for the vector in index N. The reference
135 is part of the vectorization of STMT and the vector is associated
136 with scalar destination SCALAR_DEST. */
139 read_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
140 tree array
, unsigned HOST_WIDE_INT n
)
142 tree vect_type
, vect
, vect_name
, array_ref
;
145 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
146 vect_type
= TREE_TYPE (TREE_TYPE (array
));
147 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
148 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
149 build_int_cst (size_type_node
, n
),
150 NULL_TREE
, NULL_TREE
);
152 new_stmt
= gimple_build_assign (vect
, array_ref
);
153 vect_name
= make_ssa_name (vect
, new_stmt
);
154 gimple_assign_set_lhs (new_stmt
, vect_name
);
155 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
160 /* ARRAY is an array of vectors created by create_vector_array.
161 Emit code to store SSA_NAME VECT in index N of the array.
162 The store is part of the vectorization of STMT. */
165 write_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
166 tree array
, unsigned HOST_WIDE_INT n
)
171 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
172 build_int_cst (size_type_node
, n
),
173 NULL_TREE
, NULL_TREE
);
175 new_stmt
= gimple_build_assign (array_ref
, vect
);
176 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
179 /* PTR is a pointer to an array of type TYPE. Return a representation
180 of *PTR. The memory reference replaces those in FIRST_DR
184 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
186 tree mem_ref
, alias_ptr_type
;
188 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
189 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
190 /* Arrays have the same alignment as their type. */
191 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
195 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
197 /* Function vect_mark_relevant.
199 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
202 vect_mark_relevant (vec
<gimple
> *worklist
, gimple stmt
,
203 enum vect_relevant relevant
, bool live_p
,
204 bool used_in_pattern
)
206 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
207 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
208 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
211 if (dump_enabled_p ())
212 dump_printf_loc (MSG_NOTE
, vect_location
,
213 "mark relevant %d, live %d.\n", relevant
, live_p
);
215 /* If this stmt is an original stmt in a pattern, we might need to mark its
216 related pattern stmt instead of the original stmt. However, such stmts
217 may have their own uses that are not in any pattern, in such cases the
218 stmt itself should be marked. */
219 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
222 if (!used_in_pattern
)
224 imm_use_iterator imm_iter
;
228 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
229 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
231 if (is_gimple_assign (stmt
))
232 lhs
= gimple_assign_lhs (stmt
);
234 lhs
= gimple_call_lhs (stmt
);
236 /* This use is out of pattern use, if LHS has other uses that are
237 pattern uses, we should mark the stmt itself, and not the pattern
239 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
240 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
242 if (is_gimple_debug (USE_STMT (use_p
)))
244 use_stmt
= USE_STMT (use_p
);
246 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
249 if (vinfo_for_stmt (use_stmt
)
250 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
260 /* This is the last stmt in a sequence that was detected as a
261 pattern that can potentially be vectorized. Don't mark the stmt
262 as relevant/live because it's not going to be vectorized.
263 Instead mark the pattern-stmt that replaces it. */
265 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
267 if (dump_enabled_p ())
268 dump_printf_loc (MSG_NOTE
, vect_location
,
269 "last stmt in pattern. don't mark"
270 " relevant/live.\n");
271 stmt_info
= vinfo_for_stmt (pattern_stmt
);
272 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
273 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
274 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
279 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
280 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
281 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
283 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
284 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
286 if (dump_enabled_p ())
287 dump_printf_loc (MSG_NOTE
, vect_location
,
288 "already marked relevant/live.\n");
292 worklist
->safe_push (stmt
);
296 /* Function vect_stmt_relevant_p.
298 Return true if STMT in loop that is represented by LOOP_VINFO is
299 "relevant for vectorization".
301 A stmt is considered "relevant for vectorization" if:
302 - it has uses outside the loop.
303 - it has vdefs (it alters memory).
304 - control stmts in the loop (except for the exit condition).
306 CHECKME: what other side effects would the vectorizer allow? */
309 vect_stmt_relevant_p (gimple stmt
, loop_vec_info loop_vinfo
,
310 enum vect_relevant
*relevant
, bool *live_p
)
312 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
314 imm_use_iterator imm_iter
;
318 *relevant
= vect_unused_in_scope
;
321 /* cond stmt other than loop exit cond. */
322 if (is_ctrl_stmt (stmt
)
323 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
324 != loop_exit_ctrl_vec_info_type
)
325 *relevant
= vect_used_in_scope
;
327 /* changing memory. */
328 if (gimple_code (stmt
) != GIMPLE_PHI
)
329 if (gimple_vdef (stmt
))
331 if (dump_enabled_p ())
332 dump_printf_loc (MSG_NOTE
, vect_location
,
333 "vec_stmt_relevant_p: stmt has vdefs.\n");
334 *relevant
= vect_used_in_scope
;
337 /* uses outside the loop. */
338 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
340 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
342 basic_block bb
= gimple_bb (USE_STMT (use_p
));
343 if (!flow_bb_inside_loop_p (loop
, bb
))
345 if (dump_enabled_p ())
346 dump_printf_loc (MSG_NOTE
, vect_location
,
347 "vec_stmt_relevant_p: used out of loop.\n");
349 if (is_gimple_debug (USE_STMT (use_p
)))
352 /* We expect all such uses to be in the loop exit phis
353 (because of loop closed form) */
354 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
355 gcc_assert (bb
== single_exit (loop
)->dest
);
362 return (*live_p
|| *relevant
);
366 /* Function exist_non_indexing_operands_for_use_p
368 USE is one of the uses attached to STMT. Check if USE is
369 used in STMT for anything other than indexing an array. */
372 exist_non_indexing_operands_for_use_p (tree use
, gimple stmt
)
375 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
377 /* USE corresponds to some operand in STMT. If there is no data
378 reference in STMT, then any operand that corresponds to USE
379 is not indexing an array. */
380 if (!STMT_VINFO_DATA_REF (stmt_info
))
383 /* STMT has a data_ref. FORNOW this means that its of one of
387 (This should have been verified in analyze_data_refs).
389 'var' in the second case corresponds to a def, not a use,
390 so USE cannot correspond to any operands that are not used
393 Therefore, all we need to check is if STMT falls into the
394 first case, and whether var corresponds to USE. */
396 if (!gimple_assign_copy_p (stmt
))
398 if (is_gimple_call (stmt
)
399 && gimple_call_internal_p (stmt
))
400 switch (gimple_call_internal_fn (stmt
))
403 operand
= gimple_call_arg (stmt
, 3);
408 operand
= gimple_call_arg (stmt
, 2);
418 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
420 operand
= gimple_assign_rhs1 (stmt
);
421 if (TREE_CODE (operand
) != SSA_NAME
)
432 Function process_use.
435 - a USE in STMT in a loop represented by LOOP_VINFO
436 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
437 that defined USE. This is done by calling mark_relevant and passing it
438 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
439 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
443 Generally, LIVE_P and RELEVANT are used to define the liveness and
444 relevance info of the DEF_STMT of this USE:
445 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
446 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
448 - case 1: If USE is used only for address computations (e.g. array indexing),
449 which does not need to be directly vectorized, then the liveness/relevance
450 of the respective DEF_STMT is left unchanged.
451 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
452 skip DEF_STMT cause it had already been processed.
453 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
454 be modified accordingly.
456 Return true if everything is as expected. Return false otherwise. */
459 process_use (gimple stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
460 enum vect_relevant relevant
, vec
<gimple
> *worklist
,
463 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
464 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
465 stmt_vec_info dstmt_vinfo
;
466 basic_block bb
, def_bb
;
469 enum vect_def_type dt
;
471 /* case 1: we are only interested in uses that need to be vectorized. Uses
472 that are used for address computation are not considered relevant. */
473 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
476 if (!vect_is_simple_use (use
, stmt
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
))
478 if (dump_enabled_p ())
479 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
480 "not vectorized: unsupported use in stmt.\n");
484 if (!def_stmt
|| gimple_nop_p (def_stmt
))
487 def_bb
= gimple_bb (def_stmt
);
488 if (!flow_bb_inside_loop_p (loop
, def_bb
))
490 if (dump_enabled_p ())
491 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
495 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
496 DEF_STMT must have already been processed, because this should be the
497 only way that STMT, which is a reduction-phi, was put in the worklist,
498 as there should be no other uses for DEF_STMT in the loop. So we just
499 check that everything is as expected, and we are done. */
500 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
501 bb
= gimple_bb (stmt
);
502 if (gimple_code (stmt
) == GIMPLE_PHI
503 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
504 && gimple_code (def_stmt
) != GIMPLE_PHI
505 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
506 && bb
->loop_father
== def_bb
->loop_father
)
508 if (dump_enabled_p ())
509 dump_printf_loc (MSG_NOTE
, vect_location
,
510 "reduc-stmt defining reduc-phi in the same nest.\n");
511 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
512 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
513 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
514 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
515 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
519 /* case 3a: outer-loop stmt defining an inner-loop stmt:
520 outer-loop-header-bb:
526 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
528 if (dump_enabled_p ())
529 dump_printf_loc (MSG_NOTE
, vect_location
,
530 "outer-loop def-stmt defining inner-loop stmt.\n");
534 case vect_unused_in_scope
:
535 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
536 vect_used_in_scope
: vect_unused_in_scope
;
539 case vect_used_in_outer_by_reduction
:
540 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
541 relevant
= vect_used_by_reduction
;
544 case vect_used_in_outer
:
545 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
546 relevant
= vect_used_in_scope
;
549 case vect_used_in_scope
:
557 /* case 3b: inner-loop stmt defining an outer-loop stmt:
558 outer-loop-header-bb:
562 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
564 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
566 if (dump_enabled_p ())
567 dump_printf_loc (MSG_NOTE
, vect_location
,
568 "inner-loop def-stmt defining outer-loop stmt.\n");
572 case vect_unused_in_scope
:
573 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
574 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
575 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
578 case vect_used_by_reduction
:
579 relevant
= vect_used_in_outer_by_reduction
;
582 case vect_used_in_scope
:
583 relevant
= vect_used_in_outer
;
591 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
,
592 is_pattern_stmt_p (stmt_vinfo
));
597 /* Function vect_mark_stmts_to_be_vectorized.
599 Not all stmts in the loop need to be vectorized. For example:
608 Stmt 1 and 3 do not need to be vectorized, because loop control and
609 addressing of vectorized data-refs are handled differently.
611 This pass detects such stmts. */
614 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
616 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
617 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
618 unsigned int nbbs
= loop
->num_nodes
;
619 gimple_stmt_iterator si
;
622 stmt_vec_info stmt_vinfo
;
626 enum vect_relevant relevant
, tmp_relevant
;
627 enum vect_def_type def_type
;
629 if (dump_enabled_p ())
630 dump_printf_loc (MSG_NOTE
, vect_location
,
631 "=== vect_mark_stmts_to_be_vectorized ===\n");
633 auto_vec
<gimple
, 64> worklist
;
635 /* 1. Init worklist. */
636 for (i
= 0; i
< nbbs
; i
++)
639 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
642 if (dump_enabled_p ())
644 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
645 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
646 dump_printf (MSG_NOTE
, "\n");
649 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
650 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
, false);
652 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
654 stmt
= gsi_stmt (si
);
655 if (dump_enabled_p ())
657 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
658 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
659 dump_printf (MSG_NOTE
, "\n");
662 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
663 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
, false);
667 /* 2. Process_worklist */
668 while (worklist
.length () > 0)
673 stmt
= worklist
.pop ();
674 if (dump_enabled_p ())
676 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
677 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
678 dump_printf (MSG_NOTE
, "\n");
681 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
682 (DEF_STMT) as relevant/irrelevant and live/dead according to the
683 liveness and relevance properties of STMT. */
684 stmt_vinfo
= vinfo_for_stmt (stmt
);
685 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
686 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
688 /* Generally, the liveness and relevance properties of STMT are
689 propagated as is to the DEF_STMTs of its USEs:
690 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
691 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
693 One exception is when STMT has been identified as defining a reduction
694 variable; in this case we set the liveness/relevance as follows:
696 relevant = vect_used_by_reduction
697 This is because we distinguish between two kinds of relevant stmts -
698 those that are used by a reduction computation, and those that are
699 (also) used by a regular computation. This allows us later on to
700 identify stmts that are used solely by a reduction, and therefore the
701 order of the results that they produce does not have to be kept. */
703 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
704 tmp_relevant
= relevant
;
707 case vect_reduction_def
:
708 switch (tmp_relevant
)
710 case vect_unused_in_scope
:
711 relevant
= vect_used_by_reduction
;
714 case vect_used_by_reduction
:
715 if (gimple_code (stmt
) == GIMPLE_PHI
)
720 if (dump_enabled_p ())
721 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
722 "unsupported use of reduction.\n");
729 case vect_nested_cycle
:
730 if (tmp_relevant
!= vect_unused_in_scope
731 && tmp_relevant
!= vect_used_in_outer_by_reduction
732 && tmp_relevant
!= vect_used_in_outer
)
734 if (dump_enabled_p ())
735 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
736 "unsupported use of nested cycle.\n");
744 case vect_double_reduction_def
:
745 if (tmp_relevant
!= vect_unused_in_scope
746 && tmp_relevant
!= vect_used_by_reduction
)
748 if (dump_enabled_p ())
749 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
750 "unsupported use of double reduction.\n");
762 if (is_pattern_stmt_p (stmt_vinfo
))
764 /* Pattern statements are not inserted into the code, so
765 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
766 have to scan the RHS or function arguments instead. */
767 if (is_gimple_assign (stmt
))
769 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
770 tree op
= gimple_assign_rhs1 (stmt
);
773 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
775 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
776 live_p
, relevant
, &worklist
, false)
777 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
778 live_p
, relevant
, &worklist
, false))
782 for (; i
< gimple_num_ops (stmt
); i
++)
784 op
= gimple_op (stmt
, i
);
785 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
790 else if (is_gimple_call (stmt
))
792 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
794 tree arg
= gimple_call_arg (stmt
, i
);
795 if (!process_use (stmt
, arg
, loop_vinfo
, live_p
, relevant
,
802 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
804 tree op
= USE_FROM_PTR (use_p
);
805 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
810 if (STMT_VINFO_GATHER_P (stmt_vinfo
))
813 tree decl
= vect_check_gather (stmt
, loop_vinfo
, NULL
, &off
, NULL
);
815 if (!process_use (stmt
, off
, loop_vinfo
, live_p
, relevant
,
819 } /* while worklist */
825 /* Function vect_model_simple_cost.
827 Models cost for simple operations, i.e. those that only emit ncopies of a
828 single op. Right now, this does not account for multiple insns that could
829 be generated for the single vector op. We will handle that shortly. */
832 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
833 enum vect_def_type
*dt
,
834 stmt_vector_for_cost
*prologue_cost_vec
,
835 stmt_vector_for_cost
*body_cost_vec
)
838 int inside_cost
= 0, prologue_cost
= 0;
840 /* The SLP costs were already calculated during SLP tree build. */
841 if (PURE_SLP_STMT (stmt_info
))
844 /* FORNOW: Assuming maximum 2 args per stmts. */
845 for (i
= 0; i
< 2; i
++)
846 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
847 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
848 stmt_info
, 0, vect_prologue
);
850 /* Pass the inside-of-loop statements to the target-specific cost model. */
851 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
852 stmt_info
, 0, vect_body
);
854 if (dump_enabled_p ())
855 dump_printf_loc (MSG_NOTE
, vect_location
,
856 "vect_model_simple_cost: inside_cost = %d, "
857 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
861 /* Model cost for type demotion and promotion operations. PWR is normally
862 zero for single-step promotions and demotions. It will be one if
863 two-step promotion/demotion is required, and so on. Each additional
864 step doubles the number of instructions required. */
867 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
868 enum vect_def_type
*dt
, int pwr
)
871 int inside_cost
= 0, prologue_cost
= 0;
872 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
873 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
874 void *target_cost_data
;
876 /* The SLP costs were already calculated during SLP tree build. */
877 if (PURE_SLP_STMT (stmt_info
))
881 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
883 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
885 for (i
= 0; i
< pwr
+ 1; i
++)
887 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
889 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
890 vec_promote_demote
, stmt_info
, 0,
894 /* FORNOW: Assuming maximum 2 args per stmts. */
895 for (i
= 0; i
< 2; i
++)
896 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
897 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
898 stmt_info
, 0, vect_prologue
);
900 if (dump_enabled_p ())
901 dump_printf_loc (MSG_NOTE
, vect_location
,
902 "vect_model_promotion_demotion_cost: inside_cost = %d, "
903 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
906 /* Function vect_cost_group_size
908 For grouped load or store, return the group_size only if it is the first
909 load or store of a group, else return 1. This ensures that group size is
910 only returned once per group. */
913 vect_cost_group_size (stmt_vec_info stmt_info
)
915 gimple first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
917 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
918 return GROUP_SIZE (stmt_info
);
924 /* Function vect_model_store_cost
926 Models cost for stores. In the case of grouped accesses, one access
927 has the overhead of the grouped access attributed to it. */
930 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
931 bool store_lanes_p
, enum vect_def_type dt
,
933 stmt_vector_for_cost
*prologue_cost_vec
,
934 stmt_vector_for_cost
*body_cost_vec
)
937 unsigned int inside_cost
= 0, prologue_cost
= 0;
938 struct data_reference
*first_dr
;
941 /* The SLP costs were already calculated during SLP tree build. */
942 if (PURE_SLP_STMT (stmt_info
))
945 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
946 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
947 stmt_info
, 0, vect_prologue
);
949 /* Grouped access? */
950 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
954 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
959 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
960 group_size
= vect_cost_group_size (stmt_info
);
963 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
965 /* Not a grouped access. */
969 first_dr
= STMT_VINFO_DATA_REF (stmt_info
);
972 /* We assume that the cost of a single store-lanes instruction is
973 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
974 access is instead being provided by a permute-and-store operation,
975 include the cost of the permutes. */
976 if (!store_lanes_p
&& group_size
> 1)
978 /* Uses a high and low interleave or shuffle operations for each
980 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
981 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
982 stmt_info
, 0, vect_body
);
984 if (dump_enabled_p ())
985 dump_printf_loc (MSG_NOTE
, vect_location
,
986 "vect_model_store_cost: strided group_size = %d .\n",
990 /* Costs of the stores. */
991 vect_get_store_cost (first_dr
, ncopies
, &inside_cost
, body_cost_vec
);
993 if (dump_enabled_p ())
994 dump_printf_loc (MSG_NOTE
, vect_location
,
995 "vect_model_store_cost: inside_cost = %d, "
996 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1000 /* Calculate cost of DR's memory access. */
1002 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
1003 unsigned int *inside_cost
,
1004 stmt_vector_for_cost
*body_cost_vec
)
1006 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1007 gimple stmt
= DR_STMT (dr
);
1008 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1010 switch (alignment_support_scheme
)
1014 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1015 vector_store
, stmt_info
, 0,
1018 if (dump_enabled_p ())
1019 dump_printf_loc (MSG_NOTE
, vect_location
,
1020 "vect_model_store_cost: aligned.\n");
1024 case dr_unaligned_supported
:
1026 /* Here, we assign an additional cost for the unaligned store. */
1027 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1028 unaligned_store
, stmt_info
,
1029 DR_MISALIGNMENT (dr
), vect_body
);
1030 if (dump_enabled_p ())
1031 dump_printf_loc (MSG_NOTE
, vect_location
,
1032 "vect_model_store_cost: unaligned supported by "
1037 case dr_unaligned_unsupported
:
1039 *inside_cost
= VECT_MAX_COST
;
1041 if (dump_enabled_p ())
1042 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1043 "vect_model_store_cost: unsupported access.\n");
1053 /* Function vect_model_load_cost
1055 Models cost for loads. In the case of grouped accesses, the last access
1056 has the overhead of the grouped access attributed to it. Since unaligned
1057 accesses are supported for loads, we also account for the costs of the
1058 access scheme chosen. */
1061 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1062 bool load_lanes_p
, slp_tree slp_node
,
1063 stmt_vector_for_cost
*prologue_cost_vec
,
1064 stmt_vector_for_cost
*body_cost_vec
)
1068 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
1069 unsigned int inside_cost
= 0, prologue_cost
= 0;
1071 /* The SLP costs were already calculated during SLP tree build. */
1072 if (PURE_SLP_STMT (stmt_info
))
1075 /* Grouped accesses? */
1076 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1077 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && first_stmt
&& !slp_node
)
1079 group_size
= vect_cost_group_size (stmt_info
);
1080 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1082 /* Not a grouped access. */
1089 /* We assume that the cost of a single load-lanes instruction is
1090 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1091 access is instead being provided by a load-and-permute operation,
1092 include the cost of the permutes. */
1093 if (!load_lanes_p
&& group_size
> 1)
1095 /* Uses an even and odd extract operations or shuffle operations
1096 for each needed permute. */
1097 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1098 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1099 stmt_info
, 0, vect_body
);
1101 if (dump_enabled_p ())
1102 dump_printf_loc (MSG_NOTE
, vect_location
,
1103 "vect_model_load_cost: strided group_size = %d .\n",
1107 /* The loads themselves. */
1108 if (STMT_VINFO_STRIDE_LOAD_P (stmt_info
))
1110 /* N scalar loads plus gathering them into a vector. */
1111 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1112 inside_cost
+= record_stmt_cost (body_cost_vec
,
1113 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1114 scalar_load
, stmt_info
, 0, vect_body
);
1115 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1116 stmt_info
, 0, vect_body
);
1119 vect_get_load_cost (first_dr
, ncopies
,
1120 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1121 || group_size
> 1 || slp_node
),
1122 &inside_cost
, &prologue_cost
,
1123 prologue_cost_vec
, body_cost_vec
, true);
1125 if (dump_enabled_p ())
1126 dump_printf_loc (MSG_NOTE
, vect_location
,
1127 "vect_model_load_cost: inside_cost = %d, "
1128 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1132 /* Calculate cost of DR's memory access. */
1134 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1135 bool add_realign_cost
, unsigned int *inside_cost
,
1136 unsigned int *prologue_cost
,
1137 stmt_vector_for_cost
*prologue_cost_vec
,
1138 stmt_vector_for_cost
*body_cost_vec
,
1139 bool record_prologue_costs
)
1141 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1142 gimple stmt
= DR_STMT (dr
);
1143 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1145 switch (alignment_support_scheme
)
1149 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1150 stmt_info
, 0, vect_body
);
1152 if (dump_enabled_p ())
1153 dump_printf_loc (MSG_NOTE
, vect_location
,
1154 "vect_model_load_cost: aligned.\n");
1158 case dr_unaligned_supported
:
1160 /* Here, we assign an additional cost for the unaligned load. */
1161 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1162 unaligned_load
, stmt_info
,
1163 DR_MISALIGNMENT (dr
), vect_body
);
1165 if (dump_enabled_p ())
1166 dump_printf_loc (MSG_NOTE
, vect_location
,
1167 "vect_model_load_cost: unaligned supported by "
1172 case dr_explicit_realign
:
1174 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1175 vector_load
, stmt_info
, 0, vect_body
);
1176 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1177 vec_perm
, stmt_info
, 0, vect_body
);
1179 /* FIXME: If the misalignment remains fixed across the iterations of
1180 the containing loop, the following cost should be added to the
1182 if (targetm
.vectorize
.builtin_mask_for_load
)
1183 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1184 stmt_info
, 0, vect_body
);
1186 if (dump_enabled_p ())
1187 dump_printf_loc (MSG_NOTE
, vect_location
,
1188 "vect_model_load_cost: explicit realign\n");
1192 case dr_explicit_realign_optimized
:
1194 if (dump_enabled_p ())
1195 dump_printf_loc (MSG_NOTE
, vect_location
,
1196 "vect_model_load_cost: unaligned software "
1199 /* Unaligned software pipeline has a load of an address, an initial
1200 load, and possibly a mask operation to "prime" the loop. However,
1201 if this is an access in a group of loads, which provide grouped
1202 access, then the above cost should only be considered for one
1203 access in the group. Inside the loop, there is a load op
1204 and a realignment op. */
1206 if (add_realign_cost
&& record_prologue_costs
)
1208 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1209 vector_stmt
, stmt_info
,
1211 if (targetm
.vectorize
.builtin_mask_for_load
)
1212 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1213 vector_stmt
, stmt_info
,
1217 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1218 stmt_info
, 0, vect_body
);
1219 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1220 stmt_info
, 0, vect_body
);
1222 if (dump_enabled_p ())
1223 dump_printf_loc (MSG_NOTE
, vect_location
,
1224 "vect_model_load_cost: explicit realign optimized"
1230 case dr_unaligned_unsupported
:
1232 *inside_cost
= VECT_MAX_COST
;
1234 if (dump_enabled_p ())
1235 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1236 "vect_model_load_cost: unsupported access.\n");
1245 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1246 the loop preheader for the vectorized stmt STMT. */
1249 vect_init_vector_1 (gimple stmt
, gimple new_stmt
, gimple_stmt_iterator
*gsi
)
1252 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1255 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1256 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1260 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1264 if (nested_in_vect_loop_p (loop
, stmt
))
1267 pe
= loop_preheader_edge (loop
);
1268 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1269 gcc_assert (!new_bb
);
1273 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1275 gimple_stmt_iterator gsi_bb_start
;
1277 gcc_assert (bb_vinfo
);
1278 bb
= BB_VINFO_BB (bb_vinfo
);
1279 gsi_bb_start
= gsi_after_labels (bb
);
1280 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1284 if (dump_enabled_p ())
1286 dump_printf_loc (MSG_NOTE
, vect_location
,
1287 "created new init_stmt: ");
1288 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1289 dump_printf (MSG_NOTE
, "\n");
1293 /* Function vect_init_vector.
1295 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1296 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1297 vector type a vector with all elements equal to VAL is created first.
1298 Place the initialization at BSI if it is not NULL. Otherwise, place the
1299 initialization at the loop preheader.
1300 Return the DEF of INIT_STMT.
1301 It will be used in the vectorization of STMT. */
1304 vect_init_vector (gimple stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1311 if (TREE_CODE (type
) == VECTOR_TYPE
1312 && TREE_CODE (TREE_TYPE (val
)) != VECTOR_TYPE
)
1314 if (!types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1316 if (CONSTANT_CLASS_P (val
))
1317 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (type
), val
);
1320 new_temp
= make_ssa_name (TREE_TYPE (type
), NULL
);
1321 init_stmt
= gimple_build_assign_with_ops (NOP_EXPR
,
1324 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1328 val
= build_vector_from_val (type
, val
);
1331 new_var
= vect_get_new_vect_var (type
, vect_simple_var
, "cst_");
1332 init_stmt
= gimple_build_assign (new_var
, val
);
1333 new_temp
= make_ssa_name (new_var
, init_stmt
);
1334 gimple_assign_set_lhs (init_stmt
, new_temp
);
1335 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1336 vec_oprnd
= gimple_assign_lhs (init_stmt
);
1341 /* Function vect_get_vec_def_for_operand.
1343 OP is an operand in STMT. This function returns a (vector) def that will be
1344 used in the vectorized stmt for STMT.
1346 In the case that OP is an SSA_NAME which is defined in the loop, then
1347 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1349 In case OP is an invariant or constant, a new stmt that creates a vector def
1350 needs to be introduced. */
1353 vect_get_vec_def_for_operand (tree op
, gimple stmt
, tree
*scalar_def
)
1358 stmt_vec_info def_stmt_info
= NULL
;
1359 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1360 unsigned int nunits
;
1361 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1363 enum vect_def_type dt
;
1367 if (dump_enabled_p ())
1369 dump_printf_loc (MSG_NOTE
, vect_location
,
1370 "vect_get_vec_def_for_operand: ");
1371 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1372 dump_printf (MSG_NOTE
, "\n");
1375 is_simple_use
= vect_is_simple_use (op
, stmt
, loop_vinfo
, NULL
,
1376 &def_stmt
, &def
, &dt
);
1377 gcc_assert (is_simple_use
);
1378 if (dump_enabled_p ())
1380 int loc_printed
= 0;
1383 dump_printf_loc (MSG_NOTE
, vect_location
, "def = ");
1385 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, def
);
1386 dump_printf (MSG_NOTE
, "\n");
1391 dump_printf (MSG_NOTE
, " def_stmt = ");
1393 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1394 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1395 dump_printf (MSG_NOTE
, "\n");
1401 /* Case 1: operand is a constant. */
1402 case vect_constant_def
:
1404 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1405 gcc_assert (vector_type
);
1406 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
1411 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1412 if (dump_enabled_p ())
1413 dump_printf_loc (MSG_NOTE
, vect_location
,
1414 "Create vector_cst. nunits = %d\n", nunits
);
1416 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1419 /* Case 2: operand is defined outside the loop - loop invariant. */
1420 case vect_external_def
:
1422 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (def
));
1423 gcc_assert (vector_type
);
1428 /* Create 'vec_inv = {inv,inv,..,inv}' */
1429 if (dump_enabled_p ())
1430 dump_printf_loc (MSG_NOTE
, vect_location
, "Create vector_inv.\n");
1432 return vect_init_vector (stmt
, def
, vector_type
, NULL
);
1435 /* Case 3: operand is defined inside the loop. */
1436 case vect_internal_def
:
1439 *scalar_def
= NULL
/* FIXME tuples: def_stmt*/;
1441 /* Get the def from the vectorized stmt. */
1442 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1444 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1445 /* Get vectorized pattern statement. */
1447 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1448 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1449 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1450 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1451 gcc_assert (vec_stmt
);
1452 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1453 vec_oprnd
= PHI_RESULT (vec_stmt
);
1454 else if (is_gimple_call (vec_stmt
))
1455 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1457 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1461 /* Case 4: operand is defined by a loop header phi - reduction */
1462 case vect_reduction_def
:
1463 case vect_double_reduction_def
:
1464 case vect_nested_cycle
:
1468 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1469 loop
= (gimple_bb (def_stmt
))->loop_father
;
1471 /* Get the def before the loop */
1472 op
= PHI_ARG_DEF_FROM_EDGE (def_stmt
, loop_preheader_edge (loop
));
1473 return get_initial_def_for_reduction (stmt
, op
, scalar_def
);
1476 /* Case 5: operand is defined by loop-header phi - induction. */
1477 case vect_induction_def
:
1479 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1481 /* Get the def from the vectorized stmt. */
1482 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1483 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1484 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1485 vec_oprnd
= PHI_RESULT (vec_stmt
);
1487 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1497 /* Function vect_get_vec_def_for_stmt_copy
1499 Return a vector-def for an operand. This function is used when the
1500 vectorized stmt to be created (by the caller to this function) is a "copy"
1501 created in case the vectorized result cannot fit in one vector, and several
1502 copies of the vector-stmt are required. In this case the vector-def is
1503 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1504 of the stmt that defines VEC_OPRND.
1505 DT is the type of the vector def VEC_OPRND.
1508 In case the vectorization factor (VF) is bigger than the number
1509 of elements that can fit in a vectype (nunits), we have to generate
1510 more than one vector stmt to vectorize the scalar stmt. This situation
1511 arises when there are multiple data-types operated upon in the loop; the
1512 smallest data-type determines the VF, and as a result, when vectorizing
1513 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1514 vector stmt (each computing a vector of 'nunits' results, and together
1515 computing 'VF' results in each iteration). This function is called when
1516 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1517 which VF=16 and nunits=4, so the number of copies required is 4):
1519 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1521 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1522 VS1.1: vx.1 = memref1 VS1.2
1523 VS1.2: vx.2 = memref2 VS1.3
1524 VS1.3: vx.3 = memref3
1526 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1527 VSnew.1: vz1 = vx.1 + ... VSnew.2
1528 VSnew.2: vz2 = vx.2 + ... VSnew.3
1529 VSnew.3: vz3 = vx.3 + ...
1531 The vectorization of S1 is explained in vectorizable_load.
1532 The vectorization of S2:
1533 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1534 the function 'vect_get_vec_def_for_operand' is called to
1535 get the relevant vector-def for each operand of S2. For operand x it
1536 returns the vector-def 'vx.0'.
1538 To create the remaining copies of the vector-stmt (VSnew.j), this
1539 function is called to get the relevant vector-def for each operand. It is
1540 obtained from the respective VS1.j stmt, which is recorded in the
1541 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1543 For example, to obtain the vector-def 'vx.1' in order to create the
1544 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1545 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1546 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1547 and return its def ('vx.1').
1548 Overall, to create the above sequence this function will be called 3 times:
1549 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1550 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1551 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1554 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1556 gimple vec_stmt_for_operand
;
1557 stmt_vec_info def_stmt_info
;
1559 /* Do nothing; can reuse same def. */
1560 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1563 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1564 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1565 gcc_assert (def_stmt_info
);
1566 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1567 gcc_assert (vec_stmt_for_operand
);
1568 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1569 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1570 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1572 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1577 /* Get vectorized definitions for the operands to create a copy of an original
1578 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1581 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1582 vec
<tree
> *vec_oprnds0
,
1583 vec
<tree
> *vec_oprnds1
)
1585 tree vec_oprnd
= vec_oprnds0
->pop ();
1587 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1588 vec_oprnds0
->quick_push (vec_oprnd
);
1590 if (vec_oprnds1
&& vec_oprnds1
->length ())
1592 vec_oprnd
= vec_oprnds1
->pop ();
1593 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1594 vec_oprnds1
->quick_push (vec_oprnd
);
1599 /* Get vectorized definitions for OP0 and OP1.
1600 REDUC_INDEX is the index of reduction operand in case of reduction,
1601 and -1 otherwise. */
1604 vect_get_vec_defs (tree op0
, tree op1
, gimple stmt
,
1605 vec
<tree
> *vec_oprnds0
,
1606 vec
<tree
> *vec_oprnds1
,
1607 slp_tree slp_node
, int reduc_index
)
1611 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1612 auto_vec
<tree
> ops (nops
);
1613 auto_vec
<vec
<tree
> > vec_defs (nops
);
1615 ops
.quick_push (op0
);
1617 ops
.quick_push (op1
);
1619 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1621 *vec_oprnds0
= vec_defs
[0];
1623 *vec_oprnds1
= vec_defs
[1];
1629 vec_oprnds0
->create (1);
1630 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1631 vec_oprnds0
->quick_push (vec_oprnd
);
1635 vec_oprnds1
->create (1);
1636 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
1637 vec_oprnds1
->quick_push (vec_oprnd
);
1643 /* Function vect_finish_stmt_generation.
1645 Insert a new stmt. */
1648 vect_finish_stmt_generation (gimple stmt
, gimple vec_stmt
,
1649 gimple_stmt_iterator
*gsi
)
1651 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1652 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1653 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1655 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1657 if (!gsi_end_p (*gsi
)
1658 && gimple_has_mem_ops (vec_stmt
))
1660 gimple at_stmt
= gsi_stmt (*gsi
);
1661 tree vuse
= gimple_vuse (at_stmt
);
1662 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1664 tree vdef
= gimple_vdef (at_stmt
);
1665 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1666 /* If we have an SSA vuse and insert a store, update virtual
1667 SSA form to avoid triggering the renamer. Do so only
1668 if we can easily see all uses - which is what almost always
1669 happens with the way vectorized stmts are inserted. */
1670 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1671 && ((is_gimple_assign (vec_stmt
)
1672 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1673 || (is_gimple_call (vec_stmt
)
1674 && !(gimple_call_flags (vec_stmt
)
1675 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1677 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1678 gimple_set_vdef (vec_stmt
, new_vdef
);
1679 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1683 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1685 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, loop_vinfo
,
1688 if (dump_enabled_p ())
1690 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1691 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1692 dump_printf (MSG_NOTE
, "\n");
1695 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1697 /* While EH edges will generally prevent vectorization, stmt might
1698 e.g. be in a must-not-throw region. Ensure newly created stmts
1699 that could throw are part of the same region. */
1700 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1701 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1702 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1705 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1706 a function declaration if the target has a vectorized version
1707 of the function, or NULL_TREE if the function cannot be vectorized. */
1710 vectorizable_function (gimple call
, tree vectype_out
, tree vectype_in
)
1712 tree fndecl
= gimple_call_fndecl (call
);
1714 /* We only handle functions that do not read or clobber memory -- i.e.
1715 const or novops ones. */
1716 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1720 || TREE_CODE (fndecl
) != FUNCTION_DECL
1721 || !DECL_BUILT_IN (fndecl
))
1724 return targetm
.vectorize
.builtin_vectorized_function (fndecl
, vectype_out
,
1729 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
,
1730 gimple_stmt_iterator
*);
1733 /* Function vectorizable_mask_load_store.
1735 Check if STMT performs a conditional load or store that can be vectorized.
1736 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1737 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1738 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1741 vectorizable_mask_load_store (gimple stmt
, gimple_stmt_iterator
*gsi
,
1742 gimple
*vec_stmt
, slp_tree slp_node
)
1744 tree vec_dest
= NULL
;
1745 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1746 stmt_vec_info prev_stmt_info
;
1747 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1748 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1749 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1750 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1751 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1755 tree dataref_ptr
= NULL_TREE
;
1757 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1761 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
1762 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
1763 int gather_scale
= 1;
1764 enum vect_def_type gather_dt
= vect_unknown_def_type
;
1769 enum vect_def_type dt
;
1771 if (slp_node
!= NULL
)
1774 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1775 gcc_assert (ncopies
>= 1);
1777 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1778 mask
= gimple_call_arg (stmt
, 2);
1779 if (TYPE_PRECISION (TREE_TYPE (mask
))
1780 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
))))
1783 /* FORNOW. This restriction should be relaxed. */
1784 if (nested_in_vect_loop
&& ncopies
> 1)
1786 if (dump_enabled_p ())
1787 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1788 "multiple types in nested loop.");
1792 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1795 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1798 if (!STMT_VINFO_DATA_REF (stmt_info
))
1801 elem_type
= TREE_TYPE (vectype
);
1803 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1806 if (STMT_VINFO_STRIDE_LOAD_P (stmt_info
))
1809 if (STMT_VINFO_GATHER_P (stmt_info
))
1813 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
1814 &gather_off
, &gather_scale
);
1815 gcc_assert (gather_decl
);
1816 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, NULL
,
1817 &def_stmt
, &def
, &gather_dt
,
1818 &gather_off_vectype
))
1820 if (dump_enabled_p ())
1821 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1822 "gather index use not simple.");
1826 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1828 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1829 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1831 if (dump_enabled_p ())
1832 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1833 "masked gather with integer mask not supported.");
1837 else if (tree_int_cst_compare (nested_in_vect_loop
1838 ? STMT_VINFO_DR_STEP (stmt_info
)
1839 : DR_STEP (dr
), size_zero_node
) <= 0)
1841 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1842 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
), !is_store
))
1845 if (TREE_CODE (mask
) != SSA_NAME
)
1848 if (!vect_is_simple_use (mask
, stmt
, loop_vinfo
, NULL
,
1849 &def_stmt
, &def
, &dt
))
1854 tree rhs
= gimple_call_arg (stmt
, 3);
1855 if (!vect_is_simple_use (rhs
, stmt
, loop_vinfo
, NULL
,
1856 &def_stmt
, &def
, &dt
))
1860 if (!vec_stmt
) /* transformation not required. */
1862 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1864 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1867 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1873 if (STMT_VINFO_GATHER_P (stmt_info
))
1875 tree vec_oprnd0
= NULL_TREE
, op
;
1876 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1877 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1878 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1879 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1880 tree mask_perm_mask
= NULL_TREE
;
1881 edge pe
= loop_preheader_edge (loop
);
1884 enum { NARROW
, NONE
, WIDEN
} modifier
;
1885 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
1887 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
1888 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1889 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1890 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1891 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1892 scaletype
= TREE_VALUE (arglist
);
1893 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1894 && types_compatible_p (srctype
, masktype
));
1896 if (nunits
== gather_off_nunits
)
1898 else if (nunits
== gather_off_nunits
/ 2)
1900 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1903 for (i
= 0; i
< gather_off_nunits
; ++i
)
1904 sel
[i
] = i
| nunits
;
1906 perm_mask
= vect_gen_perm_mask (gather_off_vectype
, sel
);
1907 gcc_assert (perm_mask
!= NULL_TREE
);
1909 else if (nunits
== gather_off_nunits
* 2)
1911 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1914 for (i
= 0; i
< nunits
; ++i
)
1915 sel
[i
] = i
< gather_off_nunits
1916 ? i
: i
+ nunits
- gather_off_nunits
;
1918 perm_mask
= vect_gen_perm_mask (vectype
, sel
);
1919 gcc_assert (perm_mask
!= NULL_TREE
);
1921 for (i
= 0; i
< nunits
; ++i
)
1922 sel
[i
] = i
| gather_off_nunits
;
1923 mask_perm_mask
= vect_gen_perm_mask (masktype
, sel
);
1924 gcc_assert (mask_perm_mask
!= NULL_TREE
);
1929 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1931 ptr
= fold_convert (ptrtype
, gather_base
);
1932 if (!is_gimple_min_invariant (ptr
))
1934 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1935 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1936 gcc_assert (!new_bb
);
1939 scale
= build_int_cst (scaletype
, gather_scale
);
1941 prev_stmt_info
= NULL
;
1942 for (j
= 0; j
< ncopies
; ++j
)
1944 if (modifier
== WIDEN
&& (j
& 1))
1945 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1946 perm_mask
, stmt
, gsi
);
1949 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
1952 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
1954 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1956 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1957 == TYPE_VECTOR_SUBPARTS (idxtype
));
1958 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
1959 var
= make_ssa_name (var
, NULL
);
1960 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1962 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR
, var
,
1964 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1968 if (mask_perm_mask
&& (j
& 1))
1969 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1970 mask_perm_mask
, stmt
, gsi
);
1974 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
1977 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
,
1978 &def_stmt
, &def
, &dt
);
1979 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
1983 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
1985 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
1986 == TYPE_VECTOR_SUBPARTS (masktype
));
1987 var
= vect_get_new_vect_var (masktype
, vect_simple_var
,
1989 var
= make_ssa_name (var
, NULL
);
1990 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
1992 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR
, var
,
1993 mask_op
, NULL_TREE
);
1994 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2000 = gimple_build_call (gather_decl
, 5, mask_op
, ptr
, op
, mask_op
,
2003 if (!useless_type_conversion_p (vectype
, rettype
))
2005 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
2006 == TYPE_VECTOR_SUBPARTS (rettype
));
2007 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
2008 op
= make_ssa_name (var
, new_stmt
);
2009 gimple_call_set_lhs (new_stmt
, op
);
2010 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2011 var
= make_ssa_name (vec_dest
, NULL
);
2012 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2014 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR
, var
, op
,
2019 var
= make_ssa_name (vec_dest
, new_stmt
);
2020 gimple_call_set_lhs (new_stmt
, var
);
2023 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2025 if (modifier
== NARROW
)
2032 var
= permute_vec_elements (prev_res
, var
,
2033 perm_mask
, stmt
, gsi
);
2034 new_stmt
= SSA_NAME_DEF_STMT (var
);
2037 if (prev_stmt_info
== NULL
)
2038 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2040 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2041 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2044 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2046 tree lhs
= gimple_call_lhs (stmt
);
2047 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2048 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2049 set_vinfo_for_stmt (stmt
, NULL
);
2050 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2051 gsi_replace (gsi
, new_stmt
, true);
2056 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
2057 prev_stmt_info
= NULL
;
2058 for (i
= 0; i
< ncopies
; i
++)
2060 unsigned align
, misalign
;
2064 tree rhs
= gimple_call_arg (stmt
, 3);
2065 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
, NULL
);
2066 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2067 /* We should have catched mismatched types earlier. */
2068 gcc_assert (useless_type_conversion_p (vectype
,
2069 TREE_TYPE (vec_rhs
)));
2070 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2071 NULL_TREE
, &dummy
, gsi
,
2072 &ptr_incr
, false, &inv_p
);
2073 gcc_assert (!inv_p
);
2077 vect_is_simple_use (vec_rhs
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2079 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2080 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2082 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2083 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2084 TYPE_SIZE_UNIT (vectype
));
2087 align
= TYPE_ALIGN_UNIT (vectype
);
2088 if (aligned_access_p (dr
))
2090 else if (DR_MISALIGNMENT (dr
) == -1)
2092 align
= TYPE_ALIGN_UNIT (elem_type
);
2096 misalign
= DR_MISALIGNMENT (dr
);
2097 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2100 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2101 gimple_call_arg (stmt
, 1),
2103 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2105 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2107 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2108 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2113 tree vec_mask
= NULL_TREE
;
2114 prev_stmt_info
= NULL
;
2115 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2116 for (i
= 0; i
< ncopies
; i
++)
2118 unsigned align
, misalign
;
2122 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2123 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2124 NULL_TREE
, &dummy
, gsi
,
2125 &ptr_incr
, false, &inv_p
);
2126 gcc_assert (!inv_p
);
2130 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2132 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2133 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2134 TYPE_SIZE_UNIT (vectype
));
2137 align
= TYPE_ALIGN_UNIT (vectype
);
2138 if (aligned_access_p (dr
))
2140 else if (DR_MISALIGNMENT (dr
) == -1)
2142 align
= TYPE_ALIGN_UNIT (elem_type
);
2146 misalign
= DR_MISALIGNMENT (dr
);
2147 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2150 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2151 gimple_call_arg (stmt
, 1),
2153 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
, NULL
));
2154 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2156 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2158 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2159 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2165 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2167 tree lhs
= gimple_call_lhs (stmt
);
2168 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2169 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2170 set_vinfo_for_stmt (stmt
, NULL
);
2171 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2172 gsi_replace (gsi
, new_stmt
, true);
2179 /* Function vectorizable_call.
2181 Check if STMT performs a function call that can be vectorized.
2182 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2183 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2184 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2187 vectorizable_call (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
2193 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2194 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2195 tree vectype_out
, vectype_in
;
2198 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2199 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2200 tree fndecl
, new_temp
, def
, rhs_type
;
2202 enum vect_def_type dt
[3]
2203 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2204 gimple new_stmt
= NULL
;
2206 vec
<tree
> vargs
= vNULL
;
2207 enum { NARROW
, NONE
, WIDEN
} modifier
;
2211 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2214 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2217 /* Is STMT a vectorizable call? */
2218 if (!is_gimple_call (stmt
))
2221 if (gimple_call_internal_p (stmt
)
2222 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2223 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2224 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2227 if (gimple_call_lhs (stmt
) == NULL_TREE
2228 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2231 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2233 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2235 /* Process function arguments. */
2236 rhs_type
= NULL_TREE
;
2237 vectype_in
= NULL_TREE
;
2238 nargs
= gimple_call_num_args (stmt
);
2240 /* Bail out if the function has more than three arguments, we do not have
2241 interesting builtin functions to vectorize with more than two arguments
2242 except for fma. No arguments is also not good. */
2243 if (nargs
== 0 || nargs
> 3)
2246 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2247 if (gimple_call_internal_p (stmt
)
2248 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2251 rhs_type
= unsigned_type_node
;
2254 for (i
= 0; i
< nargs
; i
++)
2258 op
= gimple_call_arg (stmt
, i
);
2260 /* We can only handle calls with arguments of the same type. */
2262 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2264 if (dump_enabled_p ())
2265 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2266 "argument types differ.\n");
2270 rhs_type
= TREE_TYPE (op
);
2272 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2273 &def_stmt
, &def
, &dt
[i
], &opvectype
))
2275 if (dump_enabled_p ())
2276 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2277 "use not simple.\n");
2282 vectype_in
= opvectype
;
2284 && opvectype
!= vectype_in
)
2286 if (dump_enabled_p ())
2287 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2288 "argument vector types differ.\n");
2292 /* If all arguments are external or constant defs use a vector type with
2293 the same size as the output vector type. */
2295 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2297 gcc_assert (vectype_in
);
2300 if (dump_enabled_p ())
2302 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2303 "no vectype for scalar type ");
2304 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2305 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2312 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2313 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2314 if (nunits_in
== nunits_out
/ 2)
2316 else if (nunits_out
== nunits_in
)
2318 else if (nunits_out
== nunits_in
/ 2)
2323 /* For now, we only vectorize functions if a target specific builtin
2324 is available. TODO -- in some cases, it might be profitable to
2325 insert the calls for pieces of the vector, in order to be able
2326 to vectorize other operations in the loop. */
2327 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
2328 if (fndecl
== NULL_TREE
)
2330 if (gimple_call_internal_p (stmt
)
2331 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
2334 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2335 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2336 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2337 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2339 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2340 { 0, 1, 2, ... vf - 1 } vector. */
2341 gcc_assert (nargs
== 0);
2345 if (dump_enabled_p ())
2346 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2347 "function is not vectorizable.\n");
2352 gcc_assert (!gimple_vuse (stmt
));
2354 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2356 else if (modifier
== NARROW
)
2357 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2359 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2361 /* Sanity check: make sure that at least one copy of the vectorized stmt
2362 needs to be generated. */
2363 gcc_assert (ncopies
>= 1);
2365 if (!vec_stmt
) /* transformation not required. */
2367 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2368 if (dump_enabled_p ())
2369 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2371 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2377 if (dump_enabled_p ())
2378 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2381 scalar_dest
= gimple_call_lhs (stmt
);
2382 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2384 prev_stmt_info
= NULL
;
2388 for (j
= 0; j
< ncopies
; ++j
)
2390 /* Build argument list for the vectorized call. */
2392 vargs
.create (nargs
);
2398 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2399 vec
<tree
> vec_oprnds0
;
2401 for (i
= 0; i
< nargs
; i
++)
2402 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2403 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2404 vec_oprnds0
= vec_defs
[0];
2406 /* Arguments are ready. Create the new vector stmt. */
2407 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2410 for (k
= 0; k
< nargs
; k
++)
2412 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2413 vargs
[k
] = vec_oprndsk
[i
];
2415 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2416 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2417 gimple_call_set_lhs (new_stmt
, new_temp
);
2418 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2419 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2422 for (i
= 0; i
< nargs
; i
++)
2424 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2425 vec_oprndsi
.release ();
2430 for (i
= 0; i
< nargs
; i
++)
2432 op
= gimple_call_arg (stmt
, i
);
2435 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2438 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2440 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2443 vargs
.quick_push (vec_oprnd0
);
2446 if (gimple_call_internal_p (stmt
)
2447 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2449 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2451 for (k
= 0; k
< nunits_out
; ++k
)
2452 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2453 tree cst
= build_vector (vectype_out
, v
);
2455 = vect_get_new_vect_var (vectype_out
, vect_simple_var
, "cst_");
2456 gimple init_stmt
= gimple_build_assign (new_var
, cst
);
2457 new_temp
= make_ssa_name (new_var
, init_stmt
);
2458 gimple_assign_set_lhs (init_stmt
, new_temp
);
2459 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2460 new_temp
= make_ssa_name (vec_dest
, NULL
);
2461 new_stmt
= gimple_build_assign (new_temp
,
2462 gimple_assign_lhs (init_stmt
));
2466 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2467 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2468 gimple_call_set_lhs (new_stmt
, new_temp
);
2470 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2473 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2475 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2477 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2483 for (j
= 0; j
< ncopies
; ++j
)
2485 /* Build argument list for the vectorized call. */
2487 vargs
.create (nargs
* 2);
2493 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2494 vec
<tree
> vec_oprnds0
;
2496 for (i
= 0; i
< nargs
; i
++)
2497 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2498 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2499 vec_oprnds0
= vec_defs
[0];
2501 /* Arguments are ready. Create the new vector stmt. */
2502 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2506 for (k
= 0; k
< nargs
; k
++)
2508 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2509 vargs
.quick_push (vec_oprndsk
[i
]);
2510 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2512 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2513 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2514 gimple_call_set_lhs (new_stmt
, new_temp
);
2515 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2516 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2519 for (i
= 0; i
< nargs
; i
++)
2521 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2522 vec_oprndsi
.release ();
2527 for (i
= 0; i
< nargs
; i
++)
2529 op
= gimple_call_arg (stmt
, i
);
2533 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2535 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2539 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2541 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2543 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2546 vargs
.quick_push (vec_oprnd0
);
2547 vargs
.quick_push (vec_oprnd1
);
2550 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2551 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2552 gimple_call_set_lhs (new_stmt
, new_temp
);
2553 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2556 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2558 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2560 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2563 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2568 /* No current target implements this case. */
2574 /* The call in STMT might prevent it from being removed in dce.
2575 We however cannot remove it here, due to the way the ssa name
2576 it defines is mapped to the new definition. So just replace
2577 rhs of the statement with something harmless. */
2582 type
= TREE_TYPE (scalar_dest
);
2583 if (is_pattern_stmt_p (stmt_info
))
2584 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2586 lhs
= gimple_call_lhs (stmt
);
2587 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2588 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2589 set_vinfo_for_stmt (stmt
, NULL
);
2590 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2591 gsi_replace (gsi
, new_stmt
, false);
2597 struct simd_call_arg_info
2601 enum vect_def_type dt
;
2602 HOST_WIDE_INT linear_step
;
2606 /* Function vectorizable_simd_clone_call.
2608 Check if STMT performs a function call that can be vectorized
2609 by calling a simd clone of the function.
2610 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2611 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2612 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2615 vectorizable_simd_clone_call (gimple stmt
, gimple_stmt_iterator
*gsi
,
2616 gimple
*vec_stmt
, slp_tree slp_node
)
2621 tree vec_oprnd0
= NULL_TREE
;
2622 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2624 unsigned int nunits
;
2625 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2626 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2627 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2628 tree fndecl
, new_temp
, def
;
2630 gimple new_stmt
= NULL
;
2632 vec
<simd_call_arg_info
> arginfo
= vNULL
;
2633 vec
<tree
> vargs
= vNULL
;
2635 tree lhs
, rtype
, ratype
;
2636 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2638 /* Is STMT a vectorizable call? */
2639 if (!is_gimple_call (stmt
))
2642 fndecl
= gimple_call_fndecl (stmt
);
2643 if (fndecl
== NULL_TREE
)
2646 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2647 if (node
== NULL
|| node
->simd_clones
== NULL
)
2650 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2653 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2656 if (gimple_call_lhs (stmt
)
2657 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2660 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2662 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2664 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2668 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2671 /* Process function arguments. */
2672 nargs
= gimple_call_num_args (stmt
);
2674 /* Bail out if the function has zero arguments. */
2678 arginfo
.create (nargs
);
2680 for (i
= 0; i
< nargs
; i
++)
2682 simd_call_arg_info thisarginfo
;
2685 thisarginfo
.linear_step
= 0;
2686 thisarginfo
.align
= 0;
2687 thisarginfo
.op
= NULL_TREE
;
2689 op
= gimple_call_arg (stmt
, i
);
2690 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2691 &def_stmt
, &def
, &thisarginfo
.dt
,
2692 &thisarginfo
.vectype
)
2693 || thisarginfo
.dt
== vect_uninitialized_def
)
2695 if (dump_enabled_p ())
2696 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2697 "use not simple.\n");
2702 if (thisarginfo
.dt
== vect_constant_def
2703 || thisarginfo
.dt
== vect_external_def
)
2704 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2706 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2708 if (thisarginfo
.dt
!= vect_constant_def
2709 && thisarginfo
.dt
!= vect_external_def
2711 && TREE_CODE (op
) == SSA_NAME
2712 && simple_iv (loop
, loop_containing_stmt (stmt
), op
, &iv
, false)
2713 && tree_fits_shwi_p (iv
.step
))
2715 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2716 thisarginfo
.op
= iv
.base
;
2718 else if ((thisarginfo
.dt
== vect_constant_def
2719 || thisarginfo
.dt
== vect_external_def
)
2720 && POINTER_TYPE_P (TREE_TYPE (op
)))
2721 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2723 arginfo
.quick_push (thisarginfo
);
2726 unsigned int badness
= 0;
2727 struct cgraph_node
*bestn
= NULL
;
2728 if (STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info
))
2729 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info
));
2731 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2732 n
= n
->simdclone
->next_clone
)
2734 unsigned int this_badness
= 0;
2735 if (n
->simdclone
->simdlen
2736 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2737 || n
->simdclone
->nargs
!= nargs
)
2739 if (n
->simdclone
->simdlen
2740 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2741 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2742 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2743 if (n
->simdclone
->inbranch
)
2744 this_badness
+= 2048;
2745 int target_badness
= targetm
.simd_clone
.usable (n
);
2746 if (target_badness
< 0)
2748 this_badness
+= target_badness
* 512;
2749 /* FORNOW: Have to add code to add the mask argument. */
2750 if (n
->simdclone
->inbranch
)
2752 for (i
= 0; i
< nargs
; i
++)
2754 switch (n
->simdclone
->args
[i
].arg_type
)
2756 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2757 if (!useless_type_conversion_p
2758 (n
->simdclone
->args
[i
].orig_type
,
2759 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2761 else if (arginfo
[i
].dt
== vect_constant_def
2762 || arginfo
[i
].dt
== vect_external_def
2763 || arginfo
[i
].linear_step
)
2766 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2767 if (arginfo
[i
].dt
!= vect_constant_def
2768 && arginfo
[i
].dt
!= vect_external_def
)
2771 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2772 if (arginfo
[i
].dt
== vect_constant_def
2773 || arginfo
[i
].dt
== vect_external_def
2774 || (arginfo
[i
].linear_step
2775 != n
->simdclone
->args
[i
].linear_step
))
2778 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2782 case SIMD_CLONE_ARG_TYPE_MASK
:
2785 if (i
== (size_t) -1)
2787 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2792 if (arginfo
[i
].align
)
2793 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2794 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2796 if (i
== (size_t) -1)
2798 if (bestn
== NULL
|| this_badness
< badness
)
2801 badness
= this_badness
;
2811 for (i
= 0; i
< nargs
; i
++)
2812 if ((arginfo
[i
].dt
== vect_constant_def
2813 || arginfo
[i
].dt
== vect_external_def
)
2814 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2817 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2819 if (arginfo
[i
].vectype
== NULL
2820 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2821 > bestn
->simdclone
->simdlen
))
2828 fndecl
= bestn
->decl
;
2829 nunits
= bestn
->simdclone
->simdlen
;
2830 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2832 /* If the function isn't const, only allow it in simd loops where user
2833 has asserted that at least nunits consecutive iterations can be
2834 performed using SIMD instructions. */
2835 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2836 && gimple_vuse (stmt
))
2842 /* Sanity check: make sure that at least one copy of the vectorized stmt
2843 needs to be generated. */
2844 gcc_assert (ncopies
>= 1);
2846 if (!vec_stmt
) /* transformation not required. */
2848 STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info
) = bestn
->decl
;
2849 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
2850 if (dump_enabled_p ())
2851 dump_printf_loc (MSG_NOTE
, vect_location
,
2852 "=== vectorizable_simd_clone_call ===\n");
2853 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2860 if (dump_enabled_p ())
2861 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2864 scalar_dest
= gimple_call_lhs (stmt
);
2865 vec_dest
= NULL_TREE
;
2870 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2871 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
2872 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
2875 rtype
= TREE_TYPE (ratype
);
2879 prev_stmt_info
= NULL
;
2880 for (j
= 0; j
< ncopies
; ++j
)
2882 /* Build argument list for the vectorized call. */
2884 vargs
.create (nargs
);
2888 for (i
= 0; i
< nargs
; i
++)
2890 unsigned int k
, l
, m
, o
;
2892 op
= gimple_call_arg (stmt
, i
);
2893 switch (bestn
->simdclone
->args
[i
].arg_type
)
2895 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2896 atype
= bestn
->simdclone
->args
[i
].vector_type
;
2897 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
2898 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
2900 if (TYPE_VECTOR_SUBPARTS (atype
)
2901 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
2903 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
2904 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2905 / TYPE_VECTOR_SUBPARTS (atype
));
2906 gcc_assert ((k
& (k
- 1)) == 0);
2909 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2912 vec_oprnd0
= arginfo
[i
].op
;
2913 if ((m
& (k
- 1)) == 0)
2915 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
2918 arginfo
[i
].op
= vec_oprnd0
;
2920 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
2922 bitsize_int ((m
& (k
- 1)) * prec
));
2924 = gimple_build_assign (make_ssa_name (atype
, NULL
),
2926 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2927 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
2931 k
= (TYPE_VECTOR_SUBPARTS (atype
)
2932 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
2933 gcc_assert ((k
& (k
- 1)) == 0);
2934 vec
<constructor_elt
, va_gc
> *ctor_elts
;
2936 vec_alloc (ctor_elts
, k
);
2939 for (l
= 0; l
< k
; l
++)
2941 if (m
== 0 && l
== 0)
2943 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2946 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
2948 arginfo
[i
].op
= vec_oprnd0
;
2951 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
2955 vargs
.safe_push (vec_oprnd0
);
2958 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
2960 = gimple_build_assign (make_ssa_name (atype
, NULL
),
2962 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2963 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
2968 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2969 vargs
.safe_push (op
);
2971 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2976 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
2981 edge pe
= loop_preheader_edge (loop
);
2982 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
2983 gcc_assert (!new_bb
);
2985 tree phi_res
= copy_ssa_name (op
, NULL
);
2986 gimple new_phi
= create_phi_node (phi_res
, loop
->header
);
2987 set_vinfo_for_stmt (new_phi
,
2988 new_stmt_vec_info (new_phi
, loop_vinfo
,
2990 add_phi_arg (new_phi
, arginfo
[i
].op
,
2991 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
2993 = POINTER_TYPE_P (TREE_TYPE (op
))
2994 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
2995 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
2996 ? sizetype
: TREE_TYPE (op
);
2998 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3000 tree tcst
= wide_int_to_tree (type
, cst
);
3001 tree phi_arg
= copy_ssa_name (op
, NULL
);
3002 new_stmt
= gimple_build_assign_with_ops (code
, phi_arg
,
3004 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3005 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3006 set_vinfo_for_stmt (new_stmt
,
3007 new_stmt_vec_info (new_stmt
, loop_vinfo
,
3009 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3011 arginfo
[i
].op
= phi_res
;
3012 vargs
.safe_push (phi_res
);
3017 = POINTER_TYPE_P (TREE_TYPE (op
))
3018 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3019 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3020 ? sizetype
: TREE_TYPE (op
);
3022 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3024 tree tcst
= wide_int_to_tree (type
, cst
);
3025 new_temp
= make_ssa_name (TREE_TYPE (op
), NULL
);
3027 = gimple_build_assign_with_ops (code
, new_temp
,
3028 arginfo
[i
].op
, tcst
);
3029 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3030 vargs
.safe_push (new_temp
);
3033 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3039 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3042 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3044 new_temp
= create_tmp_var (ratype
, NULL
);
3045 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3046 == TYPE_VECTOR_SUBPARTS (rtype
))
3047 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3049 new_temp
= make_ssa_name (rtype
, new_stmt
);
3050 gimple_call_set_lhs (new_stmt
, new_temp
);
3052 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3056 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3059 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3060 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3061 gcc_assert ((k
& (k
- 1)) == 0);
3062 for (l
= 0; l
< k
; l
++)
3067 t
= build_fold_addr_expr (new_temp
);
3068 t
= build2 (MEM_REF
, vectype
, t
,
3069 build_int_cst (TREE_TYPE (t
),
3070 l
* prec
/ BITS_PER_UNIT
));
3073 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3074 size_int (prec
), bitsize_int (l
* prec
));
3076 = gimple_build_assign (make_ssa_name (vectype
, NULL
), t
);
3077 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3078 if (j
== 0 && l
== 0)
3079 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3081 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3083 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3088 tree clobber
= build_constructor (ratype
, NULL
);
3089 TREE_THIS_VOLATILE (clobber
) = 1;
3090 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3091 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3095 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3097 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3098 / TYPE_VECTOR_SUBPARTS (rtype
));
3099 gcc_assert ((k
& (k
- 1)) == 0);
3100 if ((j
& (k
- 1)) == 0)
3101 vec_alloc (ret_ctor_elts
, k
);
3104 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3105 for (m
= 0; m
< o
; m
++)
3107 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3108 size_int (m
), NULL_TREE
, NULL_TREE
);
3110 = gimple_build_assign (make_ssa_name (rtype
, NULL
),
3112 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3113 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3114 gimple_assign_lhs (new_stmt
));
3116 tree clobber
= build_constructor (ratype
, NULL
);
3117 TREE_THIS_VOLATILE (clobber
) = 1;
3118 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3119 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3122 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3123 if ((j
& (k
- 1)) != k
- 1)
3125 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3127 = gimple_build_assign (make_ssa_name (vec_dest
, NULL
),
3129 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3131 if ((unsigned) j
== k
- 1)
3132 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3134 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3136 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3141 tree t
= build_fold_addr_expr (new_temp
);
3142 t
= build2 (MEM_REF
, vectype
, t
,
3143 build_int_cst (TREE_TYPE (t
), 0));
3145 = gimple_build_assign (make_ssa_name (vec_dest
, NULL
), t
);
3146 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3147 tree clobber
= build_constructor (ratype
, NULL
);
3148 TREE_THIS_VOLATILE (clobber
) = 1;
3149 vect_finish_stmt_generation (stmt
,
3150 gimple_build_assign (new_temp
,
3156 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3158 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3160 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3165 /* The call in STMT might prevent it from being removed in dce.
3166 We however cannot remove it here, due to the way the ssa name
3167 it defines is mapped to the new definition. So just replace
3168 rhs of the statement with something harmless. */
3175 type
= TREE_TYPE (scalar_dest
);
3176 if (is_pattern_stmt_p (stmt_info
))
3177 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3179 lhs
= gimple_call_lhs (stmt
);
3180 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3183 new_stmt
= gimple_build_nop ();
3184 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3185 set_vinfo_for_stmt (stmt
, NULL
);
3186 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3187 gsi_replace (gsi
, new_stmt
, false);
3188 unlink_stmt_vdef (stmt
);
3194 /* Function vect_gen_widened_results_half
3196 Create a vector stmt whose code, type, number of arguments, and result
3197 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3198 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3199 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3200 needs to be created (DECL is a function-decl of a target-builtin).
3201 STMT is the original scalar stmt that we are vectorizing. */
3204 vect_gen_widened_results_half (enum tree_code code
,
3206 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3207 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3213 /* Generate half of the widened result: */
3214 if (code
== CALL_EXPR
)
3216 /* Target specific support */
3217 if (op_type
== binary_op
)
3218 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3220 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3221 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3222 gimple_call_set_lhs (new_stmt
, new_temp
);
3226 /* Generic support */
3227 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3228 if (op_type
!= binary_op
)
3230 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vec_oprnd0
,
3232 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3233 gimple_assign_set_lhs (new_stmt
, new_temp
);
3235 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3241 /* Get vectorized definitions for loop-based vectorization. For the first
3242 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3243 scalar operand), and for the rest we get a copy with
3244 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3245 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3246 The vectors are collected into VEC_OPRNDS. */
3249 vect_get_loop_based_defs (tree
*oprnd
, gimple stmt
, enum vect_def_type dt
,
3250 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3254 /* Get first vector operand. */
3255 /* All the vector operands except the very first one (that is scalar oprnd)
3257 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3258 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
, NULL
);
3260 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3262 vec_oprnds
->quick_push (vec_oprnd
);
3264 /* Get second vector operand. */
3265 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3266 vec_oprnds
->quick_push (vec_oprnd
);
3270 /* For conversion in multiple steps, continue to get operands
3273 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3277 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3278 For multi-step conversions store the resulting vectors and call the function
3282 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3283 int multi_step_cvt
, gimple stmt
,
3285 gimple_stmt_iterator
*gsi
,
3286 slp_tree slp_node
, enum tree_code code
,
3287 stmt_vec_info
*prev_stmt_info
)
3290 tree vop0
, vop1
, new_tmp
, vec_dest
;
3292 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3294 vec_dest
= vec_dsts
.pop ();
3296 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3298 /* Create demotion operation. */
3299 vop0
= (*vec_oprnds
)[i
];
3300 vop1
= (*vec_oprnds
)[i
+ 1];
3301 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vop0
, vop1
);
3302 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3303 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3304 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3307 /* Store the resulting vector for next recursive call. */
3308 (*vec_oprnds
)[i
/2] = new_tmp
;
3311 /* This is the last step of the conversion sequence. Store the
3312 vectors in SLP_NODE or in vector info of the scalar statement
3313 (or in STMT_VINFO_RELATED_STMT chain). */
3315 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3318 if (!*prev_stmt_info
)
3319 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3321 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3323 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3328 /* For multi-step demotion operations we first generate demotion operations
3329 from the source type to the intermediate types, and then combine the
3330 results (stored in VEC_OPRNDS) in demotion operation to the destination
3334 /* At each level of recursion we have half of the operands we had at the
3336 vec_oprnds
->truncate ((i
+1)/2);
3337 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3338 stmt
, vec_dsts
, gsi
, slp_node
,
3339 VEC_PACK_TRUNC_EXPR
,
3343 vec_dsts
.quick_push (vec_dest
);
3347 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3348 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3349 the resulting vectors and call the function recursively. */
3352 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3353 vec
<tree
> *vec_oprnds1
,
3354 gimple stmt
, tree vec_dest
,
3355 gimple_stmt_iterator
*gsi
,
3356 enum tree_code code1
,
3357 enum tree_code code2
, tree decl1
,
3358 tree decl2
, int op_type
)
3361 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3362 gimple new_stmt1
, new_stmt2
;
3363 vec
<tree
> vec_tmp
= vNULL
;
3365 vec_tmp
.create (vec_oprnds0
->length () * 2);
3366 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3368 if (op_type
== binary_op
)
3369 vop1
= (*vec_oprnds1
)[i
];
3373 /* Generate the two halves of promotion operation. */
3374 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3375 op_type
, vec_dest
, gsi
, stmt
);
3376 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3377 op_type
, vec_dest
, gsi
, stmt
);
3378 if (is_gimple_call (new_stmt1
))
3380 new_tmp1
= gimple_call_lhs (new_stmt1
);
3381 new_tmp2
= gimple_call_lhs (new_stmt2
);
3385 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3386 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3389 /* Store the results for the next step. */
3390 vec_tmp
.quick_push (new_tmp1
);
3391 vec_tmp
.quick_push (new_tmp2
);
3394 vec_oprnds0
->release ();
3395 *vec_oprnds0
= vec_tmp
;
3399 /* Check if STMT performs a conversion operation, that can be vectorized.
3400 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3401 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3402 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3405 vectorizable_conversion (gimple stmt
, gimple_stmt_iterator
*gsi
,
3406 gimple
*vec_stmt
, slp_tree slp_node
)
3410 tree op0
, op1
= NULL_TREE
;
3411 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3412 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3413 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3414 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3415 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3416 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3420 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3421 gimple new_stmt
= NULL
;
3422 stmt_vec_info prev_stmt_info
;
3425 tree vectype_out
, vectype_in
;
3427 tree lhs_type
, rhs_type
;
3428 enum { NARROW
, NONE
, WIDEN
} modifier
;
3429 vec
<tree
> vec_oprnds0
= vNULL
;
3430 vec
<tree
> vec_oprnds1
= vNULL
;
3432 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3433 int multi_step_cvt
= 0;
3434 vec
<tree
> vec_dsts
= vNULL
;
3435 vec
<tree
> interm_types
= vNULL
;
3436 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3438 enum machine_mode rhs_mode
;
3439 unsigned short fltsz
;
3441 /* Is STMT a vectorizable conversion? */
3443 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3446 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3449 if (!is_gimple_assign (stmt
))
3452 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3455 code
= gimple_assign_rhs_code (stmt
);
3456 if (!CONVERT_EXPR_CODE_P (code
)
3457 && code
!= FIX_TRUNC_EXPR
3458 && code
!= FLOAT_EXPR
3459 && code
!= WIDEN_MULT_EXPR
3460 && code
!= WIDEN_LSHIFT_EXPR
)
3463 op_type
= TREE_CODE_LENGTH (code
);
3465 /* Check types of lhs and rhs. */
3466 scalar_dest
= gimple_assign_lhs (stmt
);
3467 lhs_type
= TREE_TYPE (scalar_dest
);
3468 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3470 op0
= gimple_assign_rhs1 (stmt
);
3471 rhs_type
= TREE_TYPE (op0
);
3473 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3474 && !((INTEGRAL_TYPE_P (lhs_type
)
3475 && INTEGRAL_TYPE_P (rhs_type
))
3476 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3477 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3480 if ((INTEGRAL_TYPE_P (lhs_type
)
3481 && (TYPE_PRECISION (lhs_type
)
3482 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3483 || (INTEGRAL_TYPE_P (rhs_type
)
3484 && (TYPE_PRECISION (rhs_type
)
3485 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
)))))
3487 if (dump_enabled_p ())
3488 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3489 "type conversion to/from bit-precision unsupported."
3494 /* Check the operands of the operation. */
3495 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
3496 &def_stmt
, &def
, &dt
[0], &vectype_in
))
3498 if (dump_enabled_p ())
3499 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3500 "use not simple.\n");
3503 if (op_type
== binary_op
)
3507 op1
= gimple_assign_rhs2 (stmt
);
3508 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3509 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3511 if (CONSTANT_CLASS_P (op0
))
3512 ok
= vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
,
3513 &def_stmt
, &def
, &dt
[1], &vectype_in
);
3515 ok
= vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
3520 if (dump_enabled_p ())
3521 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3522 "use not simple.\n");
3527 /* If op0 is an external or constant defs use a vector type of
3528 the same size as the output vector type. */
3530 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3532 gcc_assert (vectype_in
);
3535 if (dump_enabled_p ())
3537 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3538 "no vectype for scalar type ");
3539 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3540 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3546 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3547 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3548 if (nunits_in
< nunits_out
)
3550 else if (nunits_out
== nunits_in
)
3555 /* Multiple types in SLP are handled by creating the appropriate number of
3556 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3558 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
3560 else if (modifier
== NARROW
)
3561 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3563 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3565 /* Sanity check: make sure that at least one copy of the vectorized stmt
3566 needs to be generated. */
3567 gcc_assert (ncopies
>= 1);
3569 /* Supportable by target? */
3573 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3575 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3580 if (dump_enabled_p ())
3581 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3582 "conversion not supported by target.\n");
3586 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3587 &code1
, &code2
, &multi_step_cvt
,
3590 /* Binary widening operation can only be supported directly by the
3592 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3596 if (code
!= FLOAT_EXPR
3597 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3598 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3601 rhs_mode
= TYPE_MODE (rhs_type
);
3602 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3603 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3604 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3605 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3608 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3609 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3610 if (cvt_type
== NULL_TREE
)
3613 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3615 if (!supportable_convert_operation (code
, vectype_out
,
3616 cvt_type
, &decl1
, &codecvt1
))
3619 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3620 cvt_type
, &codecvt1
,
3621 &codecvt2
, &multi_step_cvt
,
3625 gcc_assert (multi_step_cvt
== 0);
3627 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3628 vectype_in
, &code1
, &code2
,
3629 &multi_step_cvt
, &interm_types
))
3633 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3636 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3637 codecvt2
= ERROR_MARK
;
3641 interm_types
.safe_push (cvt_type
);
3642 cvt_type
= NULL_TREE
;
3647 gcc_assert (op_type
== unary_op
);
3648 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3649 &code1
, &multi_step_cvt
,
3653 if (code
!= FIX_TRUNC_EXPR
3654 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3655 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3658 rhs_mode
= TYPE_MODE (rhs_type
);
3660 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3661 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3662 if (cvt_type
== NULL_TREE
)
3664 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3667 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3668 &code1
, &multi_step_cvt
,
3677 if (!vec_stmt
) /* transformation not required. */
3679 if (dump_enabled_p ())
3680 dump_printf_loc (MSG_NOTE
, vect_location
,
3681 "=== vectorizable_conversion ===\n");
3682 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3684 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3685 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3687 else if (modifier
== NARROW
)
3689 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3690 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3694 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3695 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3697 interm_types
.release ();
3702 if (dump_enabled_p ())
3703 dump_printf_loc (MSG_NOTE
, vect_location
,
3704 "transform conversion. ncopies = %d.\n", ncopies
);
3706 if (op_type
== binary_op
)
3708 if (CONSTANT_CLASS_P (op0
))
3709 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3710 else if (CONSTANT_CLASS_P (op1
))
3711 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3714 /* In case of multi-step conversion, we first generate conversion operations
3715 to the intermediate types, and then from that types to the final one.
3716 We create vector destinations for the intermediate type (TYPES) received
3717 from supportable_*_operation, and store them in the correct order
3718 for future use in vect_create_vectorized_*_stmts (). */
3719 vec_dsts
.create (multi_step_cvt
+ 1);
3720 vec_dest
= vect_create_destination_var (scalar_dest
,
3721 (cvt_type
&& modifier
== WIDEN
)
3722 ? cvt_type
: vectype_out
);
3723 vec_dsts
.quick_push (vec_dest
);
3727 for (i
= interm_types
.length () - 1;
3728 interm_types
.iterate (i
, &intermediate_type
); i
--)
3730 vec_dest
= vect_create_destination_var (scalar_dest
,
3732 vec_dsts
.quick_push (vec_dest
);
3737 vec_dest
= vect_create_destination_var (scalar_dest
,
3739 ? vectype_out
: cvt_type
);
3743 if (modifier
== WIDEN
)
3745 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3746 if (op_type
== binary_op
)
3747 vec_oprnds1
.create (1);
3749 else if (modifier
== NARROW
)
3750 vec_oprnds0
.create (
3751 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3753 else if (code
== WIDEN_LSHIFT_EXPR
)
3754 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3757 prev_stmt_info
= NULL
;
3761 for (j
= 0; j
< ncopies
; j
++)
3764 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3767 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3769 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3771 /* Arguments are ready, create the new vector stmt. */
3772 if (code1
== CALL_EXPR
)
3774 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3775 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3776 gimple_call_set_lhs (new_stmt
, new_temp
);
3780 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3781 new_stmt
= gimple_build_assign_with_ops (code1
, vec_dest
,
3783 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3784 gimple_assign_set_lhs (new_stmt
, new_temp
);
3787 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3789 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3793 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3795 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3796 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3801 /* In case the vectorization factor (VF) is bigger than the number
3802 of elements that we can fit in a vectype (nunits), we have to
3803 generate more than one vector stmt - i.e - we need to "unroll"
3804 the vector stmt by a factor VF/nunits. */
3805 for (j
= 0; j
< ncopies
; j
++)
3812 if (code
== WIDEN_LSHIFT_EXPR
)
3817 /* Store vec_oprnd1 for every vector stmt to be created
3818 for SLP_NODE. We check during the analysis that all
3819 the shift arguments are the same. */
3820 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
3821 vec_oprnds1
.quick_push (vec_oprnd1
);
3823 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3827 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
3828 &vec_oprnds1
, slp_node
, -1);
3832 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
3833 vec_oprnds0
.quick_push (vec_oprnd0
);
3834 if (op_type
== binary_op
)
3836 if (code
== WIDEN_LSHIFT_EXPR
)
3839 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
,
3841 vec_oprnds1
.quick_push (vec_oprnd1
);
3847 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
3848 vec_oprnds0
.truncate (0);
3849 vec_oprnds0
.quick_push (vec_oprnd0
);
3850 if (op_type
== binary_op
)
3852 if (code
== WIDEN_LSHIFT_EXPR
)
3855 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
3857 vec_oprnds1
.truncate (0);
3858 vec_oprnds1
.quick_push (vec_oprnd1
);
3862 /* Arguments are ready. Create the new vector stmts. */
3863 for (i
= multi_step_cvt
; i
>= 0; i
--)
3865 tree this_dest
= vec_dsts
[i
];
3866 enum tree_code c1
= code1
, c2
= code2
;
3867 if (i
== 0 && codecvt2
!= ERROR_MARK
)
3872 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
3874 stmt
, this_dest
, gsi
,
3875 c1
, c2
, decl1
, decl2
,
3879 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3883 if (codecvt1
== CALL_EXPR
)
3885 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3886 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3887 gimple_call_set_lhs (new_stmt
, new_temp
);
3891 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
3892 new_temp
= make_ssa_name (vec_dest
, NULL
);
3893 new_stmt
= gimple_build_assign_with_ops (codecvt1
,
3898 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3901 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
3904 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3907 if (!prev_stmt_info
)
3908 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3910 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3911 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3916 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3920 /* In case the vectorization factor (VF) is bigger than the number
3921 of elements that we can fit in a vectype (nunits), we have to
3922 generate more than one vector stmt - i.e - we need to "unroll"
3923 the vector stmt by a factor VF/nunits. */
3924 for (j
= 0; j
< ncopies
; j
++)
3928 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3932 vec_oprnds0
.truncate (0);
3933 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
3934 vect_pow2 (multi_step_cvt
) - 1);
3937 /* Arguments are ready. Create the new vector stmts. */
3939 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3941 if (codecvt1
== CALL_EXPR
)
3943 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3944 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3945 gimple_call_set_lhs (new_stmt
, new_temp
);
3949 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
3950 new_temp
= make_ssa_name (vec_dest
, NULL
);
3951 new_stmt
= gimple_build_assign_with_ops (codecvt1
, new_temp
,
3955 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3956 vec_oprnds0
[i
] = new_temp
;
3959 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
3960 stmt
, vec_dsts
, gsi
,
3965 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3969 vec_oprnds0
.release ();
3970 vec_oprnds1
.release ();
3971 vec_dsts
.release ();
3972 interm_types
.release ();
3978 /* Function vectorizable_assignment.
3980 Check if STMT performs an assignment (copy) that can be vectorized.
3981 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3982 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3983 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3986 vectorizable_assignment (gimple stmt
, gimple_stmt_iterator
*gsi
,
3987 gimple
*vec_stmt
, slp_tree slp_node
)
3992 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3993 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3994 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3998 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3999 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4002 vec
<tree
> vec_oprnds
= vNULL
;
4004 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4005 gimple new_stmt
= NULL
;
4006 stmt_vec_info prev_stmt_info
= NULL
;
4007 enum tree_code code
;
4010 /* Multiple types in SLP are handled by creating the appropriate number of
4011 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4013 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4016 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4018 gcc_assert (ncopies
>= 1);
4020 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4023 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4026 /* Is vectorizable assignment? */
4027 if (!is_gimple_assign (stmt
))
4030 scalar_dest
= gimple_assign_lhs (stmt
);
4031 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4034 code
= gimple_assign_rhs_code (stmt
);
4035 if (gimple_assign_single_p (stmt
)
4036 || code
== PAREN_EXPR
4037 || CONVERT_EXPR_CODE_P (code
))
4038 op
= gimple_assign_rhs1 (stmt
);
4042 if (code
== VIEW_CONVERT_EXPR
)
4043 op
= TREE_OPERAND (op
, 0);
4045 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
4046 &def_stmt
, &def
, &dt
[0], &vectype_in
))
4048 if (dump_enabled_p ())
4049 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4050 "use not simple.\n");
4054 /* We can handle NOP_EXPR conversions that do not change the number
4055 of elements or the vector size. */
4056 if ((CONVERT_EXPR_CODE_P (code
)
4057 || code
== VIEW_CONVERT_EXPR
)
4059 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4060 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4061 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4064 /* We do not handle bit-precision changes. */
4065 if ((CONVERT_EXPR_CODE_P (code
)
4066 || code
== VIEW_CONVERT_EXPR
)
4067 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4068 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4069 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4070 || ((TYPE_PRECISION (TREE_TYPE (op
))
4071 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4072 /* But a conversion that does not change the bit-pattern is ok. */
4073 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4074 > TYPE_PRECISION (TREE_TYPE (op
)))
4075 && TYPE_UNSIGNED (TREE_TYPE (op
))))
4077 if (dump_enabled_p ())
4078 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4079 "type conversion to/from bit-precision "
4084 if (!vec_stmt
) /* transformation not required. */
4086 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4087 if (dump_enabled_p ())
4088 dump_printf_loc (MSG_NOTE
, vect_location
,
4089 "=== vectorizable_assignment ===\n");
4090 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4095 if (dump_enabled_p ())
4096 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4099 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4102 for (j
= 0; j
< ncopies
; j
++)
4106 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4108 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4110 /* Arguments are ready. create the new vector stmt. */
4111 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4113 if (CONVERT_EXPR_CODE_P (code
)
4114 || code
== VIEW_CONVERT_EXPR
)
4115 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4116 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4117 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4118 gimple_assign_set_lhs (new_stmt
, new_temp
);
4119 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4121 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4128 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4130 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4132 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4135 vec_oprnds
.release ();
4140 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4141 either as shift by a scalar or by a vector. */
4144 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4147 enum machine_mode vec_mode
;
4152 vectype
= get_vectype_for_scalar_type (scalar_type
);
4156 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4158 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4160 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4162 || (optab_handler (optab
, TYPE_MODE (vectype
))
4163 == CODE_FOR_nothing
))
4167 vec_mode
= TYPE_MODE (vectype
);
4168 icode
= (int) optab_handler (optab
, vec_mode
);
4169 if (icode
== CODE_FOR_nothing
)
4176 /* Function vectorizable_shift.
4178 Check if STMT performs a shift operation that can be vectorized.
4179 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4180 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4181 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4184 vectorizable_shift (gimple stmt
, gimple_stmt_iterator
*gsi
,
4185 gimple
*vec_stmt
, slp_tree slp_node
)
4189 tree op0
, op1
= NULL
;
4190 tree vec_oprnd1
= NULL_TREE
;
4191 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4193 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4194 enum tree_code code
;
4195 enum machine_mode vec_mode
;
4199 enum machine_mode optab_op2_mode
;
4202 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4203 gimple new_stmt
= NULL
;
4204 stmt_vec_info prev_stmt_info
;
4211 vec
<tree
> vec_oprnds0
= vNULL
;
4212 vec
<tree
> vec_oprnds1
= vNULL
;
4215 bool scalar_shift_arg
= true;
4216 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4219 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4222 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4225 /* Is STMT a vectorizable binary/unary operation? */
4226 if (!is_gimple_assign (stmt
))
4229 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4232 code
= gimple_assign_rhs_code (stmt
);
4234 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4235 || code
== RROTATE_EXPR
))
4238 scalar_dest
= gimple_assign_lhs (stmt
);
4239 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4240 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4241 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4243 if (dump_enabled_p ())
4244 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4245 "bit-precision shifts not supported.\n");
4249 op0
= gimple_assign_rhs1 (stmt
);
4250 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4251 &def_stmt
, &def
, &dt
[0], &vectype
))
4253 if (dump_enabled_p ())
4254 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4255 "use not simple.\n");
4258 /* If op0 is an external or constant def use a vector type with
4259 the same size as the output vector type. */
4261 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4263 gcc_assert (vectype
);
4266 if (dump_enabled_p ())
4267 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4268 "no vectype for scalar type\n");
4272 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4273 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4274 if (nunits_out
!= nunits_in
)
4277 op1
= gimple_assign_rhs2 (stmt
);
4278 if (!vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4279 &def
, &dt
[1], &op1_vectype
))
4281 if (dump_enabled_p ())
4282 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4283 "use not simple.\n");
4288 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4292 /* Multiple types in SLP are handled by creating the appropriate number of
4293 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4295 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4298 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4300 gcc_assert (ncopies
>= 1);
4302 /* Determine whether the shift amount is a vector, or scalar. If the
4303 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4305 if (dt
[1] == vect_internal_def
&& !slp_node
)
4306 scalar_shift_arg
= false;
4307 else if (dt
[1] == vect_constant_def
4308 || dt
[1] == vect_external_def
4309 || dt
[1] == vect_internal_def
)
4311 /* In SLP, need to check whether the shift count is the same,
4312 in loops if it is a constant or invariant, it is always
4316 vec
<gimple
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4319 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4320 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4321 scalar_shift_arg
= false;
4326 if (dump_enabled_p ())
4327 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4328 "operand mode requires invariant argument.\n");
4332 /* Vector shifted by vector. */
4333 if (!scalar_shift_arg
)
4335 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4336 if (dump_enabled_p ())
4337 dump_printf_loc (MSG_NOTE
, vect_location
,
4338 "vector/vector shift/rotate found.\n");
4341 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4342 if (op1_vectype
== NULL_TREE
4343 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4345 if (dump_enabled_p ())
4346 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4347 "unusable type for last operand in"
4348 " vector/vector shift/rotate.\n");
4352 /* See if the machine has a vector shifted by scalar insn and if not
4353 then see if it has a vector shifted by vector insn. */
4356 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4358 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4360 if (dump_enabled_p ())
4361 dump_printf_loc (MSG_NOTE
, vect_location
,
4362 "vector/scalar shift/rotate found.\n");
4366 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4368 && (optab_handler (optab
, TYPE_MODE (vectype
))
4369 != CODE_FOR_nothing
))
4371 scalar_shift_arg
= false;
4373 if (dump_enabled_p ())
4374 dump_printf_loc (MSG_NOTE
, vect_location
,
4375 "vector/vector shift/rotate found.\n");
4377 /* Unlike the other binary operators, shifts/rotates have
4378 the rhs being int, instead of the same type as the lhs,
4379 so make sure the scalar is the right type if we are
4380 dealing with vectors of long long/long/short/char. */
4381 if (dt
[1] == vect_constant_def
)
4382 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4383 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4387 && TYPE_MODE (TREE_TYPE (vectype
))
4388 != TYPE_MODE (TREE_TYPE (op1
)))
4390 if (dump_enabled_p ())
4391 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4392 "unusable type for last operand in"
4393 " vector/vector shift/rotate.\n");
4396 if (vec_stmt
&& !slp_node
)
4398 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4399 op1
= vect_init_vector (stmt
, op1
,
4400 TREE_TYPE (vectype
), NULL
);
4407 /* Supportable by target? */
4410 if (dump_enabled_p ())
4411 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4415 vec_mode
= TYPE_MODE (vectype
);
4416 icode
= (int) optab_handler (optab
, vec_mode
);
4417 if (icode
== CODE_FOR_nothing
)
4419 if (dump_enabled_p ())
4420 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4421 "op not supported by target.\n");
4422 /* Check only during analysis. */
4423 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4424 || (vf
< vect_min_worthwhile_factor (code
)
4427 if (dump_enabled_p ())
4428 dump_printf_loc (MSG_NOTE
, vect_location
,
4429 "proceeding using word mode.\n");
4432 /* Worthwhile without SIMD support? Check only during analysis. */
4433 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4434 && vf
< vect_min_worthwhile_factor (code
)
4437 if (dump_enabled_p ())
4438 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4439 "not worthwhile without SIMD support.\n");
4443 if (!vec_stmt
) /* transformation not required. */
4445 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4446 if (dump_enabled_p ())
4447 dump_printf_loc (MSG_NOTE
, vect_location
,
4448 "=== vectorizable_shift ===\n");
4449 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4455 if (dump_enabled_p ())
4456 dump_printf_loc (MSG_NOTE
, vect_location
,
4457 "transform binary/unary operation.\n");
4460 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4462 prev_stmt_info
= NULL
;
4463 for (j
= 0; j
< ncopies
; j
++)
4468 if (scalar_shift_arg
)
4470 /* Vector shl and shr insn patterns can be defined with scalar
4471 operand 2 (shift operand). In this case, use constant or loop
4472 invariant op1 directly, without extending it to vector mode
4474 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4475 if (!VECTOR_MODE_P (optab_op2_mode
))
4477 if (dump_enabled_p ())
4478 dump_printf_loc (MSG_NOTE
, vect_location
,
4479 "operand 1 using scalar mode.\n");
4481 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4482 vec_oprnds1
.quick_push (vec_oprnd1
);
4485 /* Store vec_oprnd1 for every vector stmt to be created
4486 for SLP_NODE. We check during the analysis that all
4487 the shift arguments are the same.
4488 TODO: Allow different constants for different vector
4489 stmts generated for an SLP instance. */
4490 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4491 vec_oprnds1
.quick_push (vec_oprnd1
);
4496 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4497 (a special case for certain kind of vector shifts); otherwise,
4498 operand 1 should be of a vector type (the usual case). */
4500 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4503 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4507 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4509 /* Arguments are ready. Create the new vector stmt. */
4510 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4512 vop1
= vec_oprnds1
[i
];
4513 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vop0
, vop1
);
4514 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4515 gimple_assign_set_lhs (new_stmt
, new_temp
);
4516 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4518 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4525 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4527 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4528 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4531 vec_oprnds0
.release ();
4532 vec_oprnds1
.release ();
4538 /* Function vectorizable_operation.
4540 Check if STMT performs a binary, unary or ternary operation that can
4542 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4543 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4544 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4547 vectorizable_operation (gimple stmt
, gimple_stmt_iterator
*gsi
,
4548 gimple
*vec_stmt
, slp_tree slp_node
)
4552 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4553 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4555 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4556 enum tree_code code
;
4557 enum machine_mode vec_mode
;
4564 enum vect_def_type dt
[3]
4565 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4566 gimple new_stmt
= NULL
;
4567 stmt_vec_info prev_stmt_info
;
4573 vec
<tree
> vec_oprnds0
= vNULL
;
4574 vec
<tree
> vec_oprnds1
= vNULL
;
4575 vec
<tree
> vec_oprnds2
= vNULL
;
4576 tree vop0
, vop1
, vop2
;
4577 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4580 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4583 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4586 /* Is STMT a vectorizable binary/unary operation? */
4587 if (!is_gimple_assign (stmt
))
4590 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4593 code
= gimple_assign_rhs_code (stmt
);
4595 /* For pointer addition, we should use the normal plus for
4596 the vector addition. */
4597 if (code
== POINTER_PLUS_EXPR
)
4600 /* Support only unary or binary operations. */
4601 op_type
= TREE_CODE_LENGTH (code
);
4602 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4604 if (dump_enabled_p ())
4605 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4606 "num. args = %d (not unary/binary/ternary op).\n",
4611 scalar_dest
= gimple_assign_lhs (stmt
);
4612 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4614 /* Most operations cannot handle bit-precision types without extra
4616 if ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4617 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4618 /* Exception are bitwise binary operations. */
4619 && code
!= BIT_IOR_EXPR
4620 && code
!= BIT_XOR_EXPR
4621 && code
!= BIT_AND_EXPR
)
4623 if (dump_enabled_p ())
4624 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4625 "bit-precision arithmetic not supported.\n");
4629 op0
= gimple_assign_rhs1 (stmt
);
4630 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4631 &def_stmt
, &def
, &dt
[0], &vectype
))
4633 if (dump_enabled_p ())
4634 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4635 "use not simple.\n");
4638 /* If op0 is an external or constant def use a vector type with
4639 the same size as the output vector type. */
4641 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4643 gcc_assert (vectype
);
4646 if (dump_enabled_p ())
4648 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4649 "no vectype for scalar type ");
4650 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4652 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4658 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4659 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4660 if (nunits_out
!= nunits_in
)
4663 if (op_type
== binary_op
|| op_type
== ternary_op
)
4665 op1
= gimple_assign_rhs2 (stmt
);
4666 if (!vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4669 if (dump_enabled_p ())
4670 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4671 "use not simple.\n");
4675 if (op_type
== ternary_op
)
4677 op2
= gimple_assign_rhs3 (stmt
);
4678 if (!vect_is_simple_use (op2
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4681 if (dump_enabled_p ())
4682 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4683 "use not simple.\n");
4689 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4693 /* Multiple types in SLP are handled by creating the appropriate number of
4694 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4696 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4699 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4701 gcc_assert (ncopies
>= 1);
4703 /* Shifts are handled in vectorizable_shift (). */
4704 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4705 || code
== RROTATE_EXPR
)
4708 /* Supportable by target? */
4710 vec_mode
= TYPE_MODE (vectype
);
4711 if (code
== MULT_HIGHPART_EXPR
)
4713 if (can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
)))
4714 icode
= LAST_INSN_CODE
;
4716 icode
= CODE_FOR_nothing
;
4720 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4723 if (dump_enabled_p ())
4724 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4728 icode
= (int) optab_handler (optab
, vec_mode
);
4731 if (icode
== CODE_FOR_nothing
)
4733 if (dump_enabled_p ())
4734 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4735 "op not supported by target.\n");
4736 /* Check only during analysis. */
4737 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4738 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4740 if (dump_enabled_p ())
4741 dump_printf_loc (MSG_NOTE
, vect_location
,
4742 "proceeding using word mode.\n");
4745 /* Worthwhile without SIMD support? Check only during analysis. */
4746 if (!VECTOR_MODE_P (vec_mode
)
4748 && vf
< vect_min_worthwhile_factor (code
))
4750 if (dump_enabled_p ())
4751 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4752 "not worthwhile without SIMD support.\n");
4756 if (!vec_stmt
) /* transformation not required. */
4758 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4759 if (dump_enabled_p ())
4760 dump_printf_loc (MSG_NOTE
, vect_location
,
4761 "=== vectorizable_operation ===\n");
4762 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4768 if (dump_enabled_p ())
4769 dump_printf_loc (MSG_NOTE
, vect_location
,
4770 "transform binary/unary operation.\n");
4773 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4775 /* In case the vectorization factor (VF) is bigger than the number
4776 of elements that we can fit in a vectype (nunits), we have to generate
4777 more than one vector stmt - i.e - we need to "unroll" the
4778 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4779 from one copy of the vector stmt to the next, in the field
4780 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4781 stages to find the correct vector defs to be used when vectorizing
4782 stmts that use the defs of the current stmt. The example below
4783 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4784 we need to create 4 vectorized stmts):
4786 before vectorization:
4787 RELATED_STMT VEC_STMT
4791 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4793 RELATED_STMT VEC_STMT
4794 VS1_0: vx0 = memref0 VS1_1 -
4795 VS1_1: vx1 = memref1 VS1_2 -
4796 VS1_2: vx2 = memref2 VS1_3 -
4797 VS1_3: vx3 = memref3 - -
4798 S1: x = load - VS1_0
4801 step2: vectorize stmt S2 (done here):
4802 To vectorize stmt S2 we first need to find the relevant vector
4803 def for the first operand 'x'. This is, as usual, obtained from
4804 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4805 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4806 relevant vector def 'vx0'. Having found 'vx0' we can generate
4807 the vector stmt VS2_0, and as usual, record it in the
4808 STMT_VINFO_VEC_STMT of stmt S2.
4809 When creating the second copy (VS2_1), we obtain the relevant vector
4810 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4811 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4812 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4813 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4814 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4815 chain of stmts and pointers:
4816 RELATED_STMT VEC_STMT
4817 VS1_0: vx0 = memref0 VS1_1 -
4818 VS1_1: vx1 = memref1 VS1_2 -
4819 VS1_2: vx2 = memref2 VS1_3 -
4820 VS1_3: vx3 = memref3 - -
4821 S1: x = load - VS1_0
4822 VS2_0: vz0 = vx0 + v1 VS2_1 -
4823 VS2_1: vz1 = vx1 + v1 VS2_2 -
4824 VS2_2: vz2 = vx2 + v1 VS2_3 -
4825 VS2_3: vz3 = vx3 + v1 - -
4826 S2: z = x + 1 - VS2_0 */
4828 prev_stmt_info
= NULL
;
4829 for (j
= 0; j
< ncopies
; j
++)
4834 if (op_type
== binary_op
|| op_type
== ternary_op
)
4835 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4838 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4840 if (op_type
== ternary_op
)
4842 vec_oprnds2
.create (1);
4843 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
4850 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4851 if (op_type
== ternary_op
)
4853 tree vec_oprnd
= vec_oprnds2
.pop ();
4854 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
4859 /* Arguments are ready. Create the new vector stmt. */
4860 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4862 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
4863 ? vec_oprnds1
[i
] : NULL_TREE
);
4864 vop2
= ((op_type
== ternary_op
)
4865 ? vec_oprnds2
[i
] : NULL_TREE
);
4866 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
,
4868 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4869 gimple_assign_set_lhs (new_stmt
, new_temp
);
4870 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4872 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4879 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4881 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4882 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4885 vec_oprnds0
.release ();
4886 vec_oprnds1
.release ();
4887 vec_oprnds2
.release ();
4892 /* A helper function to ensure data reference DR's base alignment
4896 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
4901 if (((dataref_aux
*)dr
->aux
)->base_misaligned
)
4903 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4904 tree base_decl
= ((dataref_aux
*)dr
->aux
)->base_decl
;
4906 DECL_ALIGN (base_decl
) = TYPE_ALIGN (vectype
);
4907 DECL_USER_ALIGN (base_decl
) = 1;
4908 ((dataref_aux
*)dr
->aux
)->base_misaligned
= false;
4913 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
4914 reversal of the vector elements. If that is impossible to do,
4918 perm_mask_for_reverse (tree vectype
)
4923 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4924 sel
= XALLOCAVEC (unsigned char, nunits
);
4926 for (i
= 0; i
< nunits
; ++i
)
4927 sel
[i
] = nunits
- 1 - i
;
4929 return vect_gen_perm_mask (vectype
, sel
);
4932 /* Function vectorizable_store.
4934 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
4936 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4937 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4938 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4941 vectorizable_store (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
4947 tree vec_oprnd
= NULL_TREE
;
4948 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4949 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
4950 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4952 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4953 struct loop
*loop
= NULL
;
4954 enum machine_mode vec_mode
;
4956 enum dr_alignment_support alignment_support_scheme
;
4959 enum vect_def_type dt
;
4960 stmt_vec_info prev_stmt_info
= NULL
;
4961 tree dataref_ptr
= NULL_TREE
;
4962 tree dataref_offset
= NULL_TREE
;
4963 gimple ptr_incr
= NULL
;
4964 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4967 gimple next_stmt
, first_stmt
= NULL
;
4968 bool grouped_store
= false;
4969 bool store_lanes_p
= false;
4970 unsigned int group_size
, i
;
4971 vec
<tree
> dr_chain
= vNULL
;
4972 vec
<tree
> oprnds
= vNULL
;
4973 vec
<tree
> result_chain
= vNULL
;
4975 bool negative
= false;
4976 tree offset
= NULL_TREE
;
4977 vec
<tree
> vec_oprnds
= vNULL
;
4978 bool slp
= (slp_node
!= NULL
);
4979 unsigned int vec_num
;
4980 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4984 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
4986 /* Multiple types in SLP are handled by creating the appropriate number of
4987 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4989 if (slp
|| PURE_SLP_STMT (stmt_info
))
4992 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4994 gcc_assert (ncopies
>= 1);
4996 /* FORNOW. This restriction should be relaxed. */
4997 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
4999 if (dump_enabled_p ())
5000 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5001 "multiple types in nested loop.\n");
5005 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5008 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5011 /* Is vectorizable store? */
5013 if (!is_gimple_assign (stmt
))
5016 scalar_dest
= gimple_assign_lhs (stmt
);
5017 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5018 && is_pattern_stmt_p (stmt_info
))
5019 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5020 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5021 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5022 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5023 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5024 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5025 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5026 && TREE_CODE (scalar_dest
) != MEM_REF
)
5029 gcc_assert (gimple_assign_single_p (stmt
));
5030 op
= gimple_assign_rhs1 (stmt
);
5031 if (!vect_is_simple_use (op
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5034 if (dump_enabled_p ())
5035 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5036 "use not simple.\n");
5040 elem_type
= TREE_TYPE (vectype
);
5041 vec_mode
= TYPE_MODE (vectype
);
5043 /* FORNOW. In some cases can vectorize even if data-type not supported
5044 (e.g. - array initialization with 0). */
5045 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5048 if (!STMT_VINFO_DATA_REF (stmt_info
))
5052 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5053 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5054 size_zero_node
) < 0;
5055 if (negative
&& ncopies
> 1)
5057 if (dump_enabled_p ())
5058 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5059 "multiple types with negative step.\n");
5065 gcc_assert (!grouped_store
);
5066 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5067 if (alignment_support_scheme
!= dr_aligned
5068 && alignment_support_scheme
!= dr_unaligned_supported
)
5070 if (dump_enabled_p ())
5071 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5072 "negative step but alignment required.\n");
5075 if (dt
!= vect_constant_def
5076 && dt
!= vect_external_def
5077 && !perm_mask_for_reverse (vectype
))
5079 if (dump_enabled_p ())
5080 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5081 "negative step and reversing not supported.\n");
5086 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5088 grouped_store
= true;
5089 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5090 if (!slp
&& !PURE_SLP_STMT (stmt_info
))
5092 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5093 if (vect_store_lanes_supported (vectype
, group_size
))
5094 store_lanes_p
= true;
5095 else if (!vect_grouped_store_supported (vectype
, group_size
))
5099 if (first_stmt
== stmt
)
5101 /* STMT is the leader of the group. Check the operands of all the
5102 stmts of the group. */
5103 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5106 gcc_assert (gimple_assign_single_p (next_stmt
));
5107 op
= gimple_assign_rhs1 (next_stmt
);
5108 if (!vect_is_simple_use (op
, next_stmt
, loop_vinfo
, bb_vinfo
,
5109 &def_stmt
, &def
, &dt
))
5111 if (dump_enabled_p ())
5112 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5113 "use not simple.\n");
5116 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5121 if (!vec_stmt
) /* transformation not required. */
5123 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5124 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5131 ensure_base_align (stmt_info
, dr
);
5135 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5136 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5138 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5141 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5143 /* We vectorize all the stmts of the interleaving group when we
5144 reach the last stmt in the group. */
5145 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5146 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5155 grouped_store
= false;
5156 /* VEC_NUM is the number of vect stmts to be created for this
5158 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5159 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5160 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5161 op
= gimple_assign_rhs1 (first_stmt
);
5164 /* VEC_NUM is the number of vect stmts to be created for this
5166 vec_num
= group_size
;
5172 group_size
= vec_num
= 1;
5175 if (dump_enabled_p ())
5176 dump_printf_loc (MSG_NOTE
, vect_location
,
5177 "transform store. ncopies = %d\n", ncopies
);
5179 dr_chain
.create (group_size
);
5180 oprnds
.create (group_size
);
5182 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5183 gcc_assert (alignment_support_scheme
);
5184 /* Targets with store-lane instructions must not require explicit
5186 gcc_assert (!store_lanes_p
5187 || alignment_support_scheme
== dr_aligned
5188 || alignment_support_scheme
== dr_unaligned_supported
);
5191 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5194 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5196 aggr_type
= vectype
;
5198 /* In case the vectorization factor (VF) is bigger than the number
5199 of elements that we can fit in a vectype (nunits), we have to generate
5200 more than one vector stmt - i.e - we need to "unroll" the
5201 vector stmt by a factor VF/nunits. For more details see documentation in
5202 vect_get_vec_def_for_copy_stmt. */
5204 /* In case of interleaving (non-unit grouped access):
5211 We create vectorized stores starting from base address (the access of the
5212 first stmt in the chain (S2 in the above example), when the last store stmt
5213 of the chain (S4) is reached:
5216 VS2: &base + vec_size*1 = vx0
5217 VS3: &base + vec_size*2 = vx1
5218 VS4: &base + vec_size*3 = vx3
5220 Then permutation statements are generated:
5222 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5223 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5226 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5227 (the order of the data-refs in the output of vect_permute_store_chain
5228 corresponds to the order of scalar stmts in the interleaving chain - see
5229 the documentation of vect_permute_store_chain()).
5231 In case of both multiple types and interleaving, above vector stores and
5232 permutation stmts are created for every copy. The result vector stmts are
5233 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5234 STMT_VINFO_RELATED_STMT for the next copies.
5237 prev_stmt_info
= NULL
;
5238 for (j
= 0; j
< ncopies
; j
++)
5246 /* Get vectorized arguments for SLP_NODE. */
5247 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5248 NULL
, slp_node
, -1);
5250 vec_oprnd
= vec_oprnds
[0];
5254 /* For interleaved stores we collect vectorized defs for all the
5255 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5256 used as an input to vect_permute_store_chain(), and OPRNDS as
5257 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5259 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5260 OPRNDS are of size 1. */
5261 next_stmt
= first_stmt
;
5262 for (i
= 0; i
< group_size
; i
++)
5264 /* Since gaps are not supported for interleaved stores,
5265 GROUP_SIZE is the exact number of stmts in the chain.
5266 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5267 there is no interleaving, GROUP_SIZE is 1, and only one
5268 iteration of the loop will be executed. */
5269 gcc_assert (next_stmt
5270 && gimple_assign_single_p (next_stmt
));
5271 op
= gimple_assign_rhs1 (next_stmt
);
5273 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
5275 dr_chain
.quick_push (vec_oprnd
);
5276 oprnds
.quick_push (vec_oprnd
);
5277 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5281 /* We should have catched mismatched types earlier. */
5282 gcc_assert (useless_type_conversion_p (vectype
,
5283 TREE_TYPE (vec_oprnd
)));
5284 bool simd_lane_access_p
5285 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5286 if (simd_lane_access_p
5287 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5288 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5289 && integer_zerop (DR_OFFSET (first_dr
))
5290 && integer_zerop (DR_INIT (first_dr
))
5291 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5292 get_alias_set (DR_REF (first_dr
))))
5294 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5295 dataref_offset
= build_int_cst (reference_alias_ptr_type
5296 (DR_REF (first_dr
)), 0);
5301 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5302 simd_lane_access_p
? loop
: NULL
,
5303 offset
, &dummy
, gsi
, &ptr_incr
,
5304 simd_lane_access_p
, &inv_p
);
5305 gcc_assert (bb_vinfo
|| !inv_p
);
5309 /* For interleaved stores we created vectorized defs for all the
5310 defs stored in OPRNDS in the previous iteration (previous copy).
5311 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5312 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5314 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5315 OPRNDS are of size 1. */
5316 for (i
= 0; i
< group_size
; i
++)
5319 vect_is_simple_use (op
, NULL
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5321 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5322 dr_chain
[i
] = vec_oprnd
;
5323 oprnds
[i
] = vec_oprnd
;
5327 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5328 TYPE_SIZE_UNIT (aggr_type
));
5330 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5331 TYPE_SIZE_UNIT (aggr_type
));
5338 /* Combine all the vectors into an array. */
5339 vec_array
= create_vector_array (vectype
, vec_num
);
5340 for (i
= 0; i
< vec_num
; i
++)
5342 vec_oprnd
= dr_chain
[i
];
5343 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5347 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5348 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5349 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5350 gimple_call_set_lhs (new_stmt
, data_ref
);
5351 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5359 result_chain
.create (group_size
);
5361 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5365 next_stmt
= first_stmt
;
5366 for (i
= 0; i
< vec_num
; i
++)
5368 unsigned align
, misalign
;
5371 /* Bump the vector pointer. */
5372 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5376 vec_oprnd
= vec_oprnds
[i
];
5377 else if (grouped_store
)
5378 /* For grouped stores vectorized defs are interleaved in
5379 vect_permute_store_chain(). */
5380 vec_oprnd
= result_chain
[i
];
5382 data_ref
= build2 (MEM_REF
, TREE_TYPE (vec_oprnd
), dataref_ptr
,
5385 : build_int_cst (reference_alias_ptr_type
5386 (DR_REF (first_dr
)), 0));
5387 align
= TYPE_ALIGN_UNIT (vectype
);
5388 if (aligned_access_p (first_dr
))
5390 else if (DR_MISALIGNMENT (first_dr
) == -1)
5392 TREE_TYPE (data_ref
)
5393 = build_aligned_type (TREE_TYPE (data_ref
),
5394 TYPE_ALIGN (elem_type
));
5395 align
= TYPE_ALIGN_UNIT (elem_type
);
5400 TREE_TYPE (data_ref
)
5401 = build_aligned_type (TREE_TYPE (data_ref
),
5402 TYPE_ALIGN (elem_type
));
5403 misalign
= DR_MISALIGNMENT (first_dr
);
5405 if (dataref_offset
== NULL_TREE
)
5406 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
5410 && dt
!= vect_constant_def
5411 && dt
!= vect_external_def
)
5413 tree perm_mask
= perm_mask_for_reverse (vectype
);
5415 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
5417 tree new_temp
= make_ssa_name (perm_dest
, NULL
);
5419 /* Generate the permute statement. */
5421 = gimple_build_assign_with_ops (VEC_PERM_EXPR
, new_temp
,
5422 vec_oprnd
, vec_oprnd
,
5424 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5426 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
5427 vec_oprnd
= new_temp
;
5430 /* Arguments are ready. Create the new vector stmt. */
5431 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
5432 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5437 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5445 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5447 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5448 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5452 dr_chain
.release ();
5454 result_chain
.release ();
5455 vec_oprnds
.release ();
5460 /* Given a vector type VECTYPE and permutation SEL returns
5461 the VECTOR_CST mask that implements the permutation of the
5462 vector elements. If that is impossible to do, returns NULL. */
5465 vect_gen_perm_mask (tree vectype
, unsigned char *sel
)
5467 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
5470 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5472 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
5475 mask_elt_type
= lang_hooks
.types
.type_for_mode
5476 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
5477 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
5479 mask_elts
= XALLOCAVEC (tree
, nunits
);
5480 for (i
= nunits
- 1; i
>= 0; i
--)
5481 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
5482 mask_vec
= build_vector (mask_type
, mask_elts
);
5487 /* Given a vector variable X and Y, that was generated for the scalar
5488 STMT, generate instructions to permute the vector elements of X and Y
5489 using permutation mask MASK_VEC, insert them at *GSI and return the
5490 permuted vector variable. */
5493 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple stmt
,
5494 gimple_stmt_iterator
*gsi
)
5496 tree vectype
= TREE_TYPE (x
);
5497 tree perm_dest
, data_ref
;
5500 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
5501 data_ref
= make_ssa_name (perm_dest
, NULL
);
5503 /* Generate the permute statement. */
5504 perm_stmt
= gimple_build_assign_with_ops (VEC_PERM_EXPR
, data_ref
,
5506 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5511 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5512 inserting them on the loops preheader edge. Returns true if we
5513 were successful in doing so (and thus STMT can be moved then),
5514 otherwise returns false. */
5517 hoist_defs_of_uses (gimple stmt
, struct loop
*loop
)
5523 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5525 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5526 if (!gimple_nop_p (def_stmt
)
5527 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5529 /* Make sure we don't need to recurse. While we could do
5530 so in simple cases when there are more complex use webs
5531 we don't have an easy way to preserve stmt order to fulfil
5532 dependencies within them. */
5535 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
5537 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
5539 gimple def_stmt2
= SSA_NAME_DEF_STMT (op2
);
5540 if (!gimple_nop_p (def_stmt2
)
5541 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
5551 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5553 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5554 if (!gimple_nop_p (def_stmt
)
5555 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5557 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
5558 gsi_remove (&gsi
, false);
5559 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
5566 /* vectorizable_load.
5568 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
5570 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5571 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5572 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5575 vectorizable_load (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5576 slp_tree slp_node
, slp_instance slp_node_instance
)
5579 tree vec_dest
= NULL
;
5580 tree data_ref
= NULL
;
5581 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5582 stmt_vec_info prev_stmt_info
;
5583 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5584 struct loop
*loop
= NULL
;
5585 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
5586 bool nested_in_vect_loop
= false;
5587 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5588 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5591 enum machine_mode mode
;
5592 gimple new_stmt
= NULL
;
5594 enum dr_alignment_support alignment_support_scheme
;
5595 tree dataref_ptr
= NULL_TREE
;
5596 tree dataref_offset
= NULL_TREE
;
5597 gimple ptr_incr
= NULL
;
5598 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5600 int i
, j
, group_size
, group_gap
;
5601 tree msq
= NULL_TREE
, lsq
;
5602 tree offset
= NULL_TREE
;
5603 tree realignment_token
= NULL_TREE
;
5605 vec
<tree
> dr_chain
= vNULL
;
5606 bool grouped_load
= false;
5607 bool load_lanes_p
= false;
5610 bool negative
= false;
5611 bool compute_in_loop
= false;
5612 struct loop
*at_loop
;
5614 bool slp
= (slp_node
!= NULL
);
5615 bool slp_perm
= false;
5616 enum tree_code code
;
5617 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5620 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
5621 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
5622 int gather_scale
= 1;
5623 enum vect_def_type gather_dt
= vect_unknown_def_type
;
5627 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5628 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
5629 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5634 /* Multiple types in SLP are handled by creating the appropriate number of
5635 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5637 if (slp
|| PURE_SLP_STMT (stmt_info
))
5640 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5642 gcc_assert (ncopies
>= 1);
5644 /* FORNOW. This restriction should be relaxed. */
5645 if (nested_in_vect_loop
&& ncopies
> 1)
5647 if (dump_enabled_p ())
5648 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5649 "multiple types in nested loop.\n");
5653 /* Invalidate assumptions made by dependence analysis when vectorization
5654 on the unrolled body effectively re-orders stmts. */
5656 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5657 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5658 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5660 if (dump_enabled_p ())
5661 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5662 "cannot perform implicit CSE when unrolling "
5663 "with negative dependence distance\n");
5667 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5670 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5673 /* Is vectorizable load? */
5674 if (!is_gimple_assign (stmt
))
5677 scalar_dest
= gimple_assign_lhs (stmt
);
5678 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
5681 code
= gimple_assign_rhs_code (stmt
);
5682 if (code
!= ARRAY_REF
5683 && code
!= BIT_FIELD_REF
5684 && code
!= INDIRECT_REF
5685 && code
!= COMPONENT_REF
5686 && code
!= IMAGPART_EXPR
5687 && code
!= REALPART_EXPR
5689 && TREE_CODE_CLASS (code
) != tcc_declaration
)
5692 if (!STMT_VINFO_DATA_REF (stmt_info
))
5695 elem_type
= TREE_TYPE (vectype
);
5696 mode
= TYPE_MODE (vectype
);
5698 /* FORNOW. In some cases can vectorize even if data-type not supported
5699 (e.g. - data copies). */
5700 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
5702 if (dump_enabled_p ())
5703 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5704 "Aligned load, but unsupported type.\n");
5708 /* Check if the load is a part of an interleaving chain. */
5709 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5711 grouped_load
= true;
5713 gcc_assert (! nested_in_vect_loop
&& !STMT_VINFO_GATHER_P (stmt_info
));
5715 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5716 if (!slp
&& !PURE_SLP_STMT (stmt_info
))
5718 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5719 if (vect_load_lanes_supported (vectype
, group_size
))
5720 load_lanes_p
= true;
5721 else if (!vect_grouped_load_supported (vectype
, group_size
))
5725 /* Invalidate assumptions made by dependence analysis when vectorization
5726 on the unrolled body effectively re-orders stmts. */
5727 if (!PURE_SLP_STMT (stmt_info
)
5728 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5729 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5730 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5732 if (dump_enabled_p ())
5733 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5734 "cannot perform implicit CSE when performing "
5735 "group loads with negative dependence distance\n");
5741 if (STMT_VINFO_GATHER_P (stmt_info
))
5745 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
5746 &gather_off
, &gather_scale
);
5747 gcc_assert (gather_decl
);
5748 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, bb_vinfo
,
5749 &def_stmt
, &def
, &gather_dt
,
5750 &gather_off_vectype
))
5752 if (dump_enabled_p ())
5753 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5754 "gather index use not simple.\n");
5758 else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info
))
5762 negative
= tree_int_cst_compare (nested_in_vect_loop
5763 ? STMT_VINFO_DR_STEP (stmt_info
)
5765 size_zero_node
) < 0;
5766 if (negative
&& ncopies
> 1)
5768 if (dump_enabled_p ())
5769 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5770 "multiple types with negative step.\n");
5778 if (dump_enabled_p ())
5779 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5780 "negative step for group load not supported"
5784 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5785 if (alignment_support_scheme
!= dr_aligned
5786 && alignment_support_scheme
!= dr_unaligned_supported
)
5788 if (dump_enabled_p ())
5789 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5790 "negative step but alignment required.\n");
5793 if (!perm_mask_for_reverse (vectype
))
5795 if (dump_enabled_p ())
5796 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5797 "negative step and reversing not supported."
5804 if (!vec_stmt
) /* transformation not required. */
5806 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
5807 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
, NULL
, NULL
, NULL
);
5811 if (dump_enabled_p ())
5812 dump_printf_loc (MSG_NOTE
, vect_location
,
5813 "transform load. ncopies = %d\n", ncopies
);
5817 ensure_base_align (stmt_info
, dr
);
5819 if (STMT_VINFO_GATHER_P (stmt_info
))
5821 tree vec_oprnd0
= NULL_TREE
, op
;
5822 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
5823 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
5824 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
5825 edge pe
= loop_preheader_edge (loop
);
5828 enum { NARROW
, NONE
, WIDEN
} modifier
;
5829 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
5831 if (nunits
== gather_off_nunits
)
5833 else if (nunits
== gather_off_nunits
/ 2)
5835 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
5838 for (i
= 0; i
< gather_off_nunits
; ++i
)
5839 sel
[i
] = i
| nunits
;
5841 perm_mask
= vect_gen_perm_mask (gather_off_vectype
, sel
);
5842 gcc_assert (perm_mask
!= NULL_TREE
);
5844 else if (nunits
== gather_off_nunits
* 2)
5846 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
5849 for (i
= 0; i
< nunits
; ++i
)
5850 sel
[i
] = i
< gather_off_nunits
5851 ? i
: i
+ nunits
- gather_off_nunits
;
5853 perm_mask
= vect_gen_perm_mask (vectype
, sel
);
5854 gcc_assert (perm_mask
!= NULL_TREE
);
5860 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
5861 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5862 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5863 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5864 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5865 scaletype
= TREE_VALUE (arglist
);
5866 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
5868 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
5870 ptr
= fold_convert (ptrtype
, gather_base
);
5871 if (!is_gimple_min_invariant (ptr
))
5873 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
5874 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
5875 gcc_assert (!new_bb
);
5878 /* Currently we support only unconditional gather loads,
5879 so mask should be all ones. */
5880 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
5881 mask
= build_int_cst (masktype
, -1);
5882 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
5884 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
5885 mask
= build_vector_from_val (masktype
, mask
);
5886 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
5888 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
5892 for (j
= 0; j
< 6; ++j
)
5894 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
5895 mask
= build_real (TREE_TYPE (masktype
), r
);
5896 mask
= build_vector_from_val (masktype
, mask
);
5897 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
5902 scale
= build_int_cst (scaletype
, gather_scale
);
5904 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
5905 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
5906 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
5910 for (j
= 0; j
< 6; ++j
)
5912 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
5913 merge
= build_real (TREE_TYPE (rettype
), r
);
5917 merge
= build_vector_from_val (rettype
, merge
);
5918 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
5920 prev_stmt_info
= NULL
;
5921 for (j
= 0; j
< ncopies
; ++j
)
5923 if (modifier
== WIDEN
&& (j
& 1))
5924 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
5925 perm_mask
, stmt
, gsi
);
5928 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
5931 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
5933 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
5935 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
5936 == TYPE_VECTOR_SUBPARTS (idxtype
));
5937 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
5938 var
= make_ssa_name (var
, NULL
);
5939 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
5941 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR
, var
,
5943 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5948 = gimple_build_call (gather_decl
, 5, merge
, ptr
, op
, mask
, scale
);
5950 if (!useless_type_conversion_p (vectype
, rettype
))
5952 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
5953 == TYPE_VECTOR_SUBPARTS (rettype
));
5954 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
5955 op
= make_ssa_name (var
, new_stmt
);
5956 gimple_call_set_lhs (new_stmt
, op
);
5957 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5958 var
= make_ssa_name (vec_dest
, NULL
);
5959 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
5961 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR
, var
, op
,
5966 var
= make_ssa_name (vec_dest
, new_stmt
);
5967 gimple_call_set_lhs (new_stmt
, var
);
5970 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5972 if (modifier
== NARROW
)
5979 var
= permute_vec_elements (prev_res
, var
,
5980 perm_mask
, stmt
, gsi
);
5981 new_stmt
= SSA_NAME_DEF_STMT (var
);
5984 if (prev_stmt_info
== NULL
)
5985 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5987 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5988 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5992 else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info
))
5994 gimple_stmt_iterator incr_gsi
;
6000 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6001 gimple_seq stmts
= NULL
;
6002 tree stride_base
, stride_step
, alias_off
;
6004 gcc_assert (!nested_in_vect_loop
);
6007 = fold_build_pointer_plus
6008 (unshare_expr (DR_BASE_ADDRESS (dr
)),
6009 size_binop (PLUS_EXPR
,
6010 convert_to_ptrofftype (unshare_expr (DR_OFFSET (dr
))),
6011 convert_to_ptrofftype (DR_INIT (dr
))));
6012 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (dr
)));
6014 /* For a load with loop-invariant (but other than power-of-2)
6015 stride (i.e. not a grouped access) like so:
6017 for (i = 0; i < n; i += stride)
6020 we generate a new induction variable and new accesses to
6021 form a new vector (or vectors, depending on ncopies):
6023 for (j = 0; ; j += VF*stride)
6025 tmp2 = array[j + stride];
6027 vectemp = {tmp1, tmp2, ...}
6030 ivstep
= stride_step
;
6031 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
6032 build_int_cst (TREE_TYPE (ivstep
), vf
));
6034 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6036 create_iv (stride_base
, ivstep
, NULL
,
6037 loop
, &incr_gsi
, insert_after
,
6039 incr
= gsi_stmt (incr_gsi
);
6040 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
6042 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
6044 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6046 prev_stmt_info
= NULL
;
6047 running_off
= offvar
;
6048 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (dr
)), 0);
6049 for (j
= 0; j
< ncopies
; j
++)
6053 vec_alloc (v
, nunits
);
6054 for (i
= 0; i
< nunits
; i
++)
6056 tree newref
, newoff
;
6058 newref
= build2 (MEM_REF
, TREE_TYPE (vectype
),
6059 running_off
, alias_off
);
6061 newref
= force_gimple_operand_gsi (gsi
, newref
, true,
6064 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, newref
);
6065 newoff
= copy_ssa_name (running_off
, NULL
);
6066 incr
= gimple_build_assign_with_ops (POINTER_PLUS_EXPR
, newoff
,
6067 running_off
, stride_step
);
6068 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6070 running_off
= newoff
;
6073 vec_inv
= build_constructor (vectype
, v
);
6074 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6075 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6078 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6080 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6081 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6088 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6090 && !SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ()
6091 && first_stmt
!= SLP_TREE_SCALAR_STMTS (slp_node
)[0])
6092 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6094 /* Check if the chain of loads is already vectorized. */
6095 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6096 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6097 ??? But we can only do so if there is exactly one
6098 as we have no way to get at the rest. Leave the CSE
6100 ??? With the group load eventually participating
6101 in multiple different permutations (having multiple
6102 slp nodes which refer to the same group) the CSE
6103 is even wrong code. See PR56270. */
6106 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6109 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6110 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6112 /* VEC_NUM is the number of vect stmts to be created for this group. */
6115 grouped_load
= false;
6116 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6117 if (SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6119 group_gap
= GROUP_GAP (vinfo_for_stmt (first_stmt
));
6123 vec_num
= group_size
;
6131 group_size
= vec_num
= 1;
6135 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6136 gcc_assert (alignment_support_scheme
);
6137 /* Targets with load-lane instructions must not require explicit
6139 gcc_assert (!load_lanes_p
6140 || alignment_support_scheme
== dr_aligned
6141 || alignment_support_scheme
== dr_unaligned_supported
);
6143 /* In case the vectorization factor (VF) is bigger than the number
6144 of elements that we can fit in a vectype (nunits), we have to generate
6145 more than one vector stmt - i.e - we need to "unroll" the
6146 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6147 from one copy of the vector stmt to the next, in the field
6148 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6149 stages to find the correct vector defs to be used when vectorizing
6150 stmts that use the defs of the current stmt. The example below
6151 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6152 need to create 4 vectorized stmts):
6154 before vectorization:
6155 RELATED_STMT VEC_STMT
6159 step 1: vectorize stmt S1:
6160 We first create the vector stmt VS1_0, and, as usual, record a
6161 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6162 Next, we create the vector stmt VS1_1, and record a pointer to
6163 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6164 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6166 RELATED_STMT VEC_STMT
6167 VS1_0: vx0 = memref0 VS1_1 -
6168 VS1_1: vx1 = memref1 VS1_2 -
6169 VS1_2: vx2 = memref2 VS1_3 -
6170 VS1_3: vx3 = memref3 - -
6171 S1: x = load - VS1_0
6174 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6175 information we recorded in RELATED_STMT field is used to vectorize
6178 /* In case of interleaving (non-unit grouped access):
6185 Vectorized loads are created in the order of memory accesses
6186 starting from the access of the first stmt of the chain:
6189 VS2: vx1 = &base + vec_size*1
6190 VS3: vx3 = &base + vec_size*2
6191 VS4: vx4 = &base + vec_size*3
6193 Then permutation statements are generated:
6195 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6196 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6199 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6200 (the order of the data-refs in the output of vect_permute_load_chain
6201 corresponds to the order of scalar stmts in the interleaving chain - see
6202 the documentation of vect_permute_load_chain()).
6203 The generation of permutation stmts and recording them in
6204 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6206 In case of both multiple types and interleaving, the vector loads and
6207 permutation stmts above are created for every copy. The result vector
6208 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6209 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6211 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6212 on a target that supports unaligned accesses (dr_unaligned_supported)
6213 we generate the following code:
6217 p = p + indx * vectype_size;
6222 Otherwise, the data reference is potentially unaligned on a target that
6223 does not support unaligned accesses (dr_explicit_realign_optimized) -
6224 then generate the following code, in which the data in each iteration is
6225 obtained by two vector loads, one from the previous iteration, and one
6226 from the current iteration:
6228 msq_init = *(floor(p1))
6229 p2 = initial_addr + VS - 1;
6230 realignment_token = call target_builtin;
6233 p2 = p2 + indx * vectype_size
6235 vec_dest = realign_load (msq, lsq, realignment_token)
6240 /* If the misalignment remains the same throughout the execution of the
6241 loop, we can create the init_addr and permutation mask at the loop
6242 preheader. Otherwise, it needs to be created inside the loop.
6243 This can only occur when vectorizing memory accesses in the inner-loop
6244 nested within an outer-loop that is being vectorized. */
6246 if (nested_in_vect_loop
6247 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6248 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6250 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6251 compute_in_loop
= true;
6254 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6255 || alignment_support_scheme
== dr_explicit_realign
)
6256 && !compute_in_loop
)
6258 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6259 alignment_support_scheme
, NULL_TREE
,
6261 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6263 phi
= SSA_NAME_DEF_STMT (msq
);
6264 offset
= size_int (TYPE_VECTOR_SUBPARTS (vectype
) - 1);
6271 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6274 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6276 aggr_type
= vectype
;
6278 prev_stmt_info
= NULL
;
6279 for (j
= 0; j
< ncopies
; j
++)
6281 /* 1. Create the vector or array pointer update chain. */
6284 bool simd_lane_access_p
6285 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6286 if (simd_lane_access_p
6287 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6288 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6289 && integer_zerop (DR_OFFSET (first_dr
))
6290 && integer_zerop (DR_INIT (first_dr
))
6291 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6292 get_alias_set (DR_REF (first_dr
)))
6293 && (alignment_support_scheme
== dr_aligned
6294 || alignment_support_scheme
== dr_unaligned_supported
))
6296 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6297 dataref_offset
= build_int_cst (reference_alias_ptr_type
6298 (DR_REF (first_dr
)), 0);
6303 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
6304 offset
, &dummy
, gsi
, &ptr_incr
,
6305 simd_lane_access_p
, &inv_p
);
6307 else if (dataref_offset
)
6308 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
6309 TYPE_SIZE_UNIT (aggr_type
));
6311 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6312 TYPE_SIZE_UNIT (aggr_type
));
6314 if (grouped_load
|| slp_perm
)
6315 dr_chain
.create (vec_num
);
6321 vec_array
= create_vector_array (vectype
, vec_num
);
6324 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6325 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
6326 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
6327 gimple_call_set_lhs (new_stmt
, vec_array
);
6328 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6330 /* Extract each vector into an SSA_NAME. */
6331 for (i
= 0; i
< vec_num
; i
++)
6333 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
6335 dr_chain
.quick_push (new_temp
);
6338 /* Record the mapping between SSA_NAMEs and statements. */
6339 vect_record_grouped_load_vectors (stmt
, dr_chain
);
6343 for (i
= 0; i
< vec_num
; i
++)
6346 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6349 /* 2. Create the vector-load in the loop. */
6350 switch (alignment_support_scheme
)
6353 case dr_unaligned_supported
:
6355 unsigned int align
, misalign
;
6358 = build2 (MEM_REF
, vectype
, dataref_ptr
,
6361 : build_int_cst (reference_alias_ptr_type
6362 (DR_REF (first_dr
)), 0));
6363 align
= TYPE_ALIGN_UNIT (vectype
);
6364 if (alignment_support_scheme
== dr_aligned
)
6366 gcc_assert (aligned_access_p (first_dr
));
6369 else if (DR_MISALIGNMENT (first_dr
) == -1)
6371 TREE_TYPE (data_ref
)
6372 = build_aligned_type (TREE_TYPE (data_ref
),
6373 TYPE_ALIGN (elem_type
));
6374 align
= TYPE_ALIGN_UNIT (elem_type
);
6379 TREE_TYPE (data_ref
)
6380 = build_aligned_type (TREE_TYPE (data_ref
),
6381 TYPE_ALIGN (elem_type
));
6382 misalign
= DR_MISALIGNMENT (first_dr
);
6384 if (dataref_offset
== NULL_TREE
)
6385 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
6389 case dr_explicit_realign
:
6394 vs_minus_1
= size_int (TYPE_VECTOR_SUBPARTS (vectype
) - 1);
6396 if (compute_in_loop
)
6397 msq
= vect_setup_realignment (first_stmt
, gsi
,
6399 dr_explicit_realign
,
6402 ptr
= copy_ssa_name (dataref_ptr
, NULL
);
6403 new_stmt
= gimple_build_assign_with_ops
6404 (BIT_AND_EXPR
, ptr
, dataref_ptr
,
6406 (TREE_TYPE (dataref_ptr
),
6407 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6408 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6410 = build2 (MEM_REF
, vectype
, ptr
,
6411 build_int_cst (reference_alias_ptr_type
6412 (DR_REF (first_dr
)), 0));
6413 vec_dest
= vect_create_destination_var (scalar_dest
,
6415 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6416 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6417 gimple_assign_set_lhs (new_stmt
, new_temp
);
6418 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
6419 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
6420 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6423 bump
= size_binop (MULT_EXPR
, vs_minus_1
,
6424 TYPE_SIZE_UNIT (elem_type
));
6425 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
6426 new_stmt
= gimple_build_assign_with_ops
6427 (BIT_AND_EXPR
, NULL_TREE
, ptr
,
6430 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6431 ptr
= copy_ssa_name (dataref_ptr
, new_stmt
);
6432 gimple_assign_set_lhs (new_stmt
, ptr
);
6433 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6435 = build2 (MEM_REF
, vectype
, ptr
,
6436 build_int_cst (reference_alias_ptr_type
6437 (DR_REF (first_dr
)), 0));
6440 case dr_explicit_realign_optimized
:
6441 new_temp
= copy_ssa_name (dataref_ptr
, NULL
);
6442 new_stmt
= gimple_build_assign_with_ops
6443 (BIT_AND_EXPR
, new_temp
, dataref_ptr
,
6445 (TREE_TYPE (dataref_ptr
),
6446 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6447 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6449 = build2 (MEM_REF
, vectype
, new_temp
,
6450 build_int_cst (reference_alias_ptr_type
6451 (DR_REF (first_dr
)), 0));
6456 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6457 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6458 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6459 gimple_assign_set_lhs (new_stmt
, new_temp
);
6460 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6462 /* 3. Handle explicit realignment if necessary/supported.
6464 vec_dest = realign_load (msq, lsq, realignment_token) */
6465 if (alignment_support_scheme
== dr_explicit_realign_optimized
6466 || alignment_support_scheme
== dr_explicit_realign
)
6468 lsq
= gimple_assign_lhs (new_stmt
);
6469 if (!realignment_token
)
6470 realignment_token
= dataref_ptr
;
6471 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6473 = gimple_build_assign_with_ops (REALIGN_LOAD_EXPR
,
6476 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6477 gimple_assign_set_lhs (new_stmt
, new_temp
);
6478 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6480 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6483 if (i
== vec_num
- 1 && j
== ncopies
- 1)
6484 add_phi_arg (phi
, lsq
,
6485 loop_latch_edge (containing_loop
),
6491 /* 4. Handle invariant-load. */
6492 if (inv_p
&& !bb_vinfo
)
6494 gcc_assert (!grouped_load
);
6495 /* If we have versioned for aliasing or the loop doesn't
6496 have any data dependencies that would preclude this,
6497 then we are sure this is a loop invariant load and
6498 thus we can insert it on the preheader edge. */
6499 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
6500 && !nested_in_vect_loop
6501 && hoist_defs_of_uses (stmt
, loop
))
6503 if (dump_enabled_p ())
6505 dump_printf_loc (MSG_NOTE
, vect_location
,
6506 "hoisting out of the vectorized "
6508 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6509 dump_printf (MSG_NOTE
, "\n");
6511 tree tem
= copy_ssa_name (scalar_dest
, NULL
);
6512 gsi_insert_on_edge_immediate
6513 (loop_preheader_edge (loop
),
6514 gimple_build_assign (tem
,
6516 (gimple_assign_rhs1 (stmt
))));
6517 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
6521 gimple_stmt_iterator gsi2
= *gsi
;
6523 new_temp
= vect_init_vector (stmt
, scalar_dest
,
6526 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6527 set_vinfo_for_stmt (new_stmt
,
6528 new_stmt_vec_info (new_stmt
, loop_vinfo
,
6534 tree perm_mask
= perm_mask_for_reverse (vectype
);
6535 new_temp
= permute_vec_elements (new_temp
, new_temp
,
6536 perm_mask
, stmt
, gsi
);
6537 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6540 /* Collect vector loads and later create their permutation in
6541 vect_transform_grouped_load (). */
6542 if (grouped_load
|| slp_perm
)
6543 dr_chain
.quick_push (new_temp
);
6545 /* Store vector loads in the corresponding SLP_NODE. */
6546 if (slp
&& !slp_perm
)
6547 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6549 /* Bump the vector pointer to account for a gap. */
6550 if (slp
&& group_gap
!= 0)
6552 tree bump
= size_binop (MULT_EXPR
,
6553 TYPE_SIZE_UNIT (elem_type
),
6554 size_int (group_gap
));
6555 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6560 if (slp
&& !slp_perm
)
6565 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6566 slp_node_instance
, false))
6568 dr_chain
.release ();
6577 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
6578 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6583 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6585 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6586 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6589 dr_chain
.release ();
6595 /* Function vect_is_simple_cond.
6598 LOOP - the loop that is being vectorized.
6599 COND - Condition that is checked for simple use.
6602 *COMP_VECTYPE - the vector type for the comparison.
6604 Returns whether a COND can be vectorized. Checks whether
6605 condition operands are supportable using vec_is_simple_use. */
6608 vect_is_simple_cond (tree cond
, gimple stmt
, loop_vec_info loop_vinfo
,
6609 bb_vec_info bb_vinfo
, tree
*comp_vectype
)
6613 enum vect_def_type dt
;
6614 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
6616 if (!COMPARISON_CLASS_P (cond
))
6619 lhs
= TREE_OPERAND (cond
, 0);
6620 rhs
= TREE_OPERAND (cond
, 1);
6622 if (TREE_CODE (lhs
) == SSA_NAME
)
6624 gimple lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
6625 if (!vect_is_simple_use_1 (lhs
, stmt
, loop_vinfo
, bb_vinfo
,
6626 &lhs_def_stmt
, &def
, &dt
, &vectype1
))
6629 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
6630 && TREE_CODE (lhs
) != FIXED_CST
)
6633 if (TREE_CODE (rhs
) == SSA_NAME
)
6635 gimple rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
6636 if (!vect_is_simple_use_1 (rhs
, stmt
, loop_vinfo
, bb_vinfo
,
6637 &rhs_def_stmt
, &def
, &dt
, &vectype2
))
6640 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
6641 && TREE_CODE (rhs
) != FIXED_CST
)
6644 *comp_vectype
= vectype1
? vectype1
: vectype2
;
6648 /* vectorizable_condition.
6650 Check if STMT is conditional modify expression that can be vectorized.
6651 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6652 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
6655 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
6656 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
6657 else caluse if it is 2).
6659 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6662 vectorizable_condition (gimple stmt
, gimple_stmt_iterator
*gsi
,
6663 gimple
*vec_stmt
, tree reduc_def
, int reduc_index
,
6666 tree scalar_dest
= NULL_TREE
;
6667 tree vec_dest
= NULL_TREE
;
6668 tree cond_expr
, then_clause
, else_clause
;
6669 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6670 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6671 tree comp_vectype
= NULL_TREE
;
6672 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
6673 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
6674 tree vec_compare
, vec_cond_expr
;
6676 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6678 enum vect_def_type dt
, dts
[4];
6679 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6681 enum tree_code code
;
6682 stmt_vec_info prev_stmt_info
= NULL
;
6684 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6685 vec
<tree
> vec_oprnds0
= vNULL
;
6686 vec
<tree
> vec_oprnds1
= vNULL
;
6687 vec
<tree
> vec_oprnds2
= vNULL
;
6688 vec
<tree
> vec_oprnds3
= vNULL
;
6691 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
6694 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
6696 gcc_assert (ncopies
>= 1);
6697 if (reduc_index
&& ncopies
> 1)
6698 return false; /* FORNOW */
6700 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
6703 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6706 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6707 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
6711 /* FORNOW: not yet supported. */
6712 if (STMT_VINFO_LIVE_P (stmt_info
))
6714 if (dump_enabled_p ())
6715 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6716 "value used after loop.\n");
6720 /* Is vectorizable conditional operation? */
6721 if (!is_gimple_assign (stmt
))
6724 code
= gimple_assign_rhs_code (stmt
);
6726 if (code
!= COND_EXPR
)
6729 cond_expr
= gimple_assign_rhs1 (stmt
);
6730 then_clause
= gimple_assign_rhs2 (stmt
);
6731 else_clause
= gimple_assign_rhs3 (stmt
);
6733 if (!vect_is_simple_cond (cond_expr
, stmt
, loop_vinfo
, bb_vinfo
,
6738 if (TREE_CODE (then_clause
) == SSA_NAME
)
6740 gimple then_def_stmt
= SSA_NAME_DEF_STMT (then_clause
);
6741 if (!vect_is_simple_use (then_clause
, stmt
, loop_vinfo
, bb_vinfo
,
6742 &then_def_stmt
, &def
, &dt
))
6745 else if (TREE_CODE (then_clause
) != INTEGER_CST
6746 && TREE_CODE (then_clause
) != REAL_CST
6747 && TREE_CODE (then_clause
) != FIXED_CST
)
6750 if (TREE_CODE (else_clause
) == SSA_NAME
)
6752 gimple else_def_stmt
= SSA_NAME_DEF_STMT (else_clause
);
6753 if (!vect_is_simple_use (else_clause
, stmt
, loop_vinfo
, bb_vinfo
,
6754 &else_def_stmt
, &def
, &dt
))
6757 else if (TREE_CODE (else_clause
) != INTEGER_CST
6758 && TREE_CODE (else_clause
) != REAL_CST
6759 && TREE_CODE (else_clause
) != FIXED_CST
)
6762 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
)));
6763 /* The result of a vector comparison should be signed type. */
6764 tree cmp_type
= build_nonstandard_integer_type (prec
, 0);
6765 vec_cmp_type
= get_same_sized_vectype (cmp_type
, vectype
);
6766 if (vec_cmp_type
== NULL_TREE
)
6771 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
6772 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
6779 vec_oprnds0
.create (1);
6780 vec_oprnds1
.create (1);
6781 vec_oprnds2
.create (1);
6782 vec_oprnds3
.create (1);
6786 scalar_dest
= gimple_assign_lhs (stmt
);
6787 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6789 /* Handle cond expr. */
6790 for (j
= 0; j
< ncopies
; j
++)
6792 gimple new_stmt
= NULL
;
6797 auto_vec
<tree
, 4> ops
;
6798 auto_vec
<vec
<tree
>, 4> vec_defs
;
6800 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
6801 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
6802 ops
.safe_push (then_clause
);
6803 ops
.safe_push (else_clause
);
6804 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
6805 vec_oprnds3
= vec_defs
.pop ();
6806 vec_oprnds2
= vec_defs
.pop ();
6807 vec_oprnds1
= vec_defs
.pop ();
6808 vec_oprnds0
= vec_defs
.pop ();
6811 vec_defs
.release ();
6817 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
6819 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0), stmt
,
6820 loop_vinfo
, NULL
, >emp
, &def
, &dts
[0]);
6823 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
6825 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1), stmt
,
6826 loop_vinfo
, NULL
, >emp
, &def
, &dts
[1]);
6827 if (reduc_index
== 1)
6828 vec_then_clause
= reduc_def
;
6831 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
6833 vect_is_simple_use (then_clause
, stmt
, loop_vinfo
,
6834 NULL
, >emp
, &def
, &dts
[2]);
6836 if (reduc_index
== 2)
6837 vec_else_clause
= reduc_def
;
6840 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
6842 vect_is_simple_use (else_clause
, stmt
, loop_vinfo
,
6843 NULL
, >emp
, &def
, &dts
[3]);
6849 vec_cond_lhs
= vect_get_vec_def_for_stmt_copy (dts
[0],
6850 vec_oprnds0
.pop ());
6851 vec_cond_rhs
= vect_get_vec_def_for_stmt_copy (dts
[1],
6852 vec_oprnds1
.pop ());
6853 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
6854 vec_oprnds2
.pop ());
6855 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
6856 vec_oprnds3
.pop ());
6861 vec_oprnds0
.quick_push (vec_cond_lhs
);
6862 vec_oprnds1
.quick_push (vec_cond_rhs
);
6863 vec_oprnds2
.quick_push (vec_then_clause
);
6864 vec_oprnds3
.quick_push (vec_else_clause
);
6867 /* Arguments are ready. Create the new vector stmt. */
6868 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
6870 vec_cond_rhs
= vec_oprnds1
[i
];
6871 vec_then_clause
= vec_oprnds2
[i
];
6872 vec_else_clause
= vec_oprnds3
[i
];
6874 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
6875 vec_cond_lhs
, vec_cond_rhs
);
6876 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
6877 vec_compare
, vec_then_clause
, vec_else_clause
);
6879 new_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
6880 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6881 gimple_assign_set_lhs (new_stmt
, new_temp
);
6882 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6884 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6891 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6893 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6895 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6898 vec_oprnds0
.release ();
6899 vec_oprnds1
.release ();
6900 vec_oprnds2
.release ();
6901 vec_oprnds3
.release ();
6907 /* Make sure the statement is vectorizable. */
6910 vect_analyze_stmt (gimple stmt
, bool *need_to_vectorize
, slp_tree node
)
6912 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6913 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6914 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
6916 tree scalar_type
, vectype
;
6917 gimple pattern_stmt
;
6918 gimple_seq pattern_def_seq
;
6920 if (dump_enabled_p ())
6922 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
6923 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6924 dump_printf (MSG_NOTE
, "\n");
6927 if (gimple_has_volatile_ops (stmt
))
6929 if (dump_enabled_p ())
6930 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6931 "not vectorized: stmt has volatile operands\n");
6936 /* Skip stmts that do not need to be vectorized. In loops this is expected
6938 - the COND_EXPR which is the loop exit condition
6939 - any LABEL_EXPRs in the loop
6940 - computations that are used only for array indexing or loop control.
6941 In basic blocks we only analyze statements that are a part of some SLP
6942 instance, therefore, all the statements are relevant.
6944 Pattern statement needs to be analyzed instead of the original statement
6945 if the original statement is not relevant. Otherwise, we analyze both
6946 statements. In basic blocks we are called from some SLP instance
6947 traversal, don't analyze pattern stmts instead, the pattern stmts
6948 already will be part of SLP instance. */
6950 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
6951 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
6952 && !STMT_VINFO_LIVE_P (stmt_info
))
6954 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
6956 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
6957 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
6959 /* Analyze PATTERN_STMT instead of the original stmt. */
6960 stmt
= pattern_stmt
;
6961 stmt_info
= vinfo_for_stmt (pattern_stmt
);
6962 if (dump_enabled_p ())
6964 dump_printf_loc (MSG_NOTE
, vect_location
,
6965 "==> examining pattern statement: ");
6966 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6967 dump_printf (MSG_NOTE
, "\n");
6972 if (dump_enabled_p ())
6973 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
6978 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
6981 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
6982 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
6984 /* Analyze PATTERN_STMT too. */
6985 if (dump_enabled_p ())
6987 dump_printf_loc (MSG_NOTE
, vect_location
,
6988 "==> examining pattern statement: ");
6989 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6990 dump_printf (MSG_NOTE
, "\n");
6993 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
6997 if (is_pattern_stmt_p (stmt_info
)
6999 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
7001 gimple_stmt_iterator si
;
7003 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
7005 gimple pattern_def_stmt
= gsi_stmt (si
);
7006 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
7007 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
7009 /* Analyze def stmt of STMT if it's a pattern stmt. */
7010 if (dump_enabled_p ())
7012 dump_printf_loc (MSG_NOTE
, vect_location
,
7013 "==> examining pattern def statement: ");
7014 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
7015 dump_printf (MSG_NOTE
, "\n");
7018 if (!vect_analyze_stmt (pattern_def_stmt
,
7019 need_to_vectorize
, node
))
7025 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
7027 case vect_internal_def
:
7030 case vect_reduction_def
:
7031 case vect_nested_cycle
:
7032 gcc_assert (!bb_vinfo
&& (relevance
== vect_used_in_outer
7033 || relevance
== vect_used_in_outer_by_reduction
7034 || relevance
== vect_unused_in_scope
));
7037 case vect_induction_def
:
7038 case vect_constant_def
:
7039 case vect_external_def
:
7040 case vect_unknown_def_type
:
7047 gcc_assert (PURE_SLP_STMT (stmt_info
));
7049 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
7050 if (dump_enabled_p ())
7052 dump_printf_loc (MSG_NOTE
, vect_location
,
7053 "get vectype for scalar type: ");
7054 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
7055 dump_printf (MSG_NOTE
, "\n");
7058 vectype
= get_vectype_for_scalar_type (scalar_type
);
7061 if (dump_enabled_p ())
7063 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7064 "not SLPed: unsupported data-type ");
7065 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
7067 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7072 if (dump_enabled_p ())
7074 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
7075 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
7076 dump_printf (MSG_NOTE
, "\n");
7079 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
7082 if (STMT_VINFO_RELEVANT_P (stmt_info
))
7084 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
7085 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
7086 || (is_gimple_call (stmt
)
7087 && gimple_call_lhs (stmt
) == NULL_TREE
));
7088 *need_to_vectorize
= true;
7093 && (STMT_VINFO_RELEVANT_P (stmt_info
)
7094 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
7095 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, NULL
)
7096 || vectorizable_conversion (stmt
, NULL
, NULL
, NULL
)
7097 || vectorizable_shift (stmt
, NULL
, NULL
, NULL
)
7098 || vectorizable_operation (stmt
, NULL
, NULL
, NULL
)
7099 || vectorizable_assignment (stmt
, NULL
, NULL
, NULL
)
7100 || vectorizable_load (stmt
, NULL
, NULL
, NULL
, NULL
)
7101 || vectorizable_call (stmt
, NULL
, NULL
, NULL
)
7102 || vectorizable_store (stmt
, NULL
, NULL
, NULL
)
7103 || vectorizable_reduction (stmt
, NULL
, NULL
, NULL
)
7104 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, NULL
));
7108 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7109 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7110 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7111 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7112 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7113 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7114 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7115 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7116 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7121 if (dump_enabled_p ())
7123 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7124 "not vectorized: relevant stmt not ");
7125 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7126 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7127 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7136 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7137 need extra handling, except for vectorizable reductions. */
7138 if (STMT_VINFO_LIVE_P (stmt_info
)
7139 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7140 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
7144 if (dump_enabled_p ())
7146 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7147 "not vectorized: live stmt not ");
7148 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7149 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7150 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7160 /* Function vect_transform_stmt.
7162 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7165 vect_transform_stmt (gimple stmt
, gimple_stmt_iterator
*gsi
,
7166 bool *grouped_store
, slp_tree slp_node
,
7167 slp_instance slp_node_instance
)
7169 bool is_store
= false;
7170 gimple vec_stmt
= NULL
;
7171 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7174 switch (STMT_VINFO_TYPE (stmt_info
))
7176 case type_demotion_vec_info_type
:
7177 case type_promotion_vec_info_type
:
7178 case type_conversion_vec_info_type
:
7179 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
7183 case induc_vec_info_type
:
7184 gcc_assert (!slp_node
);
7185 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
7189 case shift_vec_info_type
:
7190 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
7194 case op_vec_info_type
:
7195 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
7199 case assignment_vec_info_type
:
7200 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
7204 case load_vec_info_type
:
7205 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
7210 case store_vec_info_type
:
7211 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
7213 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
7215 /* In case of interleaving, the whole chain is vectorized when the
7216 last store in the chain is reached. Store stmts before the last
7217 one are skipped, and there vec_stmt_info shouldn't be freed
7219 *grouped_store
= true;
7220 if (STMT_VINFO_VEC_STMT (stmt_info
))
7227 case condition_vec_info_type
:
7228 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
7232 case call_vec_info_type
:
7233 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7234 stmt
= gsi_stmt (*gsi
);
7235 if (is_gimple_call (stmt
)
7236 && gimple_call_internal_p (stmt
)
7237 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
7241 case call_simd_clone_vec_info_type
:
7242 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7243 stmt
= gsi_stmt (*gsi
);
7246 case reduc_vec_info_type
:
7247 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
7252 if (!STMT_VINFO_LIVE_P (stmt_info
))
7254 if (dump_enabled_p ())
7255 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7256 "stmt not supported.\n");
7261 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
7262 is being vectorized, but outside the immediately enclosing loop. */
7264 && STMT_VINFO_LOOP_VINFO (stmt_info
)
7265 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
7266 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
7267 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
7268 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
7269 || STMT_VINFO_RELEVANT (stmt_info
) ==
7270 vect_used_in_outer_by_reduction
))
7272 struct loop
*innerloop
= LOOP_VINFO_LOOP (
7273 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
7274 imm_use_iterator imm_iter
;
7275 use_operand_p use_p
;
7279 if (dump_enabled_p ())
7280 dump_printf_loc (MSG_NOTE
, vect_location
,
7281 "Record the vdef for outer-loop vectorization.\n");
7283 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
7284 (to be used when vectorizing outer-loop stmts that use the DEF of
7286 if (gimple_code (stmt
) == GIMPLE_PHI
)
7287 scalar_dest
= PHI_RESULT (stmt
);
7289 scalar_dest
= gimple_assign_lhs (stmt
);
7291 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
7293 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
7295 exit_phi
= USE_STMT (use_p
);
7296 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
7301 /* Handle stmts whose DEF is used outside the loop-nest that is
7302 being vectorized. */
7303 if (STMT_VINFO_LIVE_P (stmt_info
)
7304 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7306 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
7311 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
7317 /* Remove a group of stores (for SLP or interleaving), free their
7321 vect_remove_stores (gimple first_stmt
)
7323 gimple next
= first_stmt
;
7325 gimple_stmt_iterator next_si
;
7329 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
7331 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
7332 if (is_pattern_stmt_p (stmt_info
))
7333 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
7334 /* Free the attached stmt_vec_info and remove the stmt. */
7335 next_si
= gsi_for_stmt (next
);
7336 unlink_stmt_vdef (next
);
7337 gsi_remove (&next_si
, true);
7338 release_defs (next
);
7339 free_stmt_vec_info (next
);
7345 /* Function new_stmt_vec_info.
7347 Create and initialize a new stmt_vec_info struct for STMT. */
7350 new_stmt_vec_info (gimple stmt
, loop_vec_info loop_vinfo
,
7351 bb_vec_info bb_vinfo
)
7354 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
7356 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
7357 STMT_VINFO_STMT (res
) = stmt
;
7358 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
7359 STMT_VINFO_BB_VINFO (res
) = bb_vinfo
;
7360 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
7361 STMT_VINFO_LIVE_P (res
) = false;
7362 STMT_VINFO_VECTYPE (res
) = NULL
;
7363 STMT_VINFO_VEC_STMT (res
) = NULL
;
7364 STMT_VINFO_VECTORIZABLE (res
) = true;
7365 STMT_VINFO_IN_PATTERN_P (res
) = false;
7366 STMT_VINFO_RELATED_STMT (res
) = NULL
;
7367 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
7368 STMT_VINFO_DATA_REF (res
) = NULL
;
7370 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
7371 STMT_VINFO_DR_OFFSET (res
) = NULL
;
7372 STMT_VINFO_DR_INIT (res
) = NULL
;
7373 STMT_VINFO_DR_STEP (res
) = NULL
;
7374 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
7376 if (gimple_code (stmt
) == GIMPLE_PHI
7377 && is_loop_header_bb_p (gimple_bb (stmt
)))
7378 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
7380 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
7382 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
7383 STMT_SLP_TYPE (res
) = loop_vect
;
7384 GROUP_FIRST_ELEMENT (res
) = NULL
;
7385 GROUP_NEXT_ELEMENT (res
) = NULL
;
7386 GROUP_SIZE (res
) = 0;
7387 GROUP_STORE_COUNT (res
) = 0;
7388 GROUP_GAP (res
) = 0;
7389 GROUP_SAME_DR_STMT (res
) = NULL
;
7395 /* Create a hash table for stmt_vec_info. */
7398 init_stmt_vec_info_vec (void)
7400 gcc_assert (!stmt_vec_info_vec
.exists ());
7401 stmt_vec_info_vec
.create (50);
7405 /* Free hash table for stmt_vec_info. */
7408 free_stmt_vec_info_vec (void)
7412 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
7414 free_stmt_vec_info (STMT_VINFO_STMT ((stmt_vec_info
) info
));
7415 gcc_assert (stmt_vec_info_vec
.exists ());
7416 stmt_vec_info_vec
.release ();
7420 /* Free stmt vectorization related info. */
7423 free_stmt_vec_info (gimple stmt
)
7425 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7430 /* Check if this statement has a related "pattern stmt"
7431 (introduced by the vectorizer during the pattern recognition
7432 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
7434 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
7436 stmt_vec_info patt_info
7437 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
7440 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
7441 gimple patt_stmt
= STMT_VINFO_STMT (patt_info
);
7442 gimple_set_bb (patt_stmt
, NULL
);
7443 tree lhs
= gimple_get_lhs (patt_stmt
);
7444 if (TREE_CODE (lhs
) == SSA_NAME
)
7445 release_ssa_name (lhs
);
7448 gimple_stmt_iterator si
;
7449 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
7451 gimple seq_stmt
= gsi_stmt (si
);
7452 gimple_set_bb (seq_stmt
, NULL
);
7453 lhs
= gimple_get_lhs (patt_stmt
);
7454 if (TREE_CODE (lhs
) == SSA_NAME
)
7455 release_ssa_name (lhs
);
7456 free_stmt_vec_info (seq_stmt
);
7459 free_stmt_vec_info (patt_stmt
);
7463 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
7464 set_vinfo_for_stmt (stmt
, NULL
);
7469 /* Function get_vectype_for_scalar_type_and_size.
7471 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
7475 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
7477 enum machine_mode inner_mode
= TYPE_MODE (scalar_type
);
7478 enum machine_mode simd_mode
;
7479 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
7486 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
7487 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
7490 /* For vector types of elements whose mode precision doesn't
7491 match their types precision we use a element type of mode
7492 precision. The vectorization routines will have to make sure
7493 they support the proper result truncation/extension.
7494 We also make sure to build vector types with INTEGER_TYPE
7495 component type only. */
7496 if (INTEGRAL_TYPE_P (scalar_type
)
7497 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
7498 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
7499 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
7500 TYPE_UNSIGNED (scalar_type
));
7502 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
7503 When the component mode passes the above test simply use a type
7504 corresponding to that mode. The theory is that any use that
7505 would cause problems with this will disable vectorization anyway. */
7506 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
7507 && !INTEGRAL_TYPE_P (scalar_type
))
7508 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
7510 /* We can't build a vector type of elements with alignment bigger than
7512 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
7513 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
7514 TYPE_UNSIGNED (scalar_type
));
7516 /* If we felt back to using the mode fail if there was
7517 no scalar type for it. */
7518 if (scalar_type
== NULL_TREE
)
7521 /* If no size was supplied use the mode the target prefers. Otherwise
7522 lookup a vector mode of the specified size. */
7524 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
7526 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
7527 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
7531 vectype
= build_vector_type (scalar_type
, nunits
);
7533 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
7534 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
7540 unsigned int current_vector_size
;
7542 /* Function get_vectype_for_scalar_type.
7544 Returns the vector type corresponding to SCALAR_TYPE as supported
7548 get_vectype_for_scalar_type (tree scalar_type
)
7551 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
7552 current_vector_size
);
7554 && current_vector_size
== 0)
7555 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
7559 /* Function get_same_sized_vectype
7561 Returns a vector type corresponding to SCALAR_TYPE of size
7562 VECTOR_TYPE if supported by the target. */
7565 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
7567 return get_vectype_for_scalar_type_and_size
7568 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
7571 /* Function vect_is_simple_use.
7574 LOOP_VINFO - the vect info of the loop that is being vectorized.
7575 BB_VINFO - the vect info of the basic block that is being vectorized.
7576 OPERAND - operand of STMT in the loop or bb.
7577 DEF - the defining stmt in case OPERAND is an SSA_NAME.
7579 Returns whether a stmt with OPERAND can be vectorized.
7580 For loops, supportable operands are constants, loop invariants, and operands
7581 that are defined by the current iteration of the loop. Unsupportable
7582 operands are those that are defined by a previous iteration of the loop (as
7583 is the case in reduction/induction computations).
7584 For basic blocks, supportable operands are constants and bb invariants.
7585 For now, operands defined outside the basic block are not supported. */
7588 vect_is_simple_use (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
7589 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
7590 tree
*def
, enum vect_def_type
*dt
)
7593 stmt_vec_info stmt_vinfo
;
7594 struct loop
*loop
= NULL
;
7597 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7602 if (dump_enabled_p ())
7604 dump_printf_loc (MSG_NOTE
, vect_location
,
7605 "vect_is_simple_use: operand ");
7606 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
7607 dump_printf (MSG_NOTE
, "\n");
7610 if (CONSTANT_CLASS_P (operand
))
7612 *dt
= vect_constant_def
;
7616 if (is_gimple_min_invariant (operand
))
7619 *dt
= vect_external_def
;
7623 if (TREE_CODE (operand
) == PAREN_EXPR
)
7625 if (dump_enabled_p ())
7626 dump_printf_loc (MSG_NOTE
, vect_location
, "non-associatable copy.\n");
7627 operand
= TREE_OPERAND (operand
, 0);
7630 if (TREE_CODE (operand
) != SSA_NAME
)
7632 if (dump_enabled_p ())
7633 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7638 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
7639 if (*def_stmt
== NULL
)
7641 if (dump_enabled_p ())
7642 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7647 if (dump_enabled_p ())
7649 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
7650 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
7651 dump_printf (MSG_NOTE
, "\n");
7654 /* Empty stmt is expected only in case of a function argument.
7655 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
7656 if (gimple_nop_p (*def_stmt
))
7659 *dt
= vect_external_def
;
7663 bb
= gimple_bb (*def_stmt
);
7665 if ((loop
&& !flow_bb_inside_loop_p (loop
, bb
))
7666 || (!loop
&& bb
!= BB_VINFO_BB (bb_vinfo
))
7667 || (!loop
&& gimple_code (*def_stmt
) == GIMPLE_PHI
))
7668 *dt
= vect_external_def
;
7671 stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
7672 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
7675 if (*dt
== vect_unknown_def_type
7677 && *dt
== vect_double_reduction_def
7678 && gimple_code (stmt
) != GIMPLE_PHI
))
7680 if (dump_enabled_p ())
7681 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7682 "Unsupported pattern.\n");
7686 if (dump_enabled_p ())
7687 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: %d.\n", *dt
);
7689 switch (gimple_code (*def_stmt
))
7692 *def
= gimple_phi_result (*def_stmt
);
7696 *def
= gimple_assign_lhs (*def_stmt
);
7700 *def
= gimple_call_lhs (*def_stmt
);
7705 if (dump_enabled_p ())
7706 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7707 "unsupported defining stmt:\n");
7714 /* Function vect_is_simple_use_1.
7716 Same as vect_is_simple_use_1 but also determines the vector operand
7717 type of OPERAND and stores it to *VECTYPE. If the definition of
7718 OPERAND is vect_uninitialized_def, vect_constant_def or
7719 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
7720 is responsible to compute the best suited vector type for the
7724 vect_is_simple_use_1 (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
7725 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
7726 tree
*def
, enum vect_def_type
*dt
, tree
*vectype
)
7728 if (!vect_is_simple_use (operand
, stmt
, loop_vinfo
, bb_vinfo
, def_stmt
,
7732 /* Now get a vector type if the def is internal, otherwise supply
7733 NULL_TREE and leave it up to the caller to figure out a proper
7734 type for the use stmt. */
7735 if (*dt
== vect_internal_def
7736 || *dt
== vect_induction_def
7737 || *dt
== vect_reduction_def
7738 || *dt
== vect_double_reduction_def
7739 || *dt
== vect_nested_cycle
)
7741 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
7743 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7744 && !STMT_VINFO_RELEVANT (stmt_info
)
7745 && !STMT_VINFO_LIVE_P (stmt_info
))
7746 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
7748 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7749 gcc_assert (*vectype
!= NULL_TREE
);
7751 else if (*dt
== vect_uninitialized_def
7752 || *dt
== vect_constant_def
7753 || *dt
== vect_external_def
)
7754 *vectype
= NULL_TREE
;
7762 /* Function supportable_widening_operation
7764 Check whether an operation represented by the code CODE is a
7765 widening operation that is supported by the target platform in
7766 vector form (i.e., when operating on arguments of type VECTYPE_IN
7767 producing a result of type VECTYPE_OUT).
7769 Widening operations we currently support are NOP (CONVERT), FLOAT
7770 and WIDEN_MULT. This function checks if these operations are supported
7771 by the target platform either directly (via vector tree-codes), or via
7775 - CODE1 and CODE2 are codes of vector operations to be used when
7776 vectorizing the operation, if available.
7777 - MULTI_STEP_CVT determines the number of required intermediate steps in
7778 case of multi-step conversion (like char->short->int - in that case
7779 MULTI_STEP_CVT will be 1).
7780 - INTERM_TYPES contains the intermediate type required to perform the
7781 widening operation (short in the above example). */
7784 supportable_widening_operation (enum tree_code code
, gimple stmt
,
7785 tree vectype_out
, tree vectype_in
,
7786 enum tree_code
*code1
, enum tree_code
*code2
,
7787 int *multi_step_cvt
,
7788 vec
<tree
> *interm_types
)
7790 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7791 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7792 struct loop
*vect_loop
= NULL
;
7793 enum machine_mode vec_mode
;
7794 enum insn_code icode1
, icode2
;
7795 optab optab1
, optab2
;
7796 tree vectype
= vectype_in
;
7797 tree wide_vectype
= vectype_out
;
7798 enum tree_code c1
, c2
;
7800 tree prev_type
, intermediate_type
;
7801 enum machine_mode intermediate_mode
, prev_mode
;
7802 optab optab3
, optab4
;
7804 *multi_step_cvt
= 0;
7806 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
7810 case WIDEN_MULT_EXPR
:
7811 /* The result of a vectorized widening operation usually requires
7812 two vectors (because the widened results do not fit into one vector).
7813 The generated vector results would normally be expected to be
7814 generated in the same order as in the original scalar computation,
7815 i.e. if 8 results are generated in each vector iteration, they are
7816 to be organized as follows:
7817 vect1: [res1,res2,res3,res4],
7818 vect2: [res5,res6,res7,res8].
7820 However, in the special case that the result of the widening
7821 operation is used in a reduction computation only, the order doesn't
7822 matter (because when vectorizing a reduction we change the order of
7823 the computation). Some targets can take advantage of this and
7824 generate more efficient code. For example, targets like Altivec,
7825 that support widen_mult using a sequence of {mult_even,mult_odd}
7826 generate the following vectors:
7827 vect1: [res1,res3,res5,res7],
7828 vect2: [res2,res4,res6,res8].
7830 When vectorizing outer-loops, we execute the inner-loop sequentially
7831 (each vectorized inner-loop iteration contributes to VF outer-loop
7832 iterations in parallel). We therefore don't allow to change the
7833 order of the computation in the inner-loop during outer-loop
7835 /* TODO: Another case in which order doesn't *really* matter is when we
7836 widen and then contract again, e.g. (short)((int)x * y >> 8).
7837 Normally, pack_trunc performs an even/odd permute, whereas the
7838 repack from an even/odd expansion would be an interleave, which
7839 would be significantly simpler for e.g. AVX2. */
7840 /* In any case, in order to avoid duplicating the code below, recurse
7841 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
7842 are properly set up for the caller. If we fail, we'll continue with
7843 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
7845 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
7846 && !nested_in_vect_loop_p (vect_loop
, stmt
)
7847 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
7848 stmt
, vectype_out
, vectype_in
,
7849 code1
, code2
, multi_step_cvt
,
7852 /* Elements in a vector with vect_used_by_reduction property cannot
7853 be reordered if the use chain with this property does not have the
7854 same operation. One such an example is s += a * b, where elements
7855 in a and b cannot be reordered. Here we check if the vector defined
7856 by STMT is only directly used in the reduction statement. */
7857 tree lhs
= gimple_assign_lhs (stmt
);
7858 use_operand_p dummy
;
7860 stmt_vec_info use_stmt_info
= NULL
;
7861 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
7862 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
7863 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
7866 c1
= VEC_WIDEN_MULT_LO_EXPR
;
7867 c2
= VEC_WIDEN_MULT_HI_EXPR
;
7870 case VEC_WIDEN_MULT_EVEN_EXPR
:
7871 /* Support the recursion induced just above. */
7872 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
7873 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
7876 case WIDEN_LSHIFT_EXPR
:
7877 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
7878 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
7882 c1
= VEC_UNPACK_LO_EXPR
;
7883 c2
= VEC_UNPACK_HI_EXPR
;
7887 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
7888 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
7891 case FIX_TRUNC_EXPR
:
7892 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
7893 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
7894 computing the operation. */
7901 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
7903 enum tree_code ctmp
= c1
;
7908 if (code
== FIX_TRUNC_EXPR
)
7910 /* The signedness is determined from output operand. */
7911 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
7912 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
7916 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
7917 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
7920 if (!optab1
|| !optab2
)
7923 vec_mode
= TYPE_MODE (vectype
);
7924 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
7925 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
7931 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
7932 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
7935 /* Check if it's a multi-step conversion that can be done using intermediate
7938 prev_type
= vectype
;
7939 prev_mode
= vec_mode
;
7941 if (!CONVERT_EXPR_CODE_P (code
))
7944 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
7945 intermediate steps in promotion sequence. We try
7946 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
7948 interm_types
->create (MAX_INTERM_CVT_STEPS
);
7949 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
7951 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
7953 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
7954 TYPE_UNSIGNED (prev_type
));
7955 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
7956 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
7958 if (!optab3
|| !optab4
7959 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
7960 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
7961 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
7962 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
7963 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
7964 == CODE_FOR_nothing
)
7965 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
7966 == CODE_FOR_nothing
))
7969 interm_types
->quick_push (intermediate_type
);
7970 (*multi_step_cvt
)++;
7972 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
7973 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
7976 prev_type
= intermediate_type
;
7977 prev_mode
= intermediate_mode
;
7980 interm_types
->release ();
7985 /* Function supportable_narrowing_operation
7987 Check whether an operation represented by the code CODE is a
7988 narrowing operation that is supported by the target platform in
7989 vector form (i.e., when operating on arguments of type VECTYPE_IN
7990 and producing a result of type VECTYPE_OUT).
7992 Narrowing operations we currently support are NOP (CONVERT) and
7993 FIX_TRUNC. This function checks if these operations are supported by
7994 the target platform directly via vector tree-codes.
7997 - CODE1 is the code of a vector operation to be used when
7998 vectorizing the operation, if available.
7999 - MULTI_STEP_CVT determines the number of required intermediate steps in
8000 case of multi-step conversion (like int->short->char - in that case
8001 MULTI_STEP_CVT will be 1).
8002 - INTERM_TYPES contains the intermediate type required to perform the
8003 narrowing operation (short in the above example). */
8006 supportable_narrowing_operation (enum tree_code code
,
8007 tree vectype_out
, tree vectype_in
,
8008 enum tree_code
*code1
, int *multi_step_cvt
,
8009 vec
<tree
> *interm_types
)
8011 enum machine_mode vec_mode
;
8012 enum insn_code icode1
;
8013 optab optab1
, interm_optab
;
8014 tree vectype
= vectype_in
;
8015 tree narrow_vectype
= vectype_out
;
8017 tree intermediate_type
;
8018 enum machine_mode intermediate_mode
, prev_mode
;
8022 *multi_step_cvt
= 0;
8026 c1
= VEC_PACK_TRUNC_EXPR
;
8029 case FIX_TRUNC_EXPR
:
8030 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
8034 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8035 tree code and optabs used for computing the operation. */
8042 if (code
== FIX_TRUNC_EXPR
)
8043 /* The signedness is determined from output operand. */
8044 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8046 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8051 vec_mode
= TYPE_MODE (vectype
);
8052 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
8057 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8060 /* Check if it's a multi-step conversion that can be done using intermediate
8062 prev_mode
= vec_mode
;
8063 if (code
== FIX_TRUNC_EXPR
)
8064 uns
= TYPE_UNSIGNED (vectype_out
);
8066 uns
= TYPE_UNSIGNED (vectype
);
8068 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8069 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8070 costly than signed. */
8071 if (code
== FIX_TRUNC_EXPR
&& uns
)
8073 enum insn_code icode2
;
8076 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
8078 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8079 if (interm_optab
!= unknown_optab
8080 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
8081 && insn_data
[icode1
].operand
[0].mode
8082 == insn_data
[icode2
].operand
[0].mode
)
8085 optab1
= interm_optab
;
8090 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8091 intermediate steps in promotion sequence. We try
8092 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8093 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8094 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8096 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8098 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
8100 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
8103 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
8104 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8105 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
8106 == CODE_FOR_nothing
))
8109 interm_types
->quick_push (intermediate_type
);
8110 (*multi_step_cvt
)++;
8112 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8115 prev_mode
= intermediate_mode
;
8116 optab1
= interm_optab
;
8119 interm_types
->release ();