1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2015 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "fold-const.h"
32 #include "stor-layout.h"
35 #include "hard-reg-set.h"
37 #include "dominance.h"
39 #include "basic-block.h"
40 #include "gimple-pretty-print.h"
41 #include "tree-ssa-alias.h"
42 #include "internal-fn.h"
44 #include "gimple-expr.h"
48 #include "gimple-iterator.h"
49 #include "gimplify-me.h"
50 #include "gimple-ssa.h"
52 #include "tree-phinodes.h"
53 #include "ssa-iterators.h"
54 #include "stringpool.h"
55 #include "tree-ssanames.h"
56 #include "tree-ssa-loop-manip.h"
58 #include "tree-ssa-loop.h"
59 #include "tree-scalar-evolution.h"
62 #include "insn-config.h"
71 #include "recog.h" /* FIXME: for insn_data */
72 #include "insn-codes.h"
74 #include "diagnostic-core.h"
75 #include "tree-vectorizer.h"
76 #include "plugin-api.h"
81 /* For lang_hooks.types.type_for_mode. */
82 #include "langhooks.h"
84 /* Return the vectorized type for the given statement. */
87 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
89 return STMT_VINFO_VECTYPE (stmt_info
);
92 /* Return TRUE iff the given statement is in an inner loop relative to
93 the loop being vectorized. */
95 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
97 gimple stmt
= STMT_VINFO_STMT (stmt_info
);
98 basic_block bb
= gimple_bb (stmt
);
99 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
105 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
107 return (bb
->loop_father
== loop
->inner
);
110 /* Record the cost of a statement, either by directly informing the
111 target model or by saving it in a vector for later processing.
112 Return a preliminary estimate of the statement's cost. */
115 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
116 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
117 int misalign
, enum vect_cost_model_location where
)
121 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
122 add_stmt_info_to_vec (body_cost_vec
, count
, kind
,
123 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
126 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
131 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
132 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
133 void *target_cost_data
;
136 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
138 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
140 return add_stmt_cost (target_cost_data
, count
, kind
, stmt_info
,
145 /* Return a variable of type ELEM_TYPE[NELEMS]. */
148 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
150 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
154 /* ARRAY is an array of vectors created by create_vector_array.
155 Return an SSA_NAME for the vector in index N. The reference
156 is part of the vectorization of STMT and the vector is associated
157 with scalar destination SCALAR_DEST. */
160 read_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
161 tree array
, unsigned HOST_WIDE_INT n
)
163 tree vect_type
, vect
, vect_name
, array_ref
;
166 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
167 vect_type
= TREE_TYPE (TREE_TYPE (array
));
168 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
169 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
170 build_int_cst (size_type_node
, n
),
171 NULL_TREE
, NULL_TREE
);
173 new_stmt
= gimple_build_assign (vect
, array_ref
);
174 vect_name
= make_ssa_name (vect
, new_stmt
);
175 gimple_assign_set_lhs (new_stmt
, vect_name
);
176 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
181 /* ARRAY is an array of vectors created by create_vector_array.
182 Emit code to store SSA_NAME VECT in index N of the array.
183 The store is part of the vectorization of STMT. */
186 write_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
187 tree array
, unsigned HOST_WIDE_INT n
)
192 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
193 build_int_cst (size_type_node
, n
),
194 NULL_TREE
, NULL_TREE
);
196 new_stmt
= gimple_build_assign (array_ref
, vect
);
197 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
200 /* PTR is a pointer to an array of type TYPE. Return a representation
201 of *PTR. The memory reference replaces those in FIRST_DR
205 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
207 tree mem_ref
, alias_ptr_type
;
209 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
210 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
211 /* Arrays have the same alignment as their type. */
212 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
216 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
218 /* Function vect_mark_relevant.
220 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
223 vect_mark_relevant (vec
<gimple
> *worklist
, gimple stmt
,
224 enum vect_relevant relevant
, bool live_p
,
225 bool used_in_pattern
)
227 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
228 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
229 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
232 if (dump_enabled_p ())
233 dump_printf_loc (MSG_NOTE
, vect_location
,
234 "mark relevant %d, live %d.\n", relevant
, live_p
);
236 /* If this stmt is an original stmt in a pattern, we might need to mark its
237 related pattern stmt instead of the original stmt. However, such stmts
238 may have their own uses that are not in any pattern, in such cases the
239 stmt itself should be marked. */
240 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
243 if (!used_in_pattern
)
245 imm_use_iterator imm_iter
;
249 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
250 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
252 if (is_gimple_assign (stmt
))
253 lhs
= gimple_assign_lhs (stmt
);
255 lhs
= gimple_call_lhs (stmt
);
257 /* This use is out of pattern use, if LHS has other uses that are
258 pattern uses, we should mark the stmt itself, and not the pattern
260 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
261 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
263 if (is_gimple_debug (USE_STMT (use_p
)))
265 use_stmt
= USE_STMT (use_p
);
267 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
270 if (vinfo_for_stmt (use_stmt
)
271 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
281 /* This is the last stmt in a sequence that was detected as a
282 pattern that can potentially be vectorized. Don't mark the stmt
283 as relevant/live because it's not going to be vectorized.
284 Instead mark the pattern-stmt that replaces it. */
286 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
288 if (dump_enabled_p ())
289 dump_printf_loc (MSG_NOTE
, vect_location
,
290 "last stmt in pattern. don't mark"
291 " relevant/live.\n");
292 stmt_info
= vinfo_for_stmt (pattern_stmt
);
293 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
294 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
295 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
300 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
301 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
302 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
304 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
305 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
307 if (dump_enabled_p ())
308 dump_printf_loc (MSG_NOTE
, vect_location
,
309 "already marked relevant/live.\n");
313 worklist
->safe_push (stmt
);
317 /* Function vect_stmt_relevant_p.
319 Return true if STMT in loop that is represented by LOOP_VINFO is
320 "relevant for vectorization".
322 A stmt is considered "relevant for vectorization" if:
323 - it has uses outside the loop.
324 - it has vdefs (it alters memory).
325 - control stmts in the loop (except for the exit condition).
327 CHECKME: what other side effects would the vectorizer allow? */
330 vect_stmt_relevant_p (gimple stmt
, loop_vec_info loop_vinfo
,
331 enum vect_relevant
*relevant
, bool *live_p
)
333 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
335 imm_use_iterator imm_iter
;
339 *relevant
= vect_unused_in_scope
;
342 /* cond stmt other than loop exit cond. */
343 if (is_ctrl_stmt (stmt
)
344 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
345 != loop_exit_ctrl_vec_info_type
)
346 *relevant
= vect_used_in_scope
;
348 /* changing memory. */
349 if (gimple_code (stmt
) != GIMPLE_PHI
)
350 if (gimple_vdef (stmt
)
351 && !gimple_clobber_p (stmt
))
353 if (dump_enabled_p ())
354 dump_printf_loc (MSG_NOTE
, vect_location
,
355 "vec_stmt_relevant_p: stmt has vdefs.\n");
356 *relevant
= vect_used_in_scope
;
359 /* uses outside the loop. */
360 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
362 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
364 basic_block bb
= gimple_bb (USE_STMT (use_p
));
365 if (!flow_bb_inside_loop_p (loop
, bb
))
367 if (dump_enabled_p ())
368 dump_printf_loc (MSG_NOTE
, vect_location
,
369 "vec_stmt_relevant_p: used out of loop.\n");
371 if (is_gimple_debug (USE_STMT (use_p
)))
374 /* We expect all such uses to be in the loop exit phis
375 (because of loop closed form) */
376 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
377 gcc_assert (bb
== single_exit (loop
)->dest
);
384 return (*live_p
|| *relevant
);
388 /* Function exist_non_indexing_operands_for_use_p
390 USE is one of the uses attached to STMT. Check if USE is
391 used in STMT for anything other than indexing an array. */
394 exist_non_indexing_operands_for_use_p (tree use
, gimple stmt
)
397 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
399 /* USE corresponds to some operand in STMT. If there is no data
400 reference in STMT, then any operand that corresponds to USE
401 is not indexing an array. */
402 if (!STMT_VINFO_DATA_REF (stmt_info
))
405 /* STMT has a data_ref. FORNOW this means that its of one of
409 (This should have been verified in analyze_data_refs).
411 'var' in the second case corresponds to a def, not a use,
412 so USE cannot correspond to any operands that are not used
415 Therefore, all we need to check is if STMT falls into the
416 first case, and whether var corresponds to USE. */
418 if (!gimple_assign_copy_p (stmt
))
420 if (is_gimple_call (stmt
)
421 && gimple_call_internal_p (stmt
))
422 switch (gimple_call_internal_fn (stmt
))
425 operand
= gimple_call_arg (stmt
, 3);
430 operand
= gimple_call_arg (stmt
, 2);
440 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
442 operand
= gimple_assign_rhs1 (stmt
);
443 if (TREE_CODE (operand
) != SSA_NAME
)
454 Function process_use.
457 - a USE in STMT in a loop represented by LOOP_VINFO
458 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
459 that defined USE. This is done by calling mark_relevant and passing it
460 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
461 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
465 Generally, LIVE_P and RELEVANT are used to define the liveness and
466 relevance info of the DEF_STMT of this USE:
467 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
468 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
470 - case 1: If USE is used only for address computations (e.g. array indexing),
471 which does not need to be directly vectorized, then the liveness/relevance
472 of the respective DEF_STMT is left unchanged.
473 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
474 skip DEF_STMT cause it had already been processed.
475 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
476 be modified accordingly.
478 Return true if everything is as expected. Return false otherwise. */
481 process_use (gimple stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
482 enum vect_relevant relevant
, vec
<gimple
> *worklist
,
485 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
486 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
487 stmt_vec_info dstmt_vinfo
;
488 basic_block bb
, def_bb
;
491 enum vect_def_type dt
;
493 /* case 1: we are only interested in uses that need to be vectorized. Uses
494 that are used for address computation are not considered relevant. */
495 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
498 if (!vect_is_simple_use (use
, stmt
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
))
500 if (dump_enabled_p ())
501 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
502 "not vectorized: unsupported use in stmt.\n");
506 if (!def_stmt
|| gimple_nop_p (def_stmt
))
509 def_bb
= gimple_bb (def_stmt
);
510 if (!flow_bb_inside_loop_p (loop
, def_bb
))
512 if (dump_enabled_p ())
513 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
517 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
518 DEF_STMT must have already been processed, because this should be the
519 only way that STMT, which is a reduction-phi, was put in the worklist,
520 as there should be no other uses for DEF_STMT in the loop. So we just
521 check that everything is as expected, and we are done. */
522 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
523 bb
= gimple_bb (stmt
);
524 if (gimple_code (stmt
) == GIMPLE_PHI
525 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
526 && gimple_code (def_stmt
) != GIMPLE_PHI
527 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
528 && bb
->loop_father
== def_bb
->loop_father
)
530 if (dump_enabled_p ())
531 dump_printf_loc (MSG_NOTE
, vect_location
,
532 "reduc-stmt defining reduc-phi in the same nest.\n");
533 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
534 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
535 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
536 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
537 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
541 /* case 3a: outer-loop stmt defining an inner-loop stmt:
542 outer-loop-header-bb:
548 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
550 if (dump_enabled_p ())
551 dump_printf_loc (MSG_NOTE
, vect_location
,
552 "outer-loop def-stmt defining inner-loop stmt.\n");
556 case vect_unused_in_scope
:
557 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
558 vect_used_in_scope
: vect_unused_in_scope
;
561 case vect_used_in_outer_by_reduction
:
562 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
563 relevant
= vect_used_by_reduction
;
566 case vect_used_in_outer
:
567 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
568 relevant
= vect_used_in_scope
;
571 case vect_used_in_scope
:
579 /* case 3b: inner-loop stmt defining an outer-loop stmt:
580 outer-loop-header-bb:
584 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
586 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
588 if (dump_enabled_p ())
589 dump_printf_loc (MSG_NOTE
, vect_location
,
590 "inner-loop def-stmt defining outer-loop stmt.\n");
594 case vect_unused_in_scope
:
595 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
596 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
597 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
600 case vect_used_by_reduction
:
601 relevant
= vect_used_in_outer_by_reduction
;
604 case vect_used_in_scope
:
605 relevant
= vect_used_in_outer
;
613 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
,
614 is_pattern_stmt_p (stmt_vinfo
));
619 /* Function vect_mark_stmts_to_be_vectorized.
621 Not all stmts in the loop need to be vectorized. For example:
630 Stmt 1 and 3 do not need to be vectorized, because loop control and
631 addressing of vectorized data-refs are handled differently.
633 This pass detects such stmts. */
636 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
638 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
639 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
640 unsigned int nbbs
= loop
->num_nodes
;
641 gimple_stmt_iterator si
;
644 stmt_vec_info stmt_vinfo
;
648 enum vect_relevant relevant
, tmp_relevant
;
649 enum vect_def_type def_type
;
651 if (dump_enabled_p ())
652 dump_printf_loc (MSG_NOTE
, vect_location
,
653 "=== vect_mark_stmts_to_be_vectorized ===\n");
655 auto_vec
<gimple
, 64> worklist
;
657 /* 1. Init worklist. */
658 for (i
= 0; i
< nbbs
; i
++)
661 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
664 if (dump_enabled_p ())
666 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
667 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
670 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
671 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
, false);
673 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
675 stmt
= gsi_stmt (si
);
676 if (dump_enabled_p ())
678 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
679 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
682 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
683 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
, false);
687 /* 2. Process_worklist */
688 while (worklist
.length () > 0)
693 stmt
= worklist
.pop ();
694 if (dump_enabled_p ())
696 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
697 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
700 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
701 (DEF_STMT) as relevant/irrelevant and live/dead according to the
702 liveness and relevance properties of STMT. */
703 stmt_vinfo
= vinfo_for_stmt (stmt
);
704 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
705 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
707 /* Generally, the liveness and relevance properties of STMT are
708 propagated as is to the DEF_STMTs of its USEs:
709 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
710 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
712 One exception is when STMT has been identified as defining a reduction
713 variable; in this case we set the liveness/relevance as follows:
715 relevant = vect_used_by_reduction
716 This is because we distinguish between two kinds of relevant stmts -
717 those that are used by a reduction computation, and those that are
718 (also) used by a regular computation. This allows us later on to
719 identify stmts that are used solely by a reduction, and therefore the
720 order of the results that they produce does not have to be kept. */
722 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
723 tmp_relevant
= relevant
;
726 case vect_reduction_def
:
727 switch (tmp_relevant
)
729 case vect_unused_in_scope
:
730 relevant
= vect_used_by_reduction
;
733 case vect_used_by_reduction
:
734 if (gimple_code (stmt
) == GIMPLE_PHI
)
739 if (dump_enabled_p ())
740 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
741 "unsupported use of reduction.\n");
748 case vect_nested_cycle
:
749 if (tmp_relevant
!= vect_unused_in_scope
750 && tmp_relevant
!= vect_used_in_outer_by_reduction
751 && tmp_relevant
!= vect_used_in_outer
)
753 if (dump_enabled_p ())
754 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
755 "unsupported use of nested cycle.\n");
763 case vect_double_reduction_def
:
764 if (tmp_relevant
!= vect_unused_in_scope
765 && tmp_relevant
!= vect_used_by_reduction
)
767 if (dump_enabled_p ())
768 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
769 "unsupported use of double reduction.\n");
781 if (is_pattern_stmt_p (stmt_vinfo
))
783 /* Pattern statements are not inserted into the code, so
784 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
785 have to scan the RHS or function arguments instead. */
786 if (is_gimple_assign (stmt
))
788 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
789 tree op
= gimple_assign_rhs1 (stmt
);
792 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
794 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
795 live_p
, relevant
, &worklist
, false)
796 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
797 live_p
, relevant
, &worklist
, false))
801 for (; i
< gimple_num_ops (stmt
); i
++)
803 op
= gimple_op (stmt
, i
);
804 if (TREE_CODE (op
) == SSA_NAME
805 && !process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
810 else if (is_gimple_call (stmt
))
812 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
814 tree arg
= gimple_call_arg (stmt
, i
);
815 if (!process_use (stmt
, arg
, loop_vinfo
, live_p
, relevant
,
822 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
824 tree op
= USE_FROM_PTR (use_p
);
825 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
830 if (STMT_VINFO_GATHER_P (stmt_vinfo
))
833 tree decl
= vect_check_gather (stmt
, loop_vinfo
, NULL
, &off
, NULL
);
835 if (!process_use (stmt
, off
, loop_vinfo
, live_p
, relevant
,
839 } /* while worklist */
845 /* Function vect_model_simple_cost.
847 Models cost for simple operations, i.e. those that only emit ncopies of a
848 single op. Right now, this does not account for multiple insns that could
849 be generated for the single vector op. We will handle that shortly. */
852 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
853 enum vect_def_type
*dt
,
854 stmt_vector_for_cost
*prologue_cost_vec
,
855 stmt_vector_for_cost
*body_cost_vec
)
858 int inside_cost
= 0, prologue_cost
= 0;
860 /* The SLP costs were already calculated during SLP tree build. */
861 if (PURE_SLP_STMT (stmt_info
))
864 /* FORNOW: Assuming maximum 2 args per stmts. */
865 for (i
= 0; i
< 2; i
++)
866 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
867 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
868 stmt_info
, 0, vect_prologue
);
870 /* Pass the inside-of-loop statements to the target-specific cost model. */
871 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
872 stmt_info
, 0, vect_body
);
874 if (dump_enabled_p ())
875 dump_printf_loc (MSG_NOTE
, vect_location
,
876 "vect_model_simple_cost: inside_cost = %d, "
877 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
881 /* Model cost for type demotion and promotion operations. PWR is normally
882 zero for single-step promotions and demotions. It will be one if
883 two-step promotion/demotion is required, and so on. Each additional
884 step doubles the number of instructions required. */
887 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
888 enum vect_def_type
*dt
, int pwr
)
891 int inside_cost
= 0, prologue_cost
= 0;
892 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
893 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
894 void *target_cost_data
;
896 /* The SLP costs were already calculated during SLP tree build. */
897 if (PURE_SLP_STMT (stmt_info
))
901 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
903 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
905 for (i
= 0; i
< pwr
+ 1; i
++)
907 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
909 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
910 vec_promote_demote
, stmt_info
, 0,
914 /* FORNOW: Assuming maximum 2 args per stmts. */
915 for (i
= 0; i
< 2; i
++)
916 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
917 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
918 stmt_info
, 0, vect_prologue
);
920 if (dump_enabled_p ())
921 dump_printf_loc (MSG_NOTE
, vect_location
,
922 "vect_model_promotion_demotion_cost: inside_cost = %d, "
923 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
926 /* Function vect_cost_group_size
928 For grouped load or store, return the group_size only if it is the first
929 load or store of a group, else return 1. This ensures that group size is
930 only returned once per group. */
933 vect_cost_group_size (stmt_vec_info stmt_info
)
935 gimple first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
937 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
938 return GROUP_SIZE (stmt_info
);
944 /* Function vect_model_store_cost
946 Models cost for stores. In the case of grouped accesses, one access
947 has the overhead of the grouped access attributed to it. */
950 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
951 bool store_lanes_p
, enum vect_def_type dt
,
953 stmt_vector_for_cost
*prologue_cost_vec
,
954 stmt_vector_for_cost
*body_cost_vec
)
957 unsigned int inside_cost
= 0, prologue_cost
= 0;
958 struct data_reference
*first_dr
;
961 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
962 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
963 stmt_info
, 0, vect_prologue
);
965 /* Grouped access? */
966 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
970 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
975 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
976 group_size
= vect_cost_group_size (stmt_info
);
979 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
981 /* Not a grouped access. */
985 first_dr
= STMT_VINFO_DATA_REF (stmt_info
);
988 /* We assume that the cost of a single store-lanes instruction is
989 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
990 access is instead being provided by a permute-and-store operation,
991 include the cost of the permutes. */
992 if (!store_lanes_p
&& group_size
> 1
993 && !STMT_VINFO_STRIDED_P (stmt_info
))
995 /* Uses a high and low interleave or shuffle operations for each
997 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
998 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
999 stmt_info
, 0, vect_body
);
1001 if (dump_enabled_p ())
1002 dump_printf_loc (MSG_NOTE
, vect_location
,
1003 "vect_model_store_cost: strided group_size = %d .\n",
1007 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1008 /* Costs of the stores. */
1009 if (STMT_VINFO_STRIDED_P (stmt_info
)
1010 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1012 /* N scalar stores plus extracting the elements. */
1013 inside_cost
+= record_stmt_cost (body_cost_vec
,
1014 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1015 scalar_store
, stmt_info
, 0, vect_body
);
1018 vect_get_store_cost (first_dr
, ncopies
, &inside_cost
, body_cost_vec
);
1020 if (STMT_VINFO_STRIDED_P (stmt_info
))
1021 inside_cost
+= record_stmt_cost (body_cost_vec
,
1022 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1023 vec_to_scalar
, stmt_info
, 0, vect_body
);
1025 if (dump_enabled_p ())
1026 dump_printf_loc (MSG_NOTE
, vect_location
,
1027 "vect_model_store_cost: inside_cost = %d, "
1028 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1032 /* Calculate cost of DR's memory access. */
1034 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
1035 unsigned int *inside_cost
,
1036 stmt_vector_for_cost
*body_cost_vec
)
1038 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1039 gimple stmt
= DR_STMT (dr
);
1040 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1042 switch (alignment_support_scheme
)
1046 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1047 vector_store
, stmt_info
, 0,
1050 if (dump_enabled_p ())
1051 dump_printf_loc (MSG_NOTE
, vect_location
,
1052 "vect_model_store_cost: aligned.\n");
1056 case dr_unaligned_supported
:
1058 /* Here, we assign an additional cost for the unaligned store. */
1059 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1060 unaligned_store
, stmt_info
,
1061 DR_MISALIGNMENT (dr
), vect_body
);
1062 if (dump_enabled_p ())
1063 dump_printf_loc (MSG_NOTE
, vect_location
,
1064 "vect_model_store_cost: unaligned supported by "
1069 case dr_unaligned_unsupported
:
1071 *inside_cost
= VECT_MAX_COST
;
1073 if (dump_enabled_p ())
1074 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1075 "vect_model_store_cost: unsupported access.\n");
1085 /* Function vect_model_load_cost
1087 Models cost for loads. In the case of grouped accesses, the last access
1088 has the overhead of the grouped access attributed to it. Since unaligned
1089 accesses are supported for loads, we also account for the costs of the
1090 access scheme chosen. */
1093 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1094 bool load_lanes_p
, slp_tree slp_node
,
1095 stmt_vector_for_cost
*prologue_cost_vec
,
1096 stmt_vector_for_cost
*body_cost_vec
)
1100 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
1101 unsigned int inside_cost
= 0, prologue_cost
= 0;
1103 /* Grouped accesses? */
1104 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1105 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && first_stmt
&& !slp_node
)
1107 group_size
= vect_cost_group_size (stmt_info
);
1108 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1110 /* Not a grouped access. */
1117 /* We assume that the cost of a single load-lanes instruction is
1118 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1119 access is instead being provided by a load-and-permute operation,
1120 include the cost of the permutes. */
1121 if (!load_lanes_p
&& group_size
> 1
1122 && !STMT_VINFO_STRIDED_P (stmt_info
))
1124 /* Uses an even and odd extract operations or shuffle operations
1125 for each needed permute. */
1126 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1127 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1128 stmt_info
, 0, vect_body
);
1130 if (dump_enabled_p ())
1131 dump_printf_loc (MSG_NOTE
, vect_location
,
1132 "vect_model_load_cost: strided group_size = %d .\n",
1136 /* The loads themselves. */
1137 if (STMT_VINFO_STRIDED_P (stmt_info
)
1138 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1140 /* N scalar loads plus gathering them into a vector. */
1141 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1142 inside_cost
+= record_stmt_cost (body_cost_vec
,
1143 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1144 scalar_load
, stmt_info
, 0, vect_body
);
1147 vect_get_load_cost (first_dr
, ncopies
,
1148 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1149 || group_size
> 1 || slp_node
),
1150 &inside_cost
, &prologue_cost
,
1151 prologue_cost_vec
, body_cost_vec
, true);
1152 if (STMT_VINFO_STRIDED_P (stmt_info
))
1153 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1154 stmt_info
, 0, vect_body
);
1156 if (dump_enabled_p ())
1157 dump_printf_loc (MSG_NOTE
, vect_location
,
1158 "vect_model_load_cost: inside_cost = %d, "
1159 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1163 /* Calculate cost of DR's memory access. */
1165 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1166 bool add_realign_cost
, unsigned int *inside_cost
,
1167 unsigned int *prologue_cost
,
1168 stmt_vector_for_cost
*prologue_cost_vec
,
1169 stmt_vector_for_cost
*body_cost_vec
,
1170 bool record_prologue_costs
)
1172 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1173 gimple stmt
= DR_STMT (dr
);
1174 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1176 switch (alignment_support_scheme
)
1180 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1181 stmt_info
, 0, vect_body
);
1183 if (dump_enabled_p ())
1184 dump_printf_loc (MSG_NOTE
, vect_location
,
1185 "vect_model_load_cost: aligned.\n");
1189 case dr_unaligned_supported
:
1191 /* Here, we assign an additional cost for the unaligned load. */
1192 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1193 unaligned_load
, stmt_info
,
1194 DR_MISALIGNMENT (dr
), vect_body
);
1196 if (dump_enabled_p ())
1197 dump_printf_loc (MSG_NOTE
, vect_location
,
1198 "vect_model_load_cost: unaligned supported by "
1203 case dr_explicit_realign
:
1205 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1206 vector_load
, stmt_info
, 0, vect_body
);
1207 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1208 vec_perm
, stmt_info
, 0, vect_body
);
1210 /* FIXME: If the misalignment remains fixed across the iterations of
1211 the containing loop, the following cost should be added to the
1213 if (targetm
.vectorize
.builtin_mask_for_load
)
1214 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1215 stmt_info
, 0, vect_body
);
1217 if (dump_enabled_p ())
1218 dump_printf_loc (MSG_NOTE
, vect_location
,
1219 "vect_model_load_cost: explicit realign\n");
1223 case dr_explicit_realign_optimized
:
1225 if (dump_enabled_p ())
1226 dump_printf_loc (MSG_NOTE
, vect_location
,
1227 "vect_model_load_cost: unaligned software "
1230 /* Unaligned software pipeline has a load of an address, an initial
1231 load, and possibly a mask operation to "prime" the loop. However,
1232 if this is an access in a group of loads, which provide grouped
1233 access, then the above cost should only be considered for one
1234 access in the group. Inside the loop, there is a load op
1235 and a realignment op. */
1237 if (add_realign_cost
&& record_prologue_costs
)
1239 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1240 vector_stmt
, stmt_info
,
1242 if (targetm
.vectorize
.builtin_mask_for_load
)
1243 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1244 vector_stmt
, stmt_info
,
1248 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1249 stmt_info
, 0, vect_body
);
1250 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1251 stmt_info
, 0, vect_body
);
1253 if (dump_enabled_p ())
1254 dump_printf_loc (MSG_NOTE
, vect_location
,
1255 "vect_model_load_cost: explicit realign optimized"
1261 case dr_unaligned_unsupported
:
1263 *inside_cost
= VECT_MAX_COST
;
1265 if (dump_enabled_p ())
1266 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1267 "vect_model_load_cost: unsupported access.\n");
1276 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1277 the loop preheader for the vectorized stmt STMT. */
1280 vect_init_vector_1 (gimple stmt
, gimple new_stmt
, gimple_stmt_iterator
*gsi
)
1283 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1286 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1287 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1291 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1295 if (nested_in_vect_loop_p (loop
, stmt
))
1298 pe
= loop_preheader_edge (loop
);
1299 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1300 gcc_assert (!new_bb
);
1304 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1306 gimple_stmt_iterator gsi_bb_start
;
1308 gcc_assert (bb_vinfo
);
1309 bb
= BB_VINFO_BB (bb_vinfo
);
1310 gsi_bb_start
= gsi_after_labels (bb
);
1311 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1315 if (dump_enabled_p ())
1317 dump_printf_loc (MSG_NOTE
, vect_location
,
1318 "created new init_stmt: ");
1319 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1323 /* Function vect_init_vector.
1325 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1326 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1327 vector type a vector with all elements equal to VAL is created first.
1328 Place the initialization at BSI if it is not NULL. Otherwise, place the
1329 initialization at the loop preheader.
1330 Return the DEF of INIT_STMT.
1331 It will be used in the vectorization of STMT. */
1334 vect_init_vector (gimple stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1341 if (TREE_CODE (type
) == VECTOR_TYPE
1342 && TREE_CODE (TREE_TYPE (val
)) != VECTOR_TYPE
)
1344 if (!types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1346 if (CONSTANT_CLASS_P (val
))
1347 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (type
), val
);
1350 new_temp
= make_ssa_name (TREE_TYPE (type
));
1351 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1352 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1356 val
= build_vector_from_val (type
, val
);
1359 new_var
= vect_get_new_vect_var (type
, vect_simple_var
, "cst_");
1360 init_stmt
= gimple_build_assign (new_var
, val
);
1361 new_temp
= make_ssa_name (new_var
, init_stmt
);
1362 gimple_assign_set_lhs (init_stmt
, new_temp
);
1363 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1364 vec_oprnd
= gimple_assign_lhs (init_stmt
);
1369 /* Function vect_get_vec_def_for_operand.
1371 OP is an operand in STMT. This function returns a (vector) def that will be
1372 used in the vectorized stmt for STMT.
1374 In the case that OP is an SSA_NAME which is defined in the loop, then
1375 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1377 In case OP is an invariant or constant, a new stmt that creates a vector def
1378 needs to be introduced. */
1381 vect_get_vec_def_for_operand (tree op
, gimple stmt
, tree
*scalar_def
)
1386 stmt_vec_info def_stmt_info
= NULL
;
1387 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1388 unsigned int nunits
;
1389 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1391 enum vect_def_type dt
;
1395 if (dump_enabled_p ())
1397 dump_printf_loc (MSG_NOTE
, vect_location
,
1398 "vect_get_vec_def_for_operand: ");
1399 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1400 dump_printf (MSG_NOTE
, "\n");
1403 is_simple_use
= vect_is_simple_use (op
, stmt
, loop_vinfo
, NULL
,
1404 &def_stmt
, &def
, &dt
);
1405 gcc_assert (is_simple_use
);
1406 if (dump_enabled_p ())
1408 int loc_printed
= 0;
1411 dump_printf_loc (MSG_NOTE
, vect_location
, "def = ");
1413 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, def
);
1414 dump_printf (MSG_NOTE
, "\n");
1419 dump_printf (MSG_NOTE
, " def_stmt = ");
1421 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1422 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1428 /* Case 1: operand is a constant. */
1429 case vect_constant_def
:
1431 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1432 gcc_assert (vector_type
);
1433 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
1438 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1439 if (dump_enabled_p ())
1440 dump_printf_loc (MSG_NOTE
, vect_location
,
1441 "Create vector_cst. nunits = %d\n", nunits
);
1443 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1446 /* Case 2: operand is defined outside the loop - loop invariant. */
1447 case vect_external_def
:
1449 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (def
));
1450 gcc_assert (vector_type
);
1455 /* Create 'vec_inv = {inv,inv,..,inv}' */
1456 if (dump_enabled_p ())
1457 dump_printf_loc (MSG_NOTE
, vect_location
, "Create vector_inv.\n");
1459 return vect_init_vector (stmt
, def
, vector_type
, NULL
);
1462 /* Case 3: operand is defined inside the loop. */
1463 case vect_internal_def
:
1466 *scalar_def
= NULL
/* FIXME tuples: def_stmt*/;
1468 /* Get the def from the vectorized stmt. */
1469 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1471 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1472 /* Get vectorized pattern statement. */
1474 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1475 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1476 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1477 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1478 gcc_assert (vec_stmt
);
1479 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1480 vec_oprnd
= PHI_RESULT (vec_stmt
);
1481 else if (is_gimple_call (vec_stmt
))
1482 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1484 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1488 /* Case 4: operand is defined by a loop header phi - reduction */
1489 case vect_reduction_def
:
1490 case vect_double_reduction_def
:
1491 case vect_nested_cycle
:
1495 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1496 loop
= (gimple_bb (def_stmt
))->loop_father
;
1498 /* Get the def before the loop */
1499 op
= PHI_ARG_DEF_FROM_EDGE (def_stmt
, loop_preheader_edge (loop
));
1500 return get_initial_def_for_reduction (stmt
, op
, scalar_def
);
1503 /* Case 5: operand is defined by loop-header phi - induction. */
1504 case vect_induction_def
:
1506 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1508 /* Get the def from the vectorized stmt. */
1509 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1510 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1511 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1512 vec_oprnd
= PHI_RESULT (vec_stmt
);
1514 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1524 /* Function vect_get_vec_def_for_stmt_copy
1526 Return a vector-def for an operand. This function is used when the
1527 vectorized stmt to be created (by the caller to this function) is a "copy"
1528 created in case the vectorized result cannot fit in one vector, and several
1529 copies of the vector-stmt are required. In this case the vector-def is
1530 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1531 of the stmt that defines VEC_OPRND.
1532 DT is the type of the vector def VEC_OPRND.
1535 In case the vectorization factor (VF) is bigger than the number
1536 of elements that can fit in a vectype (nunits), we have to generate
1537 more than one vector stmt to vectorize the scalar stmt. This situation
1538 arises when there are multiple data-types operated upon in the loop; the
1539 smallest data-type determines the VF, and as a result, when vectorizing
1540 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1541 vector stmt (each computing a vector of 'nunits' results, and together
1542 computing 'VF' results in each iteration). This function is called when
1543 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1544 which VF=16 and nunits=4, so the number of copies required is 4):
1546 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1548 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1549 VS1.1: vx.1 = memref1 VS1.2
1550 VS1.2: vx.2 = memref2 VS1.3
1551 VS1.3: vx.3 = memref3
1553 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1554 VSnew.1: vz1 = vx.1 + ... VSnew.2
1555 VSnew.2: vz2 = vx.2 + ... VSnew.3
1556 VSnew.3: vz3 = vx.3 + ...
1558 The vectorization of S1 is explained in vectorizable_load.
1559 The vectorization of S2:
1560 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1561 the function 'vect_get_vec_def_for_operand' is called to
1562 get the relevant vector-def for each operand of S2. For operand x it
1563 returns the vector-def 'vx.0'.
1565 To create the remaining copies of the vector-stmt (VSnew.j), this
1566 function is called to get the relevant vector-def for each operand. It is
1567 obtained from the respective VS1.j stmt, which is recorded in the
1568 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1570 For example, to obtain the vector-def 'vx.1' in order to create the
1571 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1572 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1573 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1574 and return its def ('vx.1').
1575 Overall, to create the above sequence this function will be called 3 times:
1576 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1577 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1578 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1581 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1583 gimple vec_stmt_for_operand
;
1584 stmt_vec_info def_stmt_info
;
1586 /* Do nothing; can reuse same def. */
1587 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1590 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1591 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1592 gcc_assert (def_stmt_info
);
1593 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1594 gcc_assert (vec_stmt_for_operand
);
1595 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1596 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1597 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1599 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1604 /* Get vectorized definitions for the operands to create a copy of an original
1605 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1608 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1609 vec
<tree
> *vec_oprnds0
,
1610 vec
<tree
> *vec_oprnds1
)
1612 tree vec_oprnd
= vec_oprnds0
->pop ();
1614 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1615 vec_oprnds0
->quick_push (vec_oprnd
);
1617 if (vec_oprnds1
&& vec_oprnds1
->length ())
1619 vec_oprnd
= vec_oprnds1
->pop ();
1620 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1621 vec_oprnds1
->quick_push (vec_oprnd
);
1626 /* Get vectorized definitions for OP0 and OP1.
1627 REDUC_INDEX is the index of reduction operand in case of reduction,
1628 and -1 otherwise. */
1631 vect_get_vec_defs (tree op0
, tree op1
, gimple stmt
,
1632 vec
<tree
> *vec_oprnds0
,
1633 vec
<tree
> *vec_oprnds1
,
1634 slp_tree slp_node
, int reduc_index
)
1638 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1639 auto_vec
<tree
> ops (nops
);
1640 auto_vec
<vec
<tree
> > vec_defs (nops
);
1642 ops
.quick_push (op0
);
1644 ops
.quick_push (op1
);
1646 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1648 *vec_oprnds0
= vec_defs
[0];
1650 *vec_oprnds1
= vec_defs
[1];
1656 vec_oprnds0
->create (1);
1657 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1658 vec_oprnds0
->quick_push (vec_oprnd
);
1662 vec_oprnds1
->create (1);
1663 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
1664 vec_oprnds1
->quick_push (vec_oprnd
);
1670 /* Function vect_finish_stmt_generation.
1672 Insert a new stmt. */
1675 vect_finish_stmt_generation (gimple stmt
, gimple vec_stmt
,
1676 gimple_stmt_iterator
*gsi
)
1678 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1679 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1680 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1682 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1684 if (!gsi_end_p (*gsi
)
1685 && gimple_has_mem_ops (vec_stmt
))
1687 gimple at_stmt
= gsi_stmt (*gsi
);
1688 tree vuse
= gimple_vuse (at_stmt
);
1689 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1691 tree vdef
= gimple_vdef (at_stmt
);
1692 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1693 /* If we have an SSA vuse and insert a store, update virtual
1694 SSA form to avoid triggering the renamer. Do so only
1695 if we can easily see all uses - which is what almost always
1696 happens with the way vectorized stmts are inserted. */
1697 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1698 && ((is_gimple_assign (vec_stmt
)
1699 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1700 || (is_gimple_call (vec_stmt
)
1701 && !(gimple_call_flags (vec_stmt
)
1702 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1704 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1705 gimple_set_vdef (vec_stmt
, new_vdef
);
1706 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1710 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1712 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, loop_vinfo
,
1715 if (dump_enabled_p ())
1717 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1718 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1721 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1723 /* While EH edges will generally prevent vectorization, stmt might
1724 e.g. be in a must-not-throw region. Ensure newly created stmts
1725 that could throw are part of the same region. */
1726 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1727 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1728 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1731 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1732 a function declaration if the target has a vectorized version
1733 of the function, or NULL_TREE if the function cannot be vectorized. */
1736 vectorizable_function (gcall
*call
, tree vectype_out
, tree vectype_in
)
1738 tree fndecl
= gimple_call_fndecl (call
);
1740 /* We only handle functions that do not read or clobber memory -- i.e.
1741 const or novops ones. */
1742 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1746 || TREE_CODE (fndecl
) != FUNCTION_DECL
1747 || !DECL_BUILT_IN (fndecl
))
1750 return targetm
.vectorize
.builtin_vectorized_function (fndecl
, vectype_out
,
1755 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
,
1756 gimple_stmt_iterator
*);
1759 /* Function vectorizable_mask_load_store.
1761 Check if STMT performs a conditional load or store that can be vectorized.
1762 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1763 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1764 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1767 vectorizable_mask_load_store (gimple stmt
, gimple_stmt_iterator
*gsi
,
1768 gimple
*vec_stmt
, slp_tree slp_node
)
1770 tree vec_dest
= NULL
;
1771 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1772 stmt_vec_info prev_stmt_info
;
1773 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1774 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1775 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1776 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1777 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1781 tree dataref_ptr
= NULL_TREE
;
1783 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1787 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
1788 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
1789 int gather_scale
= 1;
1790 enum vect_def_type gather_dt
= vect_unknown_def_type
;
1795 enum vect_def_type dt
;
1797 if (slp_node
!= NULL
)
1800 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1801 gcc_assert (ncopies
>= 1);
1803 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1804 mask
= gimple_call_arg (stmt
, 2);
1805 if (TYPE_PRECISION (TREE_TYPE (mask
))
1806 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
))))
1809 /* FORNOW. This restriction should be relaxed. */
1810 if (nested_in_vect_loop
&& ncopies
> 1)
1812 if (dump_enabled_p ())
1813 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1814 "multiple types in nested loop.");
1818 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1821 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1824 if (!STMT_VINFO_DATA_REF (stmt_info
))
1827 elem_type
= TREE_TYPE (vectype
);
1829 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1832 if (STMT_VINFO_STRIDED_P (stmt_info
))
1835 if (STMT_VINFO_GATHER_P (stmt_info
))
1839 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
1840 &gather_off
, &gather_scale
);
1841 gcc_assert (gather_decl
);
1842 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, NULL
,
1843 &def_stmt
, &def
, &gather_dt
,
1844 &gather_off_vectype
))
1846 if (dump_enabled_p ())
1847 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1848 "gather index use not simple.");
1852 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1854 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1855 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1857 if (dump_enabled_p ())
1858 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1859 "masked gather with integer mask not supported.");
1863 else if (tree_int_cst_compare (nested_in_vect_loop
1864 ? STMT_VINFO_DR_STEP (stmt_info
)
1865 : DR_STEP (dr
), size_zero_node
) <= 0)
1867 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1868 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
), !is_store
))
1871 if (TREE_CODE (mask
) != SSA_NAME
)
1874 if (!vect_is_simple_use (mask
, stmt
, loop_vinfo
, NULL
,
1875 &def_stmt
, &def
, &dt
))
1880 tree rhs
= gimple_call_arg (stmt
, 3);
1881 if (!vect_is_simple_use (rhs
, stmt
, loop_vinfo
, NULL
,
1882 &def_stmt
, &def
, &dt
))
1886 if (!vec_stmt
) /* transformation not required. */
1888 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1890 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1893 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1899 if (STMT_VINFO_GATHER_P (stmt_info
))
1901 tree vec_oprnd0
= NULL_TREE
, op
;
1902 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1903 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1904 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1905 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1906 tree mask_perm_mask
= NULL_TREE
;
1907 edge pe
= loop_preheader_edge (loop
);
1910 enum { NARROW
, NONE
, WIDEN
} modifier
;
1911 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
1913 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
1914 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1915 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1916 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1917 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1918 scaletype
= TREE_VALUE (arglist
);
1919 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1920 && types_compatible_p (srctype
, masktype
));
1922 if (nunits
== gather_off_nunits
)
1924 else if (nunits
== gather_off_nunits
/ 2)
1926 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1929 for (i
= 0; i
< gather_off_nunits
; ++i
)
1930 sel
[i
] = i
| nunits
;
1932 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
1934 else if (nunits
== gather_off_nunits
* 2)
1936 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1939 for (i
= 0; i
< nunits
; ++i
)
1940 sel
[i
] = i
< gather_off_nunits
1941 ? i
: i
+ nunits
- gather_off_nunits
;
1943 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
1945 for (i
= 0; i
< nunits
; ++i
)
1946 sel
[i
] = i
| gather_off_nunits
;
1947 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
1952 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1954 ptr
= fold_convert (ptrtype
, gather_base
);
1955 if (!is_gimple_min_invariant (ptr
))
1957 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1958 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1959 gcc_assert (!new_bb
);
1962 scale
= build_int_cst (scaletype
, gather_scale
);
1964 prev_stmt_info
= NULL
;
1965 for (j
= 0; j
< ncopies
; ++j
)
1967 if (modifier
== WIDEN
&& (j
& 1))
1968 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1969 perm_mask
, stmt
, gsi
);
1972 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
1975 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
1977 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1979 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1980 == TYPE_VECTOR_SUBPARTS (idxtype
));
1981 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
1982 var
= make_ssa_name (var
);
1983 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1985 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1986 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1990 if (mask_perm_mask
&& (j
& 1))
1991 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1992 mask_perm_mask
, stmt
, gsi
);
1996 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
1999 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
,
2000 &def_stmt
, &def
, &dt
);
2001 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2005 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
2007 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
2008 == TYPE_VECTOR_SUBPARTS (masktype
));
2009 var
= vect_get_new_vect_var (masktype
, vect_simple_var
,
2011 var
= make_ssa_name (var
);
2012 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
2014 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
2015 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2021 = gimple_build_call (gather_decl
, 5, mask_op
, ptr
, op
, mask_op
,
2024 if (!useless_type_conversion_p (vectype
, rettype
))
2026 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
2027 == TYPE_VECTOR_SUBPARTS (rettype
));
2028 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
2029 op
= make_ssa_name (var
, new_stmt
);
2030 gimple_call_set_lhs (new_stmt
, op
);
2031 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2032 var
= make_ssa_name (vec_dest
);
2033 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2034 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2038 var
= make_ssa_name (vec_dest
, new_stmt
);
2039 gimple_call_set_lhs (new_stmt
, var
);
2042 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2044 if (modifier
== NARROW
)
2051 var
= permute_vec_elements (prev_res
, var
,
2052 perm_mask
, stmt
, gsi
);
2053 new_stmt
= SSA_NAME_DEF_STMT (var
);
2056 if (prev_stmt_info
== NULL
)
2057 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2059 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2060 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2063 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2065 tree lhs
= gimple_call_lhs (stmt
);
2066 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2067 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2068 set_vinfo_for_stmt (stmt
, NULL
);
2069 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2070 gsi_replace (gsi
, new_stmt
, true);
2075 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
2076 prev_stmt_info
= NULL
;
2077 for (i
= 0; i
< ncopies
; i
++)
2079 unsigned align
, misalign
;
2083 tree rhs
= gimple_call_arg (stmt
, 3);
2084 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
, NULL
);
2085 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2086 /* We should have catched mismatched types earlier. */
2087 gcc_assert (useless_type_conversion_p (vectype
,
2088 TREE_TYPE (vec_rhs
)));
2089 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2090 NULL_TREE
, &dummy
, gsi
,
2091 &ptr_incr
, false, &inv_p
);
2092 gcc_assert (!inv_p
);
2096 vect_is_simple_use (vec_rhs
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2098 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2099 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2101 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2102 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2103 TYPE_SIZE_UNIT (vectype
));
2106 align
= TYPE_ALIGN_UNIT (vectype
);
2107 if (aligned_access_p (dr
))
2109 else if (DR_MISALIGNMENT (dr
) == -1)
2111 align
= TYPE_ALIGN_UNIT (elem_type
);
2115 misalign
= DR_MISALIGNMENT (dr
);
2116 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2119 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2120 gimple_call_arg (stmt
, 1),
2122 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2124 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2126 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2127 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2132 tree vec_mask
= NULL_TREE
;
2133 prev_stmt_info
= NULL
;
2134 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2135 for (i
= 0; i
< ncopies
; i
++)
2137 unsigned align
, misalign
;
2141 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2142 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2143 NULL_TREE
, &dummy
, gsi
,
2144 &ptr_incr
, false, &inv_p
);
2145 gcc_assert (!inv_p
);
2149 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2151 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2152 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2153 TYPE_SIZE_UNIT (vectype
));
2156 align
= TYPE_ALIGN_UNIT (vectype
);
2157 if (aligned_access_p (dr
))
2159 else if (DR_MISALIGNMENT (dr
) == -1)
2161 align
= TYPE_ALIGN_UNIT (elem_type
);
2165 misalign
= DR_MISALIGNMENT (dr
);
2166 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2169 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2170 gimple_call_arg (stmt
, 1),
2172 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
));
2173 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2175 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2177 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2178 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2184 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2186 tree lhs
= gimple_call_lhs (stmt
);
2187 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2188 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2189 set_vinfo_for_stmt (stmt
, NULL
);
2190 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2191 gsi_replace (gsi
, new_stmt
, true);
2198 /* Function vectorizable_call.
2200 Check if GS performs a function call that can be vectorized.
2201 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2202 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2203 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2206 vectorizable_call (gimple gs
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
2213 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2214 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2215 tree vectype_out
, vectype_in
;
2218 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2219 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2220 tree fndecl
, new_temp
, def
, rhs_type
;
2222 enum vect_def_type dt
[3]
2223 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2224 gimple new_stmt
= NULL
;
2226 vec
<tree
> vargs
= vNULL
;
2227 enum { NARROW
, NONE
, WIDEN
} modifier
;
2231 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2234 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2237 /* Is GS a vectorizable call? */
2238 stmt
= dyn_cast
<gcall
*> (gs
);
2242 if (gimple_call_internal_p (stmt
)
2243 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2244 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2245 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2248 if (gimple_call_lhs (stmt
) == NULL_TREE
2249 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2252 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2254 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2256 /* Process function arguments. */
2257 rhs_type
= NULL_TREE
;
2258 vectype_in
= NULL_TREE
;
2259 nargs
= gimple_call_num_args (stmt
);
2261 /* Bail out if the function has more than three arguments, we do not have
2262 interesting builtin functions to vectorize with more than two arguments
2263 except for fma. No arguments is also not good. */
2264 if (nargs
== 0 || nargs
> 3)
2267 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2268 if (gimple_call_internal_p (stmt
)
2269 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2272 rhs_type
= unsigned_type_node
;
2275 for (i
= 0; i
< nargs
; i
++)
2279 op
= gimple_call_arg (stmt
, i
);
2281 /* We can only handle calls with arguments of the same type. */
2283 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2285 if (dump_enabled_p ())
2286 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2287 "argument types differ.\n");
2291 rhs_type
= TREE_TYPE (op
);
2293 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2294 &def_stmt
, &def
, &dt
[i
], &opvectype
))
2296 if (dump_enabled_p ())
2297 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2298 "use not simple.\n");
2303 vectype_in
= opvectype
;
2305 && opvectype
!= vectype_in
)
2307 if (dump_enabled_p ())
2308 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2309 "argument vector types differ.\n");
2313 /* If all arguments are external or constant defs use a vector type with
2314 the same size as the output vector type. */
2316 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2318 gcc_assert (vectype_in
);
2321 if (dump_enabled_p ())
2323 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2324 "no vectype for scalar type ");
2325 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2326 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2333 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2334 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2335 if (nunits_in
== nunits_out
/ 2)
2337 else if (nunits_out
== nunits_in
)
2339 else if (nunits_out
== nunits_in
/ 2)
2344 /* For now, we only vectorize functions if a target specific builtin
2345 is available. TODO -- in some cases, it might be profitable to
2346 insert the calls for pieces of the vector, in order to be able
2347 to vectorize other operations in the loop. */
2348 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
2349 if (fndecl
== NULL_TREE
)
2351 if (gimple_call_internal_p (stmt
)
2352 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
2355 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2356 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2357 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2358 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2360 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2361 { 0, 1, 2, ... vf - 1 } vector. */
2362 gcc_assert (nargs
== 0);
2366 if (dump_enabled_p ())
2367 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2368 "function is not vectorizable.\n");
2373 gcc_assert (!gimple_vuse (stmt
));
2375 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2377 else if (modifier
== NARROW
)
2378 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2380 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2382 /* Sanity check: make sure that at least one copy of the vectorized stmt
2383 needs to be generated. */
2384 gcc_assert (ncopies
>= 1);
2386 if (!vec_stmt
) /* transformation not required. */
2388 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2389 if (dump_enabled_p ())
2390 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2392 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2398 if (dump_enabled_p ())
2399 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2402 scalar_dest
= gimple_call_lhs (stmt
);
2403 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2405 prev_stmt_info
= NULL
;
2409 for (j
= 0; j
< ncopies
; ++j
)
2411 /* Build argument list for the vectorized call. */
2413 vargs
.create (nargs
);
2419 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2420 vec
<tree
> vec_oprnds0
;
2422 for (i
= 0; i
< nargs
; i
++)
2423 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2424 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2425 vec_oprnds0
= vec_defs
[0];
2427 /* Arguments are ready. Create the new vector stmt. */
2428 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2431 for (k
= 0; k
< nargs
; k
++)
2433 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2434 vargs
[k
] = vec_oprndsk
[i
];
2436 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2437 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2438 gimple_call_set_lhs (new_stmt
, new_temp
);
2439 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2440 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2443 for (i
= 0; i
< nargs
; i
++)
2445 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2446 vec_oprndsi
.release ();
2451 for (i
= 0; i
< nargs
; i
++)
2453 op
= gimple_call_arg (stmt
, i
);
2456 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2459 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2461 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2464 vargs
.quick_push (vec_oprnd0
);
2467 if (gimple_call_internal_p (stmt
)
2468 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2470 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2472 for (k
= 0; k
< nunits_out
; ++k
)
2473 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2474 tree cst
= build_vector (vectype_out
, v
);
2476 = vect_get_new_vect_var (vectype_out
, vect_simple_var
, "cst_");
2477 gimple init_stmt
= gimple_build_assign (new_var
, cst
);
2478 new_temp
= make_ssa_name (new_var
, init_stmt
);
2479 gimple_assign_set_lhs (init_stmt
, new_temp
);
2480 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2481 new_temp
= make_ssa_name (vec_dest
);
2482 new_stmt
= gimple_build_assign (new_temp
,
2483 gimple_assign_lhs (init_stmt
));
2487 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2488 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2489 gimple_call_set_lhs (new_stmt
, new_temp
);
2491 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2494 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2496 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2498 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2504 for (j
= 0; j
< ncopies
; ++j
)
2506 /* Build argument list for the vectorized call. */
2508 vargs
.create (nargs
* 2);
2514 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2515 vec
<tree
> vec_oprnds0
;
2517 for (i
= 0; i
< nargs
; i
++)
2518 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2519 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2520 vec_oprnds0
= vec_defs
[0];
2522 /* Arguments are ready. Create the new vector stmt. */
2523 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2527 for (k
= 0; k
< nargs
; k
++)
2529 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2530 vargs
.quick_push (vec_oprndsk
[i
]);
2531 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2533 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2534 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2535 gimple_call_set_lhs (new_stmt
, new_temp
);
2536 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2537 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2540 for (i
= 0; i
< nargs
; i
++)
2542 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2543 vec_oprndsi
.release ();
2548 for (i
= 0; i
< nargs
; i
++)
2550 op
= gimple_call_arg (stmt
, i
);
2554 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2556 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2560 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2562 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2564 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2567 vargs
.quick_push (vec_oprnd0
);
2568 vargs
.quick_push (vec_oprnd1
);
2571 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2572 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2573 gimple_call_set_lhs (new_stmt
, new_temp
);
2574 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2577 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2579 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2581 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2584 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2589 /* No current target implements this case. */
2595 /* The call in STMT might prevent it from being removed in dce.
2596 We however cannot remove it here, due to the way the ssa name
2597 it defines is mapped to the new definition. So just replace
2598 rhs of the statement with something harmless. */
2603 type
= TREE_TYPE (scalar_dest
);
2604 if (is_pattern_stmt_p (stmt_info
))
2605 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2607 lhs
= gimple_call_lhs (stmt
);
2608 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2609 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2610 set_vinfo_for_stmt (stmt
, NULL
);
2611 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2612 gsi_replace (gsi
, new_stmt
, false);
2618 struct simd_call_arg_info
2622 enum vect_def_type dt
;
2623 HOST_WIDE_INT linear_step
;
2627 /* Function vectorizable_simd_clone_call.
2629 Check if STMT performs a function call that can be vectorized
2630 by calling a simd clone of the function.
2631 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2632 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2633 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2636 vectorizable_simd_clone_call (gimple stmt
, gimple_stmt_iterator
*gsi
,
2637 gimple
*vec_stmt
, slp_tree slp_node
)
2642 tree vec_oprnd0
= NULL_TREE
;
2643 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2645 unsigned int nunits
;
2646 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2647 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2648 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2649 tree fndecl
, new_temp
, def
;
2651 gimple new_stmt
= NULL
;
2653 vec
<simd_call_arg_info
> arginfo
= vNULL
;
2654 vec
<tree
> vargs
= vNULL
;
2656 tree lhs
, rtype
, ratype
;
2657 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2659 /* Is STMT a vectorizable call? */
2660 if (!is_gimple_call (stmt
))
2663 fndecl
= gimple_call_fndecl (stmt
);
2664 if (fndecl
== NULL_TREE
)
2667 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2668 if (node
== NULL
|| node
->simd_clones
== NULL
)
2671 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2674 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2677 if (gimple_call_lhs (stmt
)
2678 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2681 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2683 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2685 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2689 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2692 /* Process function arguments. */
2693 nargs
= gimple_call_num_args (stmt
);
2695 /* Bail out if the function has zero arguments. */
2699 arginfo
.create (nargs
);
2701 for (i
= 0; i
< nargs
; i
++)
2703 simd_call_arg_info thisarginfo
;
2706 thisarginfo
.linear_step
= 0;
2707 thisarginfo
.align
= 0;
2708 thisarginfo
.op
= NULL_TREE
;
2710 op
= gimple_call_arg (stmt
, i
);
2711 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2712 &def_stmt
, &def
, &thisarginfo
.dt
,
2713 &thisarginfo
.vectype
)
2714 || thisarginfo
.dt
== vect_uninitialized_def
)
2716 if (dump_enabled_p ())
2717 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2718 "use not simple.\n");
2723 if (thisarginfo
.dt
== vect_constant_def
2724 || thisarginfo
.dt
== vect_external_def
)
2725 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2727 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2729 /* For linear arguments, the analyze phase should have saved
2730 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2731 if (i
* 2 + 3 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
2732 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2])
2734 gcc_assert (vec_stmt
);
2735 thisarginfo
.linear_step
2736 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2]);
2738 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 1];
2739 /* If loop has been peeled for alignment, we need to adjust it. */
2740 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
2741 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
2744 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
2745 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2];
2746 tree opt
= TREE_TYPE (thisarginfo
.op
);
2747 bias
= fold_convert (TREE_TYPE (step
), bias
);
2748 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
2750 = fold_build2 (POINTER_TYPE_P (opt
)
2751 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
2752 thisarginfo
.op
, bias
);
2756 && thisarginfo
.dt
!= vect_constant_def
2757 && thisarginfo
.dt
!= vect_external_def
2759 && TREE_CODE (op
) == SSA_NAME
2760 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
2762 && tree_fits_shwi_p (iv
.step
))
2764 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2765 thisarginfo
.op
= iv
.base
;
2767 else if ((thisarginfo
.dt
== vect_constant_def
2768 || thisarginfo
.dt
== vect_external_def
)
2769 && POINTER_TYPE_P (TREE_TYPE (op
)))
2770 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2772 arginfo
.quick_push (thisarginfo
);
2775 unsigned int badness
= 0;
2776 struct cgraph_node
*bestn
= NULL
;
2777 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
2778 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
2780 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2781 n
= n
->simdclone
->next_clone
)
2783 unsigned int this_badness
= 0;
2784 if (n
->simdclone
->simdlen
2785 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2786 || n
->simdclone
->nargs
!= nargs
)
2788 if (n
->simdclone
->simdlen
2789 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2790 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2791 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2792 if (n
->simdclone
->inbranch
)
2793 this_badness
+= 2048;
2794 int target_badness
= targetm
.simd_clone
.usable (n
);
2795 if (target_badness
< 0)
2797 this_badness
+= target_badness
* 512;
2798 /* FORNOW: Have to add code to add the mask argument. */
2799 if (n
->simdclone
->inbranch
)
2801 for (i
= 0; i
< nargs
; i
++)
2803 switch (n
->simdclone
->args
[i
].arg_type
)
2805 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2806 if (!useless_type_conversion_p
2807 (n
->simdclone
->args
[i
].orig_type
,
2808 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2810 else if (arginfo
[i
].dt
== vect_constant_def
2811 || arginfo
[i
].dt
== vect_external_def
2812 || arginfo
[i
].linear_step
)
2815 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2816 if (arginfo
[i
].dt
!= vect_constant_def
2817 && arginfo
[i
].dt
!= vect_external_def
)
2820 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2821 if (arginfo
[i
].dt
== vect_constant_def
2822 || arginfo
[i
].dt
== vect_external_def
2823 || (arginfo
[i
].linear_step
2824 != n
->simdclone
->args
[i
].linear_step
))
2827 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2831 case SIMD_CLONE_ARG_TYPE_MASK
:
2834 if (i
== (size_t) -1)
2836 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2841 if (arginfo
[i
].align
)
2842 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2843 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2845 if (i
== (size_t) -1)
2847 if (bestn
== NULL
|| this_badness
< badness
)
2850 badness
= this_badness
;
2860 for (i
= 0; i
< nargs
; i
++)
2861 if ((arginfo
[i
].dt
== vect_constant_def
2862 || arginfo
[i
].dt
== vect_external_def
)
2863 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2866 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2868 if (arginfo
[i
].vectype
== NULL
2869 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2870 > bestn
->simdclone
->simdlen
))
2877 fndecl
= bestn
->decl
;
2878 nunits
= bestn
->simdclone
->simdlen
;
2879 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2881 /* If the function isn't const, only allow it in simd loops where user
2882 has asserted that at least nunits consecutive iterations can be
2883 performed using SIMD instructions. */
2884 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2885 && gimple_vuse (stmt
))
2891 /* Sanity check: make sure that at least one copy of the vectorized stmt
2892 needs to be generated. */
2893 gcc_assert (ncopies
>= 1);
2895 if (!vec_stmt
) /* transformation not required. */
2897 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
2898 for (i
= 0; i
< nargs
; i
++)
2899 if (bestn
->simdclone
->args
[i
].arg_type
2900 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
2902 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 2
2904 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
2905 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
2906 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
2907 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
2908 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
2910 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
2911 if (dump_enabled_p ())
2912 dump_printf_loc (MSG_NOTE
, vect_location
,
2913 "=== vectorizable_simd_clone_call ===\n");
2914 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2921 if (dump_enabled_p ())
2922 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2925 scalar_dest
= gimple_call_lhs (stmt
);
2926 vec_dest
= NULL_TREE
;
2931 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2932 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
2933 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
2936 rtype
= TREE_TYPE (ratype
);
2940 prev_stmt_info
= NULL
;
2941 for (j
= 0; j
< ncopies
; ++j
)
2943 /* Build argument list for the vectorized call. */
2945 vargs
.create (nargs
);
2949 for (i
= 0; i
< nargs
; i
++)
2951 unsigned int k
, l
, m
, o
;
2953 op
= gimple_call_arg (stmt
, i
);
2954 switch (bestn
->simdclone
->args
[i
].arg_type
)
2956 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2957 atype
= bestn
->simdclone
->args
[i
].vector_type
;
2958 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
2959 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
2961 if (TYPE_VECTOR_SUBPARTS (atype
)
2962 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
2964 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
2965 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2966 / TYPE_VECTOR_SUBPARTS (atype
));
2967 gcc_assert ((k
& (k
- 1)) == 0);
2970 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2973 vec_oprnd0
= arginfo
[i
].op
;
2974 if ((m
& (k
- 1)) == 0)
2976 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
2979 arginfo
[i
].op
= vec_oprnd0
;
2981 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
2983 bitsize_int ((m
& (k
- 1)) * prec
));
2985 = gimple_build_assign (make_ssa_name (atype
),
2987 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2988 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
2992 k
= (TYPE_VECTOR_SUBPARTS (atype
)
2993 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
2994 gcc_assert ((k
& (k
- 1)) == 0);
2995 vec
<constructor_elt
, va_gc
> *ctor_elts
;
2997 vec_alloc (ctor_elts
, k
);
3000 for (l
= 0; l
< k
; l
++)
3002 if (m
== 0 && l
== 0)
3004 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
3007 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3009 arginfo
[i
].op
= vec_oprnd0
;
3012 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3016 vargs
.safe_push (vec_oprnd0
);
3019 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3021 = gimple_build_assign (make_ssa_name (atype
),
3023 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3024 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3029 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3030 vargs
.safe_push (op
);
3032 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3037 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3042 edge pe
= loop_preheader_edge (loop
);
3043 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3044 gcc_assert (!new_bb
);
3046 tree phi_res
= copy_ssa_name (op
);
3047 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3048 set_vinfo_for_stmt (new_phi
,
3049 new_stmt_vec_info (new_phi
, loop_vinfo
,
3051 add_phi_arg (new_phi
, arginfo
[i
].op
,
3052 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3054 = POINTER_TYPE_P (TREE_TYPE (op
))
3055 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3056 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3057 ? sizetype
: TREE_TYPE (op
);
3059 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3061 tree tcst
= wide_int_to_tree (type
, cst
);
3062 tree phi_arg
= copy_ssa_name (op
);
3064 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3065 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3066 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3067 set_vinfo_for_stmt (new_stmt
,
3068 new_stmt_vec_info (new_stmt
, loop_vinfo
,
3070 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3072 arginfo
[i
].op
= phi_res
;
3073 vargs
.safe_push (phi_res
);
3078 = POINTER_TYPE_P (TREE_TYPE (op
))
3079 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3080 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3081 ? sizetype
: TREE_TYPE (op
);
3083 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3085 tree tcst
= wide_int_to_tree (type
, cst
);
3086 new_temp
= make_ssa_name (TREE_TYPE (op
));
3087 new_stmt
= gimple_build_assign (new_temp
, code
,
3088 arginfo
[i
].op
, tcst
);
3089 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3090 vargs
.safe_push (new_temp
);
3093 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3099 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3102 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3104 new_temp
= create_tmp_var (ratype
);
3105 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3106 == TYPE_VECTOR_SUBPARTS (rtype
))
3107 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3109 new_temp
= make_ssa_name (rtype
, new_stmt
);
3110 gimple_call_set_lhs (new_stmt
, new_temp
);
3112 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3116 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3119 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3120 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3121 gcc_assert ((k
& (k
- 1)) == 0);
3122 for (l
= 0; l
< k
; l
++)
3127 t
= build_fold_addr_expr (new_temp
);
3128 t
= build2 (MEM_REF
, vectype
, t
,
3129 build_int_cst (TREE_TYPE (t
),
3130 l
* prec
/ BITS_PER_UNIT
));
3133 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3134 size_int (prec
), bitsize_int (l
* prec
));
3136 = gimple_build_assign (make_ssa_name (vectype
), t
);
3137 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3138 if (j
== 0 && l
== 0)
3139 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3141 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3143 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3148 tree clobber
= build_constructor (ratype
, NULL
);
3149 TREE_THIS_VOLATILE (clobber
) = 1;
3150 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3151 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3155 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3157 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3158 / TYPE_VECTOR_SUBPARTS (rtype
));
3159 gcc_assert ((k
& (k
- 1)) == 0);
3160 if ((j
& (k
- 1)) == 0)
3161 vec_alloc (ret_ctor_elts
, k
);
3164 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3165 for (m
= 0; m
< o
; m
++)
3167 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3168 size_int (m
), NULL_TREE
, NULL_TREE
);
3170 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3171 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3172 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3173 gimple_assign_lhs (new_stmt
));
3175 tree clobber
= build_constructor (ratype
, NULL
);
3176 TREE_THIS_VOLATILE (clobber
) = 1;
3177 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3178 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3181 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3182 if ((j
& (k
- 1)) != k
- 1)
3184 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3186 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3187 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3189 if ((unsigned) j
== k
- 1)
3190 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3192 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3194 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3199 tree t
= build_fold_addr_expr (new_temp
);
3200 t
= build2 (MEM_REF
, vectype
, t
,
3201 build_int_cst (TREE_TYPE (t
), 0));
3203 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3204 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3205 tree clobber
= build_constructor (ratype
, NULL
);
3206 TREE_THIS_VOLATILE (clobber
) = 1;
3207 vect_finish_stmt_generation (stmt
,
3208 gimple_build_assign (new_temp
,
3214 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3216 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3218 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3223 /* The call in STMT might prevent it from being removed in dce.
3224 We however cannot remove it here, due to the way the ssa name
3225 it defines is mapped to the new definition. So just replace
3226 rhs of the statement with something harmless. */
3233 type
= TREE_TYPE (scalar_dest
);
3234 if (is_pattern_stmt_p (stmt_info
))
3235 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3237 lhs
= gimple_call_lhs (stmt
);
3238 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3241 new_stmt
= gimple_build_nop ();
3242 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3243 set_vinfo_for_stmt (stmt
, NULL
);
3244 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3245 gsi_replace (gsi
, new_stmt
, true);
3246 unlink_stmt_vdef (stmt
);
3252 /* Function vect_gen_widened_results_half
3254 Create a vector stmt whose code, type, number of arguments, and result
3255 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3256 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3257 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3258 needs to be created (DECL is a function-decl of a target-builtin).
3259 STMT is the original scalar stmt that we are vectorizing. */
3262 vect_gen_widened_results_half (enum tree_code code
,
3264 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3265 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3271 /* Generate half of the widened result: */
3272 if (code
== CALL_EXPR
)
3274 /* Target specific support */
3275 if (op_type
== binary_op
)
3276 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3278 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3279 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3280 gimple_call_set_lhs (new_stmt
, new_temp
);
3284 /* Generic support */
3285 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3286 if (op_type
!= binary_op
)
3288 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3289 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3290 gimple_assign_set_lhs (new_stmt
, new_temp
);
3292 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3298 /* Get vectorized definitions for loop-based vectorization. For the first
3299 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3300 scalar operand), and for the rest we get a copy with
3301 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3302 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3303 The vectors are collected into VEC_OPRNDS. */
3306 vect_get_loop_based_defs (tree
*oprnd
, gimple stmt
, enum vect_def_type dt
,
3307 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3311 /* Get first vector operand. */
3312 /* All the vector operands except the very first one (that is scalar oprnd)
3314 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3315 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
, NULL
);
3317 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3319 vec_oprnds
->quick_push (vec_oprnd
);
3321 /* Get second vector operand. */
3322 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3323 vec_oprnds
->quick_push (vec_oprnd
);
3327 /* For conversion in multiple steps, continue to get operands
3330 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3334 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3335 For multi-step conversions store the resulting vectors and call the function
3339 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3340 int multi_step_cvt
, gimple stmt
,
3342 gimple_stmt_iterator
*gsi
,
3343 slp_tree slp_node
, enum tree_code code
,
3344 stmt_vec_info
*prev_stmt_info
)
3347 tree vop0
, vop1
, new_tmp
, vec_dest
;
3349 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3351 vec_dest
= vec_dsts
.pop ();
3353 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3355 /* Create demotion operation. */
3356 vop0
= (*vec_oprnds
)[i
];
3357 vop1
= (*vec_oprnds
)[i
+ 1];
3358 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3359 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3360 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3361 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3364 /* Store the resulting vector for next recursive call. */
3365 (*vec_oprnds
)[i
/2] = new_tmp
;
3368 /* This is the last step of the conversion sequence. Store the
3369 vectors in SLP_NODE or in vector info of the scalar statement
3370 (or in STMT_VINFO_RELATED_STMT chain). */
3372 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3375 if (!*prev_stmt_info
)
3376 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3378 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3380 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3385 /* For multi-step demotion operations we first generate demotion operations
3386 from the source type to the intermediate types, and then combine the
3387 results (stored in VEC_OPRNDS) in demotion operation to the destination
3391 /* At each level of recursion we have half of the operands we had at the
3393 vec_oprnds
->truncate ((i
+1)/2);
3394 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3395 stmt
, vec_dsts
, gsi
, slp_node
,
3396 VEC_PACK_TRUNC_EXPR
,
3400 vec_dsts
.quick_push (vec_dest
);
3404 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3405 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3406 the resulting vectors and call the function recursively. */
3409 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3410 vec
<tree
> *vec_oprnds1
,
3411 gimple stmt
, tree vec_dest
,
3412 gimple_stmt_iterator
*gsi
,
3413 enum tree_code code1
,
3414 enum tree_code code2
, tree decl1
,
3415 tree decl2
, int op_type
)
3418 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3419 gimple new_stmt1
, new_stmt2
;
3420 vec
<tree
> vec_tmp
= vNULL
;
3422 vec_tmp
.create (vec_oprnds0
->length () * 2);
3423 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3425 if (op_type
== binary_op
)
3426 vop1
= (*vec_oprnds1
)[i
];
3430 /* Generate the two halves of promotion operation. */
3431 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3432 op_type
, vec_dest
, gsi
, stmt
);
3433 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3434 op_type
, vec_dest
, gsi
, stmt
);
3435 if (is_gimple_call (new_stmt1
))
3437 new_tmp1
= gimple_call_lhs (new_stmt1
);
3438 new_tmp2
= gimple_call_lhs (new_stmt2
);
3442 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3443 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3446 /* Store the results for the next step. */
3447 vec_tmp
.quick_push (new_tmp1
);
3448 vec_tmp
.quick_push (new_tmp2
);
3451 vec_oprnds0
->release ();
3452 *vec_oprnds0
= vec_tmp
;
3456 /* Check if STMT performs a conversion operation, that can be vectorized.
3457 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3458 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3459 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3462 vectorizable_conversion (gimple stmt
, gimple_stmt_iterator
*gsi
,
3463 gimple
*vec_stmt
, slp_tree slp_node
)
3467 tree op0
, op1
= NULL_TREE
;
3468 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3469 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3470 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3471 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3472 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3473 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3477 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3478 gimple new_stmt
= NULL
;
3479 stmt_vec_info prev_stmt_info
;
3482 tree vectype_out
, vectype_in
;
3484 tree lhs_type
, rhs_type
;
3485 enum { NARROW
, NONE
, WIDEN
} modifier
;
3486 vec
<tree
> vec_oprnds0
= vNULL
;
3487 vec
<tree
> vec_oprnds1
= vNULL
;
3489 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3490 int multi_step_cvt
= 0;
3491 vec
<tree
> vec_dsts
= vNULL
;
3492 vec
<tree
> interm_types
= vNULL
;
3493 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3495 machine_mode rhs_mode
;
3496 unsigned short fltsz
;
3498 /* Is STMT a vectorizable conversion? */
3500 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3503 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3506 if (!is_gimple_assign (stmt
))
3509 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3512 code
= gimple_assign_rhs_code (stmt
);
3513 if (!CONVERT_EXPR_CODE_P (code
)
3514 && code
!= FIX_TRUNC_EXPR
3515 && code
!= FLOAT_EXPR
3516 && code
!= WIDEN_MULT_EXPR
3517 && code
!= WIDEN_LSHIFT_EXPR
)
3520 op_type
= TREE_CODE_LENGTH (code
);
3522 /* Check types of lhs and rhs. */
3523 scalar_dest
= gimple_assign_lhs (stmt
);
3524 lhs_type
= TREE_TYPE (scalar_dest
);
3525 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3527 op0
= gimple_assign_rhs1 (stmt
);
3528 rhs_type
= TREE_TYPE (op0
);
3530 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3531 && !((INTEGRAL_TYPE_P (lhs_type
)
3532 && INTEGRAL_TYPE_P (rhs_type
))
3533 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3534 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3537 if ((INTEGRAL_TYPE_P (lhs_type
)
3538 && (TYPE_PRECISION (lhs_type
)
3539 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3540 || (INTEGRAL_TYPE_P (rhs_type
)
3541 && (TYPE_PRECISION (rhs_type
)
3542 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
)))))
3544 if (dump_enabled_p ())
3545 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3546 "type conversion to/from bit-precision unsupported."
3551 /* Check the operands of the operation. */
3552 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
3553 &def_stmt
, &def
, &dt
[0], &vectype_in
))
3555 if (dump_enabled_p ())
3556 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3557 "use not simple.\n");
3560 if (op_type
== binary_op
)
3564 op1
= gimple_assign_rhs2 (stmt
);
3565 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3566 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3568 if (CONSTANT_CLASS_P (op0
))
3569 ok
= vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
,
3570 &def_stmt
, &def
, &dt
[1], &vectype_in
);
3572 ok
= vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
3577 if (dump_enabled_p ())
3578 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3579 "use not simple.\n");
3584 /* If op0 is an external or constant defs use a vector type of
3585 the same size as the output vector type. */
3587 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3589 gcc_assert (vectype_in
);
3592 if (dump_enabled_p ())
3594 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3595 "no vectype for scalar type ");
3596 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3597 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3603 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3604 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3605 if (nunits_in
< nunits_out
)
3607 else if (nunits_out
== nunits_in
)
3612 /* Multiple types in SLP are handled by creating the appropriate number of
3613 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3615 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
3617 else if (modifier
== NARROW
)
3618 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3620 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3622 /* Sanity check: make sure that at least one copy of the vectorized stmt
3623 needs to be generated. */
3624 gcc_assert (ncopies
>= 1);
3626 /* Supportable by target? */
3630 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3632 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3637 if (dump_enabled_p ())
3638 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3639 "conversion not supported by target.\n");
3643 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3644 &code1
, &code2
, &multi_step_cvt
,
3647 /* Binary widening operation can only be supported directly by the
3649 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3653 if (code
!= FLOAT_EXPR
3654 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3655 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3658 rhs_mode
= TYPE_MODE (rhs_type
);
3659 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3660 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3661 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3662 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3665 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3666 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3667 if (cvt_type
== NULL_TREE
)
3670 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3672 if (!supportable_convert_operation (code
, vectype_out
,
3673 cvt_type
, &decl1
, &codecvt1
))
3676 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3677 cvt_type
, &codecvt1
,
3678 &codecvt2
, &multi_step_cvt
,
3682 gcc_assert (multi_step_cvt
== 0);
3684 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3685 vectype_in
, &code1
, &code2
,
3686 &multi_step_cvt
, &interm_types
))
3690 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3693 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3694 codecvt2
= ERROR_MARK
;
3698 interm_types
.safe_push (cvt_type
);
3699 cvt_type
= NULL_TREE
;
3704 gcc_assert (op_type
== unary_op
);
3705 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3706 &code1
, &multi_step_cvt
,
3710 if (code
!= FIX_TRUNC_EXPR
3711 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3712 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3715 rhs_mode
= TYPE_MODE (rhs_type
);
3717 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3718 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3719 if (cvt_type
== NULL_TREE
)
3721 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3724 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3725 &code1
, &multi_step_cvt
,
3734 if (!vec_stmt
) /* transformation not required. */
3736 if (dump_enabled_p ())
3737 dump_printf_loc (MSG_NOTE
, vect_location
,
3738 "=== vectorizable_conversion ===\n");
3739 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3741 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3742 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3744 else if (modifier
== NARROW
)
3746 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3747 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3751 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3752 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3754 interm_types
.release ();
3759 if (dump_enabled_p ())
3760 dump_printf_loc (MSG_NOTE
, vect_location
,
3761 "transform conversion. ncopies = %d.\n", ncopies
);
3763 if (op_type
== binary_op
)
3765 if (CONSTANT_CLASS_P (op0
))
3766 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3767 else if (CONSTANT_CLASS_P (op1
))
3768 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3771 /* In case of multi-step conversion, we first generate conversion operations
3772 to the intermediate types, and then from that types to the final one.
3773 We create vector destinations for the intermediate type (TYPES) received
3774 from supportable_*_operation, and store them in the correct order
3775 for future use in vect_create_vectorized_*_stmts (). */
3776 vec_dsts
.create (multi_step_cvt
+ 1);
3777 vec_dest
= vect_create_destination_var (scalar_dest
,
3778 (cvt_type
&& modifier
== WIDEN
)
3779 ? cvt_type
: vectype_out
);
3780 vec_dsts
.quick_push (vec_dest
);
3784 for (i
= interm_types
.length () - 1;
3785 interm_types
.iterate (i
, &intermediate_type
); i
--)
3787 vec_dest
= vect_create_destination_var (scalar_dest
,
3789 vec_dsts
.quick_push (vec_dest
);
3794 vec_dest
= vect_create_destination_var (scalar_dest
,
3796 ? vectype_out
: cvt_type
);
3800 if (modifier
== WIDEN
)
3802 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3803 if (op_type
== binary_op
)
3804 vec_oprnds1
.create (1);
3806 else if (modifier
== NARROW
)
3807 vec_oprnds0
.create (
3808 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3810 else if (code
== WIDEN_LSHIFT_EXPR
)
3811 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3814 prev_stmt_info
= NULL
;
3818 for (j
= 0; j
< ncopies
; j
++)
3821 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3824 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3826 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3828 /* Arguments are ready, create the new vector stmt. */
3829 if (code1
== CALL_EXPR
)
3831 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3832 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3833 gimple_call_set_lhs (new_stmt
, new_temp
);
3837 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3838 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
3839 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3840 gimple_assign_set_lhs (new_stmt
, new_temp
);
3843 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3845 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3849 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3851 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3852 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3857 /* In case the vectorization factor (VF) is bigger than the number
3858 of elements that we can fit in a vectype (nunits), we have to
3859 generate more than one vector stmt - i.e - we need to "unroll"
3860 the vector stmt by a factor VF/nunits. */
3861 for (j
= 0; j
< ncopies
; j
++)
3868 if (code
== WIDEN_LSHIFT_EXPR
)
3873 /* Store vec_oprnd1 for every vector stmt to be created
3874 for SLP_NODE. We check during the analysis that all
3875 the shift arguments are the same. */
3876 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
3877 vec_oprnds1
.quick_push (vec_oprnd1
);
3879 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3883 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
3884 &vec_oprnds1
, slp_node
, -1);
3888 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
3889 vec_oprnds0
.quick_push (vec_oprnd0
);
3890 if (op_type
== binary_op
)
3892 if (code
== WIDEN_LSHIFT_EXPR
)
3895 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
,
3897 vec_oprnds1
.quick_push (vec_oprnd1
);
3903 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
3904 vec_oprnds0
.truncate (0);
3905 vec_oprnds0
.quick_push (vec_oprnd0
);
3906 if (op_type
== binary_op
)
3908 if (code
== WIDEN_LSHIFT_EXPR
)
3911 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
3913 vec_oprnds1
.truncate (0);
3914 vec_oprnds1
.quick_push (vec_oprnd1
);
3918 /* Arguments are ready. Create the new vector stmts. */
3919 for (i
= multi_step_cvt
; i
>= 0; i
--)
3921 tree this_dest
= vec_dsts
[i
];
3922 enum tree_code c1
= code1
, c2
= code2
;
3923 if (i
== 0 && codecvt2
!= ERROR_MARK
)
3928 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
3930 stmt
, this_dest
, gsi
,
3931 c1
, c2
, decl1
, decl2
,
3935 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3939 if (codecvt1
== CALL_EXPR
)
3941 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3942 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3943 gimple_call_set_lhs (new_stmt
, new_temp
);
3947 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
3948 new_temp
= make_ssa_name (vec_dest
);
3949 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
3953 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3956 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
3959 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3962 if (!prev_stmt_info
)
3963 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3965 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3966 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3971 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3975 /* In case the vectorization factor (VF) is bigger than the number
3976 of elements that we can fit in a vectype (nunits), we have to
3977 generate more than one vector stmt - i.e - we need to "unroll"
3978 the vector stmt by a factor VF/nunits. */
3979 for (j
= 0; j
< ncopies
; j
++)
3983 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3987 vec_oprnds0
.truncate (0);
3988 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
3989 vect_pow2 (multi_step_cvt
) - 1);
3992 /* Arguments are ready. Create the new vector stmts. */
3994 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3996 if (codecvt1
== CALL_EXPR
)
3998 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3999 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4000 gimple_call_set_lhs (new_stmt
, new_temp
);
4004 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4005 new_temp
= make_ssa_name (vec_dest
);
4006 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4010 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4011 vec_oprnds0
[i
] = new_temp
;
4014 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4015 stmt
, vec_dsts
, gsi
,
4020 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4024 vec_oprnds0
.release ();
4025 vec_oprnds1
.release ();
4026 vec_dsts
.release ();
4027 interm_types
.release ();
4033 /* Function vectorizable_assignment.
4035 Check if STMT performs an assignment (copy) that can be vectorized.
4036 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4037 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4038 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4041 vectorizable_assignment (gimple stmt
, gimple_stmt_iterator
*gsi
,
4042 gimple
*vec_stmt
, slp_tree slp_node
)
4047 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4048 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4049 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4053 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4054 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4057 vec
<tree
> vec_oprnds
= vNULL
;
4059 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4060 gimple new_stmt
= NULL
;
4061 stmt_vec_info prev_stmt_info
= NULL
;
4062 enum tree_code code
;
4065 /* Multiple types in SLP are handled by creating the appropriate number of
4066 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4068 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4071 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4073 gcc_assert (ncopies
>= 1);
4075 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4078 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4081 /* Is vectorizable assignment? */
4082 if (!is_gimple_assign (stmt
))
4085 scalar_dest
= gimple_assign_lhs (stmt
);
4086 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4089 code
= gimple_assign_rhs_code (stmt
);
4090 if (gimple_assign_single_p (stmt
)
4091 || code
== PAREN_EXPR
4092 || CONVERT_EXPR_CODE_P (code
))
4093 op
= gimple_assign_rhs1 (stmt
);
4097 if (code
== VIEW_CONVERT_EXPR
)
4098 op
= TREE_OPERAND (op
, 0);
4100 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
4101 &def_stmt
, &def
, &dt
[0], &vectype_in
))
4103 if (dump_enabled_p ())
4104 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4105 "use not simple.\n");
4109 /* We can handle NOP_EXPR conversions that do not change the number
4110 of elements or the vector size. */
4111 if ((CONVERT_EXPR_CODE_P (code
)
4112 || code
== VIEW_CONVERT_EXPR
)
4114 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4115 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4116 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4119 /* We do not handle bit-precision changes. */
4120 if ((CONVERT_EXPR_CODE_P (code
)
4121 || code
== VIEW_CONVERT_EXPR
)
4122 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4123 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4124 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4125 || ((TYPE_PRECISION (TREE_TYPE (op
))
4126 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4127 /* But a conversion that does not change the bit-pattern is ok. */
4128 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4129 > TYPE_PRECISION (TREE_TYPE (op
)))
4130 && TYPE_UNSIGNED (TREE_TYPE (op
))))
4132 if (dump_enabled_p ())
4133 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4134 "type conversion to/from bit-precision "
4139 if (!vec_stmt
) /* transformation not required. */
4141 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4142 if (dump_enabled_p ())
4143 dump_printf_loc (MSG_NOTE
, vect_location
,
4144 "=== vectorizable_assignment ===\n");
4145 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4150 if (dump_enabled_p ())
4151 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4154 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4157 for (j
= 0; j
< ncopies
; j
++)
4161 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4163 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4165 /* Arguments are ready. create the new vector stmt. */
4166 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4168 if (CONVERT_EXPR_CODE_P (code
)
4169 || code
== VIEW_CONVERT_EXPR
)
4170 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4171 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4172 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4173 gimple_assign_set_lhs (new_stmt
, new_temp
);
4174 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4176 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4183 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4185 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4187 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4190 vec_oprnds
.release ();
4195 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4196 either as shift by a scalar or by a vector. */
4199 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4202 machine_mode vec_mode
;
4207 vectype
= get_vectype_for_scalar_type (scalar_type
);
4211 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4213 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4215 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4217 || (optab_handler (optab
, TYPE_MODE (vectype
))
4218 == CODE_FOR_nothing
))
4222 vec_mode
= TYPE_MODE (vectype
);
4223 icode
= (int) optab_handler (optab
, vec_mode
);
4224 if (icode
== CODE_FOR_nothing
)
4231 /* Function vectorizable_shift.
4233 Check if STMT performs a shift operation that can be vectorized.
4234 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4235 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4236 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4239 vectorizable_shift (gimple stmt
, gimple_stmt_iterator
*gsi
,
4240 gimple
*vec_stmt
, slp_tree slp_node
)
4244 tree op0
, op1
= NULL
;
4245 tree vec_oprnd1
= NULL_TREE
;
4246 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4248 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4249 enum tree_code code
;
4250 machine_mode vec_mode
;
4254 machine_mode optab_op2_mode
;
4257 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4258 gimple new_stmt
= NULL
;
4259 stmt_vec_info prev_stmt_info
;
4266 vec
<tree
> vec_oprnds0
= vNULL
;
4267 vec
<tree
> vec_oprnds1
= vNULL
;
4270 bool scalar_shift_arg
= true;
4271 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4274 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4277 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4280 /* Is STMT a vectorizable binary/unary operation? */
4281 if (!is_gimple_assign (stmt
))
4284 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4287 code
= gimple_assign_rhs_code (stmt
);
4289 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4290 || code
== RROTATE_EXPR
))
4293 scalar_dest
= gimple_assign_lhs (stmt
);
4294 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4295 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4296 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4298 if (dump_enabled_p ())
4299 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4300 "bit-precision shifts not supported.\n");
4304 op0
= gimple_assign_rhs1 (stmt
);
4305 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4306 &def_stmt
, &def
, &dt
[0], &vectype
))
4308 if (dump_enabled_p ())
4309 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4310 "use not simple.\n");
4313 /* If op0 is an external or constant def use a vector type with
4314 the same size as the output vector type. */
4316 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4318 gcc_assert (vectype
);
4321 if (dump_enabled_p ())
4322 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4323 "no vectype for scalar type\n");
4327 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4328 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4329 if (nunits_out
!= nunits_in
)
4332 op1
= gimple_assign_rhs2 (stmt
);
4333 if (!vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4334 &def
, &dt
[1], &op1_vectype
))
4336 if (dump_enabled_p ())
4337 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4338 "use not simple.\n");
4343 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4347 /* Multiple types in SLP are handled by creating the appropriate number of
4348 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4350 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4353 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4355 gcc_assert (ncopies
>= 1);
4357 /* Determine whether the shift amount is a vector, or scalar. If the
4358 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4360 if (dt
[1] == vect_internal_def
&& !slp_node
)
4361 scalar_shift_arg
= false;
4362 else if (dt
[1] == vect_constant_def
4363 || dt
[1] == vect_external_def
4364 || dt
[1] == vect_internal_def
)
4366 /* In SLP, need to check whether the shift count is the same,
4367 in loops if it is a constant or invariant, it is always
4371 vec
<gimple
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4374 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4375 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4376 scalar_shift_arg
= false;
4381 if (dump_enabled_p ())
4382 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4383 "operand mode requires invariant argument.\n");
4387 /* Vector shifted by vector. */
4388 if (!scalar_shift_arg
)
4390 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4391 if (dump_enabled_p ())
4392 dump_printf_loc (MSG_NOTE
, vect_location
,
4393 "vector/vector shift/rotate found.\n");
4396 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4397 if (op1_vectype
== NULL_TREE
4398 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4400 if (dump_enabled_p ())
4401 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4402 "unusable type for last operand in"
4403 " vector/vector shift/rotate.\n");
4407 /* See if the machine has a vector shifted by scalar insn and if not
4408 then see if it has a vector shifted by vector insn. */
4411 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4413 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4415 if (dump_enabled_p ())
4416 dump_printf_loc (MSG_NOTE
, vect_location
,
4417 "vector/scalar shift/rotate found.\n");
4421 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4423 && (optab_handler (optab
, TYPE_MODE (vectype
))
4424 != CODE_FOR_nothing
))
4426 scalar_shift_arg
= false;
4428 if (dump_enabled_p ())
4429 dump_printf_loc (MSG_NOTE
, vect_location
,
4430 "vector/vector shift/rotate found.\n");
4432 /* Unlike the other binary operators, shifts/rotates have
4433 the rhs being int, instead of the same type as the lhs,
4434 so make sure the scalar is the right type if we are
4435 dealing with vectors of long long/long/short/char. */
4436 if (dt
[1] == vect_constant_def
)
4437 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4438 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4442 && TYPE_MODE (TREE_TYPE (vectype
))
4443 != TYPE_MODE (TREE_TYPE (op1
)))
4445 if (dump_enabled_p ())
4446 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4447 "unusable type for last operand in"
4448 " vector/vector shift/rotate.\n");
4451 if (vec_stmt
&& !slp_node
)
4453 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4454 op1
= vect_init_vector (stmt
, op1
,
4455 TREE_TYPE (vectype
), NULL
);
4462 /* Supportable by target? */
4465 if (dump_enabled_p ())
4466 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4470 vec_mode
= TYPE_MODE (vectype
);
4471 icode
= (int) optab_handler (optab
, vec_mode
);
4472 if (icode
== CODE_FOR_nothing
)
4474 if (dump_enabled_p ())
4475 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4476 "op not supported by target.\n");
4477 /* Check only during analysis. */
4478 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4479 || (vf
< vect_min_worthwhile_factor (code
)
4482 if (dump_enabled_p ())
4483 dump_printf_loc (MSG_NOTE
, vect_location
,
4484 "proceeding using word mode.\n");
4487 /* Worthwhile without SIMD support? Check only during analysis. */
4488 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4489 && vf
< vect_min_worthwhile_factor (code
)
4492 if (dump_enabled_p ())
4493 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4494 "not worthwhile without SIMD support.\n");
4498 if (!vec_stmt
) /* transformation not required. */
4500 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4501 if (dump_enabled_p ())
4502 dump_printf_loc (MSG_NOTE
, vect_location
,
4503 "=== vectorizable_shift ===\n");
4504 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4510 if (dump_enabled_p ())
4511 dump_printf_loc (MSG_NOTE
, vect_location
,
4512 "transform binary/unary operation.\n");
4515 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4517 prev_stmt_info
= NULL
;
4518 for (j
= 0; j
< ncopies
; j
++)
4523 if (scalar_shift_arg
)
4525 /* Vector shl and shr insn patterns can be defined with scalar
4526 operand 2 (shift operand). In this case, use constant or loop
4527 invariant op1 directly, without extending it to vector mode
4529 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4530 if (!VECTOR_MODE_P (optab_op2_mode
))
4532 if (dump_enabled_p ())
4533 dump_printf_loc (MSG_NOTE
, vect_location
,
4534 "operand 1 using scalar mode.\n");
4536 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4537 vec_oprnds1
.quick_push (vec_oprnd1
);
4540 /* Store vec_oprnd1 for every vector stmt to be created
4541 for SLP_NODE. We check during the analysis that all
4542 the shift arguments are the same.
4543 TODO: Allow different constants for different vector
4544 stmts generated for an SLP instance. */
4545 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4546 vec_oprnds1
.quick_push (vec_oprnd1
);
4551 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4552 (a special case for certain kind of vector shifts); otherwise,
4553 operand 1 should be of a vector type (the usual case). */
4555 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4558 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4562 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4564 /* Arguments are ready. Create the new vector stmt. */
4565 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4567 vop1
= vec_oprnds1
[i
];
4568 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4569 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4570 gimple_assign_set_lhs (new_stmt
, new_temp
);
4571 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4573 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4580 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4582 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4583 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4586 vec_oprnds0
.release ();
4587 vec_oprnds1
.release ();
4593 /* Function vectorizable_operation.
4595 Check if STMT performs a binary, unary or ternary operation that can
4597 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4598 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4599 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4602 vectorizable_operation (gimple stmt
, gimple_stmt_iterator
*gsi
,
4603 gimple
*vec_stmt
, slp_tree slp_node
)
4607 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4608 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4610 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4611 enum tree_code code
;
4612 machine_mode vec_mode
;
4619 enum vect_def_type dt
[3]
4620 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4621 gimple new_stmt
= NULL
;
4622 stmt_vec_info prev_stmt_info
;
4628 vec
<tree
> vec_oprnds0
= vNULL
;
4629 vec
<tree
> vec_oprnds1
= vNULL
;
4630 vec
<tree
> vec_oprnds2
= vNULL
;
4631 tree vop0
, vop1
, vop2
;
4632 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4635 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4638 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4641 /* Is STMT a vectorizable binary/unary operation? */
4642 if (!is_gimple_assign (stmt
))
4645 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4648 code
= gimple_assign_rhs_code (stmt
);
4650 /* For pointer addition, we should use the normal plus for
4651 the vector addition. */
4652 if (code
== POINTER_PLUS_EXPR
)
4655 /* Support only unary or binary operations. */
4656 op_type
= TREE_CODE_LENGTH (code
);
4657 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4659 if (dump_enabled_p ())
4660 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4661 "num. args = %d (not unary/binary/ternary op).\n",
4666 scalar_dest
= gimple_assign_lhs (stmt
);
4667 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4669 /* Most operations cannot handle bit-precision types without extra
4671 if ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4672 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4673 /* Exception are bitwise binary operations. */
4674 && code
!= BIT_IOR_EXPR
4675 && code
!= BIT_XOR_EXPR
4676 && code
!= BIT_AND_EXPR
)
4678 if (dump_enabled_p ())
4679 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4680 "bit-precision arithmetic not supported.\n");
4684 op0
= gimple_assign_rhs1 (stmt
);
4685 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4686 &def_stmt
, &def
, &dt
[0], &vectype
))
4688 if (dump_enabled_p ())
4689 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4690 "use not simple.\n");
4693 /* If op0 is an external or constant def use a vector type with
4694 the same size as the output vector type. */
4696 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4698 gcc_assert (vectype
);
4701 if (dump_enabled_p ())
4703 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4704 "no vectype for scalar type ");
4705 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4707 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4713 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4714 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4715 if (nunits_out
!= nunits_in
)
4718 if (op_type
== binary_op
|| op_type
== ternary_op
)
4720 op1
= gimple_assign_rhs2 (stmt
);
4721 if (!vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4724 if (dump_enabled_p ())
4725 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4726 "use not simple.\n");
4730 if (op_type
== ternary_op
)
4732 op2
= gimple_assign_rhs3 (stmt
);
4733 if (!vect_is_simple_use (op2
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4736 if (dump_enabled_p ())
4737 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4738 "use not simple.\n");
4744 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4748 /* Multiple types in SLP are handled by creating the appropriate number of
4749 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4751 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4754 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4756 gcc_assert (ncopies
>= 1);
4758 /* Shifts are handled in vectorizable_shift (). */
4759 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4760 || code
== RROTATE_EXPR
)
4763 /* Supportable by target? */
4765 vec_mode
= TYPE_MODE (vectype
);
4766 if (code
== MULT_HIGHPART_EXPR
)
4768 if (can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
)))
4769 icode
= LAST_INSN_CODE
;
4771 icode
= CODE_FOR_nothing
;
4775 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4778 if (dump_enabled_p ())
4779 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4783 icode
= (int) optab_handler (optab
, vec_mode
);
4786 if (icode
== CODE_FOR_nothing
)
4788 if (dump_enabled_p ())
4789 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4790 "op not supported by target.\n");
4791 /* Check only during analysis. */
4792 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4793 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4795 if (dump_enabled_p ())
4796 dump_printf_loc (MSG_NOTE
, vect_location
,
4797 "proceeding using word mode.\n");
4800 /* Worthwhile without SIMD support? Check only during analysis. */
4801 if (!VECTOR_MODE_P (vec_mode
)
4803 && vf
< vect_min_worthwhile_factor (code
))
4805 if (dump_enabled_p ())
4806 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4807 "not worthwhile without SIMD support.\n");
4811 if (!vec_stmt
) /* transformation not required. */
4813 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4814 if (dump_enabled_p ())
4815 dump_printf_loc (MSG_NOTE
, vect_location
,
4816 "=== vectorizable_operation ===\n");
4817 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4823 if (dump_enabled_p ())
4824 dump_printf_loc (MSG_NOTE
, vect_location
,
4825 "transform binary/unary operation.\n");
4828 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4830 /* In case the vectorization factor (VF) is bigger than the number
4831 of elements that we can fit in a vectype (nunits), we have to generate
4832 more than one vector stmt - i.e - we need to "unroll" the
4833 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4834 from one copy of the vector stmt to the next, in the field
4835 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4836 stages to find the correct vector defs to be used when vectorizing
4837 stmts that use the defs of the current stmt. The example below
4838 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4839 we need to create 4 vectorized stmts):
4841 before vectorization:
4842 RELATED_STMT VEC_STMT
4846 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4848 RELATED_STMT VEC_STMT
4849 VS1_0: vx0 = memref0 VS1_1 -
4850 VS1_1: vx1 = memref1 VS1_2 -
4851 VS1_2: vx2 = memref2 VS1_3 -
4852 VS1_3: vx3 = memref3 - -
4853 S1: x = load - VS1_0
4856 step2: vectorize stmt S2 (done here):
4857 To vectorize stmt S2 we first need to find the relevant vector
4858 def for the first operand 'x'. This is, as usual, obtained from
4859 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4860 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4861 relevant vector def 'vx0'. Having found 'vx0' we can generate
4862 the vector stmt VS2_0, and as usual, record it in the
4863 STMT_VINFO_VEC_STMT of stmt S2.
4864 When creating the second copy (VS2_1), we obtain the relevant vector
4865 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4866 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4867 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4868 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4869 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4870 chain of stmts and pointers:
4871 RELATED_STMT VEC_STMT
4872 VS1_0: vx0 = memref0 VS1_1 -
4873 VS1_1: vx1 = memref1 VS1_2 -
4874 VS1_2: vx2 = memref2 VS1_3 -
4875 VS1_3: vx3 = memref3 - -
4876 S1: x = load - VS1_0
4877 VS2_0: vz0 = vx0 + v1 VS2_1 -
4878 VS2_1: vz1 = vx1 + v1 VS2_2 -
4879 VS2_2: vz2 = vx2 + v1 VS2_3 -
4880 VS2_3: vz3 = vx3 + v1 - -
4881 S2: z = x + 1 - VS2_0 */
4883 prev_stmt_info
= NULL
;
4884 for (j
= 0; j
< ncopies
; j
++)
4889 if (op_type
== binary_op
|| op_type
== ternary_op
)
4890 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4893 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4895 if (op_type
== ternary_op
)
4897 vec_oprnds2
.create (1);
4898 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
4905 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4906 if (op_type
== ternary_op
)
4908 tree vec_oprnd
= vec_oprnds2
.pop ();
4909 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
4914 /* Arguments are ready. Create the new vector stmt. */
4915 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4917 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
4918 ? vec_oprnds1
[i
] : NULL_TREE
);
4919 vop2
= ((op_type
== ternary_op
)
4920 ? vec_oprnds2
[i
] : NULL_TREE
);
4921 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
4922 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4923 gimple_assign_set_lhs (new_stmt
, new_temp
);
4924 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4926 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4933 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4935 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4936 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4939 vec_oprnds0
.release ();
4940 vec_oprnds1
.release ();
4941 vec_oprnds2
.release ();
4946 /* A helper function to ensure data reference DR's base alignment
4950 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
4955 if (((dataref_aux
*)dr
->aux
)->base_misaligned
)
4957 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4958 tree base_decl
= ((dataref_aux
*)dr
->aux
)->base_decl
;
4960 if (decl_in_symtab_p (base_decl
))
4961 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
4964 DECL_ALIGN (base_decl
) = TYPE_ALIGN (vectype
);
4965 DECL_USER_ALIGN (base_decl
) = 1;
4967 ((dataref_aux
*)dr
->aux
)->base_misaligned
= false;
4972 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
4973 reversal of the vector elements. If that is impossible to do,
4977 perm_mask_for_reverse (tree vectype
)
4982 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4983 sel
= XALLOCAVEC (unsigned char, nunits
);
4985 for (i
= 0; i
< nunits
; ++i
)
4986 sel
[i
] = nunits
- 1 - i
;
4988 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
4990 return vect_gen_perm_mask_checked (vectype
, sel
);
4993 /* Function vectorizable_store.
4995 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
4997 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4998 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4999 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5002 vectorizable_store (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5008 tree vec_oprnd
= NULL_TREE
;
5009 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5010 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5011 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5013 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5014 struct loop
*loop
= NULL
;
5015 machine_mode vec_mode
;
5017 enum dr_alignment_support alignment_support_scheme
;
5020 enum vect_def_type dt
;
5021 stmt_vec_info prev_stmt_info
= NULL
;
5022 tree dataref_ptr
= NULL_TREE
;
5023 tree dataref_offset
= NULL_TREE
;
5024 gimple ptr_incr
= NULL
;
5025 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5028 gimple next_stmt
, first_stmt
= NULL
;
5029 bool grouped_store
= false;
5030 bool store_lanes_p
= false;
5031 unsigned int group_size
, i
;
5032 vec
<tree
> dr_chain
= vNULL
;
5033 vec
<tree
> oprnds
= vNULL
;
5034 vec
<tree
> result_chain
= vNULL
;
5036 bool negative
= false;
5037 tree offset
= NULL_TREE
;
5038 vec
<tree
> vec_oprnds
= vNULL
;
5039 bool slp
= (slp_node
!= NULL
);
5040 unsigned int vec_num
;
5041 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5045 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5047 /* Multiple types in SLP are handled by creating the appropriate number of
5048 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5050 if (slp
|| PURE_SLP_STMT (stmt_info
))
5053 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5055 gcc_assert (ncopies
>= 1);
5057 /* FORNOW. This restriction should be relaxed. */
5058 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5060 if (dump_enabled_p ())
5061 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5062 "multiple types in nested loop.\n");
5066 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5069 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5072 /* Is vectorizable store? */
5074 if (!is_gimple_assign (stmt
))
5077 scalar_dest
= gimple_assign_lhs (stmt
);
5078 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5079 && is_pattern_stmt_p (stmt_info
))
5080 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5081 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5082 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5083 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5084 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5085 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5086 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5087 && TREE_CODE (scalar_dest
) != MEM_REF
)
5090 gcc_assert (gimple_assign_single_p (stmt
));
5091 op
= gimple_assign_rhs1 (stmt
);
5092 if (!vect_is_simple_use (op
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5095 if (dump_enabled_p ())
5096 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5097 "use not simple.\n");
5101 elem_type
= TREE_TYPE (vectype
);
5102 vec_mode
= TYPE_MODE (vectype
);
5104 /* FORNOW. In some cases can vectorize even if data-type not supported
5105 (e.g. - array initialization with 0). */
5106 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5109 if (!STMT_VINFO_DATA_REF (stmt_info
))
5112 if (!STMT_VINFO_STRIDED_P (stmt_info
))
5115 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5116 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5117 size_zero_node
) < 0;
5118 if (negative
&& ncopies
> 1)
5120 if (dump_enabled_p ())
5121 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5122 "multiple types with negative step.\n");
5127 gcc_assert (!grouped_store
);
5128 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5129 if (alignment_support_scheme
!= dr_aligned
5130 && alignment_support_scheme
!= dr_unaligned_supported
)
5132 if (dump_enabled_p ())
5133 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5134 "negative step but alignment required.\n");
5137 if (dt
!= vect_constant_def
5138 && dt
!= vect_external_def
5139 && !perm_mask_for_reverse (vectype
))
5141 if (dump_enabled_p ())
5142 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5143 "negative step and reversing not supported.\n");
5149 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5151 grouped_store
= true;
5152 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5153 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5155 && !PURE_SLP_STMT (stmt_info
)
5156 && !STMT_VINFO_STRIDED_P (stmt_info
))
5158 if (vect_store_lanes_supported (vectype
, group_size
))
5159 store_lanes_p
= true;
5160 else if (!vect_grouped_store_supported (vectype
, group_size
))
5164 if (STMT_VINFO_STRIDED_P (stmt_info
)
5165 && (slp
|| PURE_SLP_STMT (stmt_info
))
5166 && (group_size
> nunits
5167 || nunits
% group_size
!= 0))
5169 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5170 "unhandled strided group store\n");
5174 if (first_stmt
== stmt
)
5176 /* STMT is the leader of the group. Check the operands of all the
5177 stmts of the group. */
5178 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5181 gcc_assert (gimple_assign_single_p (next_stmt
));
5182 op
= gimple_assign_rhs1 (next_stmt
);
5183 if (!vect_is_simple_use (op
, next_stmt
, loop_vinfo
, bb_vinfo
,
5184 &def_stmt
, &def
, &dt
))
5186 if (dump_enabled_p ())
5187 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5188 "use not simple.\n");
5191 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5196 if (!vec_stmt
) /* transformation not required. */
5198 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5199 /* The SLP costs are calculated during SLP analysis. */
5200 if (!PURE_SLP_STMT (stmt_info
))
5201 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5208 ensure_base_align (stmt_info
, dr
);
5212 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5213 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5215 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5218 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5220 /* We vectorize all the stmts of the interleaving group when we
5221 reach the last stmt in the group. */
5222 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5223 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5232 grouped_store
= false;
5233 /* VEC_NUM is the number of vect stmts to be created for this
5235 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5236 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5237 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5238 op
= gimple_assign_rhs1 (first_stmt
);
5241 /* VEC_NUM is the number of vect stmts to be created for this
5243 vec_num
= group_size
;
5249 group_size
= vec_num
= 1;
5252 if (dump_enabled_p ())
5253 dump_printf_loc (MSG_NOTE
, vect_location
,
5254 "transform store. ncopies = %d\n", ncopies
);
5256 if (STMT_VINFO_STRIDED_P (stmt_info
))
5258 gimple_stmt_iterator incr_gsi
;
5264 gimple_seq stmts
= NULL
;
5265 tree stride_base
, stride_step
, alias_off
;
5268 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5271 = fold_build_pointer_plus
5272 (unshare_expr (DR_BASE_ADDRESS (dr
)),
5273 size_binop (PLUS_EXPR
,
5274 convert_to_ptrofftype (unshare_expr (DR_OFFSET (dr
))),
5275 convert_to_ptrofftype (DR_INIT(dr
))));
5276 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (dr
)));
5278 /* For a store with loop-invariant (but other than power-of-2)
5279 stride (i.e. not a grouped access) like so:
5281 for (i = 0; i < n; i += stride)
5284 we generate a new induction variable and new stores from
5285 the components of the (vectorized) rhs:
5287 for (j = 0; ; j += VF*stride)
5292 array[j + stride] = tmp2;
5296 unsigned nstores
= nunits
;
5297 tree ltype
= elem_type
;
5300 nstores
= nunits
/ group_size
;
5301 if (group_size
< nunits
)
5302 ltype
= build_vector_type (elem_type
, group_size
);
5305 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
5306 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5309 ivstep
= stride_step
;
5310 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
5311 build_int_cst (TREE_TYPE (ivstep
),
5312 ncopies
* nstores
));
5314 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
5316 create_iv (stride_base
, ivstep
, NULL
,
5317 loop
, &incr_gsi
, insert_after
,
5319 incr
= gsi_stmt (incr_gsi
);
5320 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
5322 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
5324 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
5326 prev_stmt_info
= NULL
;
5327 running_off
= offvar
;
5328 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (dr
)), 0);
5329 for (j
= 0; j
< ncopies
; j
++)
5331 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5332 and first_stmt == stmt. */
5337 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
, NULL
,
5339 vec_oprnd
= vec_oprnds
[0];
5342 vec_oprnd
= vect_get_vec_def_for_operand (op
, first_stmt
, NULL
);
5347 vec_oprnd
= vec_oprnds
[j
];
5349 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
5352 for (i
= 0; i
< nstores
; i
++)
5354 tree newref
, newoff
;
5355 gimple incr
, assign
;
5356 tree size
= TYPE_SIZE (ltype
);
5357 /* Extract the i'th component. */
5358 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (i
),
5360 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
5363 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
5367 newref
= build2 (MEM_REF
, ltype
,
5368 running_off
, alias_off
);
5370 /* And store it to *running_off. */
5371 assign
= gimple_build_assign (newref
, elem
);
5372 vect_finish_stmt_generation (stmt
, assign
, gsi
);
5374 newoff
= copy_ssa_name (running_off
, NULL
);
5375 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5376 running_off
, stride_step
);
5377 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5379 running_off
= newoff
;
5380 if (j
== 0 && i
== 0)
5381 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= assign
;
5383 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
5384 prev_stmt_info
= vinfo_for_stmt (assign
);
5390 dr_chain
.create (group_size
);
5391 oprnds
.create (group_size
);
5393 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5394 gcc_assert (alignment_support_scheme
);
5395 /* Targets with store-lane instructions must not require explicit
5397 gcc_assert (!store_lanes_p
5398 || alignment_support_scheme
== dr_aligned
5399 || alignment_support_scheme
== dr_unaligned_supported
);
5402 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5405 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5407 aggr_type
= vectype
;
5409 /* In case the vectorization factor (VF) is bigger than the number
5410 of elements that we can fit in a vectype (nunits), we have to generate
5411 more than one vector stmt - i.e - we need to "unroll" the
5412 vector stmt by a factor VF/nunits. For more details see documentation in
5413 vect_get_vec_def_for_copy_stmt. */
5415 /* In case of interleaving (non-unit grouped access):
5422 We create vectorized stores starting from base address (the access of the
5423 first stmt in the chain (S2 in the above example), when the last store stmt
5424 of the chain (S4) is reached:
5427 VS2: &base + vec_size*1 = vx0
5428 VS3: &base + vec_size*2 = vx1
5429 VS4: &base + vec_size*3 = vx3
5431 Then permutation statements are generated:
5433 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5434 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5437 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5438 (the order of the data-refs in the output of vect_permute_store_chain
5439 corresponds to the order of scalar stmts in the interleaving chain - see
5440 the documentation of vect_permute_store_chain()).
5442 In case of both multiple types and interleaving, above vector stores and
5443 permutation stmts are created for every copy. The result vector stmts are
5444 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5445 STMT_VINFO_RELATED_STMT for the next copies.
5448 prev_stmt_info
= NULL
;
5449 for (j
= 0; j
< ncopies
; j
++)
5457 /* Get vectorized arguments for SLP_NODE. */
5458 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5459 NULL
, slp_node
, -1);
5461 vec_oprnd
= vec_oprnds
[0];
5465 /* For interleaved stores we collect vectorized defs for all the
5466 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5467 used as an input to vect_permute_store_chain(), and OPRNDS as
5468 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5470 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5471 OPRNDS are of size 1. */
5472 next_stmt
= first_stmt
;
5473 for (i
= 0; i
< group_size
; i
++)
5475 /* Since gaps are not supported for interleaved stores,
5476 GROUP_SIZE is the exact number of stmts in the chain.
5477 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5478 there is no interleaving, GROUP_SIZE is 1, and only one
5479 iteration of the loop will be executed. */
5480 gcc_assert (next_stmt
5481 && gimple_assign_single_p (next_stmt
));
5482 op
= gimple_assign_rhs1 (next_stmt
);
5484 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
5486 dr_chain
.quick_push (vec_oprnd
);
5487 oprnds
.quick_push (vec_oprnd
);
5488 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5492 /* We should have catched mismatched types earlier. */
5493 gcc_assert (useless_type_conversion_p (vectype
,
5494 TREE_TYPE (vec_oprnd
)));
5495 bool simd_lane_access_p
5496 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5497 if (simd_lane_access_p
5498 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5499 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5500 && integer_zerop (DR_OFFSET (first_dr
))
5501 && integer_zerop (DR_INIT (first_dr
))
5502 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5503 get_alias_set (DR_REF (first_dr
))))
5505 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5506 dataref_offset
= build_int_cst (reference_alias_ptr_type
5507 (DR_REF (first_dr
)), 0);
5512 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5513 simd_lane_access_p
? loop
: NULL
,
5514 offset
, &dummy
, gsi
, &ptr_incr
,
5515 simd_lane_access_p
, &inv_p
);
5516 gcc_assert (bb_vinfo
|| !inv_p
);
5520 /* For interleaved stores we created vectorized defs for all the
5521 defs stored in OPRNDS in the previous iteration (previous copy).
5522 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5523 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5525 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5526 OPRNDS are of size 1. */
5527 for (i
= 0; i
< group_size
; i
++)
5530 vect_is_simple_use (op
, NULL
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5532 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5533 dr_chain
[i
] = vec_oprnd
;
5534 oprnds
[i
] = vec_oprnd
;
5538 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5539 TYPE_SIZE_UNIT (aggr_type
));
5541 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5542 TYPE_SIZE_UNIT (aggr_type
));
5549 /* Combine all the vectors into an array. */
5550 vec_array
= create_vector_array (vectype
, vec_num
);
5551 for (i
= 0; i
< vec_num
; i
++)
5553 vec_oprnd
= dr_chain
[i
];
5554 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5558 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5559 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5560 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5561 gimple_call_set_lhs (new_stmt
, data_ref
);
5562 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5570 result_chain
.create (group_size
);
5572 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5576 next_stmt
= first_stmt
;
5577 for (i
= 0; i
< vec_num
; i
++)
5579 unsigned align
, misalign
;
5582 /* Bump the vector pointer. */
5583 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5587 vec_oprnd
= vec_oprnds
[i
];
5588 else if (grouped_store
)
5589 /* For grouped stores vectorized defs are interleaved in
5590 vect_permute_store_chain(). */
5591 vec_oprnd
= result_chain
[i
];
5593 data_ref
= fold_build2 (MEM_REF
, TREE_TYPE (vec_oprnd
),
5597 : build_int_cst (reference_alias_ptr_type
5598 (DR_REF (first_dr
)), 0));
5599 align
= TYPE_ALIGN_UNIT (vectype
);
5600 if (aligned_access_p (first_dr
))
5602 else if (DR_MISALIGNMENT (first_dr
) == -1)
5604 TREE_TYPE (data_ref
)
5605 = build_aligned_type (TREE_TYPE (data_ref
),
5606 TYPE_ALIGN (elem_type
));
5607 align
= TYPE_ALIGN_UNIT (elem_type
);
5612 TREE_TYPE (data_ref
)
5613 = build_aligned_type (TREE_TYPE (data_ref
),
5614 TYPE_ALIGN (elem_type
));
5615 misalign
= DR_MISALIGNMENT (first_dr
);
5617 if (dataref_offset
== NULL_TREE
5618 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
5619 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
5623 && dt
!= vect_constant_def
5624 && dt
!= vect_external_def
)
5626 tree perm_mask
= perm_mask_for_reverse (vectype
);
5628 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
5630 tree new_temp
= make_ssa_name (perm_dest
);
5632 /* Generate the permute statement. */
5634 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
5635 vec_oprnd
, perm_mask
);
5636 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5638 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
5639 vec_oprnd
= new_temp
;
5642 /* Arguments are ready. Create the new vector stmt. */
5643 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
5644 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5649 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5657 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5659 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5660 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5664 dr_chain
.release ();
5666 result_chain
.release ();
5667 vec_oprnds
.release ();
5672 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
5673 VECTOR_CST mask. No checks are made that the target platform supports the
5674 mask, so callers may wish to test can_vec_perm_p separately, or use
5675 vect_gen_perm_mask_checked. */
5678 vect_gen_perm_mask_any (tree vectype
, const unsigned char *sel
)
5680 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
5683 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5685 mask_elt_type
= lang_hooks
.types
.type_for_mode
5686 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
5687 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
5689 mask_elts
= XALLOCAVEC (tree
, nunits
);
5690 for (i
= nunits
- 1; i
>= 0; i
--)
5691 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
5692 mask_vec
= build_vector (mask_type
, mask_elts
);
5697 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
5698 i.e. that the target supports the pattern _for arbitrary input vectors_. */
5701 vect_gen_perm_mask_checked (tree vectype
, const unsigned char *sel
)
5703 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, sel
));
5704 return vect_gen_perm_mask_any (vectype
, sel
);
5707 /* Given a vector variable X and Y, that was generated for the scalar
5708 STMT, generate instructions to permute the vector elements of X and Y
5709 using permutation mask MASK_VEC, insert them at *GSI and return the
5710 permuted vector variable. */
5713 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple stmt
,
5714 gimple_stmt_iterator
*gsi
)
5716 tree vectype
= TREE_TYPE (x
);
5717 tree perm_dest
, data_ref
;
5720 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
5721 data_ref
= make_ssa_name (perm_dest
);
5723 /* Generate the permute statement. */
5724 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
5725 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5730 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5731 inserting them on the loops preheader edge. Returns true if we
5732 were successful in doing so (and thus STMT can be moved then),
5733 otherwise returns false. */
5736 hoist_defs_of_uses (gimple stmt
, struct loop
*loop
)
5742 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5744 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5745 if (!gimple_nop_p (def_stmt
)
5746 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5748 /* Make sure we don't need to recurse. While we could do
5749 so in simple cases when there are more complex use webs
5750 we don't have an easy way to preserve stmt order to fulfil
5751 dependencies within them. */
5754 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
5756 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
5758 gimple def_stmt2
= SSA_NAME_DEF_STMT (op2
);
5759 if (!gimple_nop_p (def_stmt2
)
5760 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
5770 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5772 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5773 if (!gimple_nop_p (def_stmt
)
5774 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5776 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
5777 gsi_remove (&gsi
, false);
5778 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
5785 /* vectorizable_load.
5787 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
5789 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5790 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5791 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5794 vectorizable_load (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5795 slp_tree slp_node
, slp_instance slp_node_instance
)
5798 tree vec_dest
= NULL
;
5799 tree data_ref
= NULL
;
5800 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5801 stmt_vec_info prev_stmt_info
;
5802 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5803 struct loop
*loop
= NULL
;
5804 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
5805 bool nested_in_vect_loop
= false;
5806 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5807 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5811 gimple new_stmt
= NULL
;
5813 enum dr_alignment_support alignment_support_scheme
;
5814 tree dataref_ptr
= NULL_TREE
;
5815 tree dataref_offset
= NULL_TREE
;
5816 gimple ptr_incr
= NULL
;
5817 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5819 int i
, j
, group_size
= -1, group_gap_adj
;
5820 tree msq
= NULL_TREE
, lsq
;
5821 tree offset
= NULL_TREE
;
5822 tree byte_offset
= NULL_TREE
;
5823 tree realignment_token
= NULL_TREE
;
5825 vec
<tree
> dr_chain
= vNULL
;
5826 bool grouped_load
= false;
5827 bool load_lanes_p
= false;
5830 bool negative
= false;
5831 bool compute_in_loop
= false;
5832 struct loop
*at_loop
;
5834 bool slp
= (slp_node
!= NULL
);
5835 bool slp_perm
= false;
5836 enum tree_code code
;
5837 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5840 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
5841 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
5842 int gather_scale
= 1;
5843 enum vect_def_type gather_dt
= vect_unknown_def_type
;
5847 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5848 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
5849 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5854 /* Multiple types in SLP are handled by creating the appropriate number of
5855 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5857 if (slp
|| PURE_SLP_STMT (stmt_info
))
5860 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5862 gcc_assert (ncopies
>= 1);
5864 /* FORNOW. This restriction should be relaxed. */
5865 if (nested_in_vect_loop
&& ncopies
> 1)
5867 if (dump_enabled_p ())
5868 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5869 "multiple types in nested loop.\n");
5873 /* Invalidate assumptions made by dependence analysis when vectorization
5874 on the unrolled body effectively re-orders stmts. */
5876 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5877 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5878 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5880 if (dump_enabled_p ())
5881 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5882 "cannot perform implicit CSE when unrolling "
5883 "with negative dependence distance\n");
5887 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5890 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5893 /* Is vectorizable load? */
5894 if (!is_gimple_assign (stmt
))
5897 scalar_dest
= gimple_assign_lhs (stmt
);
5898 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
5901 code
= gimple_assign_rhs_code (stmt
);
5902 if (code
!= ARRAY_REF
5903 && code
!= BIT_FIELD_REF
5904 && code
!= INDIRECT_REF
5905 && code
!= COMPONENT_REF
5906 && code
!= IMAGPART_EXPR
5907 && code
!= REALPART_EXPR
5909 && TREE_CODE_CLASS (code
) != tcc_declaration
)
5912 if (!STMT_VINFO_DATA_REF (stmt_info
))
5915 elem_type
= TREE_TYPE (vectype
);
5916 mode
= TYPE_MODE (vectype
);
5918 /* FORNOW. In some cases can vectorize even if data-type not supported
5919 (e.g. - data copies). */
5920 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
5922 if (dump_enabled_p ())
5923 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5924 "Aligned load, but unsupported type.\n");
5928 /* Check if the load is a part of an interleaving chain. */
5929 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5931 grouped_load
= true;
5933 gcc_assert (! nested_in_vect_loop
&& !STMT_VINFO_GATHER_P (stmt_info
));
5935 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5937 /* If this is single-element interleaving with an element distance
5938 that leaves unused vector loads around punt - we at least create
5939 very sub-optimal code in that case (and blow up memory,
5941 if (first_stmt
== stmt
5942 && !GROUP_NEXT_ELEMENT (stmt_info
)
5943 && GROUP_SIZE (stmt_info
) > TYPE_VECTOR_SUBPARTS (vectype
))
5945 if (dump_enabled_p ())
5946 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5947 "single-element interleaving not supported "
5948 "for not adjacent vector loads\n");
5952 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
5955 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5957 && !PURE_SLP_STMT (stmt_info
)
5958 && !STMT_VINFO_STRIDED_P (stmt_info
))
5960 if (vect_load_lanes_supported (vectype
, group_size
))
5961 load_lanes_p
= true;
5962 else if (!vect_grouped_load_supported (vectype
, group_size
))
5966 /* Invalidate assumptions made by dependence analysis when vectorization
5967 on the unrolled body effectively re-orders stmts. */
5968 if (!PURE_SLP_STMT (stmt_info
)
5969 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5970 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5971 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5973 if (dump_enabled_p ())
5974 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5975 "cannot perform implicit CSE when performing "
5976 "group loads with negative dependence distance\n");
5980 /* Similarly when the stmt is a load that is both part of a SLP
5981 instance and a loop vectorized stmt via the same-dr mechanism
5982 we have to give up. */
5983 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
5984 && (STMT_SLP_TYPE (stmt_info
)
5985 != STMT_SLP_TYPE (vinfo_for_stmt
5986 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
5988 if (dump_enabled_p ())
5989 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5990 "conflicting SLP types for CSEd load\n");
5996 if (STMT_VINFO_GATHER_P (stmt_info
))
6000 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
6001 &gather_off
, &gather_scale
);
6002 gcc_assert (gather_decl
);
6003 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, bb_vinfo
,
6004 &def_stmt
, &def
, &gather_dt
,
6005 &gather_off_vectype
))
6007 if (dump_enabled_p ())
6008 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6009 "gather index use not simple.\n");
6013 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6016 && (slp
|| PURE_SLP_STMT (stmt_info
)))
6017 && (group_size
> nunits
6018 || nunits
% group_size
!= 0))
6020 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6021 "unhandled strided group load\n");
6027 negative
= tree_int_cst_compare (nested_in_vect_loop
6028 ? STMT_VINFO_DR_STEP (stmt_info
)
6030 size_zero_node
) < 0;
6031 if (negative
&& ncopies
> 1)
6033 if (dump_enabled_p ())
6034 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6035 "multiple types with negative step.\n");
6043 if (dump_enabled_p ())
6044 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6045 "negative step for group load not supported"
6049 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
6050 if (alignment_support_scheme
!= dr_aligned
6051 && alignment_support_scheme
!= dr_unaligned_supported
)
6053 if (dump_enabled_p ())
6054 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6055 "negative step but alignment required.\n");
6058 if (!perm_mask_for_reverse (vectype
))
6060 if (dump_enabled_p ())
6061 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6062 "negative step and reversing not supported."
6069 if (!vec_stmt
) /* transformation not required. */
6071 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6072 /* The SLP costs are calculated during SLP analysis. */
6073 if (!PURE_SLP_STMT (stmt_info
))
6074 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
,
6079 if (dump_enabled_p ())
6080 dump_printf_loc (MSG_NOTE
, vect_location
,
6081 "transform load. ncopies = %d\n", ncopies
);
6085 ensure_base_align (stmt_info
, dr
);
6087 if (STMT_VINFO_GATHER_P (stmt_info
))
6089 tree vec_oprnd0
= NULL_TREE
, op
;
6090 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
6091 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6092 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6093 edge pe
= loop_preheader_edge (loop
);
6096 enum { NARROW
, NONE
, WIDEN
} modifier
;
6097 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
6099 if (nunits
== gather_off_nunits
)
6101 else if (nunits
== gather_off_nunits
/ 2)
6103 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
6106 for (i
= 0; i
< gather_off_nunits
; ++i
)
6107 sel
[i
] = i
| nunits
;
6109 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
6111 else if (nunits
== gather_off_nunits
* 2)
6113 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
6116 for (i
= 0; i
< nunits
; ++i
)
6117 sel
[i
] = i
< gather_off_nunits
6118 ? i
: i
+ nunits
- gather_off_nunits
;
6120 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6126 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
6127 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6128 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6129 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6130 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6131 scaletype
= TREE_VALUE (arglist
);
6132 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6134 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6136 ptr
= fold_convert (ptrtype
, gather_base
);
6137 if (!is_gimple_min_invariant (ptr
))
6139 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6140 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6141 gcc_assert (!new_bb
);
6144 /* Currently we support only unconditional gather loads,
6145 so mask should be all ones. */
6146 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6147 mask
= build_int_cst (masktype
, -1);
6148 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6150 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6151 mask
= build_vector_from_val (masktype
, mask
);
6152 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6154 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6158 for (j
= 0; j
< 6; ++j
)
6160 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6161 mask
= build_real (TREE_TYPE (masktype
), r
);
6162 mask
= build_vector_from_val (masktype
, mask
);
6163 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6168 scale
= build_int_cst (scaletype
, gather_scale
);
6170 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6171 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6172 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6176 for (j
= 0; j
< 6; ++j
)
6178 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6179 merge
= build_real (TREE_TYPE (rettype
), r
);
6183 merge
= build_vector_from_val (rettype
, merge
);
6184 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6186 prev_stmt_info
= NULL
;
6187 for (j
= 0; j
< ncopies
; ++j
)
6189 if (modifier
== WIDEN
&& (j
& 1))
6190 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6191 perm_mask
, stmt
, gsi
);
6194 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
6197 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
6199 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6201 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6202 == TYPE_VECTOR_SUBPARTS (idxtype
));
6203 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
6204 var
= make_ssa_name (var
);
6205 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6207 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6208 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6213 = gimple_build_call (gather_decl
, 5, merge
, ptr
, op
, mask
, scale
);
6215 if (!useless_type_conversion_p (vectype
, rettype
))
6217 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6218 == TYPE_VECTOR_SUBPARTS (rettype
));
6219 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
6220 op
= make_ssa_name (var
, new_stmt
);
6221 gimple_call_set_lhs (new_stmt
, op
);
6222 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6223 var
= make_ssa_name (vec_dest
);
6224 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6226 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6230 var
= make_ssa_name (vec_dest
, new_stmt
);
6231 gimple_call_set_lhs (new_stmt
, var
);
6234 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6236 if (modifier
== NARROW
)
6243 var
= permute_vec_elements (prev_res
, var
,
6244 perm_mask
, stmt
, gsi
);
6245 new_stmt
= SSA_NAME_DEF_STMT (var
);
6248 if (prev_stmt_info
== NULL
)
6249 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6251 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6252 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6256 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6258 gimple_stmt_iterator incr_gsi
;
6264 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6265 gimple_seq stmts
= NULL
;
6266 tree stride_base
, stride_step
, alias_off
;
6268 gcc_assert (!nested_in_vect_loop
);
6271 first_dr
= STMT_VINFO_DATA_REF
6272 (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info
)));
6277 = fold_build_pointer_plus
6278 (DR_BASE_ADDRESS (first_dr
),
6279 size_binop (PLUS_EXPR
,
6280 convert_to_ptrofftype (DR_OFFSET (first_dr
)),
6281 convert_to_ptrofftype (DR_INIT (first_dr
))));
6282 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr
));
6284 /* For a load with loop-invariant (but other than power-of-2)
6285 stride (i.e. not a grouped access) like so:
6287 for (i = 0; i < n; i += stride)
6290 we generate a new induction variable and new accesses to
6291 form a new vector (or vectors, depending on ncopies):
6293 for (j = 0; ; j += VF*stride)
6295 tmp2 = array[j + stride];
6297 vectemp = {tmp1, tmp2, ...}
6300 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
6301 build_int_cst (TREE_TYPE (stride_step
), vf
));
6303 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6305 create_iv (unshare_expr (stride_base
), unshare_expr (ivstep
), NULL
,
6306 loop
, &incr_gsi
, insert_after
,
6308 incr
= gsi_stmt (incr_gsi
);
6309 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
6311 stride_step
= force_gimple_operand (unshare_expr (stride_step
),
6312 &stmts
, true, NULL_TREE
);
6314 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6316 prev_stmt_info
= NULL
;
6317 running_off
= offvar
;
6318 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
6319 int nloads
= nunits
;
6320 tree ltype
= TREE_TYPE (vectype
);
6321 auto_vec
<tree
> dr_chain
;
6324 nloads
= nunits
/ group_size
;
6325 if (group_size
< nunits
)
6326 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
6329 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
6330 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6332 dr_chain
.create (ncopies
);
6334 for (j
= 0; j
< ncopies
; j
++)
6340 vec_alloc (v
, nloads
);
6341 for (i
= 0; i
< nloads
; i
++)
6343 tree newref
, newoff
;
6345 newref
= build2 (MEM_REF
, ltype
, running_off
, alias_off
);
6347 newref
= force_gimple_operand_gsi (gsi
, newref
, true,
6350 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, newref
);
6351 newoff
= copy_ssa_name (running_off
);
6352 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6353 running_off
, stride_step
);
6354 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6356 running_off
= newoff
;
6359 vec_inv
= build_constructor (vectype
, v
);
6360 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6361 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6365 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
6366 build2 (MEM_REF
, ltype
,
6367 running_off
, alias_off
));
6368 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6370 tree newoff
= copy_ssa_name (running_off
);
6371 gimple incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6372 running_off
, stride_step
);
6373 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6375 running_off
= newoff
;
6380 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6382 dr_chain
.quick_push (gimple_assign_lhs (new_stmt
));
6385 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6387 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6388 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6391 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6392 slp_node_instance
, false);
6398 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6400 && !SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ()
6401 && first_stmt
!= SLP_TREE_SCALAR_STMTS (slp_node
)[0])
6402 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6404 /* Check if the chain of loads is already vectorized. */
6405 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6406 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6407 ??? But we can only do so if there is exactly one
6408 as we have no way to get at the rest. Leave the CSE
6410 ??? With the group load eventually participating
6411 in multiple different permutations (having multiple
6412 slp nodes which refer to the same group) the CSE
6413 is even wrong code. See PR56270. */
6416 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6419 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6420 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6423 /* VEC_NUM is the number of vect stmts to be created for this group. */
6426 grouped_load
= false;
6427 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6428 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
6431 vec_num
= group_size
;
6437 group_size
= vec_num
= 1;
6441 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6442 gcc_assert (alignment_support_scheme
);
6443 /* Targets with load-lane instructions must not require explicit
6445 gcc_assert (!load_lanes_p
6446 || alignment_support_scheme
== dr_aligned
6447 || alignment_support_scheme
== dr_unaligned_supported
);
6449 /* In case the vectorization factor (VF) is bigger than the number
6450 of elements that we can fit in a vectype (nunits), we have to generate
6451 more than one vector stmt - i.e - we need to "unroll" the
6452 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6453 from one copy of the vector stmt to the next, in the field
6454 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6455 stages to find the correct vector defs to be used when vectorizing
6456 stmts that use the defs of the current stmt. The example below
6457 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6458 need to create 4 vectorized stmts):
6460 before vectorization:
6461 RELATED_STMT VEC_STMT
6465 step 1: vectorize stmt S1:
6466 We first create the vector stmt VS1_0, and, as usual, record a
6467 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6468 Next, we create the vector stmt VS1_1, and record a pointer to
6469 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6470 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6472 RELATED_STMT VEC_STMT
6473 VS1_0: vx0 = memref0 VS1_1 -
6474 VS1_1: vx1 = memref1 VS1_2 -
6475 VS1_2: vx2 = memref2 VS1_3 -
6476 VS1_3: vx3 = memref3 - -
6477 S1: x = load - VS1_0
6480 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6481 information we recorded in RELATED_STMT field is used to vectorize
6484 /* In case of interleaving (non-unit grouped access):
6491 Vectorized loads are created in the order of memory accesses
6492 starting from the access of the first stmt of the chain:
6495 VS2: vx1 = &base + vec_size*1
6496 VS3: vx3 = &base + vec_size*2
6497 VS4: vx4 = &base + vec_size*3
6499 Then permutation statements are generated:
6501 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6502 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6505 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6506 (the order of the data-refs in the output of vect_permute_load_chain
6507 corresponds to the order of scalar stmts in the interleaving chain - see
6508 the documentation of vect_permute_load_chain()).
6509 The generation of permutation stmts and recording them in
6510 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6512 In case of both multiple types and interleaving, the vector loads and
6513 permutation stmts above are created for every copy. The result vector
6514 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6515 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6517 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6518 on a target that supports unaligned accesses (dr_unaligned_supported)
6519 we generate the following code:
6523 p = p + indx * vectype_size;
6528 Otherwise, the data reference is potentially unaligned on a target that
6529 does not support unaligned accesses (dr_explicit_realign_optimized) -
6530 then generate the following code, in which the data in each iteration is
6531 obtained by two vector loads, one from the previous iteration, and one
6532 from the current iteration:
6534 msq_init = *(floor(p1))
6535 p2 = initial_addr + VS - 1;
6536 realignment_token = call target_builtin;
6539 p2 = p2 + indx * vectype_size
6541 vec_dest = realign_load (msq, lsq, realignment_token)
6546 /* If the misalignment remains the same throughout the execution of the
6547 loop, we can create the init_addr and permutation mask at the loop
6548 preheader. Otherwise, it needs to be created inside the loop.
6549 This can only occur when vectorizing memory accesses in the inner-loop
6550 nested within an outer-loop that is being vectorized. */
6552 if (nested_in_vect_loop
6553 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6554 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6556 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6557 compute_in_loop
= true;
6560 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6561 || alignment_support_scheme
== dr_explicit_realign
)
6562 && !compute_in_loop
)
6564 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6565 alignment_support_scheme
, NULL_TREE
,
6567 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6569 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
6570 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
6578 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6581 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6583 aggr_type
= vectype
;
6585 prev_stmt_info
= NULL
;
6586 for (j
= 0; j
< ncopies
; j
++)
6588 /* 1. Create the vector or array pointer update chain. */
6591 bool simd_lane_access_p
6592 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6593 if (simd_lane_access_p
6594 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6595 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6596 && integer_zerop (DR_OFFSET (first_dr
))
6597 && integer_zerop (DR_INIT (first_dr
))
6598 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6599 get_alias_set (DR_REF (first_dr
)))
6600 && (alignment_support_scheme
== dr_aligned
6601 || alignment_support_scheme
== dr_unaligned_supported
))
6603 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6604 dataref_offset
= build_int_cst (reference_alias_ptr_type
6605 (DR_REF (first_dr
)), 0);
6610 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
6611 offset
, &dummy
, gsi
, &ptr_incr
,
6612 simd_lane_access_p
, &inv_p
,
6615 else if (dataref_offset
)
6616 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
6617 TYPE_SIZE_UNIT (aggr_type
));
6619 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6620 TYPE_SIZE_UNIT (aggr_type
));
6622 if (grouped_load
|| slp_perm
)
6623 dr_chain
.create (vec_num
);
6629 vec_array
= create_vector_array (vectype
, vec_num
);
6632 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6633 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
6634 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
6635 gimple_call_set_lhs (new_stmt
, vec_array
);
6636 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6638 /* Extract each vector into an SSA_NAME. */
6639 for (i
= 0; i
< vec_num
; i
++)
6641 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
6643 dr_chain
.quick_push (new_temp
);
6646 /* Record the mapping between SSA_NAMEs and statements. */
6647 vect_record_grouped_load_vectors (stmt
, dr_chain
);
6651 for (i
= 0; i
< vec_num
; i
++)
6654 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6657 /* 2. Create the vector-load in the loop. */
6658 switch (alignment_support_scheme
)
6661 case dr_unaligned_supported
:
6663 unsigned int align
, misalign
;
6666 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
6669 : build_int_cst (reference_alias_ptr_type
6670 (DR_REF (first_dr
)), 0));
6671 align
= TYPE_ALIGN_UNIT (vectype
);
6672 if (alignment_support_scheme
== dr_aligned
)
6674 gcc_assert (aligned_access_p (first_dr
));
6677 else if (DR_MISALIGNMENT (first_dr
) == -1)
6679 TREE_TYPE (data_ref
)
6680 = build_aligned_type (TREE_TYPE (data_ref
),
6681 TYPE_ALIGN (elem_type
));
6682 align
= TYPE_ALIGN_UNIT (elem_type
);
6687 TREE_TYPE (data_ref
)
6688 = build_aligned_type (TREE_TYPE (data_ref
),
6689 TYPE_ALIGN (elem_type
));
6690 misalign
= DR_MISALIGNMENT (first_dr
);
6692 if (dataref_offset
== NULL_TREE
6693 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
6694 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
6698 case dr_explicit_realign
:
6702 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
6704 if (compute_in_loop
)
6705 msq
= vect_setup_realignment (first_stmt
, gsi
,
6707 dr_explicit_realign
,
6710 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
6711 ptr
= copy_ssa_name (dataref_ptr
);
6713 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
6714 new_stmt
= gimple_build_assign
6715 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
6717 (TREE_TYPE (dataref_ptr
),
6718 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6719 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6721 = build2 (MEM_REF
, vectype
, ptr
,
6722 build_int_cst (reference_alias_ptr_type
6723 (DR_REF (first_dr
)), 0));
6724 vec_dest
= vect_create_destination_var (scalar_dest
,
6726 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6727 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6728 gimple_assign_set_lhs (new_stmt
, new_temp
);
6729 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
6730 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
6731 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6734 bump
= size_binop (MULT_EXPR
, vs
,
6735 TYPE_SIZE_UNIT (elem_type
));
6736 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
6737 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
6738 new_stmt
= gimple_build_assign
6739 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
6742 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6743 ptr
= copy_ssa_name (ptr
, new_stmt
);
6744 gimple_assign_set_lhs (new_stmt
, ptr
);
6745 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6747 = build2 (MEM_REF
, vectype
, ptr
,
6748 build_int_cst (reference_alias_ptr_type
6749 (DR_REF (first_dr
)), 0));
6752 case dr_explicit_realign_optimized
:
6753 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
6754 new_temp
= copy_ssa_name (dataref_ptr
);
6756 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
6757 new_stmt
= gimple_build_assign
6758 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
6760 (TREE_TYPE (dataref_ptr
),
6761 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6762 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6764 = build2 (MEM_REF
, vectype
, new_temp
,
6765 build_int_cst (reference_alias_ptr_type
6766 (DR_REF (first_dr
)), 0));
6771 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6772 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6773 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6774 gimple_assign_set_lhs (new_stmt
, new_temp
);
6775 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6777 /* 3. Handle explicit realignment if necessary/supported.
6779 vec_dest = realign_load (msq, lsq, realignment_token) */
6780 if (alignment_support_scheme
== dr_explicit_realign_optimized
6781 || alignment_support_scheme
== dr_explicit_realign
)
6783 lsq
= gimple_assign_lhs (new_stmt
);
6784 if (!realignment_token
)
6785 realignment_token
= dataref_ptr
;
6786 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6787 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
6788 msq
, lsq
, realignment_token
);
6789 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6790 gimple_assign_set_lhs (new_stmt
, new_temp
);
6791 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6793 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6796 if (i
== vec_num
- 1 && j
== ncopies
- 1)
6797 add_phi_arg (phi
, lsq
,
6798 loop_latch_edge (containing_loop
),
6804 /* 4. Handle invariant-load. */
6805 if (inv_p
&& !bb_vinfo
)
6807 gcc_assert (!grouped_load
);
6808 /* If we have versioned for aliasing or the loop doesn't
6809 have any data dependencies that would preclude this,
6810 then we are sure this is a loop invariant load and
6811 thus we can insert it on the preheader edge. */
6812 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
6813 && !nested_in_vect_loop
6814 && hoist_defs_of_uses (stmt
, loop
))
6816 if (dump_enabled_p ())
6818 dump_printf_loc (MSG_NOTE
, vect_location
,
6819 "hoisting out of the vectorized "
6821 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6823 tree tem
= copy_ssa_name (scalar_dest
);
6824 gsi_insert_on_edge_immediate
6825 (loop_preheader_edge (loop
),
6826 gimple_build_assign (tem
,
6828 (gimple_assign_rhs1 (stmt
))));
6829 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
6833 gimple_stmt_iterator gsi2
= *gsi
;
6835 new_temp
= vect_init_vector (stmt
, scalar_dest
,
6838 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6839 set_vinfo_for_stmt (new_stmt
,
6840 new_stmt_vec_info (new_stmt
, loop_vinfo
,
6846 tree perm_mask
= perm_mask_for_reverse (vectype
);
6847 new_temp
= permute_vec_elements (new_temp
, new_temp
,
6848 perm_mask
, stmt
, gsi
);
6849 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6852 /* Collect vector loads and later create their permutation in
6853 vect_transform_grouped_load (). */
6854 if (grouped_load
|| slp_perm
)
6855 dr_chain
.quick_push (new_temp
);
6857 /* Store vector loads in the corresponding SLP_NODE. */
6858 if (slp
&& !slp_perm
)
6859 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6861 /* Bump the vector pointer to account for a gap or for excess
6862 elements loaded for a permuted SLP load. */
6863 if (group_gap_adj
!= 0)
6867 = wide_int_to_tree (sizetype
,
6868 wi::smul (TYPE_SIZE_UNIT (elem_type
),
6869 group_gap_adj
, &ovf
));
6870 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6875 if (slp
&& !slp_perm
)
6880 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6881 slp_node_instance
, false))
6883 dr_chain
.release ();
6892 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
6893 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6898 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6900 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6901 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6904 dr_chain
.release ();
6910 /* Function vect_is_simple_cond.
6913 LOOP - the loop that is being vectorized.
6914 COND - Condition that is checked for simple use.
6917 *COMP_VECTYPE - the vector type for the comparison.
6919 Returns whether a COND can be vectorized. Checks whether
6920 condition operands are supportable using vec_is_simple_use. */
6923 vect_is_simple_cond (tree cond
, gimple stmt
, loop_vec_info loop_vinfo
,
6924 bb_vec_info bb_vinfo
, tree
*comp_vectype
)
6928 enum vect_def_type dt
;
6929 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
6931 if (!COMPARISON_CLASS_P (cond
))
6934 lhs
= TREE_OPERAND (cond
, 0);
6935 rhs
= TREE_OPERAND (cond
, 1);
6937 if (TREE_CODE (lhs
) == SSA_NAME
)
6939 gimple lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
6940 if (!vect_is_simple_use_1 (lhs
, stmt
, loop_vinfo
, bb_vinfo
,
6941 &lhs_def_stmt
, &def
, &dt
, &vectype1
))
6944 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
6945 && TREE_CODE (lhs
) != FIXED_CST
)
6948 if (TREE_CODE (rhs
) == SSA_NAME
)
6950 gimple rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
6951 if (!vect_is_simple_use_1 (rhs
, stmt
, loop_vinfo
, bb_vinfo
,
6952 &rhs_def_stmt
, &def
, &dt
, &vectype2
))
6955 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
6956 && TREE_CODE (rhs
) != FIXED_CST
)
6959 *comp_vectype
= vectype1
? vectype1
: vectype2
;
6963 /* vectorizable_condition.
6965 Check if STMT is conditional modify expression that can be vectorized.
6966 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6967 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
6970 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
6971 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
6972 else caluse if it is 2).
6974 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6977 vectorizable_condition (gimple stmt
, gimple_stmt_iterator
*gsi
,
6978 gimple
*vec_stmt
, tree reduc_def
, int reduc_index
,
6981 tree scalar_dest
= NULL_TREE
;
6982 tree vec_dest
= NULL_TREE
;
6983 tree cond_expr
, then_clause
, else_clause
;
6984 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6985 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6986 tree comp_vectype
= NULL_TREE
;
6987 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
6988 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
6989 tree vec_compare
, vec_cond_expr
;
6991 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6993 enum vect_def_type dt
, dts
[4];
6994 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6996 enum tree_code code
;
6997 stmt_vec_info prev_stmt_info
= NULL
;
6999 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7000 vec
<tree
> vec_oprnds0
= vNULL
;
7001 vec
<tree
> vec_oprnds1
= vNULL
;
7002 vec
<tree
> vec_oprnds2
= vNULL
;
7003 vec
<tree
> vec_oprnds3
= vNULL
;
7006 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
7009 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7011 gcc_assert (ncopies
>= 1);
7012 if (reduc_index
&& ncopies
> 1)
7013 return false; /* FORNOW */
7015 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
7018 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7021 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7022 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7026 /* FORNOW: not yet supported. */
7027 if (STMT_VINFO_LIVE_P (stmt_info
))
7029 if (dump_enabled_p ())
7030 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7031 "value used after loop.\n");
7035 /* Is vectorizable conditional operation? */
7036 if (!is_gimple_assign (stmt
))
7039 code
= gimple_assign_rhs_code (stmt
);
7041 if (code
!= COND_EXPR
)
7044 cond_expr
= gimple_assign_rhs1 (stmt
);
7045 then_clause
= gimple_assign_rhs2 (stmt
);
7046 else_clause
= gimple_assign_rhs3 (stmt
);
7048 if (!vect_is_simple_cond (cond_expr
, stmt
, loop_vinfo
, bb_vinfo
,
7053 if (TREE_CODE (then_clause
) == SSA_NAME
)
7055 gimple then_def_stmt
= SSA_NAME_DEF_STMT (then_clause
);
7056 if (!vect_is_simple_use (then_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7057 &then_def_stmt
, &def
, &dt
))
7060 else if (TREE_CODE (then_clause
) != INTEGER_CST
7061 && TREE_CODE (then_clause
) != REAL_CST
7062 && TREE_CODE (then_clause
) != FIXED_CST
)
7065 if (TREE_CODE (else_clause
) == SSA_NAME
)
7067 gimple else_def_stmt
= SSA_NAME_DEF_STMT (else_clause
);
7068 if (!vect_is_simple_use (else_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7069 &else_def_stmt
, &def
, &dt
))
7072 else if (TREE_CODE (else_clause
) != INTEGER_CST
7073 && TREE_CODE (else_clause
) != REAL_CST
7074 && TREE_CODE (else_clause
) != FIXED_CST
)
7077 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
)));
7078 /* The result of a vector comparison should be signed type. */
7079 tree cmp_type
= build_nonstandard_integer_type (prec
, 0);
7080 vec_cmp_type
= get_same_sized_vectype (cmp_type
, vectype
);
7081 if (vec_cmp_type
== NULL_TREE
)
7086 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
7087 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
7094 vec_oprnds0
.create (1);
7095 vec_oprnds1
.create (1);
7096 vec_oprnds2
.create (1);
7097 vec_oprnds3
.create (1);
7101 scalar_dest
= gimple_assign_lhs (stmt
);
7102 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7104 /* Handle cond expr. */
7105 for (j
= 0; j
< ncopies
; j
++)
7107 gassign
*new_stmt
= NULL
;
7112 auto_vec
<tree
, 4> ops
;
7113 auto_vec
<vec
<tree
>, 4> vec_defs
;
7115 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
7116 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
7117 ops
.safe_push (then_clause
);
7118 ops
.safe_push (else_clause
);
7119 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7120 vec_oprnds3
= vec_defs
.pop ();
7121 vec_oprnds2
= vec_defs
.pop ();
7122 vec_oprnds1
= vec_defs
.pop ();
7123 vec_oprnds0
= vec_defs
.pop ();
7126 vec_defs
.release ();
7132 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
7134 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0), stmt
,
7135 loop_vinfo
, NULL
, >emp
, &def
, &dts
[0]);
7138 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
7140 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1), stmt
,
7141 loop_vinfo
, NULL
, >emp
, &def
, &dts
[1]);
7142 if (reduc_index
== 1)
7143 vec_then_clause
= reduc_def
;
7146 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
7148 vect_is_simple_use (then_clause
, stmt
, loop_vinfo
,
7149 NULL
, >emp
, &def
, &dts
[2]);
7151 if (reduc_index
== 2)
7152 vec_else_clause
= reduc_def
;
7155 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
7157 vect_is_simple_use (else_clause
, stmt
, loop_vinfo
,
7158 NULL
, >emp
, &def
, &dts
[3]);
7164 vec_cond_lhs
= vect_get_vec_def_for_stmt_copy (dts
[0],
7165 vec_oprnds0
.pop ());
7166 vec_cond_rhs
= vect_get_vec_def_for_stmt_copy (dts
[1],
7167 vec_oprnds1
.pop ());
7168 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
7169 vec_oprnds2
.pop ());
7170 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
7171 vec_oprnds3
.pop ());
7176 vec_oprnds0
.quick_push (vec_cond_lhs
);
7177 vec_oprnds1
.quick_push (vec_cond_rhs
);
7178 vec_oprnds2
.quick_push (vec_then_clause
);
7179 vec_oprnds3
.quick_push (vec_else_clause
);
7182 /* Arguments are ready. Create the new vector stmt. */
7183 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
7185 vec_cond_rhs
= vec_oprnds1
[i
];
7186 vec_then_clause
= vec_oprnds2
[i
];
7187 vec_else_clause
= vec_oprnds3
[i
];
7189 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
7190 vec_cond_lhs
, vec_cond_rhs
);
7191 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
7192 vec_compare
, vec_then_clause
, vec_else_clause
);
7194 new_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
7195 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7196 gimple_assign_set_lhs (new_stmt
, new_temp
);
7197 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7199 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7206 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7208 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7210 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7213 vec_oprnds0
.release ();
7214 vec_oprnds1
.release ();
7215 vec_oprnds2
.release ();
7216 vec_oprnds3
.release ();
7222 /* Make sure the statement is vectorizable. */
7225 vect_analyze_stmt (gimple stmt
, bool *need_to_vectorize
, slp_tree node
)
7227 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7228 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7229 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
7231 tree scalar_type
, vectype
;
7232 gimple pattern_stmt
;
7233 gimple_seq pattern_def_seq
;
7235 if (dump_enabled_p ())
7237 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
7238 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7241 if (gimple_has_volatile_ops (stmt
))
7243 if (dump_enabled_p ())
7244 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7245 "not vectorized: stmt has volatile operands\n");
7250 /* Skip stmts that do not need to be vectorized. In loops this is expected
7252 - the COND_EXPR which is the loop exit condition
7253 - any LABEL_EXPRs in the loop
7254 - computations that are used only for array indexing or loop control.
7255 In basic blocks we only analyze statements that are a part of some SLP
7256 instance, therefore, all the statements are relevant.
7258 Pattern statement needs to be analyzed instead of the original statement
7259 if the original statement is not relevant. Otherwise, we analyze both
7260 statements. In basic blocks we are called from some SLP instance
7261 traversal, don't analyze pattern stmts instead, the pattern stmts
7262 already will be part of SLP instance. */
7264 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
7265 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
7266 && !STMT_VINFO_LIVE_P (stmt_info
))
7268 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7270 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7271 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7273 /* Analyze PATTERN_STMT instead of the original stmt. */
7274 stmt
= pattern_stmt
;
7275 stmt_info
= vinfo_for_stmt (pattern_stmt
);
7276 if (dump_enabled_p ())
7278 dump_printf_loc (MSG_NOTE
, vect_location
,
7279 "==> examining pattern statement: ");
7280 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7285 if (dump_enabled_p ())
7286 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
7291 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7294 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7295 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7297 /* Analyze PATTERN_STMT too. */
7298 if (dump_enabled_p ())
7300 dump_printf_loc (MSG_NOTE
, vect_location
,
7301 "==> examining pattern statement: ");
7302 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7305 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
7309 if (is_pattern_stmt_p (stmt_info
)
7311 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
7313 gimple_stmt_iterator si
;
7315 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
7317 gimple pattern_def_stmt
= gsi_stmt (si
);
7318 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
7319 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
7321 /* Analyze def stmt of STMT if it's a pattern stmt. */
7322 if (dump_enabled_p ())
7324 dump_printf_loc (MSG_NOTE
, vect_location
,
7325 "==> examining pattern def statement: ");
7326 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
7329 if (!vect_analyze_stmt (pattern_def_stmt
,
7330 need_to_vectorize
, node
))
7336 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
7338 case vect_internal_def
:
7341 case vect_reduction_def
:
7342 case vect_nested_cycle
:
7343 gcc_assert (!bb_vinfo
7344 && (relevance
== vect_used_in_outer
7345 || relevance
== vect_used_in_outer_by_reduction
7346 || relevance
== vect_used_by_reduction
7347 || relevance
== vect_unused_in_scope
));
7350 case vect_induction_def
:
7351 case vect_constant_def
:
7352 case vect_external_def
:
7353 case vect_unknown_def_type
:
7360 gcc_assert (PURE_SLP_STMT (stmt_info
));
7362 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
7363 if (dump_enabled_p ())
7365 dump_printf_loc (MSG_NOTE
, vect_location
,
7366 "get vectype for scalar type: ");
7367 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
7368 dump_printf (MSG_NOTE
, "\n");
7371 vectype
= get_vectype_for_scalar_type (scalar_type
);
7374 if (dump_enabled_p ())
7376 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7377 "not SLPed: unsupported data-type ");
7378 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
7380 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7385 if (dump_enabled_p ())
7387 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
7388 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
7389 dump_printf (MSG_NOTE
, "\n");
7392 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
7395 if (STMT_VINFO_RELEVANT_P (stmt_info
))
7397 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
7398 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
7399 || (is_gimple_call (stmt
)
7400 && gimple_call_lhs (stmt
) == NULL_TREE
));
7401 *need_to_vectorize
= true;
7404 if (PURE_SLP_STMT (stmt_info
) && !node
)
7406 dump_printf_loc (MSG_NOTE
, vect_location
,
7407 "handled only by SLP analysis\n");
7413 && (STMT_VINFO_RELEVANT_P (stmt_info
)
7414 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
7415 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7416 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7417 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7418 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7419 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7420 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7421 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7422 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7423 || vectorizable_reduction (stmt
, NULL
, NULL
, node
)
7424 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7428 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7429 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7430 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7431 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7432 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7433 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7434 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7435 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7436 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7441 if (dump_enabled_p ())
7443 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7444 "not vectorized: relevant stmt not ");
7445 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7446 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7455 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7456 need extra handling, except for vectorizable reductions. */
7457 if (STMT_VINFO_LIVE_P (stmt_info
)
7458 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7459 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
7463 if (dump_enabled_p ())
7465 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7466 "not vectorized: live stmt not ");
7467 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7468 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7478 /* Function vect_transform_stmt.
7480 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7483 vect_transform_stmt (gimple stmt
, gimple_stmt_iterator
*gsi
,
7484 bool *grouped_store
, slp_tree slp_node
,
7485 slp_instance slp_node_instance
)
7487 bool is_store
= false;
7488 gimple vec_stmt
= NULL
;
7489 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7492 switch (STMT_VINFO_TYPE (stmt_info
))
7494 case type_demotion_vec_info_type
:
7495 case type_promotion_vec_info_type
:
7496 case type_conversion_vec_info_type
:
7497 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
7501 case induc_vec_info_type
:
7502 gcc_assert (!slp_node
);
7503 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
7507 case shift_vec_info_type
:
7508 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
7512 case op_vec_info_type
:
7513 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
7517 case assignment_vec_info_type
:
7518 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
7522 case load_vec_info_type
:
7523 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
7528 case store_vec_info_type
:
7529 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
7531 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
7533 /* In case of interleaving, the whole chain is vectorized when the
7534 last store in the chain is reached. Store stmts before the last
7535 one are skipped, and there vec_stmt_info shouldn't be freed
7537 *grouped_store
= true;
7538 if (STMT_VINFO_VEC_STMT (stmt_info
))
7545 case condition_vec_info_type
:
7546 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
7550 case call_vec_info_type
:
7551 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7552 stmt
= gsi_stmt (*gsi
);
7553 if (is_gimple_call (stmt
)
7554 && gimple_call_internal_p (stmt
)
7555 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
7559 case call_simd_clone_vec_info_type
:
7560 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7561 stmt
= gsi_stmt (*gsi
);
7564 case reduc_vec_info_type
:
7565 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
7570 if (!STMT_VINFO_LIVE_P (stmt_info
))
7572 if (dump_enabled_p ())
7573 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7574 "stmt not supported.\n");
7579 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
7580 is being vectorized, but outside the immediately enclosing loop. */
7582 && STMT_VINFO_LOOP_VINFO (stmt_info
)
7583 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
7584 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
7585 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
7586 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
7587 || STMT_VINFO_RELEVANT (stmt_info
) ==
7588 vect_used_in_outer_by_reduction
))
7590 struct loop
*innerloop
= LOOP_VINFO_LOOP (
7591 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
7592 imm_use_iterator imm_iter
;
7593 use_operand_p use_p
;
7597 if (dump_enabled_p ())
7598 dump_printf_loc (MSG_NOTE
, vect_location
,
7599 "Record the vdef for outer-loop vectorization.\n");
7601 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
7602 (to be used when vectorizing outer-loop stmts that use the DEF of
7604 if (gimple_code (stmt
) == GIMPLE_PHI
)
7605 scalar_dest
= PHI_RESULT (stmt
);
7607 scalar_dest
= gimple_assign_lhs (stmt
);
7609 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
7611 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
7613 exit_phi
= USE_STMT (use_p
);
7614 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
7619 /* Handle stmts whose DEF is used outside the loop-nest that is
7620 being vectorized. */
7621 if (STMT_VINFO_LIVE_P (stmt_info
)
7622 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7624 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
7629 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
7635 /* Remove a group of stores (for SLP or interleaving), free their
7639 vect_remove_stores (gimple first_stmt
)
7641 gimple next
= first_stmt
;
7643 gimple_stmt_iterator next_si
;
7647 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
7649 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
7650 if (is_pattern_stmt_p (stmt_info
))
7651 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
7652 /* Free the attached stmt_vec_info and remove the stmt. */
7653 next_si
= gsi_for_stmt (next
);
7654 unlink_stmt_vdef (next
);
7655 gsi_remove (&next_si
, true);
7656 release_defs (next
);
7657 free_stmt_vec_info (next
);
7663 /* Function new_stmt_vec_info.
7665 Create and initialize a new stmt_vec_info struct for STMT. */
7668 new_stmt_vec_info (gimple stmt
, loop_vec_info loop_vinfo
,
7669 bb_vec_info bb_vinfo
)
7672 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
7674 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
7675 STMT_VINFO_STMT (res
) = stmt
;
7676 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
7677 STMT_VINFO_BB_VINFO (res
) = bb_vinfo
;
7678 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
7679 STMT_VINFO_LIVE_P (res
) = false;
7680 STMT_VINFO_VECTYPE (res
) = NULL
;
7681 STMT_VINFO_VEC_STMT (res
) = NULL
;
7682 STMT_VINFO_VECTORIZABLE (res
) = true;
7683 STMT_VINFO_IN_PATTERN_P (res
) = false;
7684 STMT_VINFO_RELATED_STMT (res
) = NULL
;
7685 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
7686 STMT_VINFO_DATA_REF (res
) = NULL
;
7688 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
7689 STMT_VINFO_DR_OFFSET (res
) = NULL
;
7690 STMT_VINFO_DR_INIT (res
) = NULL
;
7691 STMT_VINFO_DR_STEP (res
) = NULL
;
7692 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
7694 if (gimple_code (stmt
) == GIMPLE_PHI
7695 && is_loop_header_bb_p (gimple_bb (stmt
)))
7696 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
7698 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
7700 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
7701 STMT_SLP_TYPE (res
) = loop_vect
;
7702 GROUP_FIRST_ELEMENT (res
) = NULL
;
7703 GROUP_NEXT_ELEMENT (res
) = NULL
;
7704 GROUP_SIZE (res
) = 0;
7705 GROUP_STORE_COUNT (res
) = 0;
7706 GROUP_GAP (res
) = 0;
7707 GROUP_SAME_DR_STMT (res
) = NULL
;
7713 /* Create a hash table for stmt_vec_info. */
7716 init_stmt_vec_info_vec (void)
7718 gcc_assert (!stmt_vec_info_vec
.exists ());
7719 stmt_vec_info_vec
.create (50);
7723 /* Free hash table for stmt_vec_info. */
7726 free_stmt_vec_info_vec (void)
7730 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
7732 free_stmt_vec_info (STMT_VINFO_STMT ((stmt_vec_info
) info
));
7733 gcc_assert (stmt_vec_info_vec
.exists ());
7734 stmt_vec_info_vec
.release ();
7738 /* Free stmt vectorization related info. */
7741 free_stmt_vec_info (gimple stmt
)
7743 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7748 /* Check if this statement has a related "pattern stmt"
7749 (introduced by the vectorizer during the pattern recognition
7750 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
7752 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
7754 stmt_vec_info patt_info
7755 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
7758 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
7759 gimple patt_stmt
= STMT_VINFO_STMT (patt_info
);
7760 gimple_set_bb (patt_stmt
, NULL
);
7761 tree lhs
= gimple_get_lhs (patt_stmt
);
7762 if (TREE_CODE (lhs
) == SSA_NAME
)
7763 release_ssa_name (lhs
);
7766 gimple_stmt_iterator si
;
7767 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
7769 gimple seq_stmt
= gsi_stmt (si
);
7770 gimple_set_bb (seq_stmt
, NULL
);
7771 lhs
= gimple_get_lhs (patt_stmt
);
7772 if (TREE_CODE (lhs
) == SSA_NAME
)
7773 release_ssa_name (lhs
);
7774 free_stmt_vec_info (seq_stmt
);
7777 free_stmt_vec_info (patt_stmt
);
7781 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
7782 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
7783 set_vinfo_for_stmt (stmt
, NULL
);
7788 /* Function get_vectype_for_scalar_type_and_size.
7790 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
7794 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
7796 machine_mode inner_mode
= TYPE_MODE (scalar_type
);
7797 machine_mode simd_mode
;
7798 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
7805 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
7806 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
7809 /* For vector types of elements whose mode precision doesn't
7810 match their types precision we use a element type of mode
7811 precision. The vectorization routines will have to make sure
7812 they support the proper result truncation/extension.
7813 We also make sure to build vector types with INTEGER_TYPE
7814 component type only. */
7815 if (INTEGRAL_TYPE_P (scalar_type
)
7816 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
7817 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
7818 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
7819 TYPE_UNSIGNED (scalar_type
));
7821 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
7822 When the component mode passes the above test simply use a type
7823 corresponding to that mode. The theory is that any use that
7824 would cause problems with this will disable vectorization anyway. */
7825 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
7826 && !INTEGRAL_TYPE_P (scalar_type
))
7827 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
7829 /* We can't build a vector type of elements with alignment bigger than
7831 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
7832 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
7833 TYPE_UNSIGNED (scalar_type
));
7835 /* If we felt back to using the mode fail if there was
7836 no scalar type for it. */
7837 if (scalar_type
== NULL_TREE
)
7840 /* If no size was supplied use the mode the target prefers. Otherwise
7841 lookup a vector mode of the specified size. */
7843 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
7845 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
7846 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
7850 vectype
= build_vector_type (scalar_type
, nunits
);
7852 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
7853 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
7859 unsigned int current_vector_size
;
7861 /* Function get_vectype_for_scalar_type.
7863 Returns the vector type corresponding to SCALAR_TYPE as supported
7867 get_vectype_for_scalar_type (tree scalar_type
)
7870 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
7871 current_vector_size
);
7873 && current_vector_size
== 0)
7874 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
7878 /* Function get_same_sized_vectype
7880 Returns a vector type corresponding to SCALAR_TYPE of size
7881 VECTOR_TYPE if supported by the target. */
7884 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
7886 return get_vectype_for_scalar_type_and_size
7887 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
7890 /* Function vect_is_simple_use.
7893 LOOP_VINFO - the vect info of the loop that is being vectorized.
7894 BB_VINFO - the vect info of the basic block that is being vectorized.
7895 OPERAND - operand of STMT in the loop or bb.
7896 DEF - the defining stmt in case OPERAND is an SSA_NAME.
7898 Returns whether a stmt with OPERAND can be vectorized.
7899 For loops, supportable operands are constants, loop invariants, and operands
7900 that are defined by the current iteration of the loop. Unsupportable
7901 operands are those that are defined by a previous iteration of the loop (as
7902 is the case in reduction/induction computations).
7903 For basic blocks, supportable operands are constants and bb invariants.
7904 For now, operands defined outside the basic block are not supported. */
7907 vect_is_simple_use (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
7908 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
7909 tree
*def
, enum vect_def_type
*dt
)
7913 *dt
= vect_unknown_def_type
;
7915 if (dump_enabled_p ())
7917 dump_printf_loc (MSG_NOTE
, vect_location
,
7918 "vect_is_simple_use: operand ");
7919 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
7920 dump_printf (MSG_NOTE
, "\n");
7923 if (CONSTANT_CLASS_P (operand
))
7925 *dt
= vect_constant_def
;
7929 if (is_gimple_min_invariant (operand
))
7932 *dt
= vect_external_def
;
7936 if (TREE_CODE (operand
) != SSA_NAME
)
7938 if (dump_enabled_p ())
7939 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7944 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
7947 *dt
= vect_external_def
;
7951 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
7952 if (dump_enabled_p ())
7954 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
7955 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
7958 basic_block bb
= gimple_bb (*def_stmt
);
7959 if ((loop_vinfo
&& !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), bb
))
7961 && (bb
!= BB_VINFO_BB (bb_vinfo
)
7962 || gimple_code (*def_stmt
) == GIMPLE_PHI
)))
7963 *dt
= vect_external_def
;
7966 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
7967 if (bb_vinfo
&& !STMT_VINFO_VECTORIZABLE (stmt_vinfo
))
7968 *dt
= vect_external_def
;
7970 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
7973 if (dump_enabled_p ())
7975 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
7978 case vect_uninitialized_def
:
7979 dump_printf (MSG_NOTE
, "uninitialized\n");
7981 case vect_constant_def
:
7982 dump_printf (MSG_NOTE
, "constant\n");
7984 case vect_external_def
:
7985 dump_printf (MSG_NOTE
, "external\n");
7987 case vect_internal_def
:
7988 dump_printf (MSG_NOTE
, "internal\n");
7990 case vect_induction_def
:
7991 dump_printf (MSG_NOTE
, "induction\n");
7993 case vect_reduction_def
:
7994 dump_printf (MSG_NOTE
, "reduction\n");
7996 case vect_double_reduction_def
:
7997 dump_printf (MSG_NOTE
, "double reduction\n");
7999 case vect_nested_cycle
:
8000 dump_printf (MSG_NOTE
, "nested cycle\n");
8002 case vect_unknown_def_type
:
8003 dump_printf (MSG_NOTE
, "unknown\n");
8008 if (*dt
== vect_unknown_def_type
8010 && *dt
== vect_double_reduction_def
8011 && gimple_code (stmt
) != GIMPLE_PHI
))
8013 if (dump_enabled_p ())
8014 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8015 "Unsupported pattern.\n");
8019 switch (gimple_code (*def_stmt
))
8022 *def
= gimple_phi_result (*def_stmt
);
8026 *def
= gimple_assign_lhs (*def_stmt
);
8030 *def
= gimple_call_lhs (*def_stmt
);
8035 if (dump_enabled_p ())
8036 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8037 "unsupported defining stmt:\n");
8044 /* Function vect_is_simple_use_1.
8046 Same as vect_is_simple_use_1 but also determines the vector operand
8047 type of OPERAND and stores it to *VECTYPE. If the definition of
8048 OPERAND is vect_uninitialized_def, vect_constant_def or
8049 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8050 is responsible to compute the best suited vector type for the
8054 vect_is_simple_use_1 (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
8055 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
8056 tree
*def
, enum vect_def_type
*dt
, tree
*vectype
)
8058 if (!vect_is_simple_use (operand
, stmt
, loop_vinfo
, bb_vinfo
, def_stmt
,
8062 /* Now get a vector type if the def is internal, otherwise supply
8063 NULL_TREE and leave it up to the caller to figure out a proper
8064 type for the use stmt. */
8065 if (*dt
== vect_internal_def
8066 || *dt
== vect_induction_def
8067 || *dt
== vect_reduction_def
8068 || *dt
== vect_double_reduction_def
8069 || *dt
== vect_nested_cycle
)
8071 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
8073 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8074 && !STMT_VINFO_RELEVANT (stmt_info
)
8075 && !STMT_VINFO_LIVE_P (stmt_info
))
8076 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8078 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8079 gcc_assert (*vectype
!= NULL_TREE
);
8081 else if (*dt
== vect_uninitialized_def
8082 || *dt
== vect_constant_def
8083 || *dt
== vect_external_def
)
8084 *vectype
= NULL_TREE
;
8092 /* Function supportable_widening_operation
8094 Check whether an operation represented by the code CODE is a
8095 widening operation that is supported by the target platform in
8096 vector form (i.e., when operating on arguments of type VECTYPE_IN
8097 producing a result of type VECTYPE_OUT).
8099 Widening operations we currently support are NOP (CONVERT), FLOAT
8100 and WIDEN_MULT. This function checks if these operations are supported
8101 by the target platform either directly (via vector tree-codes), or via
8105 - CODE1 and CODE2 are codes of vector operations to be used when
8106 vectorizing the operation, if available.
8107 - MULTI_STEP_CVT determines the number of required intermediate steps in
8108 case of multi-step conversion (like char->short->int - in that case
8109 MULTI_STEP_CVT will be 1).
8110 - INTERM_TYPES contains the intermediate type required to perform the
8111 widening operation (short in the above example). */
8114 supportable_widening_operation (enum tree_code code
, gimple stmt
,
8115 tree vectype_out
, tree vectype_in
,
8116 enum tree_code
*code1
, enum tree_code
*code2
,
8117 int *multi_step_cvt
,
8118 vec
<tree
> *interm_types
)
8120 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8121 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8122 struct loop
*vect_loop
= NULL
;
8123 machine_mode vec_mode
;
8124 enum insn_code icode1
, icode2
;
8125 optab optab1
, optab2
;
8126 tree vectype
= vectype_in
;
8127 tree wide_vectype
= vectype_out
;
8128 enum tree_code c1
, c2
;
8130 tree prev_type
, intermediate_type
;
8131 machine_mode intermediate_mode
, prev_mode
;
8132 optab optab3
, optab4
;
8134 *multi_step_cvt
= 0;
8136 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
8140 case WIDEN_MULT_EXPR
:
8141 /* The result of a vectorized widening operation usually requires
8142 two vectors (because the widened results do not fit into one vector).
8143 The generated vector results would normally be expected to be
8144 generated in the same order as in the original scalar computation,
8145 i.e. if 8 results are generated in each vector iteration, they are
8146 to be organized as follows:
8147 vect1: [res1,res2,res3,res4],
8148 vect2: [res5,res6,res7,res8].
8150 However, in the special case that the result of the widening
8151 operation is used in a reduction computation only, the order doesn't
8152 matter (because when vectorizing a reduction we change the order of
8153 the computation). Some targets can take advantage of this and
8154 generate more efficient code. For example, targets like Altivec,
8155 that support widen_mult using a sequence of {mult_even,mult_odd}
8156 generate the following vectors:
8157 vect1: [res1,res3,res5,res7],
8158 vect2: [res2,res4,res6,res8].
8160 When vectorizing outer-loops, we execute the inner-loop sequentially
8161 (each vectorized inner-loop iteration contributes to VF outer-loop
8162 iterations in parallel). We therefore don't allow to change the
8163 order of the computation in the inner-loop during outer-loop
8165 /* TODO: Another case in which order doesn't *really* matter is when we
8166 widen and then contract again, e.g. (short)((int)x * y >> 8).
8167 Normally, pack_trunc performs an even/odd permute, whereas the
8168 repack from an even/odd expansion would be an interleave, which
8169 would be significantly simpler for e.g. AVX2. */
8170 /* In any case, in order to avoid duplicating the code below, recurse
8171 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8172 are properly set up for the caller. If we fail, we'll continue with
8173 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8175 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
8176 && !nested_in_vect_loop_p (vect_loop
, stmt
)
8177 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
8178 stmt
, vectype_out
, vectype_in
,
8179 code1
, code2
, multi_step_cvt
,
8182 /* Elements in a vector with vect_used_by_reduction property cannot
8183 be reordered if the use chain with this property does not have the
8184 same operation. One such an example is s += a * b, where elements
8185 in a and b cannot be reordered. Here we check if the vector defined
8186 by STMT is only directly used in the reduction statement. */
8187 tree lhs
= gimple_assign_lhs (stmt
);
8188 use_operand_p dummy
;
8190 stmt_vec_info use_stmt_info
= NULL
;
8191 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
8192 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
8193 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
8196 c1
= VEC_WIDEN_MULT_LO_EXPR
;
8197 c2
= VEC_WIDEN_MULT_HI_EXPR
;
8200 case VEC_WIDEN_MULT_EVEN_EXPR
:
8201 /* Support the recursion induced just above. */
8202 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
8203 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
8206 case WIDEN_LSHIFT_EXPR
:
8207 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
8208 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
8212 c1
= VEC_UNPACK_LO_EXPR
;
8213 c2
= VEC_UNPACK_HI_EXPR
;
8217 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
8218 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
8221 case FIX_TRUNC_EXPR
:
8222 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
8223 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
8224 computing the operation. */
8231 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
8233 enum tree_code ctmp
= c1
;
8238 if (code
== FIX_TRUNC_EXPR
)
8240 /* The signedness is determined from output operand. */
8241 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8242 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
8246 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8247 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
8250 if (!optab1
|| !optab2
)
8253 vec_mode
= TYPE_MODE (vectype
);
8254 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
8255 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
8261 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8262 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8265 /* Check if it's a multi-step conversion that can be done using intermediate
8268 prev_type
= vectype
;
8269 prev_mode
= vec_mode
;
8271 if (!CONVERT_EXPR_CODE_P (code
))
8274 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8275 intermediate steps in promotion sequence. We try
8276 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
8278 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8279 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8281 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8283 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
8284 TYPE_UNSIGNED (prev_type
));
8285 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8286 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
8288 if (!optab3
|| !optab4
8289 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
8290 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8291 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
8292 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
8293 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
8294 == CODE_FOR_nothing
)
8295 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
8296 == CODE_FOR_nothing
))
8299 interm_types
->quick_push (intermediate_type
);
8300 (*multi_step_cvt
)++;
8302 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8303 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8306 prev_type
= intermediate_type
;
8307 prev_mode
= intermediate_mode
;
8310 interm_types
->release ();
8315 /* Function supportable_narrowing_operation
8317 Check whether an operation represented by the code CODE is a
8318 narrowing operation that is supported by the target platform in
8319 vector form (i.e., when operating on arguments of type VECTYPE_IN
8320 and producing a result of type VECTYPE_OUT).
8322 Narrowing operations we currently support are NOP (CONVERT) and
8323 FIX_TRUNC. This function checks if these operations are supported by
8324 the target platform directly via vector tree-codes.
8327 - CODE1 is the code of a vector operation to be used when
8328 vectorizing the operation, if available.
8329 - MULTI_STEP_CVT determines the number of required intermediate steps in
8330 case of multi-step conversion (like int->short->char - in that case
8331 MULTI_STEP_CVT will be 1).
8332 - INTERM_TYPES contains the intermediate type required to perform the
8333 narrowing operation (short in the above example). */
8336 supportable_narrowing_operation (enum tree_code code
,
8337 tree vectype_out
, tree vectype_in
,
8338 enum tree_code
*code1
, int *multi_step_cvt
,
8339 vec
<tree
> *interm_types
)
8341 machine_mode vec_mode
;
8342 enum insn_code icode1
;
8343 optab optab1
, interm_optab
;
8344 tree vectype
= vectype_in
;
8345 tree narrow_vectype
= vectype_out
;
8347 tree intermediate_type
;
8348 machine_mode intermediate_mode
, prev_mode
;
8352 *multi_step_cvt
= 0;
8356 c1
= VEC_PACK_TRUNC_EXPR
;
8359 case FIX_TRUNC_EXPR
:
8360 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
8364 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8365 tree code and optabs used for computing the operation. */
8372 if (code
== FIX_TRUNC_EXPR
)
8373 /* The signedness is determined from output operand. */
8374 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8376 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8381 vec_mode
= TYPE_MODE (vectype
);
8382 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
8387 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8390 /* Check if it's a multi-step conversion that can be done using intermediate
8392 prev_mode
= vec_mode
;
8393 if (code
== FIX_TRUNC_EXPR
)
8394 uns
= TYPE_UNSIGNED (vectype_out
);
8396 uns
= TYPE_UNSIGNED (vectype
);
8398 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8399 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8400 costly than signed. */
8401 if (code
== FIX_TRUNC_EXPR
&& uns
)
8403 enum insn_code icode2
;
8406 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
8408 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8409 if (interm_optab
!= unknown_optab
8410 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
8411 && insn_data
[icode1
].operand
[0].mode
8412 == insn_data
[icode2
].operand
[0].mode
)
8415 optab1
= interm_optab
;
8420 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8421 intermediate steps in promotion sequence. We try
8422 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8423 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8424 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8426 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8428 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
8430 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
8433 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
8434 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8435 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
8436 == CODE_FOR_nothing
))
8439 interm_types
->quick_push (intermediate_type
);
8440 (*multi_step_cvt
)++;
8442 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8445 prev_mode
= intermediate_mode
;
8446 optab1
= interm_optab
;
8449 interm_types
->release ();