1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2015 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
34 #include "fold-const.h"
35 #include "stor-layout.h"
38 #include "hard-reg-set.h"
40 #include "dominance.h"
42 #include "basic-block.h"
43 #include "gimple-pretty-print.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-ssa.h"
55 #include "tree-phinodes.h"
56 #include "ssa-iterators.h"
57 #include "stringpool.h"
58 #include "tree-ssanames.h"
59 #include "tree-ssa-loop-manip.h"
61 #include "tree-ssa-loop.h"
62 #include "tree-scalar-evolution.h"
66 #include "statistics.h"
67 #include "insn-config.h"
76 #include "recog.h" /* FIXME: for insn_data */
77 #include "insn-codes.h"
79 #include "diagnostic-core.h"
80 #include "tree-vectorizer.h"
82 #include "plugin-api.h"
87 /* For lang_hooks.types.type_for_mode. */
88 #include "langhooks.h"
90 /* Return the vectorized type for the given statement. */
93 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
95 return STMT_VINFO_VECTYPE (stmt_info
);
98 /* Return TRUE iff the given statement is in an inner loop relative to
99 the loop being vectorized. */
101 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
103 gimple stmt
= STMT_VINFO_STMT (stmt_info
);
104 basic_block bb
= gimple_bb (stmt
);
105 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
111 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
113 return (bb
->loop_father
== loop
->inner
);
116 /* Record the cost of a statement, either by directly informing the
117 target model or by saving it in a vector for later processing.
118 Return a preliminary estimate of the statement's cost. */
121 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
122 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
123 int misalign
, enum vect_cost_model_location where
)
127 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
128 add_stmt_info_to_vec (body_cost_vec
, count
, kind
,
129 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
132 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
137 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
138 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
139 void *target_cost_data
;
142 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
144 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
146 return add_stmt_cost (target_cost_data
, count
, kind
, stmt_info
,
151 /* Return a variable of type ELEM_TYPE[NELEMS]. */
154 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
156 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
160 /* ARRAY is an array of vectors created by create_vector_array.
161 Return an SSA_NAME for the vector in index N. The reference
162 is part of the vectorization of STMT and the vector is associated
163 with scalar destination SCALAR_DEST. */
166 read_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
167 tree array
, unsigned HOST_WIDE_INT n
)
169 tree vect_type
, vect
, vect_name
, array_ref
;
172 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
173 vect_type
= TREE_TYPE (TREE_TYPE (array
));
174 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
175 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
176 build_int_cst (size_type_node
, n
),
177 NULL_TREE
, NULL_TREE
);
179 new_stmt
= gimple_build_assign (vect
, array_ref
);
180 vect_name
= make_ssa_name (vect
, new_stmt
);
181 gimple_assign_set_lhs (new_stmt
, vect_name
);
182 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
187 /* ARRAY is an array of vectors created by create_vector_array.
188 Emit code to store SSA_NAME VECT in index N of the array.
189 The store is part of the vectorization of STMT. */
192 write_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
193 tree array
, unsigned HOST_WIDE_INT n
)
198 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
199 build_int_cst (size_type_node
, n
),
200 NULL_TREE
, NULL_TREE
);
202 new_stmt
= gimple_build_assign (array_ref
, vect
);
203 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
206 /* PTR is a pointer to an array of type TYPE. Return a representation
207 of *PTR. The memory reference replaces those in FIRST_DR
211 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
213 tree mem_ref
, alias_ptr_type
;
215 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
216 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
217 /* Arrays have the same alignment as their type. */
218 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
222 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
224 /* Function vect_mark_relevant.
226 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
229 vect_mark_relevant (vec
<gimple
> *worklist
, gimple stmt
,
230 enum vect_relevant relevant
, bool live_p
,
231 bool used_in_pattern
)
233 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
234 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
235 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
238 if (dump_enabled_p ())
239 dump_printf_loc (MSG_NOTE
, vect_location
,
240 "mark relevant %d, live %d.\n", relevant
, live_p
);
242 /* If this stmt is an original stmt in a pattern, we might need to mark its
243 related pattern stmt instead of the original stmt. However, such stmts
244 may have their own uses that are not in any pattern, in such cases the
245 stmt itself should be marked. */
246 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
249 if (!used_in_pattern
)
251 imm_use_iterator imm_iter
;
255 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
256 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
258 if (is_gimple_assign (stmt
))
259 lhs
= gimple_assign_lhs (stmt
);
261 lhs
= gimple_call_lhs (stmt
);
263 /* This use is out of pattern use, if LHS has other uses that are
264 pattern uses, we should mark the stmt itself, and not the pattern
266 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
267 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
269 if (is_gimple_debug (USE_STMT (use_p
)))
271 use_stmt
= USE_STMT (use_p
);
273 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
276 if (vinfo_for_stmt (use_stmt
)
277 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
287 /* This is the last stmt in a sequence that was detected as a
288 pattern that can potentially be vectorized. Don't mark the stmt
289 as relevant/live because it's not going to be vectorized.
290 Instead mark the pattern-stmt that replaces it. */
292 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
294 if (dump_enabled_p ())
295 dump_printf_loc (MSG_NOTE
, vect_location
,
296 "last stmt in pattern. don't mark"
297 " relevant/live.\n");
298 stmt_info
= vinfo_for_stmt (pattern_stmt
);
299 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
300 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
301 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
306 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
307 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
308 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
310 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
311 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
313 if (dump_enabled_p ())
314 dump_printf_loc (MSG_NOTE
, vect_location
,
315 "already marked relevant/live.\n");
319 worklist
->safe_push (stmt
);
323 /* Function vect_stmt_relevant_p.
325 Return true if STMT in loop that is represented by LOOP_VINFO is
326 "relevant for vectorization".
328 A stmt is considered "relevant for vectorization" if:
329 - it has uses outside the loop.
330 - it has vdefs (it alters memory).
331 - control stmts in the loop (except for the exit condition).
333 CHECKME: what other side effects would the vectorizer allow? */
336 vect_stmt_relevant_p (gimple stmt
, loop_vec_info loop_vinfo
,
337 enum vect_relevant
*relevant
, bool *live_p
)
339 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
341 imm_use_iterator imm_iter
;
345 *relevant
= vect_unused_in_scope
;
348 /* cond stmt other than loop exit cond. */
349 if (is_ctrl_stmt (stmt
)
350 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
351 != loop_exit_ctrl_vec_info_type
)
352 *relevant
= vect_used_in_scope
;
354 /* changing memory. */
355 if (gimple_code (stmt
) != GIMPLE_PHI
)
356 if (gimple_vdef (stmt
)
357 && !gimple_clobber_p (stmt
))
359 if (dump_enabled_p ())
360 dump_printf_loc (MSG_NOTE
, vect_location
,
361 "vec_stmt_relevant_p: stmt has vdefs.\n");
362 *relevant
= vect_used_in_scope
;
365 /* uses outside the loop. */
366 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
368 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
370 basic_block bb
= gimple_bb (USE_STMT (use_p
));
371 if (!flow_bb_inside_loop_p (loop
, bb
))
373 if (dump_enabled_p ())
374 dump_printf_loc (MSG_NOTE
, vect_location
,
375 "vec_stmt_relevant_p: used out of loop.\n");
377 if (is_gimple_debug (USE_STMT (use_p
)))
380 /* We expect all such uses to be in the loop exit phis
381 (because of loop closed form) */
382 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
383 gcc_assert (bb
== single_exit (loop
)->dest
);
390 return (*live_p
|| *relevant
);
394 /* Function exist_non_indexing_operands_for_use_p
396 USE is one of the uses attached to STMT. Check if USE is
397 used in STMT for anything other than indexing an array. */
400 exist_non_indexing_operands_for_use_p (tree use
, gimple stmt
)
403 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
405 /* USE corresponds to some operand in STMT. If there is no data
406 reference in STMT, then any operand that corresponds to USE
407 is not indexing an array. */
408 if (!STMT_VINFO_DATA_REF (stmt_info
))
411 /* STMT has a data_ref. FORNOW this means that its of one of
415 (This should have been verified in analyze_data_refs).
417 'var' in the second case corresponds to a def, not a use,
418 so USE cannot correspond to any operands that are not used
421 Therefore, all we need to check is if STMT falls into the
422 first case, and whether var corresponds to USE. */
424 if (!gimple_assign_copy_p (stmt
))
426 if (is_gimple_call (stmt
)
427 && gimple_call_internal_p (stmt
))
428 switch (gimple_call_internal_fn (stmt
))
431 operand
= gimple_call_arg (stmt
, 3);
436 operand
= gimple_call_arg (stmt
, 2);
446 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
448 operand
= gimple_assign_rhs1 (stmt
);
449 if (TREE_CODE (operand
) != SSA_NAME
)
460 Function process_use.
463 - a USE in STMT in a loop represented by LOOP_VINFO
464 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
465 that defined USE. This is done by calling mark_relevant and passing it
466 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
467 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
471 Generally, LIVE_P and RELEVANT are used to define the liveness and
472 relevance info of the DEF_STMT of this USE:
473 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
474 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
476 - case 1: If USE is used only for address computations (e.g. array indexing),
477 which does not need to be directly vectorized, then the liveness/relevance
478 of the respective DEF_STMT is left unchanged.
479 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
480 skip DEF_STMT cause it had already been processed.
481 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
482 be modified accordingly.
484 Return true if everything is as expected. Return false otherwise. */
487 process_use (gimple stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
488 enum vect_relevant relevant
, vec
<gimple
> *worklist
,
491 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
492 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
493 stmt_vec_info dstmt_vinfo
;
494 basic_block bb
, def_bb
;
497 enum vect_def_type dt
;
499 /* case 1: we are only interested in uses that need to be vectorized. Uses
500 that are used for address computation are not considered relevant. */
501 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
504 if (!vect_is_simple_use (use
, stmt
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
))
506 if (dump_enabled_p ())
507 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
508 "not vectorized: unsupported use in stmt.\n");
512 if (!def_stmt
|| gimple_nop_p (def_stmt
))
515 def_bb
= gimple_bb (def_stmt
);
516 if (!flow_bb_inside_loop_p (loop
, def_bb
))
518 if (dump_enabled_p ())
519 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
523 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
524 DEF_STMT must have already been processed, because this should be the
525 only way that STMT, which is a reduction-phi, was put in the worklist,
526 as there should be no other uses for DEF_STMT in the loop. So we just
527 check that everything is as expected, and we are done. */
528 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
529 bb
= gimple_bb (stmt
);
530 if (gimple_code (stmt
) == GIMPLE_PHI
531 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
532 && gimple_code (def_stmt
) != GIMPLE_PHI
533 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
534 && bb
->loop_father
== def_bb
->loop_father
)
536 if (dump_enabled_p ())
537 dump_printf_loc (MSG_NOTE
, vect_location
,
538 "reduc-stmt defining reduc-phi in the same nest.\n");
539 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
540 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
541 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
542 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
543 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
547 /* case 3a: outer-loop stmt defining an inner-loop stmt:
548 outer-loop-header-bb:
554 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
556 if (dump_enabled_p ())
557 dump_printf_loc (MSG_NOTE
, vect_location
,
558 "outer-loop def-stmt defining inner-loop stmt.\n");
562 case vect_unused_in_scope
:
563 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
564 vect_used_in_scope
: vect_unused_in_scope
;
567 case vect_used_in_outer_by_reduction
:
568 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
569 relevant
= vect_used_by_reduction
;
572 case vect_used_in_outer
:
573 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
574 relevant
= vect_used_in_scope
;
577 case vect_used_in_scope
:
585 /* case 3b: inner-loop stmt defining an outer-loop stmt:
586 outer-loop-header-bb:
590 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
592 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
594 if (dump_enabled_p ())
595 dump_printf_loc (MSG_NOTE
, vect_location
,
596 "inner-loop def-stmt defining outer-loop stmt.\n");
600 case vect_unused_in_scope
:
601 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
602 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
603 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
606 case vect_used_by_reduction
:
607 relevant
= vect_used_in_outer_by_reduction
;
610 case vect_used_in_scope
:
611 relevant
= vect_used_in_outer
;
619 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
,
620 is_pattern_stmt_p (stmt_vinfo
));
625 /* Function vect_mark_stmts_to_be_vectorized.
627 Not all stmts in the loop need to be vectorized. For example:
636 Stmt 1 and 3 do not need to be vectorized, because loop control and
637 addressing of vectorized data-refs are handled differently.
639 This pass detects such stmts. */
642 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
644 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
645 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
646 unsigned int nbbs
= loop
->num_nodes
;
647 gimple_stmt_iterator si
;
650 stmt_vec_info stmt_vinfo
;
654 enum vect_relevant relevant
, tmp_relevant
;
655 enum vect_def_type def_type
;
657 if (dump_enabled_p ())
658 dump_printf_loc (MSG_NOTE
, vect_location
,
659 "=== vect_mark_stmts_to_be_vectorized ===\n");
661 auto_vec
<gimple
, 64> worklist
;
663 /* 1. Init worklist. */
664 for (i
= 0; i
< nbbs
; i
++)
667 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
670 if (dump_enabled_p ())
672 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
673 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
676 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
677 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
, false);
679 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
681 stmt
= gsi_stmt (si
);
682 if (dump_enabled_p ())
684 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
685 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
688 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
689 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
, false);
693 /* 2. Process_worklist */
694 while (worklist
.length () > 0)
699 stmt
= worklist
.pop ();
700 if (dump_enabled_p ())
702 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
703 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
706 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
707 (DEF_STMT) as relevant/irrelevant and live/dead according to the
708 liveness and relevance properties of STMT. */
709 stmt_vinfo
= vinfo_for_stmt (stmt
);
710 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
711 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
713 /* Generally, the liveness and relevance properties of STMT are
714 propagated as is to the DEF_STMTs of its USEs:
715 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
716 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
718 One exception is when STMT has been identified as defining a reduction
719 variable; in this case we set the liveness/relevance as follows:
721 relevant = vect_used_by_reduction
722 This is because we distinguish between two kinds of relevant stmts -
723 those that are used by a reduction computation, and those that are
724 (also) used by a regular computation. This allows us later on to
725 identify stmts that are used solely by a reduction, and therefore the
726 order of the results that they produce does not have to be kept. */
728 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
729 tmp_relevant
= relevant
;
732 case vect_reduction_def
:
733 switch (tmp_relevant
)
735 case vect_unused_in_scope
:
736 relevant
= vect_used_by_reduction
;
739 case vect_used_by_reduction
:
740 if (gimple_code (stmt
) == GIMPLE_PHI
)
745 if (dump_enabled_p ())
746 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
747 "unsupported use of reduction.\n");
754 case vect_nested_cycle
:
755 if (tmp_relevant
!= vect_unused_in_scope
756 && tmp_relevant
!= vect_used_in_outer_by_reduction
757 && tmp_relevant
!= vect_used_in_outer
)
759 if (dump_enabled_p ())
760 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
761 "unsupported use of nested cycle.\n");
769 case vect_double_reduction_def
:
770 if (tmp_relevant
!= vect_unused_in_scope
771 && tmp_relevant
!= vect_used_by_reduction
)
773 if (dump_enabled_p ())
774 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
775 "unsupported use of double reduction.\n");
787 if (is_pattern_stmt_p (stmt_vinfo
))
789 /* Pattern statements are not inserted into the code, so
790 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
791 have to scan the RHS or function arguments instead. */
792 if (is_gimple_assign (stmt
))
794 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
795 tree op
= gimple_assign_rhs1 (stmt
);
798 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
800 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
801 live_p
, relevant
, &worklist
, false)
802 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
803 live_p
, relevant
, &worklist
, false))
807 for (; i
< gimple_num_ops (stmt
); i
++)
809 op
= gimple_op (stmt
, i
);
810 if (TREE_CODE (op
) == SSA_NAME
811 && !process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
816 else if (is_gimple_call (stmt
))
818 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
820 tree arg
= gimple_call_arg (stmt
, i
);
821 if (!process_use (stmt
, arg
, loop_vinfo
, live_p
, relevant
,
828 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
830 tree op
= USE_FROM_PTR (use_p
);
831 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
836 if (STMT_VINFO_GATHER_P (stmt_vinfo
))
839 tree decl
= vect_check_gather (stmt
, loop_vinfo
, NULL
, &off
, NULL
);
841 if (!process_use (stmt
, off
, loop_vinfo
, live_p
, relevant
,
845 } /* while worklist */
851 /* Function vect_model_simple_cost.
853 Models cost for simple operations, i.e. those that only emit ncopies of a
854 single op. Right now, this does not account for multiple insns that could
855 be generated for the single vector op. We will handle that shortly. */
858 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
859 enum vect_def_type
*dt
,
860 stmt_vector_for_cost
*prologue_cost_vec
,
861 stmt_vector_for_cost
*body_cost_vec
)
864 int inside_cost
= 0, prologue_cost
= 0;
866 /* The SLP costs were already calculated during SLP tree build. */
867 if (PURE_SLP_STMT (stmt_info
))
870 /* FORNOW: Assuming maximum 2 args per stmts. */
871 for (i
= 0; i
< 2; i
++)
872 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
873 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
874 stmt_info
, 0, vect_prologue
);
876 /* Pass the inside-of-loop statements to the target-specific cost model. */
877 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
878 stmt_info
, 0, vect_body
);
880 if (dump_enabled_p ())
881 dump_printf_loc (MSG_NOTE
, vect_location
,
882 "vect_model_simple_cost: inside_cost = %d, "
883 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
887 /* Model cost for type demotion and promotion operations. PWR is normally
888 zero for single-step promotions and demotions. It will be one if
889 two-step promotion/demotion is required, and so on. Each additional
890 step doubles the number of instructions required. */
893 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
894 enum vect_def_type
*dt
, int pwr
)
897 int inside_cost
= 0, prologue_cost
= 0;
898 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
899 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
900 void *target_cost_data
;
902 /* The SLP costs were already calculated during SLP tree build. */
903 if (PURE_SLP_STMT (stmt_info
))
907 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
909 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
911 for (i
= 0; i
< pwr
+ 1; i
++)
913 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
915 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
916 vec_promote_demote
, stmt_info
, 0,
920 /* FORNOW: Assuming maximum 2 args per stmts. */
921 for (i
= 0; i
< 2; i
++)
922 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
923 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
924 stmt_info
, 0, vect_prologue
);
926 if (dump_enabled_p ())
927 dump_printf_loc (MSG_NOTE
, vect_location
,
928 "vect_model_promotion_demotion_cost: inside_cost = %d, "
929 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
932 /* Function vect_cost_group_size
934 For grouped load or store, return the group_size only if it is the first
935 load or store of a group, else return 1. This ensures that group size is
936 only returned once per group. */
939 vect_cost_group_size (stmt_vec_info stmt_info
)
941 gimple first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
943 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
944 return GROUP_SIZE (stmt_info
);
950 /* Function vect_model_store_cost
952 Models cost for stores. In the case of grouped accesses, one access
953 has the overhead of the grouped access attributed to it. */
956 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
957 bool store_lanes_p
, enum vect_def_type dt
,
959 stmt_vector_for_cost
*prologue_cost_vec
,
960 stmt_vector_for_cost
*body_cost_vec
)
963 unsigned int inside_cost
= 0, prologue_cost
= 0;
964 struct data_reference
*first_dr
;
967 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
968 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
969 stmt_info
, 0, vect_prologue
);
971 /* Grouped access? */
972 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
976 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
981 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
982 group_size
= vect_cost_group_size (stmt_info
);
985 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
987 /* Not a grouped access. */
991 first_dr
= STMT_VINFO_DATA_REF (stmt_info
);
994 /* We assume that the cost of a single store-lanes instruction is
995 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
996 access is instead being provided by a permute-and-store operation,
997 include the cost of the permutes. */
998 if (!store_lanes_p
&& group_size
> 1
999 && !STMT_VINFO_STRIDED_P (stmt_info
))
1001 /* Uses a high and low interleave or shuffle operations for each
1003 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1004 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1005 stmt_info
, 0, vect_body
);
1007 if (dump_enabled_p ())
1008 dump_printf_loc (MSG_NOTE
, vect_location
,
1009 "vect_model_store_cost: strided group_size = %d .\n",
1013 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1014 /* Costs of the stores. */
1015 if (STMT_VINFO_STRIDED_P (stmt_info
)
1016 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1018 /* N scalar stores plus extracting the elements. */
1019 inside_cost
+= record_stmt_cost (body_cost_vec
,
1020 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1021 scalar_store
, stmt_info
, 0, vect_body
);
1024 vect_get_store_cost (first_dr
, ncopies
, &inside_cost
, body_cost_vec
);
1026 if (STMT_VINFO_STRIDED_P (stmt_info
))
1027 inside_cost
+= record_stmt_cost (body_cost_vec
,
1028 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1029 vec_to_scalar
, stmt_info
, 0, vect_body
);
1031 if (dump_enabled_p ())
1032 dump_printf_loc (MSG_NOTE
, vect_location
,
1033 "vect_model_store_cost: inside_cost = %d, "
1034 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1038 /* Calculate cost of DR's memory access. */
1040 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
1041 unsigned int *inside_cost
,
1042 stmt_vector_for_cost
*body_cost_vec
)
1044 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1045 gimple stmt
= DR_STMT (dr
);
1046 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1048 switch (alignment_support_scheme
)
1052 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1053 vector_store
, stmt_info
, 0,
1056 if (dump_enabled_p ())
1057 dump_printf_loc (MSG_NOTE
, vect_location
,
1058 "vect_model_store_cost: aligned.\n");
1062 case dr_unaligned_supported
:
1064 /* Here, we assign an additional cost for the unaligned store. */
1065 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1066 unaligned_store
, stmt_info
,
1067 DR_MISALIGNMENT (dr
), vect_body
);
1068 if (dump_enabled_p ())
1069 dump_printf_loc (MSG_NOTE
, vect_location
,
1070 "vect_model_store_cost: unaligned supported by "
1075 case dr_unaligned_unsupported
:
1077 *inside_cost
= VECT_MAX_COST
;
1079 if (dump_enabled_p ())
1080 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1081 "vect_model_store_cost: unsupported access.\n");
1091 /* Function vect_model_load_cost
1093 Models cost for loads. In the case of grouped accesses, the last access
1094 has the overhead of the grouped access attributed to it. Since unaligned
1095 accesses are supported for loads, we also account for the costs of the
1096 access scheme chosen. */
1099 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1100 bool load_lanes_p
, slp_tree slp_node
,
1101 stmt_vector_for_cost
*prologue_cost_vec
,
1102 stmt_vector_for_cost
*body_cost_vec
)
1106 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
1107 unsigned int inside_cost
= 0, prologue_cost
= 0;
1109 /* Grouped accesses? */
1110 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1111 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && first_stmt
&& !slp_node
)
1113 group_size
= vect_cost_group_size (stmt_info
);
1114 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1116 /* Not a grouped access. */
1123 /* We assume that the cost of a single load-lanes instruction is
1124 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1125 access is instead being provided by a load-and-permute operation,
1126 include the cost of the permutes. */
1127 if (!load_lanes_p
&& group_size
> 1
1128 && !STMT_VINFO_STRIDED_P (stmt_info
))
1130 /* Uses an even and odd extract operations or shuffle operations
1131 for each needed permute. */
1132 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1133 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1134 stmt_info
, 0, vect_body
);
1136 if (dump_enabled_p ())
1137 dump_printf_loc (MSG_NOTE
, vect_location
,
1138 "vect_model_load_cost: strided group_size = %d .\n",
1142 /* The loads themselves. */
1143 if (STMT_VINFO_STRIDED_P (stmt_info
)
1144 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1146 /* N scalar loads plus gathering them into a vector. */
1147 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1148 inside_cost
+= record_stmt_cost (body_cost_vec
,
1149 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1150 scalar_load
, stmt_info
, 0, vect_body
);
1153 vect_get_load_cost (first_dr
, ncopies
,
1154 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1155 || group_size
> 1 || slp_node
),
1156 &inside_cost
, &prologue_cost
,
1157 prologue_cost_vec
, body_cost_vec
, true);
1158 if (STMT_VINFO_STRIDED_P (stmt_info
))
1159 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1160 stmt_info
, 0, vect_body
);
1162 if (dump_enabled_p ())
1163 dump_printf_loc (MSG_NOTE
, vect_location
,
1164 "vect_model_load_cost: inside_cost = %d, "
1165 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1169 /* Calculate cost of DR's memory access. */
1171 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1172 bool add_realign_cost
, unsigned int *inside_cost
,
1173 unsigned int *prologue_cost
,
1174 stmt_vector_for_cost
*prologue_cost_vec
,
1175 stmt_vector_for_cost
*body_cost_vec
,
1176 bool record_prologue_costs
)
1178 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1179 gimple stmt
= DR_STMT (dr
);
1180 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1182 switch (alignment_support_scheme
)
1186 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1187 stmt_info
, 0, vect_body
);
1189 if (dump_enabled_p ())
1190 dump_printf_loc (MSG_NOTE
, vect_location
,
1191 "vect_model_load_cost: aligned.\n");
1195 case dr_unaligned_supported
:
1197 /* Here, we assign an additional cost for the unaligned load. */
1198 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1199 unaligned_load
, stmt_info
,
1200 DR_MISALIGNMENT (dr
), vect_body
);
1202 if (dump_enabled_p ())
1203 dump_printf_loc (MSG_NOTE
, vect_location
,
1204 "vect_model_load_cost: unaligned supported by "
1209 case dr_explicit_realign
:
1211 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1212 vector_load
, stmt_info
, 0, vect_body
);
1213 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1214 vec_perm
, stmt_info
, 0, vect_body
);
1216 /* FIXME: If the misalignment remains fixed across the iterations of
1217 the containing loop, the following cost should be added to the
1219 if (targetm
.vectorize
.builtin_mask_for_load
)
1220 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1221 stmt_info
, 0, vect_body
);
1223 if (dump_enabled_p ())
1224 dump_printf_loc (MSG_NOTE
, vect_location
,
1225 "vect_model_load_cost: explicit realign\n");
1229 case dr_explicit_realign_optimized
:
1231 if (dump_enabled_p ())
1232 dump_printf_loc (MSG_NOTE
, vect_location
,
1233 "vect_model_load_cost: unaligned software "
1236 /* Unaligned software pipeline has a load of an address, an initial
1237 load, and possibly a mask operation to "prime" the loop. However,
1238 if this is an access in a group of loads, which provide grouped
1239 access, then the above cost should only be considered for one
1240 access in the group. Inside the loop, there is a load op
1241 and a realignment op. */
1243 if (add_realign_cost
&& record_prologue_costs
)
1245 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1246 vector_stmt
, stmt_info
,
1248 if (targetm
.vectorize
.builtin_mask_for_load
)
1249 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1250 vector_stmt
, stmt_info
,
1254 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1255 stmt_info
, 0, vect_body
);
1256 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1257 stmt_info
, 0, vect_body
);
1259 if (dump_enabled_p ())
1260 dump_printf_loc (MSG_NOTE
, vect_location
,
1261 "vect_model_load_cost: explicit realign optimized"
1267 case dr_unaligned_unsupported
:
1269 *inside_cost
= VECT_MAX_COST
;
1271 if (dump_enabled_p ())
1272 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1273 "vect_model_load_cost: unsupported access.\n");
1282 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1283 the loop preheader for the vectorized stmt STMT. */
1286 vect_init_vector_1 (gimple stmt
, gimple new_stmt
, gimple_stmt_iterator
*gsi
)
1289 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1292 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1293 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1297 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1301 if (nested_in_vect_loop_p (loop
, stmt
))
1304 pe
= loop_preheader_edge (loop
);
1305 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1306 gcc_assert (!new_bb
);
1310 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1312 gimple_stmt_iterator gsi_bb_start
;
1314 gcc_assert (bb_vinfo
);
1315 bb
= BB_VINFO_BB (bb_vinfo
);
1316 gsi_bb_start
= gsi_after_labels (bb
);
1317 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1321 if (dump_enabled_p ())
1323 dump_printf_loc (MSG_NOTE
, vect_location
,
1324 "created new init_stmt: ");
1325 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1329 /* Function vect_init_vector.
1331 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1332 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1333 vector type a vector with all elements equal to VAL is created first.
1334 Place the initialization at BSI if it is not NULL. Otherwise, place the
1335 initialization at the loop preheader.
1336 Return the DEF of INIT_STMT.
1337 It will be used in the vectorization of STMT. */
1340 vect_init_vector (gimple stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1347 if (TREE_CODE (type
) == VECTOR_TYPE
1348 && TREE_CODE (TREE_TYPE (val
)) != VECTOR_TYPE
)
1350 if (!types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1352 if (CONSTANT_CLASS_P (val
))
1353 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (type
), val
);
1356 new_temp
= make_ssa_name (TREE_TYPE (type
));
1357 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1358 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1362 val
= build_vector_from_val (type
, val
);
1365 new_var
= vect_get_new_vect_var (type
, vect_simple_var
, "cst_");
1366 init_stmt
= gimple_build_assign (new_var
, val
);
1367 new_temp
= make_ssa_name (new_var
, init_stmt
);
1368 gimple_assign_set_lhs (init_stmt
, new_temp
);
1369 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1370 vec_oprnd
= gimple_assign_lhs (init_stmt
);
1375 /* Function vect_get_vec_def_for_operand.
1377 OP is an operand in STMT. This function returns a (vector) def that will be
1378 used in the vectorized stmt for STMT.
1380 In the case that OP is an SSA_NAME which is defined in the loop, then
1381 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1383 In case OP is an invariant or constant, a new stmt that creates a vector def
1384 needs to be introduced. */
1387 vect_get_vec_def_for_operand (tree op
, gimple stmt
, tree
*scalar_def
)
1392 stmt_vec_info def_stmt_info
= NULL
;
1393 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1394 unsigned int nunits
;
1395 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1397 enum vect_def_type dt
;
1401 if (dump_enabled_p ())
1403 dump_printf_loc (MSG_NOTE
, vect_location
,
1404 "vect_get_vec_def_for_operand: ");
1405 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1406 dump_printf (MSG_NOTE
, "\n");
1409 is_simple_use
= vect_is_simple_use (op
, stmt
, loop_vinfo
, NULL
,
1410 &def_stmt
, &def
, &dt
);
1411 gcc_assert (is_simple_use
);
1412 if (dump_enabled_p ())
1414 int loc_printed
= 0;
1417 dump_printf_loc (MSG_NOTE
, vect_location
, "def = ");
1419 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, def
);
1420 dump_printf (MSG_NOTE
, "\n");
1425 dump_printf (MSG_NOTE
, " def_stmt = ");
1427 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1428 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1434 /* Case 1: operand is a constant. */
1435 case vect_constant_def
:
1437 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1438 gcc_assert (vector_type
);
1439 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
1444 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1445 if (dump_enabled_p ())
1446 dump_printf_loc (MSG_NOTE
, vect_location
,
1447 "Create vector_cst. nunits = %d\n", nunits
);
1449 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1452 /* Case 2: operand is defined outside the loop - loop invariant. */
1453 case vect_external_def
:
1455 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (def
));
1456 gcc_assert (vector_type
);
1461 /* Create 'vec_inv = {inv,inv,..,inv}' */
1462 if (dump_enabled_p ())
1463 dump_printf_loc (MSG_NOTE
, vect_location
, "Create vector_inv.\n");
1465 return vect_init_vector (stmt
, def
, vector_type
, NULL
);
1468 /* Case 3: operand is defined inside the loop. */
1469 case vect_internal_def
:
1472 *scalar_def
= NULL
/* FIXME tuples: def_stmt*/;
1474 /* Get the def from the vectorized stmt. */
1475 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1477 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1478 /* Get vectorized pattern statement. */
1480 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1481 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1482 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1483 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1484 gcc_assert (vec_stmt
);
1485 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1486 vec_oprnd
= PHI_RESULT (vec_stmt
);
1487 else if (is_gimple_call (vec_stmt
))
1488 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1490 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1494 /* Case 4: operand is defined by a loop header phi - reduction */
1495 case vect_reduction_def
:
1496 case vect_double_reduction_def
:
1497 case vect_nested_cycle
:
1501 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1502 loop
= (gimple_bb (def_stmt
))->loop_father
;
1504 /* Get the def before the loop */
1505 op
= PHI_ARG_DEF_FROM_EDGE (def_stmt
, loop_preheader_edge (loop
));
1506 return get_initial_def_for_reduction (stmt
, op
, scalar_def
);
1509 /* Case 5: operand is defined by loop-header phi - induction. */
1510 case vect_induction_def
:
1512 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1514 /* Get the def from the vectorized stmt. */
1515 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1516 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1517 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1518 vec_oprnd
= PHI_RESULT (vec_stmt
);
1520 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1530 /* Function vect_get_vec_def_for_stmt_copy
1532 Return a vector-def for an operand. This function is used when the
1533 vectorized stmt to be created (by the caller to this function) is a "copy"
1534 created in case the vectorized result cannot fit in one vector, and several
1535 copies of the vector-stmt are required. In this case the vector-def is
1536 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1537 of the stmt that defines VEC_OPRND.
1538 DT is the type of the vector def VEC_OPRND.
1541 In case the vectorization factor (VF) is bigger than the number
1542 of elements that can fit in a vectype (nunits), we have to generate
1543 more than one vector stmt to vectorize the scalar stmt. This situation
1544 arises when there are multiple data-types operated upon in the loop; the
1545 smallest data-type determines the VF, and as a result, when vectorizing
1546 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1547 vector stmt (each computing a vector of 'nunits' results, and together
1548 computing 'VF' results in each iteration). This function is called when
1549 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1550 which VF=16 and nunits=4, so the number of copies required is 4):
1552 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1554 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1555 VS1.1: vx.1 = memref1 VS1.2
1556 VS1.2: vx.2 = memref2 VS1.3
1557 VS1.3: vx.3 = memref3
1559 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1560 VSnew.1: vz1 = vx.1 + ... VSnew.2
1561 VSnew.2: vz2 = vx.2 + ... VSnew.3
1562 VSnew.3: vz3 = vx.3 + ...
1564 The vectorization of S1 is explained in vectorizable_load.
1565 The vectorization of S2:
1566 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1567 the function 'vect_get_vec_def_for_operand' is called to
1568 get the relevant vector-def for each operand of S2. For operand x it
1569 returns the vector-def 'vx.0'.
1571 To create the remaining copies of the vector-stmt (VSnew.j), this
1572 function is called to get the relevant vector-def for each operand. It is
1573 obtained from the respective VS1.j stmt, which is recorded in the
1574 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1576 For example, to obtain the vector-def 'vx.1' in order to create the
1577 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1578 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1579 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1580 and return its def ('vx.1').
1581 Overall, to create the above sequence this function will be called 3 times:
1582 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1583 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1584 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1587 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1589 gimple vec_stmt_for_operand
;
1590 stmt_vec_info def_stmt_info
;
1592 /* Do nothing; can reuse same def. */
1593 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1596 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1597 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1598 gcc_assert (def_stmt_info
);
1599 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1600 gcc_assert (vec_stmt_for_operand
);
1601 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1602 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1603 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1605 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1610 /* Get vectorized definitions for the operands to create a copy of an original
1611 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1614 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1615 vec
<tree
> *vec_oprnds0
,
1616 vec
<tree
> *vec_oprnds1
)
1618 tree vec_oprnd
= vec_oprnds0
->pop ();
1620 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1621 vec_oprnds0
->quick_push (vec_oprnd
);
1623 if (vec_oprnds1
&& vec_oprnds1
->length ())
1625 vec_oprnd
= vec_oprnds1
->pop ();
1626 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1627 vec_oprnds1
->quick_push (vec_oprnd
);
1632 /* Get vectorized definitions for OP0 and OP1.
1633 REDUC_INDEX is the index of reduction operand in case of reduction,
1634 and -1 otherwise. */
1637 vect_get_vec_defs (tree op0
, tree op1
, gimple stmt
,
1638 vec
<tree
> *vec_oprnds0
,
1639 vec
<tree
> *vec_oprnds1
,
1640 slp_tree slp_node
, int reduc_index
)
1644 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1645 auto_vec
<tree
> ops (nops
);
1646 auto_vec
<vec
<tree
> > vec_defs (nops
);
1648 ops
.quick_push (op0
);
1650 ops
.quick_push (op1
);
1652 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1654 *vec_oprnds0
= vec_defs
[0];
1656 *vec_oprnds1
= vec_defs
[1];
1662 vec_oprnds0
->create (1);
1663 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1664 vec_oprnds0
->quick_push (vec_oprnd
);
1668 vec_oprnds1
->create (1);
1669 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
1670 vec_oprnds1
->quick_push (vec_oprnd
);
1676 /* Function vect_finish_stmt_generation.
1678 Insert a new stmt. */
1681 vect_finish_stmt_generation (gimple stmt
, gimple vec_stmt
,
1682 gimple_stmt_iterator
*gsi
)
1684 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1685 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1686 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1688 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1690 if (!gsi_end_p (*gsi
)
1691 && gimple_has_mem_ops (vec_stmt
))
1693 gimple at_stmt
= gsi_stmt (*gsi
);
1694 tree vuse
= gimple_vuse (at_stmt
);
1695 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1697 tree vdef
= gimple_vdef (at_stmt
);
1698 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1699 /* If we have an SSA vuse and insert a store, update virtual
1700 SSA form to avoid triggering the renamer. Do so only
1701 if we can easily see all uses - which is what almost always
1702 happens with the way vectorized stmts are inserted. */
1703 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1704 && ((is_gimple_assign (vec_stmt
)
1705 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1706 || (is_gimple_call (vec_stmt
)
1707 && !(gimple_call_flags (vec_stmt
)
1708 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1710 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1711 gimple_set_vdef (vec_stmt
, new_vdef
);
1712 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1716 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1718 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, loop_vinfo
,
1721 if (dump_enabled_p ())
1723 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1724 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1727 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1729 /* While EH edges will generally prevent vectorization, stmt might
1730 e.g. be in a must-not-throw region. Ensure newly created stmts
1731 that could throw are part of the same region. */
1732 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1733 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1734 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1737 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1738 a function declaration if the target has a vectorized version
1739 of the function, or NULL_TREE if the function cannot be vectorized. */
1742 vectorizable_function (gcall
*call
, tree vectype_out
, tree vectype_in
)
1744 tree fndecl
= gimple_call_fndecl (call
);
1746 /* We only handle functions that do not read or clobber memory -- i.e.
1747 const or novops ones. */
1748 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1752 || TREE_CODE (fndecl
) != FUNCTION_DECL
1753 || !DECL_BUILT_IN (fndecl
))
1756 return targetm
.vectorize
.builtin_vectorized_function (fndecl
, vectype_out
,
1761 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
,
1762 gimple_stmt_iterator
*);
1765 /* Function vectorizable_mask_load_store.
1767 Check if STMT performs a conditional load or store that can be vectorized.
1768 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1769 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1770 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1773 vectorizable_mask_load_store (gimple stmt
, gimple_stmt_iterator
*gsi
,
1774 gimple
*vec_stmt
, slp_tree slp_node
)
1776 tree vec_dest
= NULL
;
1777 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1778 stmt_vec_info prev_stmt_info
;
1779 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1780 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1781 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1782 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1783 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1787 tree dataref_ptr
= NULL_TREE
;
1789 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1793 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
1794 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
1795 int gather_scale
= 1;
1796 enum vect_def_type gather_dt
= vect_unknown_def_type
;
1801 enum vect_def_type dt
;
1803 if (slp_node
!= NULL
)
1806 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1807 gcc_assert (ncopies
>= 1);
1809 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1810 mask
= gimple_call_arg (stmt
, 2);
1811 if (TYPE_PRECISION (TREE_TYPE (mask
))
1812 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
))))
1815 /* FORNOW. This restriction should be relaxed. */
1816 if (nested_in_vect_loop
&& ncopies
> 1)
1818 if (dump_enabled_p ())
1819 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1820 "multiple types in nested loop.");
1824 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1827 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1830 if (!STMT_VINFO_DATA_REF (stmt_info
))
1833 elem_type
= TREE_TYPE (vectype
);
1835 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1838 if (STMT_VINFO_STRIDED_P (stmt_info
))
1841 if (STMT_VINFO_GATHER_P (stmt_info
))
1845 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
1846 &gather_off
, &gather_scale
);
1847 gcc_assert (gather_decl
);
1848 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, NULL
,
1849 &def_stmt
, &def
, &gather_dt
,
1850 &gather_off_vectype
))
1852 if (dump_enabled_p ())
1853 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1854 "gather index use not simple.");
1858 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1860 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1861 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1863 if (dump_enabled_p ())
1864 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1865 "masked gather with integer mask not supported.");
1869 else if (tree_int_cst_compare (nested_in_vect_loop
1870 ? STMT_VINFO_DR_STEP (stmt_info
)
1871 : DR_STEP (dr
), size_zero_node
) <= 0)
1873 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1874 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
), !is_store
))
1877 if (TREE_CODE (mask
) != SSA_NAME
)
1880 if (!vect_is_simple_use (mask
, stmt
, loop_vinfo
, NULL
,
1881 &def_stmt
, &def
, &dt
))
1886 tree rhs
= gimple_call_arg (stmt
, 3);
1887 if (!vect_is_simple_use (rhs
, stmt
, loop_vinfo
, NULL
,
1888 &def_stmt
, &def
, &dt
))
1892 if (!vec_stmt
) /* transformation not required. */
1894 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1896 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1899 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1905 if (STMT_VINFO_GATHER_P (stmt_info
))
1907 tree vec_oprnd0
= NULL_TREE
, op
;
1908 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1909 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1910 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1911 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1912 tree mask_perm_mask
= NULL_TREE
;
1913 edge pe
= loop_preheader_edge (loop
);
1916 enum { NARROW
, NONE
, WIDEN
} modifier
;
1917 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
1919 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
1920 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1921 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1922 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1923 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1924 scaletype
= TREE_VALUE (arglist
);
1925 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1926 && types_compatible_p (srctype
, masktype
));
1928 if (nunits
== gather_off_nunits
)
1930 else if (nunits
== gather_off_nunits
/ 2)
1932 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1935 for (i
= 0; i
< gather_off_nunits
; ++i
)
1936 sel
[i
] = i
| nunits
;
1938 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
1940 else if (nunits
== gather_off_nunits
* 2)
1942 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1945 for (i
= 0; i
< nunits
; ++i
)
1946 sel
[i
] = i
< gather_off_nunits
1947 ? i
: i
+ nunits
- gather_off_nunits
;
1949 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
1951 for (i
= 0; i
< nunits
; ++i
)
1952 sel
[i
] = i
| gather_off_nunits
;
1953 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
1958 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1960 ptr
= fold_convert (ptrtype
, gather_base
);
1961 if (!is_gimple_min_invariant (ptr
))
1963 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1964 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1965 gcc_assert (!new_bb
);
1968 scale
= build_int_cst (scaletype
, gather_scale
);
1970 prev_stmt_info
= NULL
;
1971 for (j
= 0; j
< ncopies
; ++j
)
1973 if (modifier
== WIDEN
&& (j
& 1))
1974 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1975 perm_mask
, stmt
, gsi
);
1978 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
1981 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
1983 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1985 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1986 == TYPE_VECTOR_SUBPARTS (idxtype
));
1987 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
1988 var
= make_ssa_name (var
);
1989 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1991 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1992 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1996 if (mask_perm_mask
&& (j
& 1))
1997 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1998 mask_perm_mask
, stmt
, gsi
);
2002 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2005 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
,
2006 &def_stmt
, &def
, &dt
);
2007 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2011 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
2013 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
2014 == TYPE_VECTOR_SUBPARTS (masktype
));
2015 var
= vect_get_new_vect_var (masktype
, vect_simple_var
,
2017 var
= make_ssa_name (var
);
2018 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
2020 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
2021 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2027 = gimple_build_call (gather_decl
, 5, mask_op
, ptr
, op
, mask_op
,
2030 if (!useless_type_conversion_p (vectype
, rettype
))
2032 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
2033 == TYPE_VECTOR_SUBPARTS (rettype
));
2034 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
2035 op
= make_ssa_name (var
, new_stmt
);
2036 gimple_call_set_lhs (new_stmt
, op
);
2037 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2038 var
= make_ssa_name (vec_dest
);
2039 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2040 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2044 var
= make_ssa_name (vec_dest
, new_stmt
);
2045 gimple_call_set_lhs (new_stmt
, var
);
2048 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2050 if (modifier
== NARROW
)
2057 var
= permute_vec_elements (prev_res
, var
,
2058 perm_mask
, stmt
, gsi
);
2059 new_stmt
= SSA_NAME_DEF_STMT (var
);
2062 if (prev_stmt_info
== NULL
)
2063 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2065 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2066 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2069 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2071 tree lhs
= gimple_call_lhs (stmt
);
2072 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2073 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2074 set_vinfo_for_stmt (stmt
, NULL
);
2075 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2076 gsi_replace (gsi
, new_stmt
, true);
2081 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
2082 prev_stmt_info
= NULL
;
2083 for (i
= 0; i
< ncopies
; i
++)
2085 unsigned align
, misalign
;
2089 tree rhs
= gimple_call_arg (stmt
, 3);
2090 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
, NULL
);
2091 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2092 /* We should have catched mismatched types earlier. */
2093 gcc_assert (useless_type_conversion_p (vectype
,
2094 TREE_TYPE (vec_rhs
)));
2095 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2096 NULL_TREE
, &dummy
, gsi
,
2097 &ptr_incr
, false, &inv_p
);
2098 gcc_assert (!inv_p
);
2102 vect_is_simple_use (vec_rhs
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2104 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2105 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2107 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2108 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2109 TYPE_SIZE_UNIT (vectype
));
2112 align
= TYPE_ALIGN_UNIT (vectype
);
2113 if (aligned_access_p (dr
))
2115 else if (DR_MISALIGNMENT (dr
) == -1)
2117 align
= TYPE_ALIGN_UNIT (elem_type
);
2121 misalign
= DR_MISALIGNMENT (dr
);
2122 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2125 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2126 gimple_call_arg (stmt
, 1),
2128 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2130 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2132 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2133 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2138 tree vec_mask
= NULL_TREE
;
2139 prev_stmt_info
= NULL
;
2140 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2141 for (i
= 0; i
< ncopies
; i
++)
2143 unsigned align
, misalign
;
2147 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2148 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2149 NULL_TREE
, &dummy
, gsi
,
2150 &ptr_incr
, false, &inv_p
);
2151 gcc_assert (!inv_p
);
2155 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2157 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2158 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2159 TYPE_SIZE_UNIT (vectype
));
2162 align
= TYPE_ALIGN_UNIT (vectype
);
2163 if (aligned_access_p (dr
))
2165 else if (DR_MISALIGNMENT (dr
) == -1)
2167 align
= TYPE_ALIGN_UNIT (elem_type
);
2171 misalign
= DR_MISALIGNMENT (dr
);
2172 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2175 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2176 gimple_call_arg (stmt
, 1),
2178 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
));
2179 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2181 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2183 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2184 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2190 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2192 tree lhs
= gimple_call_lhs (stmt
);
2193 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2194 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2195 set_vinfo_for_stmt (stmt
, NULL
);
2196 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2197 gsi_replace (gsi
, new_stmt
, true);
2204 /* Function vectorizable_call.
2206 Check if GS performs a function call that can be vectorized.
2207 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2208 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2209 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2212 vectorizable_call (gimple gs
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
2219 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2220 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2221 tree vectype_out
, vectype_in
;
2224 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2225 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2226 tree fndecl
, new_temp
, def
, rhs_type
;
2228 enum vect_def_type dt
[3]
2229 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2230 gimple new_stmt
= NULL
;
2232 vec
<tree
> vargs
= vNULL
;
2233 enum { NARROW
, NONE
, WIDEN
} modifier
;
2237 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2240 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2243 /* Is GS a vectorizable call? */
2244 stmt
= dyn_cast
<gcall
*> (gs
);
2248 if (gimple_call_internal_p (stmt
)
2249 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2250 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2251 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2254 if (gimple_call_lhs (stmt
) == NULL_TREE
2255 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2258 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2260 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2262 /* Process function arguments. */
2263 rhs_type
= NULL_TREE
;
2264 vectype_in
= NULL_TREE
;
2265 nargs
= gimple_call_num_args (stmt
);
2267 /* Bail out if the function has more than three arguments, we do not have
2268 interesting builtin functions to vectorize with more than two arguments
2269 except for fma. No arguments is also not good. */
2270 if (nargs
== 0 || nargs
> 3)
2273 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2274 if (gimple_call_internal_p (stmt
)
2275 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2278 rhs_type
= unsigned_type_node
;
2281 for (i
= 0; i
< nargs
; i
++)
2285 op
= gimple_call_arg (stmt
, i
);
2287 /* We can only handle calls with arguments of the same type. */
2289 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2291 if (dump_enabled_p ())
2292 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2293 "argument types differ.\n");
2297 rhs_type
= TREE_TYPE (op
);
2299 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2300 &def_stmt
, &def
, &dt
[i
], &opvectype
))
2302 if (dump_enabled_p ())
2303 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2304 "use not simple.\n");
2309 vectype_in
= opvectype
;
2311 && opvectype
!= vectype_in
)
2313 if (dump_enabled_p ())
2314 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2315 "argument vector types differ.\n");
2319 /* If all arguments are external or constant defs use a vector type with
2320 the same size as the output vector type. */
2322 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2324 gcc_assert (vectype_in
);
2327 if (dump_enabled_p ())
2329 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2330 "no vectype for scalar type ");
2331 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2332 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2339 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2340 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2341 if (nunits_in
== nunits_out
/ 2)
2343 else if (nunits_out
== nunits_in
)
2345 else if (nunits_out
== nunits_in
/ 2)
2350 /* For now, we only vectorize functions if a target specific builtin
2351 is available. TODO -- in some cases, it might be profitable to
2352 insert the calls for pieces of the vector, in order to be able
2353 to vectorize other operations in the loop. */
2354 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
2355 if (fndecl
== NULL_TREE
)
2357 if (gimple_call_internal_p (stmt
)
2358 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
2361 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2362 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2363 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2364 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2366 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2367 { 0, 1, 2, ... vf - 1 } vector. */
2368 gcc_assert (nargs
== 0);
2372 if (dump_enabled_p ())
2373 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2374 "function is not vectorizable.\n");
2379 gcc_assert (!gimple_vuse (stmt
));
2381 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2383 else if (modifier
== NARROW
)
2384 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2386 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2388 /* Sanity check: make sure that at least one copy of the vectorized stmt
2389 needs to be generated. */
2390 gcc_assert (ncopies
>= 1);
2392 if (!vec_stmt
) /* transformation not required. */
2394 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2395 if (dump_enabled_p ())
2396 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2398 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2404 if (dump_enabled_p ())
2405 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2408 scalar_dest
= gimple_call_lhs (stmt
);
2409 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2411 prev_stmt_info
= NULL
;
2415 for (j
= 0; j
< ncopies
; ++j
)
2417 /* Build argument list for the vectorized call. */
2419 vargs
.create (nargs
);
2425 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2426 vec
<tree
> vec_oprnds0
;
2428 for (i
= 0; i
< nargs
; i
++)
2429 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2430 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2431 vec_oprnds0
= vec_defs
[0];
2433 /* Arguments are ready. Create the new vector stmt. */
2434 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2437 for (k
= 0; k
< nargs
; k
++)
2439 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2440 vargs
[k
] = vec_oprndsk
[i
];
2442 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2443 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2444 gimple_call_set_lhs (new_stmt
, new_temp
);
2445 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2446 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2449 for (i
= 0; i
< nargs
; i
++)
2451 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2452 vec_oprndsi
.release ();
2457 for (i
= 0; i
< nargs
; i
++)
2459 op
= gimple_call_arg (stmt
, i
);
2462 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2465 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2467 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2470 vargs
.quick_push (vec_oprnd0
);
2473 if (gimple_call_internal_p (stmt
)
2474 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2476 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2478 for (k
= 0; k
< nunits_out
; ++k
)
2479 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2480 tree cst
= build_vector (vectype_out
, v
);
2482 = vect_get_new_vect_var (vectype_out
, vect_simple_var
, "cst_");
2483 gimple init_stmt
= gimple_build_assign (new_var
, cst
);
2484 new_temp
= make_ssa_name (new_var
, init_stmt
);
2485 gimple_assign_set_lhs (init_stmt
, new_temp
);
2486 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2487 new_temp
= make_ssa_name (vec_dest
);
2488 new_stmt
= gimple_build_assign (new_temp
,
2489 gimple_assign_lhs (init_stmt
));
2493 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2494 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2495 gimple_call_set_lhs (new_stmt
, new_temp
);
2497 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2500 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2502 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2504 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2510 for (j
= 0; j
< ncopies
; ++j
)
2512 /* Build argument list for the vectorized call. */
2514 vargs
.create (nargs
* 2);
2520 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2521 vec
<tree
> vec_oprnds0
;
2523 for (i
= 0; i
< nargs
; i
++)
2524 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2525 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2526 vec_oprnds0
= vec_defs
[0];
2528 /* Arguments are ready. Create the new vector stmt. */
2529 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2533 for (k
= 0; k
< nargs
; k
++)
2535 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2536 vargs
.quick_push (vec_oprndsk
[i
]);
2537 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2539 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2540 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2541 gimple_call_set_lhs (new_stmt
, new_temp
);
2542 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2543 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2546 for (i
= 0; i
< nargs
; i
++)
2548 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2549 vec_oprndsi
.release ();
2554 for (i
= 0; i
< nargs
; i
++)
2556 op
= gimple_call_arg (stmt
, i
);
2560 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2562 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2566 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2568 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2570 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2573 vargs
.quick_push (vec_oprnd0
);
2574 vargs
.quick_push (vec_oprnd1
);
2577 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2578 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2579 gimple_call_set_lhs (new_stmt
, new_temp
);
2580 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2583 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2585 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2587 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2590 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2595 /* No current target implements this case. */
2601 /* The call in STMT might prevent it from being removed in dce.
2602 We however cannot remove it here, due to the way the ssa name
2603 it defines is mapped to the new definition. So just replace
2604 rhs of the statement with something harmless. */
2609 type
= TREE_TYPE (scalar_dest
);
2610 if (is_pattern_stmt_p (stmt_info
))
2611 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2613 lhs
= gimple_call_lhs (stmt
);
2614 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2615 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2616 set_vinfo_for_stmt (stmt
, NULL
);
2617 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2618 gsi_replace (gsi
, new_stmt
, false);
2624 struct simd_call_arg_info
2628 enum vect_def_type dt
;
2629 HOST_WIDE_INT linear_step
;
2633 /* Function vectorizable_simd_clone_call.
2635 Check if STMT performs a function call that can be vectorized
2636 by calling a simd clone of the function.
2637 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2638 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2639 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2642 vectorizable_simd_clone_call (gimple stmt
, gimple_stmt_iterator
*gsi
,
2643 gimple
*vec_stmt
, slp_tree slp_node
)
2648 tree vec_oprnd0
= NULL_TREE
;
2649 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2651 unsigned int nunits
;
2652 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2653 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2654 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2655 tree fndecl
, new_temp
, def
;
2657 gimple new_stmt
= NULL
;
2659 vec
<simd_call_arg_info
> arginfo
= vNULL
;
2660 vec
<tree
> vargs
= vNULL
;
2662 tree lhs
, rtype
, ratype
;
2663 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2665 /* Is STMT a vectorizable call? */
2666 if (!is_gimple_call (stmt
))
2669 fndecl
= gimple_call_fndecl (stmt
);
2670 if (fndecl
== NULL_TREE
)
2673 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2674 if (node
== NULL
|| node
->simd_clones
== NULL
)
2677 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2680 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2683 if (gimple_call_lhs (stmt
)
2684 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2687 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2689 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2691 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2695 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2698 /* Process function arguments. */
2699 nargs
= gimple_call_num_args (stmt
);
2701 /* Bail out if the function has zero arguments. */
2705 arginfo
.create (nargs
);
2707 for (i
= 0; i
< nargs
; i
++)
2709 simd_call_arg_info thisarginfo
;
2712 thisarginfo
.linear_step
= 0;
2713 thisarginfo
.align
= 0;
2714 thisarginfo
.op
= NULL_TREE
;
2716 op
= gimple_call_arg (stmt
, i
);
2717 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2718 &def_stmt
, &def
, &thisarginfo
.dt
,
2719 &thisarginfo
.vectype
)
2720 || thisarginfo
.dt
== vect_uninitialized_def
)
2722 if (dump_enabled_p ())
2723 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2724 "use not simple.\n");
2729 if (thisarginfo
.dt
== vect_constant_def
2730 || thisarginfo
.dt
== vect_external_def
)
2731 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2733 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2735 /* For linear arguments, the analyze phase should have saved
2736 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2737 if (i
* 2 + 3 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
2738 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2])
2740 gcc_assert (vec_stmt
);
2741 thisarginfo
.linear_step
2742 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2]);
2744 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 1];
2745 /* If loop has been peeled for alignment, we need to adjust it. */
2746 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
2747 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
2750 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
2751 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2];
2752 tree opt
= TREE_TYPE (thisarginfo
.op
);
2753 bias
= fold_convert (TREE_TYPE (step
), bias
);
2754 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
2756 = fold_build2 (POINTER_TYPE_P (opt
)
2757 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
2758 thisarginfo
.op
, bias
);
2762 && thisarginfo
.dt
!= vect_constant_def
2763 && thisarginfo
.dt
!= vect_external_def
2765 && TREE_CODE (op
) == SSA_NAME
2766 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
2768 && tree_fits_shwi_p (iv
.step
))
2770 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2771 thisarginfo
.op
= iv
.base
;
2773 else if ((thisarginfo
.dt
== vect_constant_def
2774 || thisarginfo
.dt
== vect_external_def
)
2775 && POINTER_TYPE_P (TREE_TYPE (op
)))
2776 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2778 arginfo
.quick_push (thisarginfo
);
2781 unsigned int badness
= 0;
2782 struct cgraph_node
*bestn
= NULL
;
2783 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
2784 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
2786 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2787 n
= n
->simdclone
->next_clone
)
2789 unsigned int this_badness
= 0;
2790 if (n
->simdclone
->simdlen
2791 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2792 || n
->simdclone
->nargs
!= nargs
)
2794 if (n
->simdclone
->simdlen
2795 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2796 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2797 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2798 if (n
->simdclone
->inbranch
)
2799 this_badness
+= 2048;
2800 int target_badness
= targetm
.simd_clone
.usable (n
);
2801 if (target_badness
< 0)
2803 this_badness
+= target_badness
* 512;
2804 /* FORNOW: Have to add code to add the mask argument. */
2805 if (n
->simdclone
->inbranch
)
2807 for (i
= 0; i
< nargs
; i
++)
2809 switch (n
->simdclone
->args
[i
].arg_type
)
2811 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2812 if (!useless_type_conversion_p
2813 (n
->simdclone
->args
[i
].orig_type
,
2814 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2816 else if (arginfo
[i
].dt
== vect_constant_def
2817 || arginfo
[i
].dt
== vect_external_def
2818 || arginfo
[i
].linear_step
)
2821 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2822 if (arginfo
[i
].dt
!= vect_constant_def
2823 && arginfo
[i
].dt
!= vect_external_def
)
2826 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2827 if (arginfo
[i
].dt
== vect_constant_def
2828 || arginfo
[i
].dt
== vect_external_def
2829 || (arginfo
[i
].linear_step
2830 != n
->simdclone
->args
[i
].linear_step
))
2833 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2837 case SIMD_CLONE_ARG_TYPE_MASK
:
2840 if (i
== (size_t) -1)
2842 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2847 if (arginfo
[i
].align
)
2848 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2849 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2851 if (i
== (size_t) -1)
2853 if (bestn
== NULL
|| this_badness
< badness
)
2856 badness
= this_badness
;
2866 for (i
= 0; i
< nargs
; i
++)
2867 if ((arginfo
[i
].dt
== vect_constant_def
2868 || arginfo
[i
].dt
== vect_external_def
)
2869 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2872 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2874 if (arginfo
[i
].vectype
== NULL
2875 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2876 > bestn
->simdclone
->simdlen
))
2883 fndecl
= bestn
->decl
;
2884 nunits
= bestn
->simdclone
->simdlen
;
2885 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2887 /* If the function isn't const, only allow it in simd loops where user
2888 has asserted that at least nunits consecutive iterations can be
2889 performed using SIMD instructions. */
2890 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2891 && gimple_vuse (stmt
))
2897 /* Sanity check: make sure that at least one copy of the vectorized stmt
2898 needs to be generated. */
2899 gcc_assert (ncopies
>= 1);
2901 if (!vec_stmt
) /* transformation not required. */
2903 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
2904 for (i
= 0; i
< nargs
; i
++)
2905 if (bestn
->simdclone
->args
[i
].arg_type
2906 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
2908 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 2
2910 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
2911 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
2912 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
2913 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
2914 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
2916 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
2917 if (dump_enabled_p ())
2918 dump_printf_loc (MSG_NOTE
, vect_location
,
2919 "=== vectorizable_simd_clone_call ===\n");
2920 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2927 if (dump_enabled_p ())
2928 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2931 scalar_dest
= gimple_call_lhs (stmt
);
2932 vec_dest
= NULL_TREE
;
2937 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2938 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
2939 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
2942 rtype
= TREE_TYPE (ratype
);
2946 prev_stmt_info
= NULL
;
2947 for (j
= 0; j
< ncopies
; ++j
)
2949 /* Build argument list for the vectorized call. */
2951 vargs
.create (nargs
);
2955 for (i
= 0; i
< nargs
; i
++)
2957 unsigned int k
, l
, m
, o
;
2959 op
= gimple_call_arg (stmt
, i
);
2960 switch (bestn
->simdclone
->args
[i
].arg_type
)
2962 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2963 atype
= bestn
->simdclone
->args
[i
].vector_type
;
2964 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
2965 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
2967 if (TYPE_VECTOR_SUBPARTS (atype
)
2968 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
2970 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
2971 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2972 / TYPE_VECTOR_SUBPARTS (atype
));
2973 gcc_assert ((k
& (k
- 1)) == 0);
2976 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2979 vec_oprnd0
= arginfo
[i
].op
;
2980 if ((m
& (k
- 1)) == 0)
2982 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
2985 arginfo
[i
].op
= vec_oprnd0
;
2987 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
2989 bitsize_int ((m
& (k
- 1)) * prec
));
2991 = gimple_build_assign (make_ssa_name (atype
),
2993 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2994 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
2998 k
= (TYPE_VECTOR_SUBPARTS (atype
)
2999 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
3000 gcc_assert ((k
& (k
- 1)) == 0);
3001 vec
<constructor_elt
, va_gc
> *ctor_elts
;
3003 vec_alloc (ctor_elts
, k
);
3006 for (l
= 0; l
< k
; l
++)
3008 if (m
== 0 && l
== 0)
3010 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
3013 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3015 arginfo
[i
].op
= vec_oprnd0
;
3018 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3022 vargs
.safe_push (vec_oprnd0
);
3025 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3027 = gimple_build_assign (make_ssa_name (atype
),
3029 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3030 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3035 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3036 vargs
.safe_push (op
);
3038 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3043 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3048 edge pe
= loop_preheader_edge (loop
);
3049 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3050 gcc_assert (!new_bb
);
3052 tree phi_res
= copy_ssa_name (op
);
3053 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3054 set_vinfo_for_stmt (new_phi
,
3055 new_stmt_vec_info (new_phi
, loop_vinfo
,
3057 add_phi_arg (new_phi
, arginfo
[i
].op
,
3058 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3060 = POINTER_TYPE_P (TREE_TYPE (op
))
3061 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3062 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3063 ? sizetype
: TREE_TYPE (op
);
3065 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3067 tree tcst
= wide_int_to_tree (type
, cst
);
3068 tree phi_arg
= copy_ssa_name (op
);
3070 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3071 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3072 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3073 set_vinfo_for_stmt (new_stmt
,
3074 new_stmt_vec_info (new_stmt
, loop_vinfo
,
3076 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3078 arginfo
[i
].op
= phi_res
;
3079 vargs
.safe_push (phi_res
);
3084 = POINTER_TYPE_P (TREE_TYPE (op
))
3085 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3086 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3087 ? sizetype
: TREE_TYPE (op
);
3089 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3091 tree tcst
= wide_int_to_tree (type
, cst
);
3092 new_temp
= make_ssa_name (TREE_TYPE (op
));
3093 new_stmt
= gimple_build_assign (new_temp
, code
,
3094 arginfo
[i
].op
, tcst
);
3095 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3096 vargs
.safe_push (new_temp
);
3099 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3105 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3108 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3110 new_temp
= create_tmp_var (ratype
);
3111 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3112 == TYPE_VECTOR_SUBPARTS (rtype
))
3113 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3115 new_temp
= make_ssa_name (rtype
, new_stmt
);
3116 gimple_call_set_lhs (new_stmt
, new_temp
);
3118 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3122 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3125 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3126 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3127 gcc_assert ((k
& (k
- 1)) == 0);
3128 for (l
= 0; l
< k
; l
++)
3133 t
= build_fold_addr_expr (new_temp
);
3134 t
= build2 (MEM_REF
, vectype
, t
,
3135 build_int_cst (TREE_TYPE (t
),
3136 l
* prec
/ BITS_PER_UNIT
));
3139 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3140 size_int (prec
), bitsize_int (l
* prec
));
3142 = gimple_build_assign (make_ssa_name (vectype
), t
);
3143 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3144 if (j
== 0 && l
== 0)
3145 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3147 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3149 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3154 tree clobber
= build_constructor (ratype
, NULL
);
3155 TREE_THIS_VOLATILE (clobber
) = 1;
3156 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3157 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3161 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3163 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3164 / TYPE_VECTOR_SUBPARTS (rtype
));
3165 gcc_assert ((k
& (k
- 1)) == 0);
3166 if ((j
& (k
- 1)) == 0)
3167 vec_alloc (ret_ctor_elts
, k
);
3170 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3171 for (m
= 0; m
< o
; m
++)
3173 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3174 size_int (m
), NULL_TREE
, NULL_TREE
);
3176 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3177 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3178 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3179 gimple_assign_lhs (new_stmt
));
3181 tree clobber
= build_constructor (ratype
, NULL
);
3182 TREE_THIS_VOLATILE (clobber
) = 1;
3183 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3184 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3187 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3188 if ((j
& (k
- 1)) != k
- 1)
3190 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3192 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3193 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3195 if ((unsigned) j
== k
- 1)
3196 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3198 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3200 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3205 tree t
= build_fold_addr_expr (new_temp
);
3206 t
= build2 (MEM_REF
, vectype
, t
,
3207 build_int_cst (TREE_TYPE (t
), 0));
3209 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3210 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3211 tree clobber
= build_constructor (ratype
, NULL
);
3212 TREE_THIS_VOLATILE (clobber
) = 1;
3213 vect_finish_stmt_generation (stmt
,
3214 gimple_build_assign (new_temp
,
3220 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3222 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3224 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3229 /* The call in STMT might prevent it from being removed in dce.
3230 We however cannot remove it here, due to the way the ssa name
3231 it defines is mapped to the new definition. So just replace
3232 rhs of the statement with something harmless. */
3239 type
= TREE_TYPE (scalar_dest
);
3240 if (is_pattern_stmt_p (stmt_info
))
3241 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3243 lhs
= gimple_call_lhs (stmt
);
3244 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3247 new_stmt
= gimple_build_nop ();
3248 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3249 set_vinfo_for_stmt (stmt
, NULL
);
3250 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3251 gsi_replace (gsi
, new_stmt
, true);
3252 unlink_stmt_vdef (stmt
);
3258 /* Function vect_gen_widened_results_half
3260 Create a vector stmt whose code, type, number of arguments, and result
3261 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3262 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3263 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3264 needs to be created (DECL is a function-decl of a target-builtin).
3265 STMT is the original scalar stmt that we are vectorizing. */
3268 vect_gen_widened_results_half (enum tree_code code
,
3270 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3271 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3277 /* Generate half of the widened result: */
3278 if (code
== CALL_EXPR
)
3280 /* Target specific support */
3281 if (op_type
== binary_op
)
3282 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3284 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3285 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3286 gimple_call_set_lhs (new_stmt
, new_temp
);
3290 /* Generic support */
3291 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3292 if (op_type
!= binary_op
)
3294 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3295 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3296 gimple_assign_set_lhs (new_stmt
, new_temp
);
3298 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3304 /* Get vectorized definitions for loop-based vectorization. For the first
3305 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3306 scalar operand), and for the rest we get a copy with
3307 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3308 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3309 The vectors are collected into VEC_OPRNDS. */
3312 vect_get_loop_based_defs (tree
*oprnd
, gimple stmt
, enum vect_def_type dt
,
3313 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3317 /* Get first vector operand. */
3318 /* All the vector operands except the very first one (that is scalar oprnd)
3320 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3321 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
, NULL
);
3323 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3325 vec_oprnds
->quick_push (vec_oprnd
);
3327 /* Get second vector operand. */
3328 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3329 vec_oprnds
->quick_push (vec_oprnd
);
3333 /* For conversion in multiple steps, continue to get operands
3336 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3340 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3341 For multi-step conversions store the resulting vectors and call the function
3345 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3346 int multi_step_cvt
, gimple stmt
,
3348 gimple_stmt_iterator
*gsi
,
3349 slp_tree slp_node
, enum tree_code code
,
3350 stmt_vec_info
*prev_stmt_info
)
3353 tree vop0
, vop1
, new_tmp
, vec_dest
;
3355 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3357 vec_dest
= vec_dsts
.pop ();
3359 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3361 /* Create demotion operation. */
3362 vop0
= (*vec_oprnds
)[i
];
3363 vop1
= (*vec_oprnds
)[i
+ 1];
3364 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3365 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3366 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3367 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3370 /* Store the resulting vector for next recursive call. */
3371 (*vec_oprnds
)[i
/2] = new_tmp
;
3374 /* This is the last step of the conversion sequence. Store the
3375 vectors in SLP_NODE or in vector info of the scalar statement
3376 (or in STMT_VINFO_RELATED_STMT chain). */
3378 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3380 if (!*prev_stmt_info
)
3381 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3383 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3385 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3389 /* For multi-step demotion operations we first generate demotion operations
3390 from the source type to the intermediate types, and then combine the
3391 results (stored in VEC_OPRNDS) in demotion operation to the destination
3395 /* At each level of recursion we have half of the operands we had at the
3397 vec_oprnds
->truncate ((i
+1)/2);
3398 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3399 stmt
, vec_dsts
, gsi
, slp_node
,
3400 VEC_PACK_TRUNC_EXPR
,
3404 vec_dsts
.quick_push (vec_dest
);
3408 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3409 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3410 the resulting vectors and call the function recursively. */
3413 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3414 vec
<tree
> *vec_oprnds1
,
3415 gimple stmt
, tree vec_dest
,
3416 gimple_stmt_iterator
*gsi
,
3417 enum tree_code code1
,
3418 enum tree_code code2
, tree decl1
,
3419 tree decl2
, int op_type
)
3422 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3423 gimple new_stmt1
, new_stmt2
;
3424 vec
<tree
> vec_tmp
= vNULL
;
3426 vec_tmp
.create (vec_oprnds0
->length () * 2);
3427 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3429 if (op_type
== binary_op
)
3430 vop1
= (*vec_oprnds1
)[i
];
3434 /* Generate the two halves of promotion operation. */
3435 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3436 op_type
, vec_dest
, gsi
, stmt
);
3437 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3438 op_type
, vec_dest
, gsi
, stmt
);
3439 if (is_gimple_call (new_stmt1
))
3441 new_tmp1
= gimple_call_lhs (new_stmt1
);
3442 new_tmp2
= gimple_call_lhs (new_stmt2
);
3446 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3447 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3450 /* Store the results for the next step. */
3451 vec_tmp
.quick_push (new_tmp1
);
3452 vec_tmp
.quick_push (new_tmp2
);
3455 vec_oprnds0
->release ();
3456 *vec_oprnds0
= vec_tmp
;
3460 /* Check if STMT performs a conversion operation, that can be vectorized.
3461 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3462 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3463 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3466 vectorizable_conversion (gimple stmt
, gimple_stmt_iterator
*gsi
,
3467 gimple
*vec_stmt
, slp_tree slp_node
)
3471 tree op0
, op1
= NULL_TREE
;
3472 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3473 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3474 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3475 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3476 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3477 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3481 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3482 gimple new_stmt
= NULL
;
3483 stmt_vec_info prev_stmt_info
;
3486 tree vectype_out
, vectype_in
;
3488 tree lhs_type
, rhs_type
;
3489 enum { NARROW
, NONE
, WIDEN
} modifier
;
3490 vec
<tree
> vec_oprnds0
= vNULL
;
3491 vec
<tree
> vec_oprnds1
= vNULL
;
3493 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3494 int multi_step_cvt
= 0;
3495 vec
<tree
> vec_dsts
= vNULL
;
3496 vec
<tree
> interm_types
= vNULL
;
3497 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3499 machine_mode rhs_mode
;
3500 unsigned short fltsz
;
3502 /* Is STMT a vectorizable conversion? */
3504 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3507 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3510 if (!is_gimple_assign (stmt
))
3513 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3516 code
= gimple_assign_rhs_code (stmt
);
3517 if (!CONVERT_EXPR_CODE_P (code
)
3518 && code
!= FIX_TRUNC_EXPR
3519 && code
!= FLOAT_EXPR
3520 && code
!= WIDEN_MULT_EXPR
3521 && code
!= WIDEN_LSHIFT_EXPR
)
3524 op_type
= TREE_CODE_LENGTH (code
);
3526 /* Check types of lhs and rhs. */
3527 scalar_dest
= gimple_assign_lhs (stmt
);
3528 lhs_type
= TREE_TYPE (scalar_dest
);
3529 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3531 op0
= gimple_assign_rhs1 (stmt
);
3532 rhs_type
= TREE_TYPE (op0
);
3534 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3535 && !((INTEGRAL_TYPE_P (lhs_type
)
3536 && INTEGRAL_TYPE_P (rhs_type
))
3537 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3538 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3541 if ((INTEGRAL_TYPE_P (lhs_type
)
3542 && (TYPE_PRECISION (lhs_type
)
3543 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3544 || (INTEGRAL_TYPE_P (rhs_type
)
3545 && (TYPE_PRECISION (rhs_type
)
3546 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
)))))
3548 if (dump_enabled_p ())
3549 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3550 "type conversion to/from bit-precision unsupported."
3555 /* Check the operands of the operation. */
3556 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
3557 &def_stmt
, &def
, &dt
[0], &vectype_in
))
3559 if (dump_enabled_p ())
3560 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3561 "use not simple.\n");
3564 if (op_type
== binary_op
)
3568 op1
= gimple_assign_rhs2 (stmt
);
3569 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3570 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3572 if (CONSTANT_CLASS_P (op0
))
3573 ok
= vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
,
3574 &def_stmt
, &def
, &dt
[1], &vectype_in
);
3576 ok
= vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
3581 if (dump_enabled_p ())
3582 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3583 "use not simple.\n");
3588 /* If op0 is an external or constant defs use a vector type of
3589 the same size as the output vector type. */
3591 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3593 gcc_assert (vectype_in
);
3596 if (dump_enabled_p ())
3598 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3599 "no vectype for scalar type ");
3600 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3601 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3607 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3608 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3609 if (nunits_in
< nunits_out
)
3611 else if (nunits_out
== nunits_in
)
3616 /* Multiple types in SLP are handled by creating the appropriate number of
3617 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3619 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
3621 else if (modifier
== NARROW
)
3622 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3624 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3626 /* Sanity check: make sure that at least one copy of the vectorized stmt
3627 needs to be generated. */
3628 gcc_assert (ncopies
>= 1);
3630 /* Supportable by target? */
3634 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3636 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3641 if (dump_enabled_p ())
3642 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3643 "conversion not supported by target.\n");
3647 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3648 &code1
, &code2
, &multi_step_cvt
,
3651 /* Binary widening operation can only be supported directly by the
3653 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3657 if (code
!= FLOAT_EXPR
3658 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3659 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3662 rhs_mode
= TYPE_MODE (rhs_type
);
3663 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3664 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3665 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3666 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3669 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3670 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3671 if (cvt_type
== NULL_TREE
)
3674 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3676 if (!supportable_convert_operation (code
, vectype_out
,
3677 cvt_type
, &decl1
, &codecvt1
))
3680 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3681 cvt_type
, &codecvt1
,
3682 &codecvt2
, &multi_step_cvt
,
3686 gcc_assert (multi_step_cvt
== 0);
3688 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3689 vectype_in
, &code1
, &code2
,
3690 &multi_step_cvt
, &interm_types
))
3694 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3697 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3698 codecvt2
= ERROR_MARK
;
3702 interm_types
.safe_push (cvt_type
);
3703 cvt_type
= NULL_TREE
;
3708 gcc_assert (op_type
== unary_op
);
3709 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3710 &code1
, &multi_step_cvt
,
3714 if (code
!= FIX_TRUNC_EXPR
3715 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3716 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3719 rhs_mode
= TYPE_MODE (rhs_type
);
3721 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3722 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3723 if (cvt_type
== NULL_TREE
)
3725 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3728 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3729 &code1
, &multi_step_cvt
,
3738 if (!vec_stmt
) /* transformation not required. */
3740 if (dump_enabled_p ())
3741 dump_printf_loc (MSG_NOTE
, vect_location
,
3742 "=== vectorizable_conversion ===\n");
3743 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3745 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3746 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3748 else if (modifier
== NARROW
)
3750 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3751 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3755 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3756 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3758 interm_types
.release ();
3763 if (dump_enabled_p ())
3764 dump_printf_loc (MSG_NOTE
, vect_location
,
3765 "transform conversion. ncopies = %d.\n", ncopies
);
3767 if (op_type
== binary_op
)
3769 if (CONSTANT_CLASS_P (op0
))
3770 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3771 else if (CONSTANT_CLASS_P (op1
))
3772 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3775 /* In case of multi-step conversion, we first generate conversion operations
3776 to the intermediate types, and then from that types to the final one.
3777 We create vector destinations for the intermediate type (TYPES) received
3778 from supportable_*_operation, and store them in the correct order
3779 for future use in vect_create_vectorized_*_stmts (). */
3780 vec_dsts
.create (multi_step_cvt
+ 1);
3781 vec_dest
= vect_create_destination_var (scalar_dest
,
3782 (cvt_type
&& modifier
== WIDEN
)
3783 ? cvt_type
: vectype_out
);
3784 vec_dsts
.quick_push (vec_dest
);
3788 for (i
= interm_types
.length () - 1;
3789 interm_types
.iterate (i
, &intermediate_type
); i
--)
3791 vec_dest
= vect_create_destination_var (scalar_dest
,
3793 vec_dsts
.quick_push (vec_dest
);
3798 vec_dest
= vect_create_destination_var (scalar_dest
,
3800 ? vectype_out
: cvt_type
);
3804 if (modifier
== WIDEN
)
3806 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3807 if (op_type
== binary_op
)
3808 vec_oprnds1
.create (1);
3810 else if (modifier
== NARROW
)
3811 vec_oprnds0
.create (
3812 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3814 else if (code
== WIDEN_LSHIFT_EXPR
)
3815 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3818 prev_stmt_info
= NULL
;
3822 for (j
= 0; j
< ncopies
; j
++)
3825 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3828 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3830 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3832 /* Arguments are ready, create the new vector stmt. */
3833 if (code1
== CALL_EXPR
)
3835 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3836 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3837 gimple_call_set_lhs (new_stmt
, new_temp
);
3841 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3842 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
3843 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3844 gimple_assign_set_lhs (new_stmt
, new_temp
);
3847 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3849 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3853 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3855 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3856 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3861 /* In case the vectorization factor (VF) is bigger than the number
3862 of elements that we can fit in a vectype (nunits), we have to
3863 generate more than one vector stmt - i.e - we need to "unroll"
3864 the vector stmt by a factor VF/nunits. */
3865 for (j
= 0; j
< ncopies
; j
++)
3872 if (code
== WIDEN_LSHIFT_EXPR
)
3877 /* Store vec_oprnd1 for every vector stmt to be created
3878 for SLP_NODE. We check during the analysis that all
3879 the shift arguments are the same. */
3880 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
3881 vec_oprnds1
.quick_push (vec_oprnd1
);
3883 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3887 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
3888 &vec_oprnds1
, slp_node
, -1);
3892 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
3893 vec_oprnds0
.quick_push (vec_oprnd0
);
3894 if (op_type
== binary_op
)
3896 if (code
== WIDEN_LSHIFT_EXPR
)
3899 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
,
3901 vec_oprnds1
.quick_push (vec_oprnd1
);
3907 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
3908 vec_oprnds0
.truncate (0);
3909 vec_oprnds0
.quick_push (vec_oprnd0
);
3910 if (op_type
== binary_op
)
3912 if (code
== WIDEN_LSHIFT_EXPR
)
3915 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
3917 vec_oprnds1
.truncate (0);
3918 vec_oprnds1
.quick_push (vec_oprnd1
);
3922 /* Arguments are ready. Create the new vector stmts. */
3923 for (i
= multi_step_cvt
; i
>= 0; i
--)
3925 tree this_dest
= vec_dsts
[i
];
3926 enum tree_code c1
= code1
, c2
= code2
;
3927 if (i
== 0 && codecvt2
!= ERROR_MARK
)
3932 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
3934 stmt
, this_dest
, gsi
,
3935 c1
, c2
, decl1
, decl2
,
3939 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3943 if (codecvt1
== CALL_EXPR
)
3945 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3946 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3947 gimple_call_set_lhs (new_stmt
, new_temp
);
3951 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
3952 new_temp
= make_ssa_name (vec_dest
);
3953 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
3957 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3960 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
3963 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3965 if (!prev_stmt_info
)
3966 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3968 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3969 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3973 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3977 /* In case the vectorization factor (VF) is bigger than the number
3978 of elements that we can fit in a vectype (nunits), we have to
3979 generate more than one vector stmt - i.e - we need to "unroll"
3980 the vector stmt by a factor VF/nunits. */
3981 for (j
= 0; j
< ncopies
; j
++)
3985 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3989 vec_oprnds0
.truncate (0);
3990 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
3991 vect_pow2 (multi_step_cvt
) - 1);
3994 /* Arguments are ready. Create the new vector stmts. */
3996 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3998 if (codecvt1
== CALL_EXPR
)
4000 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4001 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4002 gimple_call_set_lhs (new_stmt
, new_temp
);
4006 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4007 new_temp
= make_ssa_name (vec_dest
);
4008 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4012 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4013 vec_oprnds0
[i
] = new_temp
;
4016 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4017 stmt
, vec_dsts
, gsi
,
4022 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4026 vec_oprnds0
.release ();
4027 vec_oprnds1
.release ();
4028 vec_dsts
.release ();
4029 interm_types
.release ();
4035 /* Function vectorizable_assignment.
4037 Check if STMT performs an assignment (copy) that can be vectorized.
4038 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4039 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4040 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4043 vectorizable_assignment (gimple stmt
, gimple_stmt_iterator
*gsi
,
4044 gimple
*vec_stmt
, slp_tree slp_node
)
4049 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4050 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4051 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4055 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4056 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4059 vec
<tree
> vec_oprnds
= vNULL
;
4061 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4062 gimple new_stmt
= NULL
;
4063 stmt_vec_info prev_stmt_info
= NULL
;
4064 enum tree_code code
;
4067 /* Multiple types in SLP are handled by creating the appropriate number of
4068 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4070 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4073 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4075 gcc_assert (ncopies
>= 1);
4077 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4080 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4083 /* Is vectorizable assignment? */
4084 if (!is_gimple_assign (stmt
))
4087 scalar_dest
= gimple_assign_lhs (stmt
);
4088 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4091 code
= gimple_assign_rhs_code (stmt
);
4092 if (gimple_assign_single_p (stmt
)
4093 || code
== PAREN_EXPR
4094 || CONVERT_EXPR_CODE_P (code
))
4095 op
= gimple_assign_rhs1 (stmt
);
4099 if (code
== VIEW_CONVERT_EXPR
)
4100 op
= TREE_OPERAND (op
, 0);
4102 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
4103 &def_stmt
, &def
, &dt
[0], &vectype_in
))
4105 if (dump_enabled_p ())
4106 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4107 "use not simple.\n");
4111 /* We can handle NOP_EXPR conversions that do not change the number
4112 of elements or the vector size. */
4113 if ((CONVERT_EXPR_CODE_P (code
)
4114 || code
== VIEW_CONVERT_EXPR
)
4116 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4117 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4118 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4121 /* We do not handle bit-precision changes. */
4122 if ((CONVERT_EXPR_CODE_P (code
)
4123 || code
== VIEW_CONVERT_EXPR
)
4124 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4125 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4126 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4127 || ((TYPE_PRECISION (TREE_TYPE (op
))
4128 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4129 /* But a conversion that does not change the bit-pattern is ok. */
4130 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4131 > TYPE_PRECISION (TREE_TYPE (op
)))
4132 && TYPE_UNSIGNED (TREE_TYPE (op
))))
4134 if (dump_enabled_p ())
4135 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4136 "type conversion to/from bit-precision "
4141 if (!vec_stmt
) /* transformation not required. */
4143 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4144 if (dump_enabled_p ())
4145 dump_printf_loc (MSG_NOTE
, vect_location
,
4146 "=== vectorizable_assignment ===\n");
4147 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4152 if (dump_enabled_p ())
4153 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4156 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4159 for (j
= 0; j
< ncopies
; j
++)
4163 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4165 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4167 /* Arguments are ready. create the new vector stmt. */
4168 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4170 if (CONVERT_EXPR_CODE_P (code
)
4171 || code
== VIEW_CONVERT_EXPR
)
4172 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4173 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4174 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4175 gimple_assign_set_lhs (new_stmt
, new_temp
);
4176 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4178 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4185 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4187 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4189 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4192 vec_oprnds
.release ();
4197 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4198 either as shift by a scalar or by a vector. */
4201 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4204 machine_mode vec_mode
;
4209 vectype
= get_vectype_for_scalar_type (scalar_type
);
4213 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4215 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4217 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4219 || (optab_handler (optab
, TYPE_MODE (vectype
))
4220 == CODE_FOR_nothing
))
4224 vec_mode
= TYPE_MODE (vectype
);
4225 icode
= (int) optab_handler (optab
, vec_mode
);
4226 if (icode
== CODE_FOR_nothing
)
4233 /* Function vectorizable_shift.
4235 Check if STMT performs a shift operation that can be vectorized.
4236 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4237 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4238 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4241 vectorizable_shift (gimple stmt
, gimple_stmt_iterator
*gsi
,
4242 gimple
*vec_stmt
, slp_tree slp_node
)
4246 tree op0
, op1
= NULL
;
4247 tree vec_oprnd1
= NULL_TREE
;
4248 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4250 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4251 enum tree_code code
;
4252 machine_mode vec_mode
;
4256 machine_mode optab_op2_mode
;
4259 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4260 gimple new_stmt
= NULL
;
4261 stmt_vec_info prev_stmt_info
;
4268 vec
<tree
> vec_oprnds0
= vNULL
;
4269 vec
<tree
> vec_oprnds1
= vNULL
;
4272 bool scalar_shift_arg
= true;
4273 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4276 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4279 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4282 /* Is STMT a vectorizable binary/unary operation? */
4283 if (!is_gimple_assign (stmt
))
4286 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4289 code
= gimple_assign_rhs_code (stmt
);
4291 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4292 || code
== RROTATE_EXPR
))
4295 scalar_dest
= gimple_assign_lhs (stmt
);
4296 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4297 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4298 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4300 if (dump_enabled_p ())
4301 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4302 "bit-precision shifts not supported.\n");
4306 op0
= gimple_assign_rhs1 (stmt
);
4307 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4308 &def_stmt
, &def
, &dt
[0], &vectype
))
4310 if (dump_enabled_p ())
4311 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4312 "use not simple.\n");
4315 /* If op0 is an external or constant def use a vector type with
4316 the same size as the output vector type. */
4318 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4320 gcc_assert (vectype
);
4323 if (dump_enabled_p ())
4324 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4325 "no vectype for scalar type\n");
4329 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4330 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4331 if (nunits_out
!= nunits_in
)
4334 op1
= gimple_assign_rhs2 (stmt
);
4335 if (!vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4336 &def
, &dt
[1], &op1_vectype
))
4338 if (dump_enabled_p ())
4339 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4340 "use not simple.\n");
4345 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4349 /* Multiple types in SLP are handled by creating the appropriate number of
4350 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4352 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4355 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4357 gcc_assert (ncopies
>= 1);
4359 /* Determine whether the shift amount is a vector, or scalar. If the
4360 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4362 if (dt
[1] == vect_internal_def
&& !slp_node
)
4363 scalar_shift_arg
= false;
4364 else if (dt
[1] == vect_constant_def
4365 || dt
[1] == vect_external_def
4366 || dt
[1] == vect_internal_def
)
4368 /* In SLP, need to check whether the shift count is the same,
4369 in loops if it is a constant or invariant, it is always
4373 vec
<gimple
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4376 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4377 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4378 scalar_shift_arg
= false;
4383 if (dump_enabled_p ())
4384 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4385 "operand mode requires invariant argument.\n");
4389 /* Vector shifted by vector. */
4390 if (!scalar_shift_arg
)
4392 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4393 if (dump_enabled_p ())
4394 dump_printf_loc (MSG_NOTE
, vect_location
,
4395 "vector/vector shift/rotate found.\n");
4398 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4399 if (op1_vectype
== NULL_TREE
4400 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4402 if (dump_enabled_p ())
4403 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4404 "unusable type for last operand in"
4405 " vector/vector shift/rotate.\n");
4409 /* See if the machine has a vector shifted by scalar insn and if not
4410 then see if it has a vector shifted by vector insn. */
4413 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4415 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4417 if (dump_enabled_p ())
4418 dump_printf_loc (MSG_NOTE
, vect_location
,
4419 "vector/scalar shift/rotate found.\n");
4423 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4425 && (optab_handler (optab
, TYPE_MODE (vectype
))
4426 != CODE_FOR_nothing
))
4428 scalar_shift_arg
= false;
4430 if (dump_enabled_p ())
4431 dump_printf_loc (MSG_NOTE
, vect_location
,
4432 "vector/vector shift/rotate found.\n");
4434 /* Unlike the other binary operators, shifts/rotates have
4435 the rhs being int, instead of the same type as the lhs,
4436 so make sure the scalar is the right type if we are
4437 dealing with vectors of long long/long/short/char. */
4438 if (dt
[1] == vect_constant_def
)
4439 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4440 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4444 && TYPE_MODE (TREE_TYPE (vectype
))
4445 != TYPE_MODE (TREE_TYPE (op1
)))
4447 if (dump_enabled_p ())
4448 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4449 "unusable type for last operand in"
4450 " vector/vector shift/rotate.\n");
4453 if (vec_stmt
&& !slp_node
)
4455 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4456 op1
= vect_init_vector (stmt
, op1
,
4457 TREE_TYPE (vectype
), NULL
);
4464 /* Supportable by target? */
4467 if (dump_enabled_p ())
4468 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4472 vec_mode
= TYPE_MODE (vectype
);
4473 icode
= (int) optab_handler (optab
, vec_mode
);
4474 if (icode
== CODE_FOR_nothing
)
4476 if (dump_enabled_p ())
4477 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4478 "op not supported by target.\n");
4479 /* Check only during analysis. */
4480 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4481 || (vf
< vect_min_worthwhile_factor (code
)
4484 if (dump_enabled_p ())
4485 dump_printf_loc (MSG_NOTE
, vect_location
,
4486 "proceeding using word mode.\n");
4489 /* Worthwhile without SIMD support? Check only during analysis. */
4490 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4491 && vf
< vect_min_worthwhile_factor (code
)
4494 if (dump_enabled_p ())
4495 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4496 "not worthwhile without SIMD support.\n");
4500 if (!vec_stmt
) /* transformation not required. */
4502 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4503 if (dump_enabled_p ())
4504 dump_printf_loc (MSG_NOTE
, vect_location
,
4505 "=== vectorizable_shift ===\n");
4506 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4512 if (dump_enabled_p ())
4513 dump_printf_loc (MSG_NOTE
, vect_location
,
4514 "transform binary/unary operation.\n");
4517 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4519 prev_stmt_info
= NULL
;
4520 for (j
= 0; j
< ncopies
; j
++)
4525 if (scalar_shift_arg
)
4527 /* Vector shl and shr insn patterns can be defined with scalar
4528 operand 2 (shift operand). In this case, use constant or loop
4529 invariant op1 directly, without extending it to vector mode
4531 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4532 if (!VECTOR_MODE_P (optab_op2_mode
))
4534 if (dump_enabled_p ())
4535 dump_printf_loc (MSG_NOTE
, vect_location
,
4536 "operand 1 using scalar mode.\n");
4538 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4539 vec_oprnds1
.quick_push (vec_oprnd1
);
4542 /* Store vec_oprnd1 for every vector stmt to be created
4543 for SLP_NODE. We check during the analysis that all
4544 the shift arguments are the same.
4545 TODO: Allow different constants for different vector
4546 stmts generated for an SLP instance. */
4547 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4548 vec_oprnds1
.quick_push (vec_oprnd1
);
4553 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4554 (a special case for certain kind of vector shifts); otherwise,
4555 operand 1 should be of a vector type (the usual case). */
4557 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4560 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4564 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4566 /* Arguments are ready. Create the new vector stmt. */
4567 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4569 vop1
= vec_oprnds1
[i
];
4570 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4571 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4572 gimple_assign_set_lhs (new_stmt
, new_temp
);
4573 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4575 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4582 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4584 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4585 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4588 vec_oprnds0
.release ();
4589 vec_oprnds1
.release ();
4595 /* Function vectorizable_operation.
4597 Check if STMT performs a binary, unary or ternary operation that can
4599 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4600 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4601 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4604 vectorizable_operation (gimple stmt
, gimple_stmt_iterator
*gsi
,
4605 gimple
*vec_stmt
, slp_tree slp_node
)
4609 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4610 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4612 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4613 enum tree_code code
;
4614 machine_mode vec_mode
;
4621 enum vect_def_type dt
[3]
4622 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4623 gimple new_stmt
= NULL
;
4624 stmt_vec_info prev_stmt_info
;
4630 vec
<tree
> vec_oprnds0
= vNULL
;
4631 vec
<tree
> vec_oprnds1
= vNULL
;
4632 vec
<tree
> vec_oprnds2
= vNULL
;
4633 tree vop0
, vop1
, vop2
;
4634 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4637 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4640 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4643 /* Is STMT a vectorizable binary/unary operation? */
4644 if (!is_gimple_assign (stmt
))
4647 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4650 code
= gimple_assign_rhs_code (stmt
);
4652 /* For pointer addition, we should use the normal plus for
4653 the vector addition. */
4654 if (code
== POINTER_PLUS_EXPR
)
4657 /* Support only unary or binary operations. */
4658 op_type
= TREE_CODE_LENGTH (code
);
4659 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4661 if (dump_enabled_p ())
4662 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4663 "num. args = %d (not unary/binary/ternary op).\n",
4668 scalar_dest
= gimple_assign_lhs (stmt
);
4669 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4671 /* Most operations cannot handle bit-precision types without extra
4673 if ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4674 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4675 /* Exception are bitwise binary operations. */
4676 && code
!= BIT_IOR_EXPR
4677 && code
!= BIT_XOR_EXPR
4678 && code
!= BIT_AND_EXPR
)
4680 if (dump_enabled_p ())
4681 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4682 "bit-precision arithmetic not supported.\n");
4686 op0
= gimple_assign_rhs1 (stmt
);
4687 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4688 &def_stmt
, &def
, &dt
[0], &vectype
))
4690 if (dump_enabled_p ())
4691 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4692 "use not simple.\n");
4695 /* If op0 is an external or constant def use a vector type with
4696 the same size as the output vector type. */
4698 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4700 gcc_assert (vectype
);
4703 if (dump_enabled_p ())
4705 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4706 "no vectype for scalar type ");
4707 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4709 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4715 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4716 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4717 if (nunits_out
!= nunits_in
)
4720 if (op_type
== binary_op
|| op_type
== ternary_op
)
4722 op1
= gimple_assign_rhs2 (stmt
);
4723 if (!vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4726 if (dump_enabled_p ())
4727 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4728 "use not simple.\n");
4732 if (op_type
== ternary_op
)
4734 op2
= gimple_assign_rhs3 (stmt
);
4735 if (!vect_is_simple_use (op2
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4738 if (dump_enabled_p ())
4739 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4740 "use not simple.\n");
4746 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4750 /* Multiple types in SLP are handled by creating the appropriate number of
4751 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4753 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4756 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4758 gcc_assert (ncopies
>= 1);
4760 /* Shifts are handled in vectorizable_shift (). */
4761 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4762 || code
== RROTATE_EXPR
)
4765 /* Supportable by target? */
4767 vec_mode
= TYPE_MODE (vectype
);
4768 if (code
== MULT_HIGHPART_EXPR
)
4770 if (can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
)))
4771 icode
= LAST_INSN_CODE
;
4773 icode
= CODE_FOR_nothing
;
4777 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4780 if (dump_enabled_p ())
4781 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4785 icode
= (int) optab_handler (optab
, vec_mode
);
4788 if (icode
== CODE_FOR_nothing
)
4790 if (dump_enabled_p ())
4791 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4792 "op not supported by target.\n");
4793 /* Check only during analysis. */
4794 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4795 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4797 if (dump_enabled_p ())
4798 dump_printf_loc (MSG_NOTE
, vect_location
,
4799 "proceeding using word mode.\n");
4802 /* Worthwhile without SIMD support? Check only during analysis. */
4803 if (!VECTOR_MODE_P (vec_mode
)
4805 && vf
< vect_min_worthwhile_factor (code
))
4807 if (dump_enabled_p ())
4808 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4809 "not worthwhile without SIMD support.\n");
4813 if (!vec_stmt
) /* transformation not required. */
4815 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4816 if (dump_enabled_p ())
4817 dump_printf_loc (MSG_NOTE
, vect_location
,
4818 "=== vectorizable_operation ===\n");
4819 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4825 if (dump_enabled_p ())
4826 dump_printf_loc (MSG_NOTE
, vect_location
,
4827 "transform binary/unary operation.\n");
4830 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4832 /* In case the vectorization factor (VF) is bigger than the number
4833 of elements that we can fit in a vectype (nunits), we have to generate
4834 more than one vector stmt - i.e - we need to "unroll" the
4835 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4836 from one copy of the vector stmt to the next, in the field
4837 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4838 stages to find the correct vector defs to be used when vectorizing
4839 stmts that use the defs of the current stmt. The example below
4840 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4841 we need to create 4 vectorized stmts):
4843 before vectorization:
4844 RELATED_STMT VEC_STMT
4848 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4850 RELATED_STMT VEC_STMT
4851 VS1_0: vx0 = memref0 VS1_1 -
4852 VS1_1: vx1 = memref1 VS1_2 -
4853 VS1_2: vx2 = memref2 VS1_3 -
4854 VS1_3: vx3 = memref3 - -
4855 S1: x = load - VS1_0
4858 step2: vectorize stmt S2 (done here):
4859 To vectorize stmt S2 we first need to find the relevant vector
4860 def for the first operand 'x'. This is, as usual, obtained from
4861 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4862 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4863 relevant vector def 'vx0'. Having found 'vx0' we can generate
4864 the vector stmt VS2_0, and as usual, record it in the
4865 STMT_VINFO_VEC_STMT of stmt S2.
4866 When creating the second copy (VS2_1), we obtain the relevant vector
4867 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4868 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4869 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4870 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4871 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4872 chain of stmts and pointers:
4873 RELATED_STMT VEC_STMT
4874 VS1_0: vx0 = memref0 VS1_1 -
4875 VS1_1: vx1 = memref1 VS1_2 -
4876 VS1_2: vx2 = memref2 VS1_3 -
4877 VS1_3: vx3 = memref3 - -
4878 S1: x = load - VS1_0
4879 VS2_0: vz0 = vx0 + v1 VS2_1 -
4880 VS2_1: vz1 = vx1 + v1 VS2_2 -
4881 VS2_2: vz2 = vx2 + v1 VS2_3 -
4882 VS2_3: vz3 = vx3 + v1 - -
4883 S2: z = x + 1 - VS2_0 */
4885 prev_stmt_info
= NULL
;
4886 for (j
= 0; j
< ncopies
; j
++)
4891 if (op_type
== binary_op
|| op_type
== ternary_op
)
4892 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4895 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4897 if (op_type
== ternary_op
)
4899 vec_oprnds2
.create (1);
4900 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
4907 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4908 if (op_type
== ternary_op
)
4910 tree vec_oprnd
= vec_oprnds2
.pop ();
4911 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
4916 /* Arguments are ready. Create the new vector stmt. */
4917 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4919 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
4920 ? vec_oprnds1
[i
] : NULL_TREE
);
4921 vop2
= ((op_type
== ternary_op
)
4922 ? vec_oprnds2
[i
] : NULL_TREE
);
4923 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
4924 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4925 gimple_assign_set_lhs (new_stmt
, new_temp
);
4926 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4928 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4935 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4937 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4938 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4941 vec_oprnds0
.release ();
4942 vec_oprnds1
.release ();
4943 vec_oprnds2
.release ();
4948 /* A helper function to ensure data reference DR's base alignment
4952 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
4957 if (((dataref_aux
*)dr
->aux
)->base_misaligned
)
4959 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4960 tree base_decl
= ((dataref_aux
*)dr
->aux
)->base_decl
;
4962 if (decl_in_symtab_p (base_decl
))
4963 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
4966 DECL_ALIGN (base_decl
) = TYPE_ALIGN (vectype
);
4967 DECL_USER_ALIGN (base_decl
) = 1;
4969 ((dataref_aux
*)dr
->aux
)->base_misaligned
= false;
4974 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
4975 reversal of the vector elements. If that is impossible to do,
4979 perm_mask_for_reverse (tree vectype
)
4984 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4985 sel
= XALLOCAVEC (unsigned char, nunits
);
4987 for (i
= 0; i
< nunits
; ++i
)
4988 sel
[i
] = nunits
- 1 - i
;
4990 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
4992 return vect_gen_perm_mask_checked (vectype
, sel
);
4995 /* Function vectorizable_store.
4997 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
4999 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5000 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5001 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5004 vectorizable_store (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5010 tree vec_oprnd
= NULL_TREE
;
5011 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5012 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5013 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5015 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5016 struct loop
*loop
= NULL
;
5017 machine_mode vec_mode
;
5019 enum dr_alignment_support alignment_support_scheme
;
5022 enum vect_def_type dt
;
5023 stmt_vec_info prev_stmt_info
= NULL
;
5024 tree dataref_ptr
= NULL_TREE
;
5025 tree dataref_offset
= NULL_TREE
;
5026 gimple ptr_incr
= NULL
;
5027 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5030 gimple next_stmt
, first_stmt
= NULL
;
5031 bool grouped_store
= false;
5032 bool store_lanes_p
= false;
5033 unsigned int group_size
, i
;
5034 vec
<tree
> dr_chain
= vNULL
;
5035 vec
<tree
> oprnds
= vNULL
;
5036 vec
<tree
> result_chain
= vNULL
;
5038 bool negative
= false;
5039 tree offset
= NULL_TREE
;
5040 vec
<tree
> vec_oprnds
= vNULL
;
5041 bool slp
= (slp_node
!= NULL
);
5042 unsigned int vec_num
;
5043 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5047 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5049 /* Multiple types in SLP are handled by creating the appropriate number of
5050 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5052 if (slp
|| PURE_SLP_STMT (stmt_info
))
5055 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5057 gcc_assert (ncopies
>= 1);
5059 /* FORNOW. This restriction should be relaxed. */
5060 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5062 if (dump_enabled_p ())
5063 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5064 "multiple types in nested loop.\n");
5068 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5071 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5074 /* Is vectorizable store? */
5076 if (!is_gimple_assign (stmt
))
5079 scalar_dest
= gimple_assign_lhs (stmt
);
5080 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5081 && is_pattern_stmt_p (stmt_info
))
5082 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5083 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5084 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5085 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5086 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5087 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5088 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5089 && TREE_CODE (scalar_dest
) != MEM_REF
)
5092 gcc_assert (gimple_assign_single_p (stmt
));
5093 op
= gimple_assign_rhs1 (stmt
);
5094 if (!vect_is_simple_use (op
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5097 if (dump_enabled_p ())
5098 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5099 "use not simple.\n");
5103 elem_type
= TREE_TYPE (vectype
);
5104 vec_mode
= TYPE_MODE (vectype
);
5106 /* FORNOW. In some cases can vectorize even if data-type not supported
5107 (e.g. - array initialization with 0). */
5108 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5111 if (!STMT_VINFO_DATA_REF (stmt_info
))
5114 if (!STMT_VINFO_STRIDED_P (stmt_info
))
5117 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5118 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5119 size_zero_node
) < 0;
5120 if (negative
&& ncopies
> 1)
5122 if (dump_enabled_p ())
5123 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5124 "multiple types with negative step.\n");
5129 gcc_assert (!grouped_store
);
5130 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5131 if (alignment_support_scheme
!= dr_aligned
5132 && alignment_support_scheme
!= dr_unaligned_supported
)
5134 if (dump_enabled_p ())
5135 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5136 "negative step but alignment required.\n");
5139 if (dt
!= vect_constant_def
5140 && dt
!= vect_external_def
5141 && !perm_mask_for_reverse (vectype
))
5143 if (dump_enabled_p ())
5144 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5145 "negative step and reversing not supported.\n");
5151 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5153 grouped_store
= true;
5154 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5155 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5157 && !PURE_SLP_STMT (stmt_info
)
5158 && !STMT_VINFO_STRIDED_P (stmt_info
))
5160 if (vect_store_lanes_supported (vectype
, group_size
))
5161 store_lanes_p
= true;
5162 else if (!vect_grouped_store_supported (vectype
, group_size
))
5166 if (STMT_VINFO_STRIDED_P (stmt_info
)
5167 && (slp
|| PURE_SLP_STMT (stmt_info
))
5168 && (group_size
> nunits
5169 || nunits
% group_size
!= 0))
5171 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5172 "unhandled strided group store\n");
5176 if (first_stmt
== stmt
)
5178 /* STMT is the leader of the group. Check the operands of all the
5179 stmts of the group. */
5180 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5183 gcc_assert (gimple_assign_single_p (next_stmt
));
5184 op
= gimple_assign_rhs1 (next_stmt
);
5185 if (!vect_is_simple_use (op
, next_stmt
, loop_vinfo
, bb_vinfo
,
5186 &def_stmt
, &def
, &dt
))
5188 if (dump_enabled_p ())
5189 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5190 "use not simple.\n");
5193 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5198 if (!vec_stmt
) /* transformation not required. */
5200 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5201 /* The SLP costs are calculated during SLP analysis. */
5202 if (!PURE_SLP_STMT (stmt_info
))
5203 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5210 ensure_base_align (stmt_info
, dr
);
5214 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5215 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5217 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5220 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5222 /* We vectorize all the stmts of the interleaving group when we
5223 reach the last stmt in the group. */
5224 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5225 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5234 grouped_store
= false;
5235 /* VEC_NUM is the number of vect stmts to be created for this
5237 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5238 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5239 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5240 op
= gimple_assign_rhs1 (first_stmt
);
5243 /* VEC_NUM is the number of vect stmts to be created for this
5245 vec_num
= group_size
;
5251 group_size
= vec_num
= 1;
5254 if (dump_enabled_p ())
5255 dump_printf_loc (MSG_NOTE
, vect_location
,
5256 "transform store. ncopies = %d\n", ncopies
);
5258 if (STMT_VINFO_STRIDED_P (stmt_info
))
5260 gimple_stmt_iterator incr_gsi
;
5266 gimple_seq stmts
= NULL
;
5267 tree stride_base
, stride_step
, alias_off
;
5270 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5273 = fold_build_pointer_plus
5274 (unshare_expr (DR_BASE_ADDRESS (dr
)),
5275 size_binop (PLUS_EXPR
,
5276 convert_to_ptrofftype (unshare_expr (DR_OFFSET (dr
))),
5277 convert_to_ptrofftype (DR_INIT(dr
))));
5278 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (dr
)));
5280 /* For a store with loop-invariant (but other than power-of-2)
5281 stride (i.e. not a grouped access) like so:
5283 for (i = 0; i < n; i += stride)
5286 we generate a new induction variable and new stores from
5287 the components of the (vectorized) rhs:
5289 for (j = 0; ; j += VF*stride)
5294 array[j + stride] = tmp2;
5298 unsigned nstores
= nunits
;
5299 tree ltype
= elem_type
;
5302 nstores
= nunits
/ group_size
;
5303 if (group_size
< nunits
)
5304 ltype
= build_vector_type (elem_type
, group_size
);
5307 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
5308 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5311 ivstep
= stride_step
;
5312 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
5313 build_int_cst (TREE_TYPE (ivstep
),
5314 ncopies
* nstores
));
5316 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
5318 create_iv (stride_base
, ivstep
, NULL
,
5319 loop
, &incr_gsi
, insert_after
,
5321 incr
= gsi_stmt (incr_gsi
);
5322 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
5324 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
5326 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
5328 prev_stmt_info
= NULL
;
5329 running_off
= offvar
;
5330 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (dr
)), 0);
5331 for (j
= 0; j
< ncopies
; j
++)
5333 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5334 and first_stmt == stmt. */
5336 vec_oprnd
= vect_get_vec_def_for_operand (op
, first_stmt
, NULL
);
5338 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
5340 for (i
= 0; i
< nstores
; i
++)
5342 tree newref
, newoff
;
5343 gimple incr
, assign
;
5344 tree size
= TYPE_SIZE (ltype
);
5345 /* Extract the i'th component. */
5346 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (i
),
5348 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
5351 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
5355 newref
= build2 (MEM_REF
, ltype
,
5356 running_off
, alias_off
);
5358 /* And store it to *running_off. */
5359 assign
= gimple_build_assign (newref
, elem
);
5360 vect_finish_stmt_generation (stmt
, assign
, gsi
);
5362 newoff
= copy_ssa_name (running_off
, NULL
);
5363 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5364 running_off
, stride_step
);
5365 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5367 running_off
= newoff
;
5368 if (j
== 0 && i
== i
)
5369 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= assign
;
5371 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
5372 prev_stmt_info
= vinfo_for_stmt (assign
);
5378 dr_chain
.create (group_size
);
5379 oprnds
.create (group_size
);
5381 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5382 gcc_assert (alignment_support_scheme
);
5383 /* Targets with store-lane instructions must not require explicit
5385 gcc_assert (!store_lanes_p
5386 || alignment_support_scheme
== dr_aligned
5387 || alignment_support_scheme
== dr_unaligned_supported
);
5390 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5393 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5395 aggr_type
= vectype
;
5397 /* In case the vectorization factor (VF) is bigger than the number
5398 of elements that we can fit in a vectype (nunits), we have to generate
5399 more than one vector stmt - i.e - we need to "unroll" the
5400 vector stmt by a factor VF/nunits. For more details see documentation in
5401 vect_get_vec_def_for_copy_stmt. */
5403 /* In case of interleaving (non-unit grouped access):
5410 We create vectorized stores starting from base address (the access of the
5411 first stmt in the chain (S2 in the above example), when the last store stmt
5412 of the chain (S4) is reached:
5415 VS2: &base + vec_size*1 = vx0
5416 VS3: &base + vec_size*2 = vx1
5417 VS4: &base + vec_size*3 = vx3
5419 Then permutation statements are generated:
5421 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5422 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5425 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5426 (the order of the data-refs in the output of vect_permute_store_chain
5427 corresponds to the order of scalar stmts in the interleaving chain - see
5428 the documentation of vect_permute_store_chain()).
5430 In case of both multiple types and interleaving, above vector stores and
5431 permutation stmts are created for every copy. The result vector stmts are
5432 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5433 STMT_VINFO_RELATED_STMT for the next copies.
5436 prev_stmt_info
= NULL
;
5437 for (j
= 0; j
< ncopies
; j
++)
5445 /* Get vectorized arguments for SLP_NODE. */
5446 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5447 NULL
, slp_node
, -1);
5449 vec_oprnd
= vec_oprnds
[0];
5453 /* For interleaved stores we collect vectorized defs for all the
5454 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5455 used as an input to vect_permute_store_chain(), and OPRNDS as
5456 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5458 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5459 OPRNDS are of size 1. */
5460 next_stmt
= first_stmt
;
5461 for (i
= 0; i
< group_size
; i
++)
5463 /* Since gaps are not supported for interleaved stores,
5464 GROUP_SIZE is the exact number of stmts in the chain.
5465 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5466 there is no interleaving, GROUP_SIZE is 1, and only one
5467 iteration of the loop will be executed. */
5468 gcc_assert (next_stmt
5469 && gimple_assign_single_p (next_stmt
));
5470 op
= gimple_assign_rhs1 (next_stmt
);
5472 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
5474 dr_chain
.quick_push (vec_oprnd
);
5475 oprnds
.quick_push (vec_oprnd
);
5476 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5480 /* We should have catched mismatched types earlier. */
5481 gcc_assert (useless_type_conversion_p (vectype
,
5482 TREE_TYPE (vec_oprnd
)));
5483 bool simd_lane_access_p
5484 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5485 if (simd_lane_access_p
5486 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5487 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5488 && integer_zerop (DR_OFFSET (first_dr
))
5489 && integer_zerop (DR_INIT (first_dr
))
5490 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5491 get_alias_set (DR_REF (first_dr
))))
5493 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5494 dataref_offset
= build_int_cst (reference_alias_ptr_type
5495 (DR_REF (first_dr
)), 0);
5500 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5501 simd_lane_access_p
? loop
: NULL
,
5502 offset
, &dummy
, gsi
, &ptr_incr
,
5503 simd_lane_access_p
, &inv_p
);
5504 gcc_assert (bb_vinfo
|| !inv_p
);
5508 /* For interleaved stores we created vectorized defs for all the
5509 defs stored in OPRNDS in the previous iteration (previous copy).
5510 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5511 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5513 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5514 OPRNDS are of size 1. */
5515 for (i
= 0; i
< group_size
; i
++)
5518 vect_is_simple_use (op
, NULL
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5520 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5521 dr_chain
[i
] = vec_oprnd
;
5522 oprnds
[i
] = vec_oprnd
;
5526 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5527 TYPE_SIZE_UNIT (aggr_type
));
5529 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5530 TYPE_SIZE_UNIT (aggr_type
));
5537 /* Combine all the vectors into an array. */
5538 vec_array
= create_vector_array (vectype
, vec_num
);
5539 for (i
= 0; i
< vec_num
; i
++)
5541 vec_oprnd
= dr_chain
[i
];
5542 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5546 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5547 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5548 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5549 gimple_call_set_lhs (new_stmt
, data_ref
);
5550 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5558 result_chain
.create (group_size
);
5560 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5564 next_stmt
= first_stmt
;
5565 for (i
= 0; i
< vec_num
; i
++)
5567 unsigned align
, misalign
;
5570 /* Bump the vector pointer. */
5571 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5575 vec_oprnd
= vec_oprnds
[i
];
5576 else if (grouped_store
)
5577 /* For grouped stores vectorized defs are interleaved in
5578 vect_permute_store_chain(). */
5579 vec_oprnd
= result_chain
[i
];
5581 data_ref
= build2 (MEM_REF
, TREE_TYPE (vec_oprnd
), dataref_ptr
,
5584 : build_int_cst (reference_alias_ptr_type
5585 (DR_REF (first_dr
)), 0));
5586 align
= TYPE_ALIGN_UNIT (vectype
);
5587 if (aligned_access_p (first_dr
))
5589 else if (DR_MISALIGNMENT (first_dr
) == -1)
5591 TREE_TYPE (data_ref
)
5592 = build_aligned_type (TREE_TYPE (data_ref
),
5593 TYPE_ALIGN (elem_type
));
5594 align
= TYPE_ALIGN_UNIT (elem_type
);
5599 TREE_TYPE (data_ref
)
5600 = build_aligned_type (TREE_TYPE (data_ref
),
5601 TYPE_ALIGN (elem_type
));
5602 misalign
= DR_MISALIGNMENT (first_dr
);
5604 if (dataref_offset
== NULL_TREE
)
5605 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
5609 && dt
!= vect_constant_def
5610 && dt
!= vect_external_def
)
5612 tree perm_mask
= perm_mask_for_reverse (vectype
);
5614 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
5616 tree new_temp
= make_ssa_name (perm_dest
);
5618 /* Generate the permute statement. */
5620 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
5621 vec_oprnd
, perm_mask
);
5622 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5624 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
5625 vec_oprnd
= new_temp
;
5628 /* Arguments are ready. Create the new vector stmt. */
5629 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
5630 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5635 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5643 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5645 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5646 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5650 dr_chain
.release ();
5652 result_chain
.release ();
5653 vec_oprnds
.release ();
5658 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
5659 VECTOR_CST mask. No checks are made that the target platform supports the
5660 mask, so callers may wish to test can_vec_perm_p separately, or use
5661 vect_gen_perm_mask_checked. */
5664 vect_gen_perm_mask_any (tree vectype
, const unsigned char *sel
)
5666 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
5669 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5671 mask_elt_type
= lang_hooks
.types
.type_for_mode
5672 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
5673 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
5675 mask_elts
= XALLOCAVEC (tree
, nunits
);
5676 for (i
= nunits
- 1; i
>= 0; i
--)
5677 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
5678 mask_vec
= build_vector (mask_type
, mask_elts
);
5683 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
5684 i.e. that the target supports the pattern _for arbitrary input vectors_. */
5687 vect_gen_perm_mask_checked (tree vectype
, const unsigned char *sel
)
5689 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, sel
));
5690 return vect_gen_perm_mask_any (vectype
, sel
);
5693 /* Given a vector variable X and Y, that was generated for the scalar
5694 STMT, generate instructions to permute the vector elements of X and Y
5695 using permutation mask MASK_VEC, insert them at *GSI and return the
5696 permuted vector variable. */
5699 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple stmt
,
5700 gimple_stmt_iterator
*gsi
)
5702 tree vectype
= TREE_TYPE (x
);
5703 tree perm_dest
, data_ref
;
5706 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
5707 data_ref
= make_ssa_name (perm_dest
);
5709 /* Generate the permute statement. */
5710 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
5711 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5716 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5717 inserting them on the loops preheader edge. Returns true if we
5718 were successful in doing so (and thus STMT can be moved then),
5719 otherwise returns false. */
5722 hoist_defs_of_uses (gimple stmt
, struct loop
*loop
)
5728 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5730 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5731 if (!gimple_nop_p (def_stmt
)
5732 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5734 /* Make sure we don't need to recurse. While we could do
5735 so in simple cases when there are more complex use webs
5736 we don't have an easy way to preserve stmt order to fulfil
5737 dependencies within them. */
5740 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
5742 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
5744 gimple def_stmt2
= SSA_NAME_DEF_STMT (op2
);
5745 if (!gimple_nop_p (def_stmt2
)
5746 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
5756 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5758 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5759 if (!gimple_nop_p (def_stmt
)
5760 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5762 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
5763 gsi_remove (&gsi
, false);
5764 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
5771 /* vectorizable_load.
5773 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
5775 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5776 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5777 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5780 vectorizable_load (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5781 slp_tree slp_node
, slp_instance slp_node_instance
)
5784 tree vec_dest
= NULL
;
5785 tree data_ref
= NULL
;
5786 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5787 stmt_vec_info prev_stmt_info
;
5788 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5789 struct loop
*loop
= NULL
;
5790 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
5791 bool nested_in_vect_loop
= false;
5792 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5793 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5797 gimple new_stmt
= NULL
;
5799 enum dr_alignment_support alignment_support_scheme
;
5800 tree dataref_ptr
= NULL_TREE
;
5801 tree dataref_offset
= NULL_TREE
;
5802 gimple ptr_incr
= NULL
;
5803 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5805 int i
, j
, group_size
= -1, group_gap
;
5806 tree msq
= NULL_TREE
, lsq
;
5807 tree offset
= NULL_TREE
;
5808 tree byte_offset
= NULL_TREE
;
5809 tree realignment_token
= NULL_TREE
;
5811 vec
<tree
> dr_chain
= vNULL
;
5812 bool grouped_load
= false;
5813 bool load_lanes_p
= false;
5816 bool negative
= false;
5817 bool compute_in_loop
= false;
5818 struct loop
*at_loop
;
5820 bool slp
= (slp_node
!= NULL
);
5821 bool slp_perm
= false;
5822 enum tree_code code
;
5823 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5826 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
5827 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
5828 int gather_scale
= 1;
5829 enum vect_def_type gather_dt
= vect_unknown_def_type
;
5833 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5834 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
5835 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5840 /* Multiple types in SLP are handled by creating the appropriate number of
5841 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5843 if (slp
|| PURE_SLP_STMT (stmt_info
))
5846 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5848 gcc_assert (ncopies
>= 1);
5850 /* FORNOW. This restriction should be relaxed. */
5851 if (nested_in_vect_loop
&& ncopies
> 1)
5853 if (dump_enabled_p ())
5854 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5855 "multiple types in nested loop.\n");
5859 /* Invalidate assumptions made by dependence analysis when vectorization
5860 on the unrolled body effectively re-orders stmts. */
5862 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5863 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5864 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5866 if (dump_enabled_p ())
5867 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5868 "cannot perform implicit CSE when unrolling "
5869 "with negative dependence distance\n");
5873 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5876 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5879 /* Is vectorizable load? */
5880 if (!is_gimple_assign (stmt
))
5883 scalar_dest
= gimple_assign_lhs (stmt
);
5884 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
5887 code
= gimple_assign_rhs_code (stmt
);
5888 if (code
!= ARRAY_REF
5889 && code
!= BIT_FIELD_REF
5890 && code
!= INDIRECT_REF
5891 && code
!= COMPONENT_REF
5892 && code
!= IMAGPART_EXPR
5893 && code
!= REALPART_EXPR
5895 && TREE_CODE_CLASS (code
) != tcc_declaration
)
5898 if (!STMT_VINFO_DATA_REF (stmt_info
))
5901 elem_type
= TREE_TYPE (vectype
);
5902 mode
= TYPE_MODE (vectype
);
5904 /* FORNOW. In some cases can vectorize even if data-type not supported
5905 (e.g. - data copies). */
5906 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
5908 if (dump_enabled_p ())
5909 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5910 "Aligned load, but unsupported type.\n");
5914 /* Check if the load is a part of an interleaving chain. */
5915 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5917 grouped_load
= true;
5919 gcc_assert (! nested_in_vect_loop
&& !STMT_VINFO_GATHER_P (stmt_info
));
5921 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5923 /* If this is single-element interleaving with an element distance
5924 that leaves unused vector loads around punt - we at least create
5925 very sub-optimal code in that case (and blow up memory,
5927 if (first_stmt
== stmt
5928 && !GROUP_NEXT_ELEMENT (stmt_info
)
5929 && GROUP_SIZE (stmt_info
) > TYPE_VECTOR_SUBPARTS (vectype
))
5931 if (dump_enabled_p ())
5932 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5933 "single-element interleaving not supported "
5934 "for not adjacent vector loads\n");
5938 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
5941 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5943 && !PURE_SLP_STMT (stmt_info
)
5944 && !STMT_VINFO_STRIDED_P (stmt_info
))
5946 if (vect_load_lanes_supported (vectype
, group_size
))
5947 load_lanes_p
= true;
5948 else if (!vect_grouped_load_supported (vectype
, group_size
))
5952 /* Invalidate assumptions made by dependence analysis when vectorization
5953 on the unrolled body effectively re-orders stmts. */
5954 if (!PURE_SLP_STMT (stmt_info
)
5955 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5956 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5957 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5959 if (dump_enabled_p ())
5960 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5961 "cannot perform implicit CSE when performing "
5962 "group loads with negative dependence distance\n");
5966 /* Similarly when the stmt is a load that is both part of a SLP
5967 instance and a loop vectorized stmt via the same-dr mechanism
5968 we have to give up. */
5969 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
5970 && (STMT_SLP_TYPE (stmt_info
)
5971 != STMT_SLP_TYPE (vinfo_for_stmt
5972 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
5974 if (dump_enabled_p ())
5975 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5976 "conflicting SLP types for CSEd load\n");
5982 if (STMT_VINFO_GATHER_P (stmt_info
))
5986 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
5987 &gather_off
, &gather_scale
);
5988 gcc_assert (gather_decl
);
5989 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, bb_vinfo
,
5990 &def_stmt
, &def
, &gather_dt
,
5991 &gather_off_vectype
))
5993 if (dump_enabled_p ())
5994 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5995 "gather index use not simple.\n");
5999 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6002 && (slp
|| PURE_SLP_STMT (stmt_info
)))
6003 && (group_size
> nunits
6004 || nunits
% group_size
!= 0
6005 /* We don't support load permutations. */
6008 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6009 "unhandled strided group load\n");
6015 negative
= tree_int_cst_compare (nested_in_vect_loop
6016 ? STMT_VINFO_DR_STEP (stmt_info
)
6018 size_zero_node
) < 0;
6019 if (negative
&& ncopies
> 1)
6021 if (dump_enabled_p ())
6022 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6023 "multiple types with negative step.\n");
6031 if (dump_enabled_p ())
6032 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6033 "negative step for group load not supported"
6037 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
6038 if (alignment_support_scheme
!= dr_aligned
6039 && alignment_support_scheme
!= dr_unaligned_supported
)
6041 if (dump_enabled_p ())
6042 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6043 "negative step but alignment required.\n");
6046 if (!perm_mask_for_reverse (vectype
))
6048 if (dump_enabled_p ())
6049 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6050 "negative step and reversing not supported."
6057 if (!vec_stmt
) /* transformation not required. */
6059 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6060 /* The SLP costs are calculated during SLP analysis. */
6061 if (!PURE_SLP_STMT (stmt_info
))
6062 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
,
6067 if (dump_enabled_p ())
6068 dump_printf_loc (MSG_NOTE
, vect_location
,
6069 "transform load. ncopies = %d\n", ncopies
);
6073 ensure_base_align (stmt_info
, dr
);
6075 if (STMT_VINFO_GATHER_P (stmt_info
))
6077 tree vec_oprnd0
= NULL_TREE
, op
;
6078 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
6079 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6080 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6081 edge pe
= loop_preheader_edge (loop
);
6084 enum { NARROW
, NONE
, WIDEN
} modifier
;
6085 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
6087 if (nunits
== gather_off_nunits
)
6089 else if (nunits
== gather_off_nunits
/ 2)
6091 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
6094 for (i
= 0; i
< gather_off_nunits
; ++i
)
6095 sel
[i
] = i
| nunits
;
6097 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
6099 else if (nunits
== gather_off_nunits
* 2)
6101 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
6104 for (i
= 0; i
< nunits
; ++i
)
6105 sel
[i
] = i
< gather_off_nunits
6106 ? i
: i
+ nunits
- gather_off_nunits
;
6108 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6114 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
6115 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6116 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6117 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6118 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6119 scaletype
= TREE_VALUE (arglist
);
6120 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6122 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6124 ptr
= fold_convert (ptrtype
, gather_base
);
6125 if (!is_gimple_min_invariant (ptr
))
6127 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6128 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6129 gcc_assert (!new_bb
);
6132 /* Currently we support only unconditional gather loads,
6133 so mask should be all ones. */
6134 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6135 mask
= build_int_cst (masktype
, -1);
6136 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6138 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6139 mask
= build_vector_from_val (masktype
, mask
);
6140 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6142 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6146 for (j
= 0; j
< 6; ++j
)
6148 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6149 mask
= build_real (TREE_TYPE (masktype
), r
);
6150 mask
= build_vector_from_val (masktype
, mask
);
6151 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6156 scale
= build_int_cst (scaletype
, gather_scale
);
6158 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6159 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6160 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6164 for (j
= 0; j
< 6; ++j
)
6166 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6167 merge
= build_real (TREE_TYPE (rettype
), r
);
6171 merge
= build_vector_from_val (rettype
, merge
);
6172 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6174 prev_stmt_info
= NULL
;
6175 for (j
= 0; j
< ncopies
; ++j
)
6177 if (modifier
== WIDEN
&& (j
& 1))
6178 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6179 perm_mask
, stmt
, gsi
);
6182 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
6185 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
6187 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6189 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6190 == TYPE_VECTOR_SUBPARTS (idxtype
));
6191 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
6192 var
= make_ssa_name (var
);
6193 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6195 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6196 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6201 = gimple_build_call (gather_decl
, 5, merge
, ptr
, op
, mask
, scale
);
6203 if (!useless_type_conversion_p (vectype
, rettype
))
6205 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6206 == TYPE_VECTOR_SUBPARTS (rettype
));
6207 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
6208 op
= make_ssa_name (var
, new_stmt
);
6209 gimple_call_set_lhs (new_stmt
, op
);
6210 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6211 var
= make_ssa_name (vec_dest
);
6212 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6214 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6218 var
= make_ssa_name (vec_dest
, new_stmt
);
6219 gimple_call_set_lhs (new_stmt
, var
);
6222 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6224 if (modifier
== NARROW
)
6231 var
= permute_vec_elements (prev_res
, var
,
6232 perm_mask
, stmt
, gsi
);
6233 new_stmt
= SSA_NAME_DEF_STMT (var
);
6236 if (prev_stmt_info
== NULL
)
6237 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6239 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6240 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6244 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6246 gimple_stmt_iterator incr_gsi
;
6252 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6253 gimple_seq stmts
= NULL
;
6254 tree stride_base
, stride_step
, alias_off
;
6256 gcc_assert (!nested_in_vect_loop
);
6259 = fold_build_pointer_plus
6260 (unshare_expr (DR_BASE_ADDRESS (dr
)),
6261 size_binop (PLUS_EXPR
,
6262 convert_to_ptrofftype (unshare_expr (DR_OFFSET (dr
))),
6263 convert_to_ptrofftype (DR_INIT (dr
))));
6264 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (dr
)));
6266 /* For a load with loop-invariant (but other than power-of-2)
6267 stride (i.e. not a grouped access) like so:
6269 for (i = 0; i < n; i += stride)
6272 we generate a new induction variable and new accesses to
6273 form a new vector (or vectors, depending on ncopies):
6275 for (j = 0; ; j += VF*stride)
6277 tmp2 = array[j + stride];
6279 vectemp = {tmp1, tmp2, ...}
6282 ivstep
= stride_step
;
6283 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
6284 build_int_cst (TREE_TYPE (ivstep
), vf
));
6286 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6288 create_iv (stride_base
, ivstep
, NULL
,
6289 loop
, &incr_gsi
, insert_after
,
6291 incr
= gsi_stmt (incr_gsi
);
6292 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
6294 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
6296 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6298 prev_stmt_info
= NULL
;
6299 running_off
= offvar
;
6300 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (dr
)), 0);
6301 int nloads
= nunits
;
6302 tree ltype
= TREE_TYPE (vectype
);
6305 nloads
= nunits
/ group_size
;
6306 if (group_size
< nunits
)
6307 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
6310 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
6311 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6312 gcc_assert (!slp_perm
);
6314 for (j
= 0; j
< ncopies
; j
++)
6320 vec_alloc (v
, nloads
);
6321 for (i
= 0; i
< nloads
; i
++)
6323 tree newref
, newoff
;
6325 newref
= build2 (MEM_REF
, ltype
, running_off
, alias_off
);
6327 newref
= force_gimple_operand_gsi (gsi
, newref
, true,
6330 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, newref
);
6331 newoff
= copy_ssa_name (running_off
);
6332 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6333 running_off
, stride_step
);
6334 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6336 running_off
= newoff
;
6339 vec_inv
= build_constructor (vectype
, v
);
6340 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6341 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6345 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
6346 build2 (MEM_REF
, ltype
,
6347 running_off
, alias_off
));
6348 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6350 tree newoff
= copy_ssa_name (running_off
);
6351 gimple incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6352 running_off
, stride_step
);
6353 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6355 running_off
= newoff
;
6359 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6361 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6363 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6364 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6371 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6373 && !SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ()
6374 && first_stmt
!= SLP_TREE_SCALAR_STMTS (slp_node
)[0])
6375 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6377 /* Check if the chain of loads is already vectorized. */
6378 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6379 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6380 ??? But we can only do so if there is exactly one
6381 as we have no way to get at the rest. Leave the CSE
6383 ??? With the group load eventually participating
6384 in multiple different permutations (having multiple
6385 slp nodes which refer to the same group) the CSE
6386 is even wrong code. See PR56270. */
6389 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6392 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6393 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6395 /* VEC_NUM is the number of vect stmts to be created for this group. */
6398 grouped_load
= false;
6399 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6400 group_gap
= GROUP_GAP (vinfo_for_stmt (first_stmt
));
6404 vec_num
= group_size
;
6412 group_size
= vec_num
= 1;
6416 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6417 gcc_assert (alignment_support_scheme
);
6418 /* Targets with load-lane instructions must not require explicit
6420 gcc_assert (!load_lanes_p
6421 || alignment_support_scheme
== dr_aligned
6422 || alignment_support_scheme
== dr_unaligned_supported
);
6424 /* In case the vectorization factor (VF) is bigger than the number
6425 of elements that we can fit in a vectype (nunits), we have to generate
6426 more than one vector stmt - i.e - we need to "unroll" the
6427 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6428 from one copy of the vector stmt to the next, in the field
6429 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6430 stages to find the correct vector defs to be used when vectorizing
6431 stmts that use the defs of the current stmt. The example below
6432 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6433 need to create 4 vectorized stmts):
6435 before vectorization:
6436 RELATED_STMT VEC_STMT
6440 step 1: vectorize stmt S1:
6441 We first create the vector stmt VS1_0, and, as usual, record a
6442 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6443 Next, we create the vector stmt VS1_1, and record a pointer to
6444 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6445 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6447 RELATED_STMT VEC_STMT
6448 VS1_0: vx0 = memref0 VS1_1 -
6449 VS1_1: vx1 = memref1 VS1_2 -
6450 VS1_2: vx2 = memref2 VS1_3 -
6451 VS1_3: vx3 = memref3 - -
6452 S1: x = load - VS1_0
6455 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6456 information we recorded in RELATED_STMT field is used to vectorize
6459 /* In case of interleaving (non-unit grouped access):
6466 Vectorized loads are created in the order of memory accesses
6467 starting from the access of the first stmt of the chain:
6470 VS2: vx1 = &base + vec_size*1
6471 VS3: vx3 = &base + vec_size*2
6472 VS4: vx4 = &base + vec_size*3
6474 Then permutation statements are generated:
6476 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6477 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6480 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6481 (the order of the data-refs in the output of vect_permute_load_chain
6482 corresponds to the order of scalar stmts in the interleaving chain - see
6483 the documentation of vect_permute_load_chain()).
6484 The generation of permutation stmts and recording them in
6485 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6487 In case of both multiple types and interleaving, the vector loads and
6488 permutation stmts above are created for every copy. The result vector
6489 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6490 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6492 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6493 on a target that supports unaligned accesses (dr_unaligned_supported)
6494 we generate the following code:
6498 p = p + indx * vectype_size;
6503 Otherwise, the data reference is potentially unaligned on a target that
6504 does not support unaligned accesses (dr_explicit_realign_optimized) -
6505 then generate the following code, in which the data in each iteration is
6506 obtained by two vector loads, one from the previous iteration, and one
6507 from the current iteration:
6509 msq_init = *(floor(p1))
6510 p2 = initial_addr + VS - 1;
6511 realignment_token = call target_builtin;
6514 p2 = p2 + indx * vectype_size
6516 vec_dest = realign_load (msq, lsq, realignment_token)
6521 /* If the misalignment remains the same throughout the execution of the
6522 loop, we can create the init_addr and permutation mask at the loop
6523 preheader. Otherwise, it needs to be created inside the loop.
6524 This can only occur when vectorizing memory accesses in the inner-loop
6525 nested within an outer-loop that is being vectorized. */
6527 if (nested_in_vect_loop
6528 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6529 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6531 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6532 compute_in_loop
= true;
6535 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6536 || alignment_support_scheme
== dr_explicit_realign
)
6537 && !compute_in_loop
)
6539 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6540 alignment_support_scheme
, NULL_TREE
,
6542 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6544 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
6545 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
6553 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6556 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6558 aggr_type
= vectype
;
6560 prev_stmt_info
= NULL
;
6561 for (j
= 0; j
< ncopies
; j
++)
6563 /* 1. Create the vector or array pointer update chain. */
6566 bool simd_lane_access_p
6567 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6568 if (simd_lane_access_p
6569 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6570 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6571 && integer_zerop (DR_OFFSET (first_dr
))
6572 && integer_zerop (DR_INIT (first_dr
))
6573 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6574 get_alias_set (DR_REF (first_dr
)))
6575 && (alignment_support_scheme
== dr_aligned
6576 || alignment_support_scheme
== dr_unaligned_supported
))
6578 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6579 dataref_offset
= build_int_cst (reference_alias_ptr_type
6580 (DR_REF (first_dr
)), 0);
6585 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
6586 offset
, &dummy
, gsi
, &ptr_incr
,
6587 simd_lane_access_p
, &inv_p
,
6590 else if (dataref_offset
)
6591 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
6592 TYPE_SIZE_UNIT (aggr_type
));
6594 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6595 TYPE_SIZE_UNIT (aggr_type
));
6597 if (grouped_load
|| slp_perm
)
6598 dr_chain
.create (vec_num
);
6604 vec_array
= create_vector_array (vectype
, vec_num
);
6607 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6608 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
6609 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
6610 gimple_call_set_lhs (new_stmt
, vec_array
);
6611 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6613 /* Extract each vector into an SSA_NAME. */
6614 for (i
= 0; i
< vec_num
; i
++)
6616 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
6618 dr_chain
.quick_push (new_temp
);
6621 /* Record the mapping between SSA_NAMEs and statements. */
6622 vect_record_grouped_load_vectors (stmt
, dr_chain
);
6626 for (i
= 0; i
< vec_num
; i
++)
6629 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6632 /* 2. Create the vector-load in the loop. */
6633 switch (alignment_support_scheme
)
6636 case dr_unaligned_supported
:
6638 unsigned int align
, misalign
;
6641 = build2 (MEM_REF
, vectype
, dataref_ptr
,
6644 : build_int_cst (reference_alias_ptr_type
6645 (DR_REF (first_dr
)), 0));
6646 align
= TYPE_ALIGN_UNIT (vectype
);
6647 if (alignment_support_scheme
== dr_aligned
)
6649 gcc_assert (aligned_access_p (first_dr
));
6652 else if (DR_MISALIGNMENT (first_dr
) == -1)
6654 TREE_TYPE (data_ref
)
6655 = build_aligned_type (TREE_TYPE (data_ref
),
6656 TYPE_ALIGN (elem_type
));
6657 align
= TYPE_ALIGN_UNIT (elem_type
);
6662 TREE_TYPE (data_ref
)
6663 = build_aligned_type (TREE_TYPE (data_ref
),
6664 TYPE_ALIGN (elem_type
));
6665 misalign
= DR_MISALIGNMENT (first_dr
);
6667 if (dataref_offset
== NULL_TREE
)
6668 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
6672 case dr_explicit_realign
:
6676 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
6678 if (compute_in_loop
)
6679 msq
= vect_setup_realignment (first_stmt
, gsi
,
6681 dr_explicit_realign
,
6684 ptr
= copy_ssa_name (dataref_ptr
);
6685 new_stmt
= gimple_build_assign
6686 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
6688 (TREE_TYPE (dataref_ptr
),
6689 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6690 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6692 = build2 (MEM_REF
, vectype
, ptr
,
6693 build_int_cst (reference_alias_ptr_type
6694 (DR_REF (first_dr
)), 0));
6695 vec_dest
= vect_create_destination_var (scalar_dest
,
6697 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6698 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6699 gimple_assign_set_lhs (new_stmt
, new_temp
);
6700 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
6701 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
6702 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6705 bump
= size_binop (MULT_EXPR
, vs
,
6706 TYPE_SIZE_UNIT (elem_type
));
6707 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
6708 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
6709 new_stmt
= gimple_build_assign
6710 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
6713 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6714 ptr
= copy_ssa_name (dataref_ptr
, new_stmt
);
6715 gimple_assign_set_lhs (new_stmt
, ptr
);
6716 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6718 = build2 (MEM_REF
, vectype
, ptr
,
6719 build_int_cst (reference_alias_ptr_type
6720 (DR_REF (first_dr
)), 0));
6723 case dr_explicit_realign_optimized
:
6724 new_temp
= copy_ssa_name (dataref_ptr
);
6725 new_stmt
= gimple_build_assign
6726 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
6728 (TREE_TYPE (dataref_ptr
),
6729 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6730 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6732 = build2 (MEM_REF
, vectype
, new_temp
,
6733 build_int_cst (reference_alias_ptr_type
6734 (DR_REF (first_dr
)), 0));
6739 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6740 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6741 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6742 gimple_assign_set_lhs (new_stmt
, new_temp
);
6743 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6745 /* 3. Handle explicit realignment if necessary/supported.
6747 vec_dest = realign_load (msq, lsq, realignment_token) */
6748 if (alignment_support_scheme
== dr_explicit_realign_optimized
6749 || alignment_support_scheme
== dr_explicit_realign
)
6751 lsq
= gimple_assign_lhs (new_stmt
);
6752 if (!realignment_token
)
6753 realignment_token
= dataref_ptr
;
6754 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6755 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
6756 msq
, lsq
, realignment_token
);
6757 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6758 gimple_assign_set_lhs (new_stmt
, new_temp
);
6759 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6761 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6764 if (i
== vec_num
- 1 && j
== ncopies
- 1)
6765 add_phi_arg (phi
, lsq
,
6766 loop_latch_edge (containing_loop
),
6772 /* 4. Handle invariant-load. */
6773 if (inv_p
&& !bb_vinfo
)
6775 gcc_assert (!grouped_load
);
6776 /* If we have versioned for aliasing or the loop doesn't
6777 have any data dependencies that would preclude this,
6778 then we are sure this is a loop invariant load and
6779 thus we can insert it on the preheader edge. */
6780 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
6781 && !nested_in_vect_loop
6782 && hoist_defs_of_uses (stmt
, loop
))
6784 if (dump_enabled_p ())
6786 dump_printf_loc (MSG_NOTE
, vect_location
,
6787 "hoisting out of the vectorized "
6789 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6791 tree tem
= copy_ssa_name (scalar_dest
);
6792 gsi_insert_on_edge_immediate
6793 (loop_preheader_edge (loop
),
6794 gimple_build_assign (tem
,
6796 (gimple_assign_rhs1 (stmt
))));
6797 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
6801 gimple_stmt_iterator gsi2
= *gsi
;
6803 new_temp
= vect_init_vector (stmt
, scalar_dest
,
6806 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6807 set_vinfo_for_stmt (new_stmt
,
6808 new_stmt_vec_info (new_stmt
, loop_vinfo
,
6814 tree perm_mask
= perm_mask_for_reverse (vectype
);
6815 new_temp
= permute_vec_elements (new_temp
, new_temp
,
6816 perm_mask
, stmt
, gsi
);
6817 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6820 /* Collect vector loads and later create their permutation in
6821 vect_transform_grouped_load (). */
6822 if (grouped_load
|| slp_perm
)
6823 dr_chain
.quick_push (new_temp
);
6825 /* Store vector loads in the corresponding SLP_NODE. */
6826 if (slp
&& !slp_perm
)
6827 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6829 /* Bump the vector pointer to account for a gap. */
6830 if (slp
&& group_gap
!= 0)
6832 tree bump
= size_binop (MULT_EXPR
,
6833 TYPE_SIZE_UNIT (elem_type
),
6834 size_int (group_gap
));
6835 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6840 if (slp
&& !slp_perm
)
6845 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6846 slp_node_instance
, false))
6848 dr_chain
.release ();
6857 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
6858 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6863 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6865 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6866 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6869 dr_chain
.release ();
6875 /* Function vect_is_simple_cond.
6878 LOOP - the loop that is being vectorized.
6879 COND - Condition that is checked for simple use.
6882 *COMP_VECTYPE - the vector type for the comparison.
6884 Returns whether a COND can be vectorized. Checks whether
6885 condition operands are supportable using vec_is_simple_use. */
6888 vect_is_simple_cond (tree cond
, gimple stmt
, loop_vec_info loop_vinfo
,
6889 bb_vec_info bb_vinfo
, tree
*comp_vectype
)
6893 enum vect_def_type dt
;
6894 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
6896 if (!COMPARISON_CLASS_P (cond
))
6899 lhs
= TREE_OPERAND (cond
, 0);
6900 rhs
= TREE_OPERAND (cond
, 1);
6902 if (TREE_CODE (lhs
) == SSA_NAME
)
6904 gimple lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
6905 if (!vect_is_simple_use_1 (lhs
, stmt
, loop_vinfo
, bb_vinfo
,
6906 &lhs_def_stmt
, &def
, &dt
, &vectype1
))
6909 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
6910 && TREE_CODE (lhs
) != FIXED_CST
)
6913 if (TREE_CODE (rhs
) == SSA_NAME
)
6915 gimple rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
6916 if (!vect_is_simple_use_1 (rhs
, stmt
, loop_vinfo
, bb_vinfo
,
6917 &rhs_def_stmt
, &def
, &dt
, &vectype2
))
6920 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
6921 && TREE_CODE (rhs
) != FIXED_CST
)
6924 *comp_vectype
= vectype1
? vectype1
: vectype2
;
6928 /* vectorizable_condition.
6930 Check if STMT is conditional modify expression that can be vectorized.
6931 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6932 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
6935 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
6936 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
6937 else caluse if it is 2).
6939 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6942 vectorizable_condition (gimple stmt
, gimple_stmt_iterator
*gsi
,
6943 gimple
*vec_stmt
, tree reduc_def
, int reduc_index
,
6946 tree scalar_dest
= NULL_TREE
;
6947 tree vec_dest
= NULL_TREE
;
6948 tree cond_expr
, then_clause
, else_clause
;
6949 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6950 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6951 tree comp_vectype
= NULL_TREE
;
6952 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
6953 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
6954 tree vec_compare
, vec_cond_expr
;
6956 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6958 enum vect_def_type dt
, dts
[4];
6959 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6961 enum tree_code code
;
6962 stmt_vec_info prev_stmt_info
= NULL
;
6964 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6965 vec
<tree
> vec_oprnds0
= vNULL
;
6966 vec
<tree
> vec_oprnds1
= vNULL
;
6967 vec
<tree
> vec_oprnds2
= vNULL
;
6968 vec
<tree
> vec_oprnds3
= vNULL
;
6971 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
6974 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
6976 gcc_assert (ncopies
>= 1);
6977 if (reduc_index
&& ncopies
> 1)
6978 return false; /* FORNOW */
6980 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
6983 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6986 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6987 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
6991 /* FORNOW: not yet supported. */
6992 if (STMT_VINFO_LIVE_P (stmt_info
))
6994 if (dump_enabled_p ())
6995 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6996 "value used after loop.\n");
7000 /* Is vectorizable conditional operation? */
7001 if (!is_gimple_assign (stmt
))
7004 code
= gimple_assign_rhs_code (stmt
);
7006 if (code
!= COND_EXPR
)
7009 cond_expr
= gimple_assign_rhs1 (stmt
);
7010 then_clause
= gimple_assign_rhs2 (stmt
);
7011 else_clause
= gimple_assign_rhs3 (stmt
);
7013 if (!vect_is_simple_cond (cond_expr
, stmt
, loop_vinfo
, bb_vinfo
,
7018 if (TREE_CODE (then_clause
) == SSA_NAME
)
7020 gimple then_def_stmt
= SSA_NAME_DEF_STMT (then_clause
);
7021 if (!vect_is_simple_use (then_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7022 &then_def_stmt
, &def
, &dt
))
7025 else if (TREE_CODE (then_clause
) != INTEGER_CST
7026 && TREE_CODE (then_clause
) != REAL_CST
7027 && TREE_CODE (then_clause
) != FIXED_CST
)
7030 if (TREE_CODE (else_clause
) == SSA_NAME
)
7032 gimple else_def_stmt
= SSA_NAME_DEF_STMT (else_clause
);
7033 if (!vect_is_simple_use (else_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7034 &else_def_stmt
, &def
, &dt
))
7037 else if (TREE_CODE (else_clause
) != INTEGER_CST
7038 && TREE_CODE (else_clause
) != REAL_CST
7039 && TREE_CODE (else_clause
) != FIXED_CST
)
7042 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
)));
7043 /* The result of a vector comparison should be signed type. */
7044 tree cmp_type
= build_nonstandard_integer_type (prec
, 0);
7045 vec_cmp_type
= get_same_sized_vectype (cmp_type
, vectype
);
7046 if (vec_cmp_type
== NULL_TREE
)
7051 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
7052 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
7059 vec_oprnds0
.create (1);
7060 vec_oprnds1
.create (1);
7061 vec_oprnds2
.create (1);
7062 vec_oprnds3
.create (1);
7066 scalar_dest
= gimple_assign_lhs (stmt
);
7067 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7069 /* Handle cond expr. */
7070 for (j
= 0; j
< ncopies
; j
++)
7072 gassign
*new_stmt
= NULL
;
7077 auto_vec
<tree
, 4> ops
;
7078 auto_vec
<vec
<tree
>, 4> vec_defs
;
7080 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
7081 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
7082 ops
.safe_push (then_clause
);
7083 ops
.safe_push (else_clause
);
7084 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7085 vec_oprnds3
= vec_defs
.pop ();
7086 vec_oprnds2
= vec_defs
.pop ();
7087 vec_oprnds1
= vec_defs
.pop ();
7088 vec_oprnds0
= vec_defs
.pop ();
7091 vec_defs
.release ();
7097 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
7099 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0), stmt
,
7100 loop_vinfo
, NULL
, >emp
, &def
, &dts
[0]);
7103 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
7105 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1), stmt
,
7106 loop_vinfo
, NULL
, >emp
, &def
, &dts
[1]);
7107 if (reduc_index
== 1)
7108 vec_then_clause
= reduc_def
;
7111 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
7113 vect_is_simple_use (then_clause
, stmt
, loop_vinfo
,
7114 NULL
, >emp
, &def
, &dts
[2]);
7116 if (reduc_index
== 2)
7117 vec_else_clause
= reduc_def
;
7120 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
7122 vect_is_simple_use (else_clause
, stmt
, loop_vinfo
,
7123 NULL
, >emp
, &def
, &dts
[3]);
7129 vec_cond_lhs
= vect_get_vec_def_for_stmt_copy (dts
[0],
7130 vec_oprnds0
.pop ());
7131 vec_cond_rhs
= vect_get_vec_def_for_stmt_copy (dts
[1],
7132 vec_oprnds1
.pop ());
7133 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
7134 vec_oprnds2
.pop ());
7135 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
7136 vec_oprnds3
.pop ());
7141 vec_oprnds0
.quick_push (vec_cond_lhs
);
7142 vec_oprnds1
.quick_push (vec_cond_rhs
);
7143 vec_oprnds2
.quick_push (vec_then_clause
);
7144 vec_oprnds3
.quick_push (vec_else_clause
);
7147 /* Arguments are ready. Create the new vector stmt. */
7148 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
7150 vec_cond_rhs
= vec_oprnds1
[i
];
7151 vec_then_clause
= vec_oprnds2
[i
];
7152 vec_else_clause
= vec_oprnds3
[i
];
7154 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
7155 vec_cond_lhs
, vec_cond_rhs
);
7156 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
7157 vec_compare
, vec_then_clause
, vec_else_clause
);
7159 new_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
7160 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7161 gimple_assign_set_lhs (new_stmt
, new_temp
);
7162 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7164 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7171 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7173 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7175 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7178 vec_oprnds0
.release ();
7179 vec_oprnds1
.release ();
7180 vec_oprnds2
.release ();
7181 vec_oprnds3
.release ();
7187 /* Make sure the statement is vectorizable. */
7190 vect_analyze_stmt (gimple stmt
, bool *need_to_vectorize
, slp_tree node
)
7192 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7193 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7194 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
7196 tree scalar_type
, vectype
;
7197 gimple pattern_stmt
;
7198 gimple_seq pattern_def_seq
;
7200 if (dump_enabled_p ())
7202 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
7203 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7206 if (gimple_has_volatile_ops (stmt
))
7208 if (dump_enabled_p ())
7209 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7210 "not vectorized: stmt has volatile operands\n");
7215 /* Skip stmts that do not need to be vectorized. In loops this is expected
7217 - the COND_EXPR which is the loop exit condition
7218 - any LABEL_EXPRs in the loop
7219 - computations that are used only for array indexing or loop control.
7220 In basic blocks we only analyze statements that are a part of some SLP
7221 instance, therefore, all the statements are relevant.
7223 Pattern statement needs to be analyzed instead of the original statement
7224 if the original statement is not relevant. Otherwise, we analyze both
7225 statements. In basic blocks we are called from some SLP instance
7226 traversal, don't analyze pattern stmts instead, the pattern stmts
7227 already will be part of SLP instance. */
7229 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
7230 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
7231 && !STMT_VINFO_LIVE_P (stmt_info
))
7233 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7235 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7236 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7238 /* Analyze PATTERN_STMT instead of the original stmt. */
7239 stmt
= pattern_stmt
;
7240 stmt_info
= vinfo_for_stmt (pattern_stmt
);
7241 if (dump_enabled_p ())
7243 dump_printf_loc (MSG_NOTE
, vect_location
,
7244 "==> examining pattern statement: ");
7245 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7250 if (dump_enabled_p ())
7251 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
7256 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7259 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7260 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7262 /* Analyze PATTERN_STMT too. */
7263 if (dump_enabled_p ())
7265 dump_printf_loc (MSG_NOTE
, vect_location
,
7266 "==> examining pattern statement: ");
7267 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7270 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
7274 if (is_pattern_stmt_p (stmt_info
)
7276 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
7278 gimple_stmt_iterator si
;
7280 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
7282 gimple pattern_def_stmt
= gsi_stmt (si
);
7283 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
7284 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
7286 /* Analyze def stmt of STMT if it's a pattern stmt. */
7287 if (dump_enabled_p ())
7289 dump_printf_loc (MSG_NOTE
, vect_location
,
7290 "==> examining pattern def statement: ");
7291 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
7294 if (!vect_analyze_stmt (pattern_def_stmt
,
7295 need_to_vectorize
, node
))
7301 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
7303 case vect_internal_def
:
7306 case vect_reduction_def
:
7307 case vect_nested_cycle
:
7308 gcc_assert (!bb_vinfo
7309 && (relevance
== vect_used_in_outer
7310 || relevance
== vect_used_in_outer_by_reduction
7311 || relevance
== vect_used_by_reduction
7312 || relevance
== vect_unused_in_scope
));
7315 case vect_induction_def
:
7316 case vect_constant_def
:
7317 case vect_external_def
:
7318 case vect_unknown_def_type
:
7325 gcc_assert (PURE_SLP_STMT (stmt_info
));
7327 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
7328 if (dump_enabled_p ())
7330 dump_printf_loc (MSG_NOTE
, vect_location
,
7331 "get vectype for scalar type: ");
7332 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
7333 dump_printf (MSG_NOTE
, "\n");
7336 vectype
= get_vectype_for_scalar_type (scalar_type
);
7339 if (dump_enabled_p ())
7341 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7342 "not SLPed: unsupported data-type ");
7343 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
7345 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7350 if (dump_enabled_p ())
7352 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
7353 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
7354 dump_printf (MSG_NOTE
, "\n");
7357 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
7360 if (STMT_VINFO_RELEVANT_P (stmt_info
))
7362 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
7363 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
7364 || (is_gimple_call (stmt
)
7365 && gimple_call_lhs (stmt
) == NULL_TREE
));
7366 *need_to_vectorize
= true;
7369 if (PURE_SLP_STMT (stmt_info
) && !node
)
7371 dump_printf_loc (MSG_NOTE
, vect_location
,
7372 "handled only by SLP analysis\n");
7378 && (STMT_VINFO_RELEVANT_P (stmt_info
)
7379 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
7380 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7381 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7382 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7383 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7384 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7385 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7386 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7387 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7388 || vectorizable_reduction (stmt
, NULL
, NULL
, node
)
7389 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7393 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7394 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7395 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7396 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7397 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7398 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7399 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7400 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7401 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7406 if (dump_enabled_p ())
7408 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7409 "not vectorized: relevant stmt not ");
7410 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7411 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7420 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7421 need extra handling, except for vectorizable reductions. */
7422 if (STMT_VINFO_LIVE_P (stmt_info
)
7423 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7424 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
7428 if (dump_enabled_p ())
7430 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7431 "not vectorized: live stmt not ");
7432 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7433 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7443 /* Function vect_transform_stmt.
7445 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7448 vect_transform_stmt (gimple stmt
, gimple_stmt_iterator
*gsi
,
7449 bool *grouped_store
, slp_tree slp_node
,
7450 slp_instance slp_node_instance
)
7452 bool is_store
= false;
7453 gimple vec_stmt
= NULL
;
7454 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7457 switch (STMT_VINFO_TYPE (stmt_info
))
7459 case type_demotion_vec_info_type
:
7460 case type_promotion_vec_info_type
:
7461 case type_conversion_vec_info_type
:
7462 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
7466 case induc_vec_info_type
:
7467 gcc_assert (!slp_node
);
7468 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
7472 case shift_vec_info_type
:
7473 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
7477 case op_vec_info_type
:
7478 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
7482 case assignment_vec_info_type
:
7483 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
7487 case load_vec_info_type
:
7488 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
7493 case store_vec_info_type
:
7494 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
7496 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
7498 /* In case of interleaving, the whole chain is vectorized when the
7499 last store in the chain is reached. Store stmts before the last
7500 one are skipped, and there vec_stmt_info shouldn't be freed
7502 *grouped_store
= true;
7503 if (STMT_VINFO_VEC_STMT (stmt_info
))
7510 case condition_vec_info_type
:
7511 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
7515 case call_vec_info_type
:
7516 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7517 stmt
= gsi_stmt (*gsi
);
7518 if (is_gimple_call (stmt
)
7519 && gimple_call_internal_p (stmt
)
7520 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
7524 case call_simd_clone_vec_info_type
:
7525 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7526 stmt
= gsi_stmt (*gsi
);
7529 case reduc_vec_info_type
:
7530 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
7535 if (!STMT_VINFO_LIVE_P (stmt_info
))
7537 if (dump_enabled_p ())
7538 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7539 "stmt not supported.\n");
7544 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
7545 is being vectorized, but outside the immediately enclosing loop. */
7547 && STMT_VINFO_LOOP_VINFO (stmt_info
)
7548 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
7549 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
7550 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
7551 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
7552 || STMT_VINFO_RELEVANT (stmt_info
) ==
7553 vect_used_in_outer_by_reduction
))
7555 struct loop
*innerloop
= LOOP_VINFO_LOOP (
7556 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
7557 imm_use_iterator imm_iter
;
7558 use_operand_p use_p
;
7562 if (dump_enabled_p ())
7563 dump_printf_loc (MSG_NOTE
, vect_location
,
7564 "Record the vdef for outer-loop vectorization.\n");
7566 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
7567 (to be used when vectorizing outer-loop stmts that use the DEF of
7569 if (gimple_code (stmt
) == GIMPLE_PHI
)
7570 scalar_dest
= PHI_RESULT (stmt
);
7572 scalar_dest
= gimple_assign_lhs (stmt
);
7574 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
7576 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
7578 exit_phi
= USE_STMT (use_p
);
7579 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
7584 /* Handle stmts whose DEF is used outside the loop-nest that is
7585 being vectorized. */
7586 if (STMT_VINFO_LIVE_P (stmt_info
)
7587 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7589 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
7594 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
7600 /* Remove a group of stores (for SLP or interleaving), free their
7604 vect_remove_stores (gimple first_stmt
)
7606 gimple next
= first_stmt
;
7608 gimple_stmt_iterator next_si
;
7612 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
7614 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
7615 if (is_pattern_stmt_p (stmt_info
))
7616 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
7617 /* Free the attached stmt_vec_info and remove the stmt. */
7618 next_si
= gsi_for_stmt (next
);
7619 unlink_stmt_vdef (next
);
7620 gsi_remove (&next_si
, true);
7621 release_defs (next
);
7622 free_stmt_vec_info (next
);
7628 /* Function new_stmt_vec_info.
7630 Create and initialize a new stmt_vec_info struct for STMT. */
7633 new_stmt_vec_info (gimple stmt
, loop_vec_info loop_vinfo
,
7634 bb_vec_info bb_vinfo
)
7637 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
7639 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
7640 STMT_VINFO_STMT (res
) = stmt
;
7641 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
7642 STMT_VINFO_BB_VINFO (res
) = bb_vinfo
;
7643 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
7644 STMT_VINFO_LIVE_P (res
) = false;
7645 STMT_VINFO_VECTYPE (res
) = NULL
;
7646 STMT_VINFO_VEC_STMT (res
) = NULL
;
7647 STMT_VINFO_VECTORIZABLE (res
) = true;
7648 STMT_VINFO_IN_PATTERN_P (res
) = false;
7649 STMT_VINFO_RELATED_STMT (res
) = NULL
;
7650 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
7651 STMT_VINFO_DATA_REF (res
) = NULL
;
7653 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
7654 STMT_VINFO_DR_OFFSET (res
) = NULL
;
7655 STMT_VINFO_DR_INIT (res
) = NULL
;
7656 STMT_VINFO_DR_STEP (res
) = NULL
;
7657 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
7659 if (gimple_code (stmt
) == GIMPLE_PHI
7660 && is_loop_header_bb_p (gimple_bb (stmt
)))
7661 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
7663 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
7665 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
7666 STMT_SLP_TYPE (res
) = loop_vect
;
7667 GROUP_FIRST_ELEMENT (res
) = NULL
;
7668 GROUP_NEXT_ELEMENT (res
) = NULL
;
7669 GROUP_SIZE (res
) = 0;
7670 GROUP_STORE_COUNT (res
) = 0;
7671 GROUP_GAP (res
) = 0;
7672 GROUP_SAME_DR_STMT (res
) = NULL
;
7678 /* Create a hash table for stmt_vec_info. */
7681 init_stmt_vec_info_vec (void)
7683 gcc_assert (!stmt_vec_info_vec
.exists ());
7684 stmt_vec_info_vec
.create (50);
7688 /* Free hash table for stmt_vec_info. */
7691 free_stmt_vec_info_vec (void)
7695 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
7697 free_stmt_vec_info (STMT_VINFO_STMT ((stmt_vec_info
) info
));
7698 gcc_assert (stmt_vec_info_vec
.exists ());
7699 stmt_vec_info_vec
.release ();
7703 /* Free stmt vectorization related info. */
7706 free_stmt_vec_info (gimple stmt
)
7708 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7713 /* Check if this statement has a related "pattern stmt"
7714 (introduced by the vectorizer during the pattern recognition
7715 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
7717 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
7719 stmt_vec_info patt_info
7720 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
7723 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
7724 gimple patt_stmt
= STMT_VINFO_STMT (patt_info
);
7725 gimple_set_bb (patt_stmt
, NULL
);
7726 tree lhs
= gimple_get_lhs (patt_stmt
);
7727 if (TREE_CODE (lhs
) == SSA_NAME
)
7728 release_ssa_name (lhs
);
7731 gimple_stmt_iterator si
;
7732 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
7734 gimple seq_stmt
= gsi_stmt (si
);
7735 gimple_set_bb (seq_stmt
, NULL
);
7736 lhs
= gimple_get_lhs (patt_stmt
);
7737 if (TREE_CODE (lhs
) == SSA_NAME
)
7738 release_ssa_name (lhs
);
7739 free_stmt_vec_info (seq_stmt
);
7742 free_stmt_vec_info (patt_stmt
);
7746 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
7747 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
7748 set_vinfo_for_stmt (stmt
, NULL
);
7753 /* Function get_vectype_for_scalar_type_and_size.
7755 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
7759 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
7761 machine_mode inner_mode
= TYPE_MODE (scalar_type
);
7762 machine_mode simd_mode
;
7763 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
7770 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
7771 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
7774 /* For vector types of elements whose mode precision doesn't
7775 match their types precision we use a element type of mode
7776 precision. The vectorization routines will have to make sure
7777 they support the proper result truncation/extension.
7778 We also make sure to build vector types with INTEGER_TYPE
7779 component type only. */
7780 if (INTEGRAL_TYPE_P (scalar_type
)
7781 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
7782 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
7783 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
7784 TYPE_UNSIGNED (scalar_type
));
7786 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
7787 When the component mode passes the above test simply use a type
7788 corresponding to that mode. The theory is that any use that
7789 would cause problems with this will disable vectorization anyway. */
7790 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
7791 && !INTEGRAL_TYPE_P (scalar_type
))
7792 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
7794 /* We can't build a vector type of elements with alignment bigger than
7796 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
7797 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
7798 TYPE_UNSIGNED (scalar_type
));
7800 /* If we felt back to using the mode fail if there was
7801 no scalar type for it. */
7802 if (scalar_type
== NULL_TREE
)
7805 /* If no size was supplied use the mode the target prefers. Otherwise
7806 lookup a vector mode of the specified size. */
7808 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
7810 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
7811 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
7815 vectype
= build_vector_type (scalar_type
, nunits
);
7817 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
7818 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
7824 unsigned int current_vector_size
;
7826 /* Function get_vectype_for_scalar_type.
7828 Returns the vector type corresponding to SCALAR_TYPE as supported
7832 get_vectype_for_scalar_type (tree scalar_type
)
7835 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
7836 current_vector_size
);
7838 && current_vector_size
== 0)
7839 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
7843 /* Function get_same_sized_vectype
7845 Returns a vector type corresponding to SCALAR_TYPE of size
7846 VECTOR_TYPE if supported by the target. */
7849 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
7851 return get_vectype_for_scalar_type_and_size
7852 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
7855 /* Function vect_is_simple_use.
7858 LOOP_VINFO - the vect info of the loop that is being vectorized.
7859 BB_VINFO - the vect info of the basic block that is being vectorized.
7860 OPERAND - operand of STMT in the loop or bb.
7861 DEF - the defining stmt in case OPERAND is an SSA_NAME.
7863 Returns whether a stmt with OPERAND can be vectorized.
7864 For loops, supportable operands are constants, loop invariants, and operands
7865 that are defined by the current iteration of the loop. Unsupportable
7866 operands are those that are defined by a previous iteration of the loop (as
7867 is the case in reduction/induction computations).
7868 For basic blocks, supportable operands are constants and bb invariants.
7869 For now, operands defined outside the basic block are not supported. */
7872 vect_is_simple_use (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
7873 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
7874 tree
*def
, enum vect_def_type
*dt
)
7878 *dt
= vect_unknown_def_type
;
7880 if (dump_enabled_p ())
7882 dump_printf_loc (MSG_NOTE
, vect_location
,
7883 "vect_is_simple_use: operand ");
7884 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
7885 dump_printf (MSG_NOTE
, "\n");
7888 if (CONSTANT_CLASS_P (operand
))
7890 *dt
= vect_constant_def
;
7894 if (is_gimple_min_invariant (operand
))
7897 *dt
= vect_external_def
;
7901 if (TREE_CODE (operand
) != SSA_NAME
)
7903 if (dump_enabled_p ())
7904 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7909 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
7912 *dt
= vect_external_def
;
7916 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
7917 if (dump_enabled_p ())
7919 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
7920 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
7923 basic_block bb
= gimple_bb (*def_stmt
);
7924 if ((loop_vinfo
&& !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), bb
))
7926 && (bb
!= BB_VINFO_BB (bb_vinfo
)
7927 || gimple_code (*def_stmt
) == GIMPLE_PHI
)))
7928 *dt
= vect_external_def
;
7931 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
7932 if (bb_vinfo
&& !STMT_VINFO_VECTORIZABLE (stmt_vinfo
))
7933 *dt
= vect_external_def
;
7935 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
7938 if (dump_enabled_p ())
7940 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
7943 case vect_uninitialized_def
:
7944 dump_printf (MSG_NOTE
, "uninitialized\n");
7946 case vect_constant_def
:
7947 dump_printf (MSG_NOTE
, "constant\n");
7949 case vect_external_def
:
7950 dump_printf (MSG_NOTE
, "external\n");
7952 case vect_internal_def
:
7953 dump_printf (MSG_NOTE
, "internal\n");
7955 case vect_induction_def
:
7956 dump_printf (MSG_NOTE
, "induction\n");
7958 case vect_reduction_def
:
7959 dump_printf (MSG_NOTE
, "reduction\n");
7961 case vect_double_reduction_def
:
7962 dump_printf (MSG_NOTE
, "double reduction\n");
7964 case vect_nested_cycle
:
7965 dump_printf (MSG_NOTE
, "nested cycle\n");
7967 case vect_unknown_def_type
:
7968 dump_printf (MSG_NOTE
, "unknown\n");
7973 if (*dt
== vect_unknown_def_type
7975 && *dt
== vect_double_reduction_def
7976 && gimple_code (stmt
) != GIMPLE_PHI
))
7978 if (dump_enabled_p ())
7979 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7980 "Unsupported pattern.\n");
7984 switch (gimple_code (*def_stmt
))
7987 *def
= gimple_phi_result (*def_stmt
);
7991 *def
= gimple_assign_lhs (*def_stmt
);
7995 *def
= gimple_call_lhs (*def_stmt
);
8000 if (dump_enabled_p ())
8001 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8002 "unsupported defining stmt:\n");
8009 /* Function vect_is_simple_use_1.
8011 Same as vect_is_simple_use_1 but also determines the vector operand
8012 type of OPERAND and stores it to *VECTYPE. If the definition of
8013 OPERAND is vect_uninitialized_def, vect_constant_def or
8014 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8015 is responsible to compute the best suited vector type for the
8019 vect_is_simple_use_1 (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
8020 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
8021 tree
*def
, enum vect_def_type
*dt
, tree
*vectype
)
8023 if (!vect_is_simple_use (operand
, stmt
, loop_vinfo
, bb_vinfo
, def_stmt
,
8027 /* Now get a vector type if the def is internal, otherwise supply
8028 NULL_TREE and leave it up to the caller to figure out a proper
8029 type for the use stmt. */
8030 if (*dt
== vect_internal_def
8031 || *dt
== vect_induction_def
8032 || *dt
== vect_reduction_def
8033 || *dt
== vect_double_reduction_def
8034 || *dt
== vect_nested_cycle
)
8036 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
8038 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8039 && !STMT_VINFO_RELEVANT (stmt_info
)
8040 && !STMT_VINFO_LIVE_P (stmt_info
))
8041 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8043 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8044 gcc_assert (*vectype
!= NULL_TREE
);
8046 else if (*dt
== vect_uninitialized_def
8047 || *dt
== vect_constant_def
8048 || *dt
== vect_external_def
)
8049 *vectype
= NULL_TREE
;
8057 /* Function supportable_widening_operation
8059 Check whether an operation represented by the code CODE is a
8060 widening operation that is supported by the target platform in
8061 vector form (i.e., when operating on arguments of type VECTYPE_IN
8062 producing a result of type VECTYPE_OUT).
8064 Widening operations we currently support are NOP (CONVERT), FLOAT
8065 and WIDEN_MULT. This function checks if these operations are supported
8066 by the target platform either directly (via vector tree-codes), or via
8070 - CODE1 and CODE2 are codes of vector operations to be used when
8071 vectorizing the operation, if available.
8072 - MULTI_STEP_CVT determines the number of required intermediate steps in
8073 case of multi-step conversion (like char->short->int - in that case
8074 MULTI_STEP_CVT will be 1).
8075 - INTERM_TYPES contains the intermediate type required to perform the
8076 widening operation (short in the above example). */
8079 supportable_widening_operation (enum tree_code code
, gimple stmt
,
8080 tree vectype_out
, tree vectype_in
,
8081 enum tree_code
*code1
, enum tree_code
*code2
,
8082 int *multi_step_cvt
,
8083 vec
<tree
> *interm_types
)
8085 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8086 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8087 struct loop
*vect_loop
= NULL
;
8088 machine_mode vec_mode
;
8089 enum insn_code icode1
, icode2
;
8090 optab optab1
, optab2
;
8091 tree vectype
= vectype_in
;
8092 tree wide_vectype
= vectype_out
;
8093 enum tree_code c1
, c2
;
8095 tree prev_type
, intermediate_type
;
8096 machine_mode intermediate_mode
, prev_mode
;
8097 optab optab3
, optab4
;
8099 *multi_step_cvt
= 0;
8101 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
8105 case WIDEN_MULT_EXPR
:
8106 /* The result of a vectorized widening operation usually requires
8107 two vectors (because the widened results do not fit into one vector).
8108 The generated vector results would normally be expected to be
8109 generated in the same order as in the original scalar computation,
8110 i.e. if 8 results are generated in each vector iteration, they are
8111 to be organized as follows:
8112 vect1: [res1,res2,res3,res4],
8113 vect2: [res5,res6,res7,res8].
8115 However, in the special case that the result of the widening
8116 operation is used in a reduction computation only, the order doesn't
8117 matter (because when vectorizing a reduction we change the order of
8118 the computation). Some targets can take advantage of this and
8119 generate more efficient code. For example, targets like Altivec,
8120 that support widen_mult using a sequence of {mult_even,mult_odd}
8121 generate the following vectors:
8122 vect1: [res1,res3,res5,res7],
8123 vect2: [res2,res4,res6,res8].
8125 When vectorizing outer-loops, we execute the inner-loop sequentially
8126 (each vectorized inner-loop iteration contributes to VF outer-loop
8127 iterations in parallel). We therefore don't allow to change the
8128 order of the computation in the inner-loop during outer-loop
8130 /* TODO: Another case in which order doesn't *really* matter is when we
8131 widen and then contract again, e.g. (short)((int)x * y >> 8).
8132 Normally, pack_trunc performs an even/odd permute, whereas the
8133 repack from an even/odd expansion would be an interleave, which
8134 would be significantly simpler for e.g. AVX2. */
8135 /* In any case, in order to avoid duplicating the code below, recurse
8136 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8137 are properly set up for the caller. If we fail, we'll continue with
8138 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8140 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
8141 && !nested_in_vect_loop_p (vect_loop
, stmt
)
8142 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
8143 stmt
, vectype_out
, vectype_in
,
8144 code1
, code2
, multi_step_cvt
,
8147 /* Elements in a vector with vect_used_by_reduction property cannot
8148 be reordered if the use chain with this property does not have the
8149 same operation. One such an example is s += a * b, where elements
8150 in a and b cannot be reordered. Here we check if the vector defined
8151 by STMT is only directly used in the reduction statement. */
8152 tree lhs
= gimple_assign_lhs (stmt
);
8153 use_operand_p dummy
;
8155 stmt_vec_info use_stmt_info
= NULL
;
8156 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
8157 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
8158 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
8161 c1
= VEC_WIDEN_MULT_LO_EXPR
;
8162 c2
= VEC_WIDEN_MULT_HI_EXPR
;
8165 case VEC_WIDEN_MULT_EVEN_EXPR
:
8166 /* Support the recursion induced just above. */
8167 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
8168 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
8171 case WIDEN_LSHIFT_EXPR
:
8172 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
8173 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
8177 c1
= VEC_UNPACK_LO_EXPR
;
8178 c2
= VEC_UNPACK_HI_EXPR
;
8182 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
8183 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
8186 case FIX_TRUNC_EXPR
:
8187 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
8188 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
8189 computing the operation. */
8196 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
8198 enum tree_code ctmp
= c1
;
8203 if (code
== FIX_TRUNC_EXPR
)
8205 /* The signedness is determined from output operand. */
8206 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8207 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
8211 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8212 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
8215 if (!optab1
|| !optab2
)
8218 vec_mode
= TYPE_MODE (vectype
);
8219 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
8220 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
8226 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8227 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8230 /* Check if it's a multi-step conversion that can be done using intermediate
8233 prev_type
= vectype
;
8234 prev_mode
= vec_mode
;
8236 if (!CONVERT_EXPR_CODE_P (code
))
8239 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8240 intermediate steps in promotion sequence. We try
8241 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
8243 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8244 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8246 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8248 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
8249 TYPE_UNSIGNED (prev_type
));
8250 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8251 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
8253 if (!optab3
|| !optab4
8254 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
8255 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8256 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
8257 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
8258 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
8259 == CODE_FOR_nothing
)
8260 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
8261 == CODE_FOR_nothing
))
8264 interm_types
->quick_push (intermediate_type
);
8265 (*multi_step_cvt
)++;
8267 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8268 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8271 prev_type
= intermediate_type
;
8272 prev_mode
= intermediate_mode
;
8275 interm_types
->release ();
8280 /* Function supportable_narrowing_operation
8282 Check whether an operation represented by the code CODE is a
8283 narrowing operation that is supported by the target platform in
8284 vector form (i.e., when operating on arguments of type VECTYPE_IN
8285 and producing a result of type VECTYPE_OUT).
8287 Narrowing operations we currently support are NOP (CONVERT) and
8288 FIX_TRUNC. This function checks if these operations are supported by
8289 the target platform directly via vector tree-codes.
8292 - CODE1 is the code of a vector operation to be used when
8293 vectorizing the operation, if available.
8294 - MULTI_STEP_CVT determines the number of required intermediate steps in
8295 case of multi-step conversion (like int->short->char - in that case
8296 MULTI_STEP_CVT will be 1).
8297 - INTERM_TYPES contains the intermediate type required to perform the
8298 narrowing operation (short in the above example). */
8301 supportable_narrowing_operation (enum tree_code code
,
8302 tree vectype_out
, tree vectype_in
,
8303 enum tree_code
*code1
, int *multi_step_cvt
,
8304 vec
<tree
> *interm_types
)
8306 machine_mode vec_mode
;
8307 enum insn_code icode1
;
8308 optab optab1
, interm_optab
;
8309 tree vectype
= vectype_in
;
8310 tree narrow_vectype
= vectype_out
;
8312 tree intermediate_type
;
8313 machine_mode intermediate_mode
, prev_mode
;
8317 *multi_step_cvt
= 0;
8321 c1
= VEC_PACK_TRUNC_EXPR
;
8324 case FIX_TRUNC_EXPR
:
8325 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
8329 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8330 tree code and optabs used for computing the operation. */
8337 if (code
== FIX_TRUNC_EXPR
)
8338 /* The signedness is determined from output operand. */
8339 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8341 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8346 vec_mode
= TYPE_MODE (vectype
);
8347 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
8352 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8355 /* Check if it's a multi-step conversion that can be done using intermediate
8357 prev_mode
= vec_mode
;
8358 if (code
== FIX_TRUNC_EXPR
)
8359 uns
= TYPE_UNSIGNED (vectype_out
);
8361 uns
= TYPE_UNSIGNED (vectype
);
8363 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8364 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8365 costly than signed. */
8366 if (code
== FIX_TRUNC_EXPR
&& uns
)
8368 enum insn_code icode2
;
8371 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
8373 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8374 if (interm_optab
!= unknown_optab
8375 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
8376 && insn_data
[icode1
].operand
[0].mode
8377 == insn_data
[icode2
].operand
[0].mode
)
8380 optab1
= interm_optab
;
8385 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8386 intermediate steps in promotion sequence. We try
8387 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8388 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8389 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8391 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8393 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
8395 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
8398 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
8399 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8400 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
8401 == CODE_FOR_nothing
))
8404 interm_types
->quick_push (intermediate_type
);
8405 (*multi_step_cvt
)++;
8407 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8410 prev_mode
= intermediate_mode
;
8411 optab1
= interm_optab
;
8414 interm_types
->release ();