1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2018 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
37 #include "fold-const.h"
38 #include "stor-layout.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
44 #include "tree-ssa-loop-manip.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
50 #include "internal-fn.h"
51 #include "tree-vector-builder.h"
52 #include "vec-perm-indices.h"
53 #include "tree-ssa-loop-niter.h"
54 #include "gimple-fold.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* Return the vectorized type for the given statement. */
62 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
64 return STMT_VINFO_VECTYPE (stmt_info
);
67 /* Return TRUE iff the given statement is in an inner loop relative to
68 the loop being vectorized. */
70 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
72 gimple
*stmt
= STMT_VINFO_STMT (stmt_info
);
73 basic_block bb
= gimple_bb (stmt
);
74 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
80 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
82 return (bb
->loop_father
== loop
->inner
);
85 /* Record the cost of a statement, either by directly informing the
86 target model or by saving it in a vector for later processing.
87 Return a preliminary estimate of the statement's cost. */
90 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
91 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
92 int misalign
, enum vect_cost_model_location where
)
94 if ((kind
== vector_load
|| kind
== unaligned_load
)
95 && STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
96 kind
= vector_gather_load
;
97 if ((kind
== vector_store
|| kind
== unaligned_store
)
98 && STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
99 kind
= vector_scatter_store
;
101 stmt_info_for_cost si
= { count
, kind
, where
, stmt_info
, misalign
};
102 body_cost_vec
->safe_push (si
);
104 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
106 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
109 /* Return a variable of type ELEM_TYPE[NELEMS]. */
112 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
114 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
118 /* ARRAY is an array of vectors created by create_vector_array.
119 Return an SSA_NAME for the vector in index N. The reference
120 is part of the vectorization of STMT_INFO and the vector is associated
121 with scalar destination SCALAR_DEST. */
124 read_vector_array (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
125 tree scalar_dest
, tree array
, unsigned HOST_WIDE_INT n
)
127 tree vect_type
, vect
, vect_name
, array_ref
;
130 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
131 vect_type
= TREE_TYPE (TREE_TYPE (array
));
132 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
133 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
134 build_int_cst (size_type_node
, n
),
135 NULL_TREE
, NULL_TREE
);
137 new_stmt
= gimple_build_assign (vect
, array_ref
);
138 vect_name
= make_ssa_name (vect
, new_stmt
);
139 gimple_assign_set_lhs (new_stmt
, vect_name
);
140 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
145 /* ARRAY is an array of vectors created by create_vector_array.
146 Emit code to store SSA_NAME VECT in index N of the array.
147 The store is part of the vectorization of STMT_INFO. */
150 write_vector_array (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
151 tree vect
, tree array
, unsigned HOST_WIDE_INT n
)
156 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
157 build_int_cst (size_type_node
, n
),
158 NULL_TREE
, NULL_TREE
);
160 new_stmt
= gimple_build_assign (array_ref
, vect
);
161 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
164 /* PTR is a pointer to an array of type TYPE. Return a representation
165 of *PTR. The memory reference replaces those in FIRST_DR
169 create_array_ref (tree type
, tree ptr
, tree alias_ptr_type
)
173 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
174 /* Arrays have the same alignment as their type. */
175 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
179 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
180 Emit the clobber before *GSI. */
183 vect_clobber_variable (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
186 tree clobber
= build_clobber (TREE_TYPE (var
));
187 gimple
*new_stmt
= gimple_build_assign (var
, clobber
);
188 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
191 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
193 /* Function vect_mark_relevant.
195 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
198 vect_mark_relevant (vec
<stmt_vec_info
> *worklist
, stmt_vec_info stmt_info
,
199 enum vect_relevant relevant
, bool live_p
)
201 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
202 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
204 if (dump_enabled_p ())
206 dump_printf_loc (MSG_NOTE
, vect_location
,
207 "mark relevant %d, live %d: ", relevant
, live_p
);
208 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt_info
->stmt
, 0);
211 /* If this stmt is an original stmt in a pattern, we might need to mark its
212 related pattern stmt instead of the original stmt. However, such stmts
213 may have their own uses that are not in any pattern, in such cases the
214 stmt itself should be marked. */
215 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
217 /* This is the last stmt in a sequence that was detected as a
218 pattern that can potentially be vectorized. Don't mark the stmt
219 as relevant/live because it's not going to be vectorized.
220 Instead mark the pattern-stmt that replaces it. */
222 if (dump_enabled_p ())
223 dump_printf_loc (MSG_NOTE
, vect_location
,
224 "last stmt in pattern. don't mark"
225 " relevant/live.\n");
226 stmt_vec_info old_stmt_info
= stmt_info
;
227 stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
228 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == old_stmt_info
);
229 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
230 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
233 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
234 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
235 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
237 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
238 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
240 if (dump_enabled_p ())
241 dump_printf_loc (MSG_NOTE
, vect_location
,
242 "already marked relevant/live.\n");
246 worklist
->safe_push (stmt_info
);
250 /* Function is_simple_and_all_uses_invariant
252 Return true if STMT_INFO is simple and all uses of it are invariant. */
255 is_simple_and_all_uses_invariant (stmt_vec_info stmt_info
,
256 loop_vec_info loop_vinfo
)
261 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
265 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
267 enum vect_def_type dt
= vect_uninitialized_def
;
269 if (!vect_is_simple_use (op
, loop_vinfo
, &dt
))
271 if (dump_enabled_p ())
272 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
273 "use not simple.\n");
277 if (dt
!= vect_external_def
&& dt
!= vect_constant_def
)
283 /* Function vect_stmt_relevant_p.
285 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
286 is "relevant for vectorization".
288 A stmt is considered "relevant for vectorization" if:
289 - it has uses outside the loop.
290 - it has vdefs (it alters memory).
291 - control stmts in the loop (except for the exit condition).
293 CHECKME: what other side effects would the vectorizer allow? */
296 vect_stmt_relevant_p (stmt_vec_info stmt_info
, loop_vec_info loop_vinfo
,
297 enum vect_relevant
*relevant
, bool *live_p
)
299 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
301 imm_use_iterator imm_iter
;
305 *relevant
= vect_unused_in_scope
;
308 /* cond stmt other than loop exit cond. */
309 if (is_ctrl_stmt (stmt_info
->stmt
)
310 && STMT_VINFO_TYPE (stmt_info
) != loop_exit_ctrl_vec_info_type
)
311 *relevant
= vect_used_in_scope
;
313 /* changing memory. */
314 if (gimple_code (stmt_info
->stmt
) != GIMPLE_PHI
)
315 if (gimple_vdef (stmt_info
->stmt
)
316 && !gimple_clobber_p (stmt_info
->stmt
))
318 if (dump_enabled_p ())
319 dump_printf_loc (MSG_NOTE
, vect_location
,
320 "vec_stmt_relevant_p: stmt has vdefs.\n");
321 *relevant
= vect_used_in_scope
;
324 /* uses outside the loop. */
325 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt_info
->stmt
, op_iter
, SSA_OP_DEF
)
327 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
329 basic_block bb
= gimple_bb (USE_STMT (use_p
));
330 if (!flow_bb_inside_loop_p (loop
, bb
))
332 if (dump_enabled_p ())
333 dump_printf_loc (MSG_NOTE
, vect_location
,
334 "vec_stmt_relevant_p: used out of loop.\n");
336 if (is_gimple_debug (USE_STMT (use_p
)))
339 /* We expect all such uses to be in the loop exit phis
340 (because of loop closed form) */
341 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
342 gcc_assert (bb
== single_exit (loop
)->dest
);
349 if (*live_p
&& *relevant
== vect_unused_in_scope
350 && !is_simple_and_all_uses_invariant (stmt_info
, loop_vinfo
))
352 if (dump_enabled_p ())
353 dump_printf_loc (MSG_NOTE
, vect_location
,
354 "vec_stmt_relevant_p: stmt live but not relevant.\n");
355 *relevant
= vect_used_only_live
;
358 return (*live_p
|| *relevant
);
362 /* Function exist_non_indexing_operands_for_use_p
364 USE is one of the uses attached to STMT_INFO. Check if USE is
365 used in STMT_INFO for anything other than indexing an array. */
368 exist_non_indexing_operands_for_use_p (tree use
, stmt_vec_info stmt_info
)
372 /* USE corresponds to some operand in STMT. If there is no data
373 reference in STMT, then any operand that corresponds to USE
374 is not indexing an array. */
375 if (!STMT_VINFO_DATA_REF (stmt_info
))
378 /* STMT has a data_ref. FORNOW this means that its of one of
382 (This should have been verified in analyze_data_refs).
384 'var' in the second case corresponds to a def, not a use,
385 so USE cannot correspond to any operands that are not used
388 Therefore, all we need to check is if STMT falls into the
389 first case, and whether var corresponds to USE. */
391 gassign
*assign
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
392 if (!assign
|| !gimple_assign_copy_p (assign
))
394 gcall
*call
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
395 if (call
&& gimple_call_internal_p (call
))
397 internal_fn ifn
= gimple_call_internal_fn (call
);
398 int mask_index
= internal_fn_mask_index (ifn
);
400 && use
== gimple_call_arg (call
, mask_index
))
402 int stored_value_index
= internal_fn_stored_value_index (ifn
);
403 if (stored_value_index
>= 0
404 && use
== gimple_call_arg (call
, stored_value_index
))
406 if (internal_gather_scatter_fn_p (ifn
)
407 && use
== gimple_call_arg (call
, 1))
413 if (TREE_CODE (gimple_assign_lhs (assign
)) == SSA_NAME
)
415 operand
= gimple_assign_rhs1 (assign
);
416 if (TREE_CODE (operand
) != SSA_NAME
)
427 Function process_use.
430 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
431 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
432 that defined USE. This is done by calling mark_relevant and passing it
433 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
434 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
438 Generally, LIVE_P and RELEVANT are used to define the liveness and
439 relevance info of the DEF_STMT of this USE:
440 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
441 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
443 - case 1: If USE is used only for address computations (e.g. array indexing),
444 which does not need to be directly vectorized, then the liveness/relevance
445 of the respective DEF_STMT is left unchanged.
446 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
447 we skip DEF_STMT cause it had already been processed.
448 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
449 "relevant" will be modified accordingly.
451 Return true if everything is as expected. Return false otherwise. */
454 process_use (stmt_vec_info stmt_vinfo
, tree use
, loop_vec_info loop_vinfo
,
455 enum vect_relevant relevant
, vec
<stmt_vec_info
> *worklist
,
458 stmt_vec_info dstmt_vinfo
;
459 basic_block bb
, def_bb
;
460 enum vect_def_type dt
;
462 /* case 1: we are only interested in uses that need to be vectorized. Uses
463 that are used for address computation are not considered relevant. */
464 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt_vinfo
))
467 if (!vect_is_simple_use (use
, loop_vinfo
, &dt
, &dstmt_vinfo
))
469 if (dump_enabled_p ())
470 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
471 "not vectorized: unsupported use in stmt.\n");
478 def_bb
= gimple_bb (dstmt_vinfo
->stmt
);
480 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
481 DSTMT_VINFO must have already been processed, because this should be the
482 only way that STMT, which is a reduction-phi, was put in the worklist,
483 as there should be no other uses for DSTMT_VINFO in the loop. So we just
484 check that everything is as expected, and we are done. */
485 bb
= gimple_bb (stmt_vinfo
->stmt
);
486 if (gimple_code (stmt_vinfo
->stmt
) == GIMPLE_PHI
487 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
488 && gimple_code (dstmt_vinfo
->stmt
) != GIMPLE_PHI
489 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
490 && bb
->loop_father
== def_bb
->loop_father
)
492 if (dump_enabled_p ())
493 dump_printf_loc (MSG_NOTE
, vect_location
,
494 "reduc-stmt defining reduc-phi in the same nest.\n");
495 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
496 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
497 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
501 /* case 3a: outer-loop stmt defining an inner-loop stmt:
502 outer-loop-header-bb:
508 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
510 if (dump_enabled_p ())
511 dump_printf_loc (MSG_NOTE
, vect_location
,
512 "outer-loop def-stmt defining inner-loop stmt.\n");
516 case vect_unused_in_scope
:
517 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
518 vect_used_in_scope
: vect_unused_in_scope
;
521 case vect_used_in_outer_by_reduction
:
522 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
523 relevant
= vect_used_by_reduction
;
526 case vect_used_in_outer
:
527 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
528 relevant
= vect_used_in_scope
;
531 case vect_used_in_scope
:
539 /* case 3b: inner-loop stmt defining an outer-loop stmt:
540 outer-loop-header-bb:
544 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
546 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
548 if (dump_enabled_p ())
549 dump_printf_loc (MSG_NOTE
, vect_location
,
550 "inner-loop def-stmt defining outer-loop stmt.\n");
554 case vect_unused_in_scope
:
555 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
556 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
557 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
560 case vect_used_by_reduction
:
561 case vect_used_only_live
:
562 relevant
= vect_used_in_outer_by_reduction
;
565 case vect_used_in_scope
:
566 relevant
= vect_used_in_outer
;
573 /* We are also not interested in uses on loop PHI backedges that are
574 inductions. Otherwise we'll needlessly vectorize the IV increment
575 and cause hybrid SLP for SLP inductions. Unless the PHI is live
577 else if (gimple_code (stmt_vinfo
->stmt
) == GIMPLE_PHI
578 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_induction_def
579 && ! STMT_VINFO_LIVE_P (stmt_vinfo
)
580 && (PHI_ARG_DEF_FROM_EDGE (stmt_vinfo
->stmt
,
581 loop_latch_edge (bb
->loop_father
))
584 if (dump_enabled_p ())
585 dump_printf_loc (MSG_NOTE
, vect_location
,
586 "induction value on backedge.\n");
591 vect_mark_relevant (worklist
, dstmt_vinfo
, relevant
, false);
596 /* Function vect_mark_stmts_to_be_vectorized.
598 Not all stmts in the loop need to be vectorized. For example:
607 Stmt 1 and 3 do not need to be vectorized, because loop control and
608 addressing of vectorized data-refs are handled differently.
610 This pass detects such stmts. */
613 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
615 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
616 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
617 unsigned int nbbs
= loop
->num_nodes
;
618 gimple_stmt_iterator si
;
622 enum vect_relevant relevant
;
624 DUMP_VECT_SCOPE ("vect_mark_stmts_to_be_vectorized");
626 auto_vec
<stmt_vec_info
, 64> worklist
;
628 /* 1. Init worklist. */
629 for (i
= 0; i
< nbbs
; i
++)
632 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
634 stmt_vec_info phi_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
635 if (dump_enabled_p ())
637 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
638 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi_info
->stmt
, 0);
641 if (vect_stmt_relevant_p (phi_info
, loop_vinfo
, &relevant
, &live_p
))
642 vect_mark_relevant (&worklist
, phi_info
, relevant
, live_p
);
644 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
646 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
647 if (dump_enabled_p ())
649 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
650 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt_info
->stmt
, 0);
653 if (vect_stmt_relevant_p (stmt_info
, loop_vinfo
, &relevant
, &live_p
))
654 vect_mark_relevant (&worklist
, stmt_info
, relevant
, live_p
);
658 /* 2. Process_worklist */
659 while (worklist
.length () > 0)
664 stmt_vec_info stmt_vinfo
= worklist
.pop ();
665 if (dump_enabled_p ())
667 dump_printf_loc (MSG_NOTE
, vect_location
,
668 "worklist: examine stmt: ");
669 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt_vinfo
->stmt
, 0);
672 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
673 (DEF_STMT) as relevant/irrelevant according to the relevance property
675 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
677 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
678 propagated as is to the DEF_STMTs of its USEs.
680 One exception is when STMT has been identified as defining a reduction
681 variable; in this case we set the relevance to vect_used_by_reduction.
682 This is because we distinguish between two kinds of relevant stmts -
683 those that are used by a reduction computation, and those that are
684 (also) used by a regular computation. This allows us later on to
685 identify stmts that are used solely by a reduction, and therefore the
686 order of the results that they produce does not have to be kept. */
688 switch (STMT_VINFO_DEF_TYPE (stmt_vinfo
))
690 case vect_reduction_def
:
691 gcc_assert (relevant
!= vect_unused_in_scope
);
692 if (relevant
!= vect_unused_in_scope
693 && relevant
!= vect_used_in_scope
694 && relevant
!= vect_used_by_reduction
695 && relevant
!= vect_used_only_live
)
697 if (dump_enabled_p ())
698 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
699 "unsupported use of reduction.\n");
704 case vect_nested_cycle
:
705 if (relevant
!= vect_unused_in_scope
706 && relevant
!= vect_used_in_outer_by_reduction
707 && relevant
!= vect_used_in_outer
)
709 if (dump_enabled_p ())
710 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
711 "unsupported use of nested cycle.\n");
717 case vect_double_reduction_def
:
718 if (relevant
!= vect_unused_in_scope
719 && relevant
!= vect_used_by_reduction
720 && relevant
!= vect_used_only_live
)
722 if (dump_enabled_p ())
723 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
724 "unsupported use of double reduction.\n");
734 if (is_pattern_stmt_p (stmt_vinfo
))
736 /* Pattern statements are not inserted into the code, so
737 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
738 have to scan the RHS or function arguments instead. */
739 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt_vinfo
->stmt
))
741 enum tree_code rhs_code
= gimple_assign_rhs_code (assign
);
742 tree op
= gimple_assign_rhs1 (assign
);
745 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
747 if (!process_use (stmt_vinfo
, TREE_OPERAND (op
, 0),
748 loop_vinfo
, relevant
, &worklist
, false)
749 || !process_use (stmt_vinfo
, TREE_OPERAND (op
, 1),
750 loop_vinfo
, relevant
, &worklist
, false))
754 for (; i
< gimple_num_ops (assign
); i
++)
756 op
= gimple_op (assign
, i
);
757 if (TREE_CODE (op
) == SSA_NAME
758 && !process_use (stmt_vinfo
, op
, loop_vinfo
, relevant
,
763 else if (gcall
*call
= dyn_cast
<gcall
*> (stmt_vinfo
->stmt
))
765 for (i
= 0; i
< gimple_call_num_args (call
); i
++)
767 tree arg
= gimple_call_arg (call
, i
);
768 if (!process_use (stmt_vinfo
, arg
, loop_vinfo
, relevant
,
775 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt_vinfo
->stmt
, iter
, SSA_OP_USE
)
777 tree op
= USE_FROM_PTR (use_p
);
778 if (!process_use (stmt_vinfo
, op
, loop_vinfo
, relevant
,
783 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo
))
785 gather_scatter_info gs_info
;
786 if (!vect_check_gather_scatter (stmt_vinfo
, loop_vinfo
, &gs_info
))
788 if (!process_use (stmt_vinfo
, gs_info
.offset
, loop_vinfo
, relevant
,
792 } /* while worklist */
797 /* Compute the prologue cost for invariant or constant operands. */
800 vect_prologue_cost_for_slp_op (slp_tree node
, stmt_vec_info stmt_info
,
801 unsigned opno
, enum vect_def_type dt
,
802 stmt_vector_for_cost
*cost_vec
)
804 gimple
*stmt
= SLP_TREE_SCALAR_STMTS (node
)[0]->stmt
;
805 tree op
= gimple_op (stmt
, opno
);
806 unsigned prologue_cost
= 0;
808 /* Without looking at the actual initializer a vector of
809 constants can be implemented as load from the constant pool.
810 When all elements are the same we can use a splat. */
811 tree vectype
= get_vectype_for_scalar_type (TREE_TYPE (op
));
812 unsigned group_size
= SLP_TREE_SCALAR_STMTS (node
).length ();
813 unsigned num_vects_to_check
;
814 unsigned HOST_WIDE_INT const_nunits
;
816 if (TYPE_VECTOR_SUBPARTS (vectype
).is_constant (&const_nunits
)
817 && ! multiple_p (const_nunits
, group_size
))
819 num_vects_to_check
= SLP_TREE_NUMBER_OF_VEC_STMTS (node
);
820 nelt_limit
= const_nunits
;
824 /* If either the vector has variable length or the vectors
825 are composed of repeated whole groups we only need to
826 cost construction once. All vectors will be the same. */
827 num_vects_to_check
= 1;
828 nelt_limit
= group_size
;
830 tree elt
= NULL_TREE
;
832 for (unsigned j
= 0; j
< num_vects_to_check
* nelt_limit
; ++j
)
834 unsigned si
= j
% group_size
;
836 elt
= gimple_op (SLP_TREE_SCALAR_STMTS (node
)[si
]->stmt
, opno
);
837 /* ??? We're just tracking whether all operands of a single
838 vector initializer are the same, ideally we'd check if
839 we emitted the same one already. */
840 else if (elt
!= gimple_op (SLP_TREE_SCALAR_STMTS (node
)[si
]->stmt
,
844 if (nelt
== nelt_limit
)
846 /* ??? We need to pass down stmt_info for a vector type
847 even if it points to the wrong stmt. */
848 prologue_cost
+= record_stmt_cost
850 dt
== vect_external_def
851 ? (elt
? scalar_to_vec
: vec_construct
)
853 stmt_info
, 0, vect_prologue
);
858 return prologue_cost
;
861 /* Function vect_model_simple_cost.
863 Models cost for simple operations, i.e. those that only emit ncopies of a
864 single op. Right now, this does not account for multiple insns that could
865 be generated for the single vector op. We will handle that shortly. */
868 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
869 enum vect_def_type
*dt
,
872 stmt_vector_for_cost
*cost_vec
)
874 int inside_cost
= 0, prologue_cost
= 0;
876 gcc_assert (cost_vec
!= NULL
);
878 /* ??? Somehow we need to fix this at the callers. */
880 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (node
);
884 /* Scan operands and account for prologue cost of constants/externals.
885 ??? This over-estimates cost for multiple uses and should be
887 gimple
*stmt
= SLP_TREE_SCALAR_STMTS (node
)[0]->stmt
;
888 tree lhs
= gimple_get_lhs (stmt
);
889 for (unsigned i
= 0; i
< gimple_num_ops (stmt
); ++i
)
891 tree op
= gimple_op (stmt
, i
);
892 enum vect_def_type dt
;
893 if (!op
|| op
== lhs
)
895 if (vect_is_simple_use (op
, stmt_info
->vinfo
, &dt
)
896 && (dt
== vect_constant_def
|| dt
== vect_external_def
))
897 prologue_cost
+= vect_prologue_cost_for_slp_op (node
, stmt_info
,
902 /* Cost the "broadcast" of a scalar operand in to a vector operand.
903 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
905 for (int i
= 0; i
< ndts
; i
++)
906 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
907 prologue_cost
+= record_stmt_cost (cost_vec
, 1, scalar_to_vec
,
908 stmt_info
, 0, vect_prologue
);
910 /* Adjust for two-operator SLP nodes. */
911 if (node
&& SLP_TREE_TWO_OPERATORS (node
))
914 inside_cost
+= record_stmt_cost (cost_vec
, ncopies
, vec_perm
,
915 stmt_info
, 0, vect_body
);
918 /* Pass the inside-of-loop statements to the target-specific cost model. */
919 inside_cost
+= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
920 stmt_info
, 0, vect_body
);
922 if (dump_enabled_p ())
923 dump_printf_loc (MSG_NOTE
, vect_location
,
924 "vect_model_simple_cost: inside_cost = %d, "
925 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
929 /* Model cost for type demotion and promotion operations. PWR is normally
930 zero for single-step promotions and demotions. It will be one if
931 two-step promotion/demotion is required, and so on. Each additional
932 step doubles the number of instructions required. */
935 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
936 enum vect_def_type
*dt
, int pwr
,
937 stmt_vector_for_cost
*cost_vec
)
940 int inside_cost
= 0, prologue_cost
= 0;
942 for (i
= 0; i
< pwr
+ 1; i
++)
944 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
946 inside_cost
+= record_stmt_cost (cost_vec
, vect_pow2 (tmp
),
947 vec_promote_demote
, stmt_info
, 0,
951 /* FORNOW: Assuming maximum 2 args per stmts. */
952 for (i
= 0; i
< 2; i
++)
953 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
954 prologue_cost
+= record_stmt_cost (cost_vec
, 1, vector_stmt
,
955 stmt_info
, 0, vect_prologue
);
957 if (dump_enabled_p ())
958 dump_printf_loc (MSG_NOTE
, vect_location
,
959 "vect_model_promotion_demotion_cost: inside_cost = %d, "
960 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
963 /* Function vect_model_store_cost
965 Models cost for stores. In the case of grouped accesses, one access
966 has the overhead of the grouped access attributed to it. */
969 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
970 enum vect_def_type dt
,
971 vect_memory_access_type memory_access_type
,
972 vec_load_store_type vls_type
, slp_tree slp_node
,
973 stmt_vector_for_cost
*cost_vec
)
975 unsigned int inside_cost
= 0, prologue_cost
= 0;
976 stmt_vec_info first_stmt_info
= stmt_info
;
977 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
979 /* ??? Somehow we need to fix this at the callers. */
981 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
983 if (vls_type
== VLS_STORE_INVARIANT
)
986 prologue_cost
+= vect_prologue_cost_for_slp_op (slp_node
, stmt_info
,
989 prologue_cost
+= record_stmt_cost (cost_vec
, 1, scalar_to_vec
,
990 stmt_info
, 0, vect_prologue
);
993 /* Grouped stores update all elements in the group at once,
994 so we want the DR for the first statement. */
995 if (!slp_node
&& grouped_access_p
)
996 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
998 /* True if we should include any once-per-group costs as well as
999 the cost of the statement itself. For SLP we only get called
1000 once per group anyhow. */
1001 bool first_stmt_p
= (first_stmt_info
== stmt_info
);
1003 /* We assume that the cost of a single store-lanes instruction is
1004 equivalent to the cost of DR_GROUP_SIZE separate stores. If a grouped
1005 access is instead being provided by a permute-and-store operation,
1006 include the cost of the permutes. */
1008 && memory_access_type
== VMAT_CONTIGUOUS_PERMUTE
)
1010 /* Uses a high and low interleave or shuffle operations for each
1012 int group_size
= DR_GROUP_SIZE (first_stmt_info
);
1013 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1014 inside_cost
= record_stmt_cost (cost_vec
, nstmts
, vec_perm
,
1015 stmt_info
, 0, vect_body
);
1017 if (dump_enabled_p ())
1018 dump_printf_loc (MSG_NOTE
, vect_location
,
1019 "vect_model_store_cost: strided group_size = %d .\n",
1023 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1024 /* Costs of the stores. */
1025 if (memory_access_type
== VMAT_ELEMENTWISE
1026 || memory_access_type
== VMAT_GATHER_SCATTER
)
1028 /* N scalar stores plus extracting the elements. */
1029 unsigned int assumed_nunits
= vect_nunits_for_cost (vectype
);
1030 inside_cost
+= record_stmt_cost (cost_vec
,
1031 ncopies
* assumed_nunits
,
1032 scalar_store
, stmt_info
, 0, vect_body
);
1035 vect_get_store_cost (stmt_info
, ncopies
, &inside_cost
, cost_vec
);
1037 if (memory_access_type
== VMAT_ELEMENTWISE
1038 || memory_access_type
== VMAT_STRIDED_SLP
)
1040 /* N scalar stores plus extracting the elements. */
1041 unsigned int assumed_nunits
= vect_nunits_for_cost (vectype
);
1042 inside_cost
+= record_stmt_cost (cost_vec
,
1043 ncopies
* assumed_nunits
,
1044 vec_to_scalar
, stmt_info
, 0, vect_body
);
1047 if (dump_enabled_p ())
1048 dump_printf_loc (MSG_NOTE
, vect_location
,
1049 "vect_model_store_cost: inside_cost = %d, "
1050 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1054 /* Calculate cost of DR's memory access. */
1056 vect_get_store_cost (stmt_vec_info stmt_info
, int ncopies
,
1057 unsigned int *inside_cost
,
1058 stmt_vector_for_cost
*body_cost_vec
)
1060 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
);
1061 int alignment_support_scheme
1062 = vect_supportable_dr_alignment (dr_info
, false);
1064 switch (alignment_support_scheme
)
1068 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1069 vector_store
, stmt_info
, 0,
1072 if (dump_enabled_p ())
1073 dump_printf_loc (MSG_NOTE
, vect_location
,
1074 "vect_model_store_cost: aligned.\n");
1078 case dr_unaligned_supported
:
1080 /* Here, we assign an additional cost for the unaligned store. */
1081 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1082 unaligned_store
, stmt_info
,
1083 DR_MISALIGNMENT (dr_info
),
1085 if (dump_enabled_p ())
1086 dump_printf_loc (MSG_NOTE
, vect_location
,
1087 "vect_model_store_cost: unaligned supported by "
1092 case dr_unaligned_unsupported
:
1094 *inside_cost
= VECT_MAX_COST
;
1096 if (dump_enabled_p ())
1097 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1098 "vect_model_store_cost: unsupported access.\n");
1108 /* Function vect_model_load_cost
1110 Models cost for loads. In the case of grouped accesses, one access has
1111 the overhead of the grouped access attributed to it. Since unaligned
1112 accesses are supported for loads, we also account for the costs of the
1113 access scheme chosen. */
1116 vect_model_load_cost (stmt_vec_info stmt_info
, unsigned ncopies
,
1117 vect_memory_access_type memory_access_type
,
1118 slp_instance instance
,
1120 stmt_vector_for_cost
*cost_vec
)
1122 unsigned int inside_cost
= 0, prologue_cost
= 0;
1123 bool grouped_access_p
= STMT_VINFO_GROUPED_ACCESS (stmt_info
);
1125 gcc_assert (cost_vec
);
1127 /* ??? Somehow we need to fix this at the callers. */
1129 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
1131 if (slp_node
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
1133 /* If the load is permuted then the alignment is determined by
1134 the first group element not by the first scalar stmt DR. */
1135 stmt_vec_info first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
1136 /* Record the cost for the permutation. */
1138 unsigned assumed_nunits
1139 = vect_nunits_for_cost (STMT_VINFO_VECTYPE (first_stmt_info
));
1140 unsigned slp_vf
= (ncopies
* assumed_nunits
) / instance
->group_size
;
1141 vect_transform_slp_perm_load (slp_node
, vNULL
, NULL
,
1142 slp_vf
, instance
, true,
1144 inside_cost
+= record_stmt_cost (cost_vec
, n_perms
, vec_perm
,
1145 first_stmt_info
, 0, vect_body
);
1146 /* And adjust the number of loads performed. This handles
1147 redundancies as well as loads that are later dead. */
1148 auto_sbitmap
perm (DR_GROUP_SIZE (first_stmt_info
));
1149 bitmap_clear (perm
);
1150 for (unsigned i
= 0;
1151 i
< SLP_TREE_LOAD_PERMUTATION (slp_node
).length (); ++i
)
1152 bitmap_set_bit (perm
, SLP_TREE_LOAD_PERMUTATION (slp_node
)[i
]);
1154 bool load_seen
= false;
1155 for (unsigned i
= 0; i
< DR_GROUP_SIZE (first_stmt_info
); ++i
)
1157 if (i
% assumed_nunits
== 0)
1163 if (bitmap_bit_p (perm
, i
))
1169 <= (DR_GROUP_SIZE (first_stmt_info
)
1170 - DR_GROUP_GAP (first_stmt_info
)
1171 + assumed_nunits
- 1) / assumed_nunits
);
1174 /* Grouped loads read all elements in the group at once,
1175 so we want the DR for the first statement. */
1176 stmt_vec_info first_stmt_info
= stmt_info
;
1177 if (!slp_node
&& grouped_access_p
)
1178 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
1180 /* True if we should include any once-per-group costs as well as
1181 the cost of the statement itself. For SLP we only get called
1182 once per group anyhow. */
1183 bool first_stmt_p
= (first_stmt_info
== stmt_info
);
1185 /* We assume that the cost of a single load-lanes instruction is
1186 equivalent to the cost of DR_GROUP_SIZE separate loads. If a grouped
1187 access is instead being provided by a load-and-permute operation,
1188 include the cost of the permutes. */
1190 && memory_access_type
== VMAT_CONTIGUOUS_PERMUTE
)
1192 /* Uses an even and odd extract operations or shuffle operations
1193 for each needed permute. */
1194 int group_size
= DR_GROUP_SIZE (first_stmt_info
);
1195 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1196 inside_cost
+= record_stmt_cost (cost_vec
, nstmts
, vec_perm
,
1197 stmt_info
, 0, vect_body
);
1199 if (dump_enabled_p ())
1200 dump_printf_loc (MSG_NOTE
, vect_location
,
1201 "vect_model_load_cost: strided group_size = %d .\n",
1205 /* The loads themselves. */
1206 if (memory_access_type
== VMAT_ELEMENTWISE
1207 || memory_access_type
== VMAT_GATHER_SCATTER
)
1209 /* N scalar loads plus gathering them into a vector. */
1210 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1211 unsigned int assumed_nunits
= vect_nunits_for_cost (vectype
);
1212 inside_cost
+= record_stmt_cost (cost_vec
,
1213 ncopies
* assumed_nunits
,
1214 scalar_load
, stmt_info
, 0, vect_body
);
1217 vect_get_load_cost (stmt_info
, ncopies
, first_stmt_p
,
1218 &inside_cost
, &prologue_cost
,
1219 cost_vec
, cost_vec
, true);
1220 if (memory_access_type
== VMAT_ELEMENTWISE
1221 || memory_access_type
== VMAT_STRIDED_SLP
)
1222 inside_cost
+= record_stmt_cost (cost_vec
, ncopies
, vec_construct
,
1223 stmt_info
, 0, vect_body
);
1225 if (dump_enabled_p ())
1226 dump_printf_loc (MSG_NOTE
, vect_location
,
1227 "vect_model_load_cost: inside_cost = %d, "
1228 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1232 /* Calculate cost of DR's memory access. */
1234 vect_get_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1235 bool add_realign_cost
, unsigned int *inside_cost
,
1236 unsigned int *prologue_cost
,
1237 stmt_vector_for_cost
*prologue_cost_vec
,
1238 stmt_vector_for_cost
*body_cost_vec
,
1239 bool record_prologue_costs
)
1241 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
);
1242 int alignment_support_scheme
1243 = vect_supportable_dr_alignment (dr_info
, false);
1245 switch (alignment_support_scheme
)
1249 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1250 stmt_info
, 0, vect_body
);
1252 if (dump_enabled_p ())
1253 dump_printf_loc (MSG_NOTE
, vect_location
,
1254 "vect_model_load_cost: aligned.\n");
1258 case dr_unaligned_supported
:
1260 /* Here, we assign an additional cost for the unaligned load. */
1261 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1262 unaligned_load
, stmt_info
,
1263 DR_MISALIGNMENT (dr_info
),
1266 if (dump_enabled_p ())
1267 dump_printf_loc (MSG_NOTE
, vect_location
,
1268 "vect_model_load_cost: unaligned supported by "
1273 case dr_explicit_realign
:
1275 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1276 vector_load
, stmt_info
, 0, vect_body
);
1277 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1278 vec_perm
, stmt_info
, 0, vect_body
);
1280 /* FIXME: If the misalignment remains fixed across the iterations of
1281 the containing loop, the following cost should be added to the
1283 if (targetm
.vectorize
.builtin_mask_for_load
)
1284 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1285 stmt_info
, 0, vect_body
);
1287 if (dump_enabled_p ())
1288 dump_printf_loc (MSG_NOTE
, vect_location
,
1289 "vect_model_load_cost: explicit realign\n");
1293 case dr_explicit_realign_optimized
:
1295 if (dump_enabled_p ())
1296 dump_printf_loc (MSG_NOTE
, vect_location
,
1297 "vect_model_load_cost: unaligned software "
1300 /* Unaligned software pipeline has a load of an address, an initial
1301 load, and possibly a mask operation to "prime" the loop. However,
1302 if this is an access in a group of loads, which provide grouped
1303 access, then the above cost should only be considered for one
1304 access in the group. Inside the loop, there is a load op
1305 and a realignment op. */
1307 if (add_realign_cost
&& record_prologue_costs
)
1309 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1310 vector_stmt
, stmt_info
,
1312 if (targetm
.vectorize
.builtin_mask_for_load
)
1313 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1314 vector_stmt
, stmt_info
,
1318 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1319 stmt_info
, 0, vect_body
);
1320 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1321 stmt_info
, 0, vect_body
);
1323 if (dump_enabled_p ())
1324 dump_printf_loc (MSG_NOTE
, vect_location
,
1325 "vect_model_load_cost: explicit realign optimized"
1331 case dr_unaligned_unsupported
:
1333 *inside_cost
= VECT_MAX_COST
;
1335 if (dump_enabled_p ())
1336 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1337 "vect_model_load_cost: unsupported access.\n");
1346 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1347 the loop preheader for the vectorized stmt STMT_VINFO. */
1350 vect_init_vector_1 (stmt_vec_info stmt_vinfo
, gimple
*new_stmt
,
1351 gimple_stmt_iterator
*gsi
)
1354 vect_finish_stmt_generation (stmt_vinfo
, new_stmt
, gsi
);
1357 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1361 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1365 if (nested_in_vect_loop_p (loop
, stmt_vinfo
))
1368 pe
= loop_preheader_edge (loop
);
1369 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1370 gcc_assert (!new_bb
);
1374 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1376 gimple_stmt_iterator gsi_bb_start
;
1378 gcc_assert (bb_vinfo
);
1379 bb
= BB_VINFO_BB (bb_vinfo
);
1380 gsi_bb_start
= gsi_after_labels (bb
);
1381 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1385 if (dump_enabled_p ())
1387 dump_printf_loc (MSG_NOTE
, vect_location
,
1388 "created new init_stmt: ");
1389 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1393 /* Function vect_init_vector.
1395 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1396 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1397 vector type a vector with all elements equal to VAL is created first.
1398 Place the initialization at BSI if it is not NULL. Otherwise, place the
1399 initialization at the loop preheader.
1400 Return the DEF of INIT_STMT.
1401 It will be used in the vectorization of STMT_INFO. */
1404 vect_init_vector (stmt_vec_info stmt_info
, tree val
, tree type
,
1405 gimple_stmt_iterator
*gsi
)
1410 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1411 if (! useless_type_conversion_p (type
, TREE_TYPE (val
)))
1413 gcc_assert (TREE_CODE (type
) == VECTOR_TYPE
);
1414 if (! types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1416 /* Scalar boolean value should be transformed into
1417 all zeros or all ones value before building a vector. */
1418 if (VECTOR_BOOLEAN_TYPE_P (type
))
1420 tree true_val
= build_all_ones_cst (TREE_TYPE (type
));
1421 tree false_val
= build_zero_cst (TREE_TYPE (type
));
1423 if (CONSTANT_CLASS_P (val
))
1424 val
= integer_zerop (val
) ? false_val
: true_val
;
1427 new_temp
= make_ssa_name (TREE_TYPE (type
));
1428 init_stmt
= gimple_build_assign (new_temp
, COND_EXPR
,
1429 val
, true_val
, false_val
);
1430 vect_init_vector_1 (stmt_info
, init_stmt
, gsi
);
1434 else if (CONSTANT_CLASS_P (val
))
1435 val
= fold_convert (TREE_TYPE (type
), val
);
1438 new_temp
= make_ssa_name (TREE_TYPE (type
));
1439 if (! INTEGRAL_TYPE_P (TREE_TYPE (val
)))
1440 init_stmt
= gimple_build_assign (new_temp
,
1441 fold_build1 (VIEW_CONVERT_EXPR
,
1445 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1446 vect_init_vector_1 (stmt_info
, init_stmt
, gsi
);
1450 val
= build_vector_from_val (type
, val
);
1453 new_temp
= vect_get_new_ssa_name (type
, vect_simple_var
, "cst_");
1454 init_stmt
= gimple_build_assign (new_temp
, val
);
1455 vect_init_vector_1 (stmt_info
, init_stmt
, gsi
);
1459 /* Function vect_get_vec_def_for_operand_1.
1461 For a defining stmt DEF_STMT_INFO of a scalar stmt, return a vector def
1462 with type DT that will be used in the vectorized stmt. */
1465 vect_get_vec_def_for_operand_1 (stmt_vec_info def_stmt_info
,
1466 enum vect_def_type dt
)
1469 stmt_vec_info vec_stmt_info
;
1473 /* operand is a constant or a loop invariant. */
1474 case vect_constant_def
:
1475 case vect_external_def
:
1476 /* Code should use vect_get_vec_def_for_operand. */
1479 /* operand is defined inside the loop. */
1480 case vect_internal_def
:
1482 /* Get the def from the vectorized stmt. */
1483 vec_stmt_info
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1484 /* Get vectorized pattern statement. */
1486 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1487 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1488 vec_stmt_info
= (STMT_VINFO_VEC_STMT
1489 (STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1490 gcc_assert (vec_stmt_info
);
1491 if (gphi
*phi
= dyn_cast
<gphi
*> (vec_stmt_info
->stmt
))
1492 vec_oprnd
= PHI_RESULT (phi
);
1494 vec_oprnd
= gimple_get_lhs (vec_stmt_info
->stmt
);
1498 /* operand is defined by a loop header phi. */
1499 case vect_reduction_def
:
1500 case vect_double_reduction_def
:
1501 case vect_nested_cycle
:
1502 case vect_induction_def
:
1504 gcc_assert (gimple_code (def_stmt_info
->stmt
) == GIMPLE_PHI
);
1506 /* Get the def from the vectorized stmt. */
1507 vec_stmt_info
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1508 if (gphi
*phi
= dyn_cast
<gphi
*> (vec_stmt_info
->stmt
))
1509 vec_oprnd
= PHI_RESULT (phi
);
1511 vec_oprnd
= gimple_get_lhs (vec_stmt_info
->stmt
);
1521 /* Function vect_get_vec_def_for_operand.
1523 OP is an operand in STMT_VINFO. This function returns a (vector) def
1524 that will be used in the vectorized stmt for STMT_VINFO.
1526 In the case that OP is an SSA_NAME which is defined in the loop, then
1527 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1529 In case OP is an invariant or constant, a new stmt that creates a vector def
1530 needs to be introduced. VECTYPE may be used to specify a required type for
1531 vector invariant. */
1534 vect_get_vec_def_for_operand (tree op
, stmt_vec_info stmt_vinfo
, tree vectype
)
1537 enum vect_def_type dt
;
1539 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1541 if (dump_enabled_p ())
1543 dump_printf_loc (MSG_NOTE
, vect_location
,
1544 "vect_get_vec_def_for_operand: ");
1545 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1546 dump_printf (MSG_NOTE
, "\n");
1549 stmt_vec_info def_stmt_info
;
1550 is_simple_use
= vect_is_simple_use (op
, loop_vinfo
, &dt
,
1551 &def_stmt_info
, &def_stmt
);
1552 gcc_assert (is_simple_use
);
1553 if (def_stmt
&& dump_enabled_p ())
1555 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1556 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1559 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
1561 tree stmt_vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
1565 vector_type
= vectype
;
1566 else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op
))
1567 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype
))
1568 vector_type
= build_same_sized_truth_vector_type (stmt_vectype
);
1570 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1572 gcc_assert (vector_type
);
1573 return vect_init_vector (stmt_vinfo
, op
, vector_type
, NULL
);
1576 return vect_get_vec_def_for_operand_1 (def_stmt_info
, dt
);
1580 /* Function vect_get_vec_def_for_stmt_copy
1582 Return a vector-def for an operand. This function is used when the
1583 vectorized stmt to be created (by the caller to this function) is a "copy"
1584 created in case the vectorized result cannot fit in one vector, and several
1585 copies of the vector-stmt are required. In this case the vector-def is
1586 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1587 of the stmt that defines VEC_OPRND. VINFO describes the vectorization.
1590 In case the vectorization factor (VF) is bigger than the number
1591 of elements that can fit in a vectype (nunits), we have to generate
1592 more than one vector stmt to vectorize the scalar stmt. This situation
1593 arises when there are multiple data-types operated upon in the loop; the
1594 smallest data-type determines the VF, and as a result, when vectorizing
1595 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1596 vector stmt (each computing a vector of 'nunits' results, and together
1597 computing 'VF' results in each iteration). This function is called when
1598 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1599 which VF=16 and nunits=4, so the number of copies required is 4):
1601 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1603 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1604 VS1.1: vx.1 = memref1 VS1.2
1605 VS1.2: vx.2 = memref2 VS1.3
1606 VS1.3: vx.3 = memref3
1608 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1609 VSnew.1: vz1 = vx.1 + ... VSnew.2
1610 VSnew.2: vz2 = vx.2 + ... VSnew.3
1611 VSnew.3: vz3 = vx.3 + ...
1613 The vectorization of S1 is explained in vectorizable_load.
1614 The vectorization of S2:
1615 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1616 the function 'vect_get_vec_def_for_operand' is called to
1617 get the relevant vector-def for each operand of S2. For operand x it
1618 returns the vector-def 'vx.0'.
1620 To create the remaining copies of the vector-stmt (VSnew.j), this
1621 function is called to get the relevant vector-def for each operand. It is
1622 obtained from the respective VS1.j stmt, which is recorded in the
1623 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1625 For example, to obtain the vector-def 'vx.1' in order to create the
1626 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1627 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1628 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1629 and return its def ('vx.1').
1630 Overall, to create the above sequence this function will be called 3 times:
1631 vx.1 = vect_get_vec_def_for_stmt_copy (vinfo, vx.0);
1632 vx.2 = vect_get_vec_def_for_stmt_copy (vinfo, vx.1);
1633 vx.3 = vect_get_vec_def_for_stmt_copy (vinfo, vx.2); */
1636 vect_get_vec_def_for_stmt_copy (vec_info
*vinfo
, tree vec_oprnd
)
1638 stmt_vec_info def_stmt_info
= vinfo
->lookup_def (vec_oprnd
);
1640 /* Do nothing; can reuse same def. */
1643 def_stmt_info
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1644 gcc_assert (def_stmt_info
);
1645 if (gphi
*phi
= dyn_cast
<gphi
*> (def_stmt_info
->stmt
))
1646 vec_oprnd
= PHI_RESULT (phi
);
1648 vec_oprnd
= gimple_get_lhs (def_stmt_info
->stmt
);
1653 /* Get vectorized definitions for the operands to create a copy of an original
1654 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1657 vect_get_vec_defs_for_stmt_copy (vec_info
*vinfo
,
1658 vec
<tree
> *vec_oprnds0
,
1659 vec
<tree
> *vec_oprnds1
)
1661 tree vec_oprnd
= vec_oprnds0
->pop ();
1663 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd
);
1664 vec_oprnds0
->quick_push (vec_oprnd
);
1666 if (vec_oprnds1
&& vec_oprnds1
->length ())
1668 vec_oprnd
= vec_oprnds1
->pop ();
1669 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd
);
1670 vec_oprnds1
->quick_push (vec_oprnd
);
1675 /* Get vectorized definitions for OP0 and OP1. */
1678 vect_get_vec_defs (tree op0
, tree op1
, stmt_vec_info stmt_info
,
1679 vec
<tree
> *vec_oprnds0
,
1680 vec
<tree
> *vec_oprnds1
,
1685 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1686 auto_vec
<tree
> ops (nops
);
1687 auto_vec
<vec
<tree
> > vec_defs (nops
);
1689 ops
.quick_push (op0
);
1691 ops
.quick_push (op1
);
1693 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
1695 *vec_oprnds0
= vec_defs
[0];
1697 *vec_oprnds1
= vec_defs
[1];
1703 vec_oprnds0
->create (1);
1704 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt_info
);
1705 vec_oprnds0
->quick_push (vec_oprnd
);
1709 vec_oprnds1
->create (1);
1710 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt_info
);
1711 vec_oprnds1
->quick_push (vec_oprnd
);
1716 /* Helper function called by vect_finish_replace_stmt and
1717 vect_finish_stmt_generation. Set the location of the new
1718 statement and create and return a stmt_vec_info for it. */
1720 static stmt_vec_info
1721 vect_finish_stmt_generation_1 (stmt_vec_info stmt_info
, gimple
*vec_stmt
)
1723 vec_info
*vinfo
= stmt_info
->vinfo
;
1725 stmt_vec_info vec_stmt_info
= vinfo
->add_stmt (vec_stmt
);
1727 if (dump_enabled_p ())
1729 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1730 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1733 gimple_set_location (vec_stmt
, gimple_location (stmt_info
->stmt
));
1735 /* While EH edges will generally prevent vectorization, stmt might
1736 e.g. be in a must-not-throw region. Ensure newly created stmts
1737 that could throw are part of the same region. */
1738 int lp_nr
= lookup_stmt_eh_lp (stmt_info
->stmt
);
1739 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1740 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1742 return vec_stmt_info
;
1745 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1746 which sets the same scalar result as STMT_INFO did. Create and return a
1747 stmt_vec_info for VEC_STMT. */
1750 vect_finish_replace_stmt (stmt_vec_info stmt_info
, gimple
*vec_stmt
)
1752 gcc_assert (gimple_get_lhs (stmt_info
->stmt
) == gimple_get_lhs (vec_stmt
));
1754 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt_info
->stmt
);
1755 gsi_replace (&gsi
, vec_stmt
, false);
1757 return vect_finish_stmt_generation_1 (stmt_info
, vec_stmt
);
1760 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1761 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1764 vect_finish_stmt_generation (stmt_vec_info stmt_info
, gimple
*vec_stmt
,
1765 gimple_stmt_iterator
*gsi
)
1767 gcc_assert (gimple_code (stmt_info
->stmt
) != GIMPLE_LABEL
);
1769 if (!gsi_end_p (*gsi
)
1770 && gimple_has_mem_ops (vec_stmt
))
1772 gimple
*at_stmt
= gsi_stmt (*gsi
);
1773 tree vuse
= gimple_vuse (at_stmt
);
1774 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1776 tree vdef
= gimple_vdef (at_stmt
);
1777 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1778 /* If we have an SSA vuse and insert a store, update virtual
1779 SSA form to avoid triggering the renamer. Do so only
1780 if we can easily see all uses - which is what almost always
1781 happens with the way vectorized stmts are inserted. */
1782 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1783 && ((is_gimple_assign (vec_stmt
)
1784 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1785 || (is_gimple_call (vec_stmt
)
1786 && !(gimple_call_flags (vec_stmt
)
1787 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1789 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1790 gimple_set_vdef (vec_stmt
, new_vdef
);
1791 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1795 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1796 return vect_finish_stmt_generation_1 (stmt_info
, vec_stmt
);
1799 /* We want to vectorize a call to combined function CFN with function
1800 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1801 as the types of all inputs. Check whether this is possible using
1802 an internal function, returning its code if so or IFN_LAST if not. */
1805 vectorizable_internal_function (combined_fn cfn
, tree fndecl
,
1806 tree vectype_out
, tree vectype_in
)
1809 if (internal_fn_p (cfn
))
1810 ifn
= as_internal_fn (cfn
);
1812 ifn
= associated_internal_fn (fndecl
);
1813 if (ifn
!= IFN_LAST
&& direct_internal_fn_p (ifn
))
1815 const direct_internal_fn_info
&info
= direct_internal_fn (ifn
);
1816 if (info
.vectorizable
)
1818 tree type0
= (info
.type0
< 0 ? vectype_out
: vectype_in
);
1819 tree type1
= (info
.type1
< 0 ? vectype_out
: vectype_in
);
1820 if (direct_internal_fn_supported_p (ifn
, tree_pair (type0
, type1
),
1821 OPTIMIZE_FOR_SPEED
))
1829 static tree
permute_vec_elements (tree
, tree
, tree
, stmt_vec_info
,
1830 gimple_stmt_iterator
*);
1832 /* Check whether a load or store statement in the loop described by
1833 LOOP_VINFO is possible in a fully-masked loop. This is testing
1834 whether the vectorizer pass has the appropriate support, as well as
1835 whether the target does.
1837 VLS_TYPE says whether the statement is a load or store and VECTYPE
1838 is the type of the vector being loaded or stored. MEMORY_ACCESS_TYPE
1839 says how the load or store is going to be implemented and GROUP_SIZE
1840 is the number of load or store statements in the containing group.
1841 If the access is a gather load or scatter store, GS_INFO describes
1844 Clear LOOP_VINFO_CAN_FULLY_MASK_P if a fully-masked loop is not
1845 supported, otherwise record the required mask types. */
1848 check_load_store_masking (loop_vec_info loop_vinfo
, tree vectype
,
1849 vec_load_store_type vls_type
, int group_size
,
1850 vect_memory_access_type memory_access_type
,
1851 gather_scatter_info
*gs_info
)
1853 /* Invariant loads need no special support. */
1854 if (memory_access_type
== VMAT_INVARIANT
)
1857 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
1858 machine_mode vecmode
= TYPE_MODE (vectype
);
1859 bool is_load
= (vls_type
== VLS_LOAD
);
1860 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
1863 ? !vect_load_lanes_supported (vectype
, group_size
, true)
1864 : !vect_store_lanes_supported (vectype
, group_size
, true))
1866 if (dump_enabled_p ())
1867 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1868 "can't use a fully-masked loop because the"
1869 " target doesn't have an appropriate masked"
1870 " load/store-lanes instruction.\n");
1871 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
1874 unsigned int ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
1875 vect_record_loop_mask (loop_vinfo
, masks
, ncopies
, vectype
);
1879 if (memory_access_type
== VMAT_GATHER_SCATTER
)
1881 internal_fn ifn
= (is_load
1882 ? IFN_MASK_GATHER_LOAD
1883 : IFN_MASK_SCATTER_STORE
);
1884 tree offset_type
= TREE_TYPE (gs_info
->offset
);
1885 if (!internal_gather_scatter_fn_supported_p (ifn
, vectype
,
1886 gs_info
->memory_type
,
1887 TYPE_SIGN (offset_type
),
1890 if (dump_enabled_p ())
1891 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1892 "can't use a fully-masked loop because the"
1893 " target doesn't have an appropriate masked"
1894 " gather load or scatter store instruction.\n");
1895 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
1898 unsigned int ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
1899 vect_record_loop_mask (loop_vinfo
, masks
, ncopies
, vectype
);
1903 if (memory_access_type
!= VMAT_CONTIGUOUS
1904 && memory_access_type
!= VMAT_CONTIGUOUS_PERMUTE
)
1906 /* Element X of the data must come from iteration i * VF + X of the
1907 scalar loop. We need more work to support other mappings. */
1908 if (dump_enabled_p ())
1909 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1910 "can't use a fully-masked loop because an access"
1911 " isn't contiguous.\n");
1912 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
1916 machine_mode mask_mode
;
1917 if (!(targetm
.vectorize
.get_mask_mode
1918 (GET_MODE_NUNITS (vecmode
),
1919 GET_MODE_SIZE (vecmode
)).exists (&mask_mode
))
1920 || !can_vec_mask_load_store_p (vecmode
, mask_mode
, is_load
))
1922 if (dump_enabled_p ())
1923 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1924 "can't use a fully-masked loop because the target"
1925 " doesn't have the appropriate masked load or"
1927 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
1930 /* We might load more scalars than we need for permuting SLP loads.
1931 We checked in get_group_load_store_type that the extra elements
1932 don't leak into a new vector. */
1933 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1934 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1935 unsigned int nvectors
;
1936 if (can_div_away_from_zero_p (group_size
* vf
, nunits
, &nvectors
))
1937 vect_record_loop_mask (loop_vinfo
, masks
, nvectors
, vectype
);
1942 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1943 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1944 that needs to be applied to all loads and stores in a vectorized loop.
1945 Return VEC_MASK if LOOP_MASK is null, otherwise return VEC_MASK & LOOP_MASK.
1947 MASK_TYPE is the type of both masks. If new statements are needed,
1948 insert them before GSI. */
1951 prepare_load_store_mask (tree mask_type
, tree loop_mask
, tree vec_mask
,
1952 gimple_stmt_iterator
*gsi
)
1954 gcc_assert (useless_type_conversion_p (mask_type
, TREE_TYPE (vec_mask
)));
1958 gcc_assert (TREE_TYPE (loop_mask
) == mask_type
);
1959 tree and_res
= make_temp_ssa_name (mask_type
, NULL
, "vec_mask_and");
1960 gimple
*and_stmt
= gimple_build_assign (and_res
, BIT_AND_EXPR
,
1961 vec_mask
, loop_mask
);
1962 gsi_insert_before (gsi
, and_stmt
, GSI_SAME_STMT
);
1966 /* Determine whether we can use a gather load or scatter store to vectorize
1967 strided load or store STMT_INFO by truncating the current offset to a
1968 smaller width. We need to be able to construct an offset vector:
1970 { 0, X, X*2, X*3, ... }
1972 without loss of precision, where X is STMT_INFO's DR_STEP.
1974 Return true if this is possible, describing the gather load or scatter
1975 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1978 vect_truncate_gather_scatter_offset (stmt_vec_info stmt_info
,
1979 loop_vec_info loop_vinfo
, bool masked_p
,
1980 gather_scatter_info
*gs_info
)
1982 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
);
1983 data_reference
*dr
= dr_info
->dr
;
1984 tree step
= DR_STEP (dr
);
1985 if (TREE_CODE (step
) != INTEGER_CST
)
1987 /* ??? Perhaps we could use range information here? */
1988 if (dump_enabled_p ())
1989 dump_printf_loc (MSG_NOTE
, vect_location
,
1990 "cannot truncate variable step.\n");
1994 /* Get the number of bits in an element. */
1995 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1996 scalar_mode element_mode
= SCALAR_TYPE_MODE (TREE_TYPE (vectype
));
1997 unsigned int element_bits
= GET_MODE_BITSIZE (element_mode
);
1999 /* Set COUNT to the upper limit on the number of elements - 1.
2000 Start with the maximum vectorization factor. */
2001 unsigned HOST_WIDE_INT count
= vect_max_vf (loop_vinfo
) - 1;
2003 /* Try lowering COUNT to the number of scalar latch iterations. */
2004 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2005 widest_int max_iters
;
2006 if (max_loop_iterations (loop
, &max_iters
)
2007 && max_iters
< count
)
2008 count
= max_iters
.to_shwi ();
2010 /* Try scales of 1 and the element size. */
2011 int scales
[] = { 1, vect_get_scalar_dr_size (dr_info
) };
2012 wi::overflow_type overflow
= wi::OVF_NONE
;
2013 for (int i
= 0; i
< 2; ++i
)
2015 int scale
= scales
[i
];
2017 if (!wi::multiple_of_p (wi::to_widest (step
), scale
, SIGNED
, &factor
))
2020 /* See whether we can calculate (COUNT - 1) * STEP / SCALE
2021 in OFFSET_BITS bits. */
2022 widest_int range
= wi::mul (count
, factor
, SIGNED
, &overflow
);
2025 signop sign
= range
>= 0 ? UNSIGNED
: SIGNED
;
2026 if (wi::min_precision (range
, sign
) > element_bits
)
2028 overflow
= wi::OVF_UNKNOWN
;
2032 /* See whether the target supports the operation. */
2033 tree memory_type
= TREE_TYPE (DR_REF (dr
));
2034 if (!vect_gather_scatter_fn_p (DR_IS_READ (dr
), masked_p
, vectype
,
2035 memory_type
, element_bits
, sign
, scale
,
2036 &gs_info
->ifn
, &gs_info
->element_type
))
2039 tree offset_type
= build_nonstandard_integer_type (element_bits
,
2042 gs_info
->decl
= NULL_TREE
;
2043 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
2044 but we don't need to store that here. */
2045 gs_info
->base
= NULL_TREE
;
2046 gs_info
->offset
= fold_convert (offset_type
, step
);
2047 gs_info
->offset_dt
= vect_constant_def
;
2048 gs_info
->offset_vectype
= NULL_TREE
;
2049 gs_info
->scale
= scale
;
2050 gs_info
->memory_type
= memory_type
;
2054 if (overflow
&& dump_enabled_p ())
2055 dump_printf_loc (MSG_NOTE
, vect_location
,
2056 "truncating gather/scatter offset to %d bits"
2057 " might change its value.\n", element_bits
);
2062 /* Return true if we can use gather/scatter internal functions to
2063 vectorize STMT_INFO, which is a grouped or strided load or store.
2064 MASKED_P is true if load or store is conditional. When returning
2065 true, fill in GS_INFO with the information required to perform the
2069 vect_use_strided_gather_scatters_p (stmt_vec_info stmt_info
,
2070 loop_vec_info loop_vinfo
, bool masked_p
,
2071 gather_scatter_info
*gs_info
)
2073 if (!vect_check_gather_scatter (stmt_info
, loop_vinfo
, gs_info
)
2075 return vect_truncate_gather_scatter_offset (stmt_info
, loop_vinfo
,
2078 scalar_mode element_mode
= SCALAR_TYPE_MODE (gs_info
->element_type
);
2079 unsigned int element_bits
= GET_MODE_BITSIZE (element_mode
);
2080 tree offset_type
= TREE_TYPE (gs_info
->offset
);
2081 unsigned int offset_bits
= TYPE_PRECISION (offset_type
);
2083 /* Enforced by vect_check_gather_scatter. */
2084 gcc_assert (element_bits
>= offset_bits
);
2086 /* If the elements are wider than the offset, convert the offset to the
2087 same width, without changing its sign. */
2088 if (element_bits
> offset_bits
)
2090 bool unsigned_p
= TYPE_UNSIGNED (offset_type
);
2091 offset_type
= build_nonstandard_integer_type (element_bits
, unsigned_p
);
2092 gs_info
->offset
= fold_convert (offset_type
, gs_info
->offset
);
2095 if (dump_enabled_p ())
2096 dump_printf_loc (MSG_NOTE
, vect_location
,
2097 "using gather/scatter for strided/grouped access,"
2098 " scale = %d\n", gs_info
->scale
);
2103 /* STMT_INFO is a non-strided load or store, meaning that it accesses
2104 elements with a known constant step. Return -1 if that step
2105 is negative, 0 if it is zero, and 1 if it is greater than zero. */
2108 compare_step_with_zero (stmt_vec_info stmt_info
)
2110 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
);
2111 return tree_int_cst_compare (vect_dr_behavior (dr_info
)->step
,
2115 /* If the target supports a permute mask that reverses the elements in
2116 a vector of type VECTYPE, return that mask, otherwise return null. */
2119 perm_mask_for_reverse (tree vectype
)
2121 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2123 /* The encoding has a single stepped pattern. */
2124 vec_perm_builder
sel (nunits
, 1, 3);
2125 for (int i
= 0; i
< 3; ++i
)
2126 sel
.quick_push (nunits
- 1 - i
);
2128 vec_perm_indices
indices (sel
, 1, nunits
);
2129 if (!can_vec_perm_const_p (TYPE_MODE (vectype
), indices
))
2131 return vect_gen_perm_mask_checked (vectype
, indices
);
2134 /* STMT_INFO is either a masked or unconditional store. Return the value
2138 vect_get_store_rhs (stmt_vec_info stmt_info
)
2140 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt_info
->stmt
))
2142 gcc_assert (gimple_assign_single_p (assign
));
2143 return gimple_assign_rhs1 (assign
);
2145 if (gcall
*call
= dyn_cast
<gcall
*> (stmt_info
->stmt
))
2147 internal_fn ifn
= gimple_call_internal_fn (call
);
2148 int index
= internal_fn_stored_value_index (ifn
);
2149 gcc_assert (index
>= 0);
2150 return gimple_call_arg (call
, index
);
2155 /* A subroutine of get_load_store_type, with a subset of the same
2156 arguments. Handle the case where STMT_INFO is part of a grouped load
2159 For stores, the statements in the group are all consecutive
2160 and there is no gap at the end. For loads, the statements in the
2161 group might not be consecutive; there can be gaps between statements
2162 as well as at the end. */
2165 get_group_load_store_type (stmt_vec_info stmt_info
, tree vectype
, bool slp
,
2166 bool masked_p
, vec_load_store_type vls_type
,
2167 vect_memory_access_type
*memory_access_type
,
2168 gather_scatter_info
*gs_info
)
2170 vec_info
*vinfo
= stmt_info
->vinfo
;
2171 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2172 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2173 stmt_vec_info first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
2174 dr_vec_info
*first_dr_info
= STMT_VINFO_DR_INFO (first_stmt_info
);
2175 unsigned int group_size
= DR_GROUP_SIZE (first_stmt_info
);
2176 bool single_element_p
= (stmt_info
== first_stmt_info
2177 && !DR_GROUP_NEXT_ELEMENT (stmt_info
));
2178 unsigned HOST_WIDE_INT gap
= DR_GROUP_GAP (first_stmt_info
);
2179 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2181 /* True if the vectorized statements would access beyond the last
2182 statement in the group. */
2183 bool overrun_p
= false;
2185 /* True if we can cope with such overrun by peeling for gaps, so that
2186 there is at least one final scalar iteration after the vector loop. */
2187 bool can_overrun_p
= (!masked_p
2188 && vls_type
== VLS_LOAD
2192 /* There can only be a gap at the end of the group if the stride is
2193 known at compile time. */
2194 gcc_assert (!STMT_VINFO_STRIDED_P (stmt_info
) || gap
== 0);
2196 /* Stores can't yet have gaps. */
2197 gcc_assert (slp
|| vls_type
== VLS_LOAD
|| gap
== 0);
2201 if (STMT_VINFO_STRIDED_P (stmt_info
))
2203 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2204 separated by the stride, until we have a complete vector.
2205 Fall back to scalar accesses if that isn't possible. */
2206 if (multiple_p (nunits
, group_size
))
2207 *memory_access_type
= VMAT_STRIDED_SLP
;
2209 *memory_access_type
= VMAT_ELEMENTWISE
;
2213 overrun_p
= loop_vinfo
&& gap
!= 0;
2214 if (overrun_p
&& vls_type
!= VLS_LOAD
)
2216 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2217 "Grouped store with gaps requires"
2218 " non-consecutive accesses\n");
2221 /* An overrun is fine if the trailing elements are smaller
2222 than the alignment boundary B. Every vector access will
2223 be a multiple of B and so we are guaranteed to access a
2224 non-gap element in the same B-sized block. */
2226 && gap
< (vect_known_alignment_in_bytes (first_dr_info
)
2227 / vect_get_scalar_dr_size (first_dr_info
)))
2229 if (overrun_p
&& !can_overrun_p
)
2231 if (dump_enabled_p ())
2232 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2233 "Peeling for outer loop is not supported\n");
2236 *memory_access_type
= VMAT_CONTIGUOUS
;
2241 /* We can always handle this case using elementwise accesses,
2242 but see if something more efficient is available. */
2243 *memory_access_type
= VMAT_ELEMENTWISE
;
2245 /* If there is a gap at the end of the group then these optimizations
2246 would access excess elements in the last iteration. */
2247 bool would_overrun_p
= (gap
!= 0);
2248 /* An overrun is fine if the trailing elements are smaller than the
2249 alignment boundary B. Every vector access will be a multiple of B
2250 and so we are guaranteed to access a non-gap element in the
2251 same B-sized block. */
2254 && gap
< (vect_known_alignment_in_bytes (first_dr_info
)
2255 / vect_get_scalar_dr_size (first_dr_info
)))
2256 would_overrun_p
= false;
2258 if (!STMT_VINFO_STRIDED_P (stmt_info
)
2259 && (can_overrun_p
|| !would_overrun_p
)
2260 && compare_step_with_zero (stmt_info
) > 0)
2262 /* First cope with the degenerate case of a single-element
2264 if (known_eq (TYPE_VECTOR_SUBPARTS (vectype
), 1U))
2265 *memory_access_type
= VMAT_CONTIGUOUS
;
2267 /* Otherwise try using LOAD/STORE_LANES. */
2268 if (*memory_access_type
== VMAT_ELEMENTWISE
2269 && (vls_type
== VLS_LOAD
2270 ? vect_load_lanes_supported (vectype
, group_size
, masked_p
)
2271 : vect_store_lanes_supported (vectype
, group_size
,
2274 *memory_access_type
= VMAT_LOAD_STORE_LANES
;
2275 overrun_p
= would_overrun_p
;
2278 /* If that fails, try using permuting loads. */
2279 if (*memory_access_type
== VMAT_ELEMENTWISE
2280 && (vls_type
== VLS_LOAD
2281 ? vect_grouped_load_supported (vectype
, single_element_p
,
2283 : vect_grouped_store_supported (vectype
, group_size
)))
2285 *memory_access_type
= VMAT_CONTIGUOUS_PERMUTE
;
2286 overrun_p
= would_overrun_p
;
2290 /* As a last resort, trying using a gather load or scatter store.
2292 ??? Although the code can handle all group sizes correctly,
2293 it probably isn't a win to use separate strided accesses based
2294 on nearby locations. Or, even if it's a win over scalar code,
2295 it might not be a win over vectorizing at a lower VF, if that
2296 allows us to use contiguous accesses. */
2297 if (*memory_access_type
== VMAT_ELEMENTWISE
2300 && vect_use_strided_gather_scatters_p (stmt_info
, loop_vinfo
,
2302 *memory_access_type
= VMAT_GATHER_SCATTER
;
2305 if (vls_type
!= VLS_LOAD
&& first_stmt_info
== stmt_info
)
2307 /* STMT is the leader of the group. Check the operands of all the
2308 stmts of the group. */
2309 stmt_vec_info next_stmt_info
= DR_GROUP_NEXT_ELEMENT (stmt_info
);
2310 while (next_stmt_info
)
2312 tree op
= vect_get_store_rhs (next_stmt_info
);
2313 enum vect_def_type dt
;
2314 if (!vect_is_simple_use (op
, vinfo
, &dt
))
2316 if (dump_enabled_p ())
2317 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2318 "use not simple.\n");
2321 next_stmt_info
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
2327 gcc_assert (can_overrun_p
);
2328 if (dump_enabled_p ())
2329 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2330 "Data access with gaps requires scalar "
2332 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) = true;
2338 /* A subroutine of get_load_store_type, with a subset of the same
2339 arguments. Handle the case where STMT_INFO is a load or store that
2340 accesses consecutive elements with a negative step. */
2342 static vect_memory_access_type
2343 get_negative_load_store_type (stmt_vec_info stmt_info
, tree vectype
,
2344 vec_load_store_type vls_type
,
2345 unsigned int ncopies
)
2347 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
);
2348 dr_alignment_support alignment_support_scheme
;
2352 if (dump_enabled_p ())
2353 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2354 "multiple types with negative step.\n");
2355 return VMAT_ELEMENTWISE
;
2358 alignment_support_scheme
= vect_supportable_dr_alignment (dr_info
, false);
2359 if (alignment_support_scheme
!= dr_aligned
2360 && alignment_support_scheme
!= dr_unaligned_supported
)
2362 if (dump_enabled_p ())
2363 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2364 "negative step but alignment required.\n");
2365 return VMAT_ELEMENTWISE
;
2368 if (vls_type
== VLS_STORE_INVARIANT
)
2370 if (dump_enabled_p ())
2371 dump_printf_loc (MSG_NOTE
, vect_location
,
2372 "negative step with invariant source;"
2373 " no permute needed.\n");
2374 return VMAT_CONTIGUOUS_DOWN
;
2377 if (!perm_mask_for_reverse (vectype
))
2379 if (dump_enabled_p ())
2380 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2381 "negative step and reversing not supported.\n");
2382 return VMAT_ELEMENTWISE
;
2385 return VMAT_CONTIGUOUS_REVERSE
;
2388 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2389 if there is a memory access type that the vectorized form can use,
2390 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2391 or scatters, fill in GS_INFO accordingly.
2393 SLP says whether we're performing SLP rather than loop vectorization.
2394 MASKED_P is true if the statement is conditional on a vectorized mask.
2395 VECTYPE is the vector type that the vectorized statements will use.
2396 NCOPIES is the number of vector statements that will be needed. */
2399 get_load_store_type (stmt_vec_info stmt_info
, tree vectype
, bool slp
,
2400 bool masked_p
, vec_load_store_type vls_type
,
2401 unsigned int ncopies
,
2402 vect_memory_access_type
*memory_access_type
,
2403 gather_scatter_info
*gs_info
)
2405 vec_info
*vinfo
= stmt_info
->vinfo
;
2406 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2407 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2408 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
2410 *memory_access_type
= VMAT_GATHER_SCATTER
;
2411 if (!vect_check_gather_scatter (stmt_info
, loop_vinfo
, gs_info
))
2413 else if (!vect_is_simple_use (gs_info
->offset
, vinfo
,
2414 &gs_info
->offset_dt
,
2415 &gs_info
->offset_vectype
))
2417 if (dump_enabled_p ())
2418 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2419 "%s index use not simple.\n",
2420 vls_type
== VLS_LOAD
? "gather" : "scatter");
2424 else if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
2426 if (!get_group_load_store_type (stmt_info
, vectype
, slp
, masked_p
,
2427 vls_type
, memory_access_type
, gs_info
))
2430 else if (STMT_VINFO_STRIDED_P (stmt_info
))
2434 && vect_use_strided_gather_scatters_p (stmt_info
, loop_vinfo
,
2436 *memory_access_type
= VMAT_GATHER_SCATTER
;
2438 *memory_access_type
= VMAT_ELEMENTWISE
;
2442 int cmp
= compare_step_with_zero (stmt_info
);
2444 *memory_access_type
= get_negative_load_store_type
2445 (stmt_info
, vectype
, vls_type
, ncopies
);
2448 gcc_assert (vls_type
== VLS_LOAD
);
2449 *memory_access_type
= VMAT_INVARIANT
;
2452 *memory_access_type
= VMAT_CONTIGUOUS
;
2455 if ((*memory_access_type
== VMAT_ELEMENTWISE
2456 || *memory_access_type
== VMAT_STRIDED_SLP
)
2457 && !nunits
.is_constant ())
2459 if (dump_enabled_p ())
2460 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2461 "Not using elementwise accesses due to variable "
2462 "vectorization factor.\n");
2466 /* FIXME: At the moment the cost model seems to underestimate the
2467 cost of using elementwise accesses. This check preserves the
2468 traditional behavior until that can be fixed. */
2469 if (*memory_access_type
== VMAT_ELEMENTWISE
2470 && !STMT_VINFO_STRIDED_P (stmt_info
)
2471 && !(stmt_info
== DR_GROUP_FIRST_ELEMENT (stmt_info
)
2472 && !DR_GROUP_NEXT_ELEMENT (stmt_info
)
2473 && !pow2p_hwi (DR_GROUP_SIZE (stmt_info
))))
2475 if (dump_enabled_p ())
2476 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2477 "not falling back to elementwise accesses\n");
2483 /* Return true if boolean argument MASK is suitable for vectorizing
2484 conditional load or store STMT_INFO. When returning true, store the type
2485 of the definition in *MASK_DT_OUT and the type of the vectorized mask
2486 in *MASK_VECTYPE_OUT. */
2489 vect_check_load_store_mask (stmt_vec_info stmt_info
, tree mask
,
2490 vect_def_type
*mask_dt_out
,
2491 tree
*mask_vectype_out
)
2493 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask
)))
2495 if (dump_enabled_p ())
2496 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2497 "mask argument is not a boolean.\n");
2501 if (TREE_CODE (mask
) != SSA_NAME
)
2503 if (dump_enabled_p ())
2504 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2505 "mask argument is not an SSA name.\n");
2509 enum vect_def_type mask_dt
;
2511 if (!vect_is_simple_use (mask
, stmt_info
->vinfo
, &mask_dt
, &mask_vectype
))
2513 if (dump_enabled_p ())
2514 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2515 "mask use not simple.\n");
2519 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2521 mask_vectype
= get_mask_type_for_scalar_type (TREE_TYPE (vectype
));
2523 if (!mask_vectype
|| !VECTOR_BOOLEAN_TYPE_P (mask_vectype
))
2525 if (dump_enabled_p ())
2526 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2527 "could not find an appropriate vector mask type.\n");
2531 if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype
),
2532 TYPE_VECTOR_SUBPARTS (vectype
)))
2534 if (dump_enabled_p ())
2536 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2537 "vector mask type ");
2538 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, mask_vectype
);
2539 dump_printf (MSG_MISSED_OPTIMIZATION
,
2540 " does not match vector data type ");
2541 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, vectype
);
2542 dump_printf (MSG_MISSED_OPTIMIZATION
, ".\n");
2547 *mask_dt_out
= mask_dt
;
2548 *mask_vectype_out
= mask_vectype
;
2552 /* Return true if stored value RHS is suitable for vectorizing store
2553 statement STMT_INFO. When returning true, store the type of the
2554 definition in *RHS_DT_OUT, the type of the vectorized store value in
2555 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2558 vect_check_store_rhs (stmt_vec_info stmt_info
, tree rhs
,
2559 vect_def_type
*rhs_dt_out
, tree
*rhs_vectype_out
,
2560 vec_load_store_type
*vls_type_out
)
2562 /* In the case this is a store from a constant make sure
2563 native_encode_expr can handle it. */
2564 if (CONSTANT_CLASS_P (rhs
) && native_encode_expr (rhs
, NULL
, 64) == 0)
2566 if (dump_enabled_p ())
2567 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2568 "cannot encode constant as a byte sequence.\n");
2572 enum vect_def_type rhs_dt
;
2574 if (!vect_is_simple_use (rhs
, stmt_info
->vinfo
, &rhs_dt
, &rhs_vectype
))
2576 if (dump_enabled_p ())
2577 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2578 "use not simple.\n");
2582 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2583 if (rhs_vectype
&& !useless_type_conversion_p (vectype
, rhs_vectype
))
2585 if (dump_enabled_p ())
2586 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2587 "incompatible vector types.\n");
2591 *rhs_dt_out
= rhs_dt
;
2592 *rhs_vectype_out
= rhs_vectype
;
2593 if (rhs_dt
== vect_constant_def
|| rhs_dt
== vect_external_def
)
2594 *vls_type_out
= VLS_STORE_INVARIANT
;
2596 *vls_type_out
= VLS_STORE
;
2600 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2601 Note that we support masks with floating-point type, in which case the
2602 floats are interpreted as a bitmask. */
2605 vect_build_all_ones_mask (stmt_vec_info stmt_info
, tree masktype
)
2607 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
2608 return build_int_cst (masktype
, -1);
2609 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
2611 tree mask
= build_int_cst (TREE_TYPE (masktype
), -1);
2612 mask
= build_vector_from_val (masktype
, mask
);
2613 return vect_init_vector (stmt_info
, mask
, masktype
, NULL
);
2615 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
2619 for (int j
= 0; j
< 6; ++j
)
2621 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
2622 tree mask
= build_real (TREE_TYPE (masktype
), r
);
2623 mask
= build_vector_from_val (masktype
, mask
);
2624 return vect_init_vector (stmt_info
, mask
, masktype
, NULL
);
2629 /* Build an all-zero merge value of type VECTYPE while vectorizing
2630 STMT_INFO as a gather load. */
2633 vect_build_zero_merge_argument (stmt_vec_info stmt_info
, tree vectype
)
2636 if (TREE_CODE (TREE_TYPE (vectype
)) == INTEGER_TYPE
)
2637 merge
= build_int_cst (TREE_TYPE (vectype
), 0);
2638 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype
)))
2642 for (int j
= 0; j
< 6; ++j
)
2644 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (vectype
)));
2645 merge
= build_real (TREE_TYPE (vectype
), r
);
2649 merge
= build_vector_from_val (vectype
, merge
);
2650 return vect_init_vector (stmt_info
, merge
, vectype
, NULL
);
2653 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2654 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2655 the gather load operation. If the load is conditional, MASK is the
2656 unvectorized condition and MASK_DT is its definition type, otherwise
2660 vect_build_gather_load_calls (stmt_vec_info stmt_info
,
2661 gimple_stmt_iterator
*gsi
,
2662 stmt_vec_info
*vec_stmt
,
2663 gather_scatter_info
*gs_info
,
2666 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2667 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2668 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2669 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2670 int ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
2671 edge pe
= loop_preheader_edge (loop
);
2672 enum { NARROW
, NONE
, WIDEN
} modifier
;
2673 poly_uint64 gather_off_nunits
2674 = TYPE_VECTOR_SUBPARTS (gs_info
->offset_vectype
);
2676 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
->decl
));
2677 tree rettype
= TREE_TYPE (TREE_TYPE (gs_info
->decl
));
2678 tree srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2679 tree ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2680 tree idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2681 tree masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
2682 tree scaletype
= TREE_VALUE (arglist
);
2683 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
2684 && (!mask
|| types_compatible_p (srctype
, masktype
)));
2686 tree perm_mask
= NULL_TREE
;
2687 tree mask_perm_mask
= NULL_TREE
;
2688 if (known_eq (nunits
, gather_off_nunits
))
2690 else if (known_eq (nunits
* 2, gather_off_nunits
))
2694 /* Currently widening gathers and scatters are only supported for
2695 fixed-length vectors. */
2696 int count
= gather_off_nunits
.to_constant ();
2697 vec_perm_builder
sel (count
, count
, 1);
2698 for (int i
= 0; i
< count
; ++i
)
2699 sel
.quick_push (i
| (count
/ 2));
2701 vec_perm_indices
indices (sel
, 1, count
);
2702 perm_mask
= vect_gen_perm_mask_checked (gs_info
->offset_vectype
,
2705 else if (known_eq (nunits
, gather_off_nunits
* 2))
2709 /* Currently narrowing gathers and scatters are only supported for
2710 fixed-length vectors. */
2711 int count
= nunits
.to_constant ();
2712 vec_perm_builder
sel (count
, count
, 1);
2713 sel
.quick_grow (count
);
2714 for (int i
= 0; i
< count
; ++i
)
2715 sel
[i
] = i
< count
/ 2 ? i
: i
+ count
/ 2;
2716 vec_perm_indices
indices (sel
, 2, count
);
2717 perm_mask
= vect_gen_perm_mask_checked (vectype
, indices
);
2723 for (int i
= 0; i
< count
; ++i
)
2724 sel
[i
] = i
| (count
/ 2);
2725 indices
.new_vector (sel
, 2, count
);
2726 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, indices
);
2732 tree scalar_dest
= gimple_get_lhs (stmt_info
->stmt
);
2733 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2735 tree ptr
= fold_convert (ptrtype
, gs_info
->base
);
2736 if (!is_gimple_min_invariant (ptr
))
2739 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
2740 basic_block new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
2741 gcc_assert (!new_bb
);
2744 tree scale
= build_int_cst (scaletype
, gs_info
->scale
);
2746 tree vec_oprnd0
= NULL_TREE
;
2747 tree vec_mask
= NULL_TREE
;
2748 tree src_op
= NULL_TREE
;
2749 tree mask_op
= NULL_TREE
;
2750 tree prev_res
= NULL_TREE
;
2751 stmt_vec_info prev_stmt_info
= NULL
;
2755 src_op
= vect_build_zero_merge_argument (stmt_info
, rettype
);
2756 mask_op
= vect_build_all_ones_mask (stmt_info
, masktype
);
2759 for (int j
= 0; j
< ncopies
; ++j
)
2762 if (modifier
== WIDEN
&& (j
& 1))
2763 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
2764 perm_mask
, stmt_info
, gsi
);
2767 = vect_get_vec_def_for_operand (gs_info
->offset
, stmt_info
);
2769 op
= vec_oprnd0
= vect_get_vec_def_for_stmt_copy (loop_vinfo
,
2772 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
2774 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
)),
2775 TYPE_VECTOR_SUBPARTS (idxtype
)));
2776 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
2777 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
2778 gassign
*new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2779 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
2785 if (mask_perm_mask
&& (j
& 1))
2786 mask_op
= permute_vec_elements (mask_op
, mask_op
,
2787 mask_perm_mask
, stmt_info
, gsi
);
2791 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt_info
);
2793 vec_mask
= vect_get_vec_def_for_stmt_copy (loop_vinfo
,
2797 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
2800 (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
)),
2801 TYPE_VECTOR_SUBPARTS (masktype
)));
2802 var
= vect_get_new_ssa_name (masktype
, vect_simple_var
);
2803 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
2805 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
2806 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
2813 gcall
*new_call
= gimple_build_call (gs_info
->decl
, 5, src_op
, ptr
, op
,
2816 stmt_vec_info new_stmt_info
;
2817 if (!useless_type_conversion_p (vectype
, rettype
))
2819 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype
),
2820 TYPE_VECTOR_SUBPARTS (rettype
)));
2821 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
2822 gimple_call_set_lhs (new_call
, op
);
2823 vect_finish_stmt_generation (stmt_info
, new_call
, gsi
);
2824 var
= make_ssa_name (vec_dest
);
2825 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2826 gassign
*new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2828 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
2832 var
= make_ssa_name (vec_dest
, new_call
);
2833 gimple_call_set_lhs (new_call
, var
);
2835 = vect_finish_stmt_generation (stmt_info
, new_call
, gsi
);
2838 if (modifier
== NARROW
)
2845 var
= permute_vec_elements (prev_res
, var
, perm_mask
,
2847 new_stmt_info
= loop_vinfo
->lookup_def (var
);
2850 if (prev_stmt_info
== NULL
)
2851 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
2853 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
2854 prev_stmt_info
= new_stmt_info
;
2858 /* Prepare the base and offset in GS_INFO for vectorization.
2859 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2860 to the vectorized offset argument for the first copy of STMT_INFO.
2861 STMT_INFO is the statement described by GS_INFO and LOOP is the
2865 vect_get_gather_scatter_ops (struct loop
*loop
, stmt_vec_info stmt_info
,
2866 gather_scatter_info
*gs_info
,
2867 tree
*dataref_ptr
, tree
*vec_offset
)
2869 gimple_seq stmts
= NULL
;
2870 *dataref_ptr
= force_gimple_operand (gs_info
->base
, &stmts
, true, NULL_TREE
);
2874 edge pe
= loop_preheader_edge (loop
);
2875 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
2876 gcc_assert (!new_bb
);
2878 tree offset_type
= TREE_TYPE (gs_info
->offset
);
2879 tree offset_vectype
= get_vectype_for_scalar_type (offset_type
);
2880 *vec_offset
= vect_get_vec_def_for_operand (gs_info
->offset
, stmt_info
,
2884 /* Prepare to implement a grouped or strided load or store using
2885 the gather load or scatter store operation described by GS_INFO.
2886 STMT_INFO is the load or store statement.
2888 Set *DATAREF_BUMP to the amount that should be added to the base
2889 address after each copy of the vectorized statement. Set *VEC_OFFSET
2890 to an invariant offset vector in which element I has the value
2891 I * DR_STEP / SCALE. */
2894 vect_get_strided_load_store_ops (stmt_vec_info stmt_info
,
2895 loop_vec_info loop_vinfo
,
2896 gather_scatter_info
*gs_info
,
2897 tree
*dataref_bump
, tree
*vec_offset
)
2899 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
2900 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2901 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2904 tree bump
= size_binop (MULT_EXPR
,
2905 fold_convert (sizetype
, DR_STEP (dr
)),
2906 size_int (TYPE_VECTOR_SUBPARTS (vectype
)));
2907 *dataref_bump
= force_gimple_operand (bump
, &stmts
, true, NULL_TREE
);
2909 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
2911 /* The offset given in GS_INFO can have pointer type, so use the element
2912 type of the vector instead. */
2913 tree offset_type
= TREE_TYPE (gs_info
->offset
);
2914 tree offset_vectype
= get_vectype_for_scalar_type (offset_type
);
2915 offset_type
= TREE_TYPE (offset_vectype
);
2917 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2918 tree step
= size_binop (EXACT_DIV_EXPR
, DR_STEP (dr
),
2919 ssize_int (gs_info
->scale
));
2920 step
= fold_convert (offset_type
, step
);
2921 step
= force_gimple_operand (step
, &stmts
, true, NULL_TREE
);
2923 /* Create {0, X, X*2, X*3, ...}. */
2924 *vec_offset
= gimple_build (&stmts
, VEC_SERIES_EXPR
, offset_vectype
,
2925 build_zero_cst (offset_type
), step
);
2927 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
2930 /* Return the amount that should be added to a vector pointer to move
2931 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
2932 being vectorized and MEMORY_ACCESS_TYPE describes the type of
2936 vect_get_data_ptr_increment (dr_vec_info
*dr_info
, tree aggr_type
,
2937 vect_memory_access_type memory_access_type
)
2939 if (memory_access_type
== VMAT_INVARIANT
)
2940 return size_zero_node
;
2942 tree iv_step
= TYPE_SIZE_UNIT (aggr_type
);
2943 tree step
= vect_dr_behavior (dr_info
)->step
;
2944 if (tree_int_cst_sgn (step
) == -1)
2945 iv_step
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (iv_step
), iv_step
);
2949 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */
2952 vectorizable_bswap (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
2953 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
2954 tree vectype_in
, stmt_vector_for_cost
*cost_vec
)
2957 gcall
*stmt
= as_a
<gcall
*> (stmt_info
->stmt
);
2958 vec_info
*vinfo
= stmt_info
->vinfo
;
2959 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2961 unsigned HOST_WIDE_INT nunits
, num_bytes
;
2963 op
= gimple_call_arg (stmt
, 0);
2964 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2966 if (!TYPE_VECTOR_SUBPARTS (vectype
).is_constant (&nunits
))
2969 /* Multiple types in SLP are handled by creating the appropriate number of
2970 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2975 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
2977 gcc_assert (ncopies
>= 1);
2979 tree char_vectype
= get_same_sized_vectype (char_type_node
, vectype_in
);
2983 if (!TYPE_VECTOR_SUBPARTS (char_vectype
).is_constant (&num_bytes
))
2986 unsigned word_bytes
= num_bytes
/ nunits
;
2988 /* The encoding uses one stepped pattern for each byte in the word. */
2989 vec_perm_builder
elts (num_bytes
, word_bytes
, 3);
2990 for (unsigned i
= 0; i
< 3; ++i
)
2991 for (unsigned j
= 0; j
< word_bytes
; ++j
)
2992 elts
.quick_push ((i
+ 1) * word_bytes
- j
- 1);
2994 vec_perm_indices
indices (elts
, 1, num_bytes
);
2995 if (!can_vec_perm_const_p (TYPE_MODE (char_vectype
), indices
))
3000 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
3001 DUMP_VECT_SCOPE ("vectorizable_bswap");
3004 record_stmt_cost (cost_vec
,
3005 1, vector_stmt
, stmt_info
, 0, vect_prologue
);
3006 record_stmt_cost (cost_vec
,
3007 ncopies
, vec_perm
, stmt_info
, 0, vect_body
);
3012 tree bswap_vconst
= vec_perm_indices_to_tree (char_vectype
, indices
);
3015 vec
<tree
> vec_oprnds
= vNULL
;
3016 stmt_vec_info new_stmt_info
= NULL
;
3017 stmt_vec_info prev_stmt_info
= NULL
;
3018 for (unsigned j
= 0; j
< ncopies
; j
++)
3022 vect_get_vec_defs (op
, NULL
, stmt_info
, &vec_oprnds
, NULL
, slp_node
);
3024 vect_get_vec_defs_for_stmt_copy (vinfo
, &vec_oprnds
, NULL
);
3026 /* Arguments are ready. create the new vector stmt. */
3029 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
3032 tree tem
= make_ssa_name (char_vectype
);
3033 new_stmt
= gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
3034 char_vectype
, vop
));
3035 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3036 tree tem2
= make_ssa_name (char_vectype
);
3037 new_stmt
= gimple_build_assign (tem2
, VEC_PERM_EXPR
,
3038 tem
, tem
, bswap_vconst
);
3039 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3040 tem
= make_ssa_name (vectype
);
3041 new_stmt
= gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
3044 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3046 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
3053 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
3055 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
3057 prev_stmt_info
= new_stmt_info
;
3060 vec_oprnds
.release ();
3064 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3065 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3066 in a single step. On success, store the binary pack code in
3070 simple_integer_narrowing (tree vectype_out
, tree vectype_in
,
3071 tree_code
*convert_code
)
3073 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out
))
3074 || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in
)))
3078 int multi_step_cvt
= 0;
3079 auto_vec
<tree
, 8> interm_types
;
3080 if (!supportable_narrowing_operation (NOP_EXPR
, vectype_out
, vectype_in
,
3081 &code
, &multi_step_cvt
,
3086 *convert_code
= code
;
3090 /* Function vectorizable_call.
3092 Check if STMT_INFO performs a function call that can be vectorized.
3093 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3094 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3095 Return true if STMT_INFO is vectorizable in this way. */
3098 vectorizable_call (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
3099 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
3100 stmt_vector_for_cost
*cost_vec
)
3106 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3107 stmt_vec_info prev_stmt_info
;
3108 tree vectype_out
, vectype_in
;
3109 poly_uint64 nunits_in
;
3110 poly_uint64 nunits_out
;
3111 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3112 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3113 vec_info
*vinfo
= stmt_info
->vinfo
;
3114 tree fndecl
, new_temp
, rhs_type
;
3115 enum vect_def_type dt
[4]
3116 = { vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
,
3117 vect_unknown_def_type
};
3118 int ndts
= ARRAY_SIZE (dt
);
3120 auto_vec
<tree
, 8> vargs
;
3121 auto_vec
<tree
, 8> orig_vargs
;
3122 enum { NARROW
, NONE
, WIDEN
} modifier
;
3126 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3129 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
3133 /* Is STMT_INFO a vectorizable call? */
3134 stmt
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
3138 if (gimple_call_internal_p (stmt
)
3139 && (internal_load_fn_p (gimple_call_internal_fn (stmt
))
3140 || internal_store_fn_p (gimple_call_internal_fn (stmt
))))
3141 /* Handled by vectorizable_load and vectorizable_store. */
3144 if (gimple_call_lhs (stmt
) == NULL_TREE
3145 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
3148 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
3150 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3152 /* Process function arguments. */
3153 rhs_type
= NULL_TREE
;
3154 vectype_in
= NULL_TREE
;
3155 nargs
= gimple_call_num_args (stmt
);
3157 /* Bail out if the function has more than three arguments, we do not have
3158 interesting builtin functions to vectorize with more than two arguments
3159 except for fma. No arguments is also not good. */
3160 if (nargs
== 0 || nargs
> 4)
3163 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
3164 combined_fn cfn
= gimple_call_combined_fn (stmt
);
3165 if (cfn
== CFN_GOMP_SIMD_LANE
)
3168 rhs_type
= unsigned_type_node
;
3172 if (internal_fn_p (cfn
))
3173 mask_opno
= internal_fn_mask_index (as_internal_fn (cfn
));
3175 for (i
= 0; i
< nargs
; i
++)
3179 op
= gimple_call_arg (stmt
, i
);
3180 if (!vect_is_simple_use (op
, vinfo
, &dt
[i
], &opvectype
))
3182 if (dump_enabled_p ())
3183 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3184 "use not simple.\n");
3188 /* Skip the mask argument to an internal function. This operand
3189 has been converted via a pattern if necessary. */
3190 if ((int) i
== mask_opno
)
3193 /* We can only handle calls with arguments of the same type. */
3195 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
3197 if (dump_enabled_p ())
3198 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3199 "argument types differ.\n");
3203 rhs_type
= TREE_TYPE (op
);
3206 vectype_in
= opvectype
;
3208 && opvectype
!= vectype_in
)
3210 if (dump_enabled_p ())
3211 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3212 "argument vector types differ.\n");
3216 /* If all arguments are external or constant defs use a vector type with
3217 the same size as the output vector type. */
3219 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3221 gcc_assert (vectype_in
);
3224 if (dump_enabled_p ())
3226 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3227 "no vectype for scalar type ");
3228 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3229 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3236 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3237 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3238 if (known_eq (nunits_in
* 2, nunits_out
))
3240 else if (known_eq (nunits_out
, nunits_in
))
3242 else if (known_eq (nunits_out
* 2, nunits_in
))
3247 /* We only handle functions that do not read or clobber memory. */
3248 if (gimple_vuse (stmt
))
3250 if (dump_enabled_p ())
3251 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3252 "function reads from or writes to memory.\n");
3256 /* For now, we only vectorize functions if a target specific builtin
3257 is available. TODO -- in some cases, it might be profitable to
3258 insert the calls for pieces of the vector, in order to be able
3259 to vectorize other operations in the loop. */
3261 internal_fn ifn
= IFN_LAST
;
3262 tree callee
= gimple_call_fndecl (stmt
);
3264 /* First try using an internal function. */
3265 tree_code convert_code
= ERROR_MARK
;
3267 && (modifier
== NONE
3268 || (modifier
== NARROW
3269 && simple_integer_narrowing (vectype_out
, vectype_in
,
3271 ifn
= vectorizable_internal_function (cfn
, callee
, vectype_out
,
3274 /* If that fails, try asking for a target-specific built-in function. */
3275 if (ifn
== IFN_LAST
)
3277 if (cfn
!= CFN_LAST
)
3278 fndecl
= targetm
.vectorize
.builtin_vectorized_function
3279 (cfn
, vectype_out
, vectype_in
);
3281 fndecl
= targetm
.vectorize
.builtin_md_vectorized_function
3282 (callee
, vectype_out
, vectype_in
);
3285 if (ifn
== IFN_LAST
&& !fndecl
)
3287 if (cfn
== CFN_GOMP_SIMD_LANE
3290 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
3291 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
3292 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
3293 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
3295 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3296 { 0, 1, 2, ... vf - 1 } vector. */
3297 gcc_assert (nargs
== 0);
3299 else if (modifier
== NONE
3300 && (gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP16
)
3301 || gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP32
)
3302 || gimple_call_builtin_p (stmt
, BUILT_IN_BSWAP64
)))
3303 return vectorizable_bswap (stmt_info
, gsi
, vec_stmt
, slp_node
,
3304 vectype_in
, cost_vec
);
3307 if (dump_enabled_p ())
3308 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3309 "function is not vectorizable.\n");
3316 else if (modifier
== NARROW
&& ifn
== IFN_LAST
)
3317 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_out
);
3319 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
3321 /* Sanity check: make sure that at least one copy of the vectorized stmt
3322 needs to be generated. */
3323 gcc_assert (ncopies
>= 1);
3325 vec_loop_masks
*masks
= (loop_vinfo
? &LOOP_VINFO_MASKS (loop_vinfo
) : NULL
);
3326 if (!vec_stmt
) /* transformation not required. */
3328 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
3329 DUMP_VECT_SCOPE ("vectorizable_call");
3330 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, slp_node
, cost_vec
);
3331 if (ifn
!= IFN_LAST
&& modifier
== NARROW
&& !slp_node
)
3332 record_stmt_cost (cost_vec
, ncopies
/ 2,
3333 vec_promote_demote
, stmt_info
, 0, vect_body
);
3335 if (loop_vinfo
&& mask_opno
>= 0)
3337 unsigned int nvectors
= (slp_node
3338 ? SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
)
3340 vect_record_loop_mask (loop_vinfo
, masks
, nvectors
, vectype_out
);
3347 if (dump_enabled_p ())
3348 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
3351 scalar_dest
= gimple_call_lhs (stmt
);
3352 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
3354 bool masked_loop_p
= loop_vinfo
&& LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
);
3356 stmt_vec_info new_stmt_info
= NULL
;
3357 prev_stmt_info
= NULL
;
3358 if (modifier
== NONE
|| ifn
!= IFN_LAST
)
3360 tree prev_res
= NULL_TREE
;
3361 vargs
.safe_grow (nargs
);
3362 orig_vargs
.safe_grow (nargs
);
3363 for (j
= 0; j
< ncopies
; ++j
)
3365 /* Build argument list for the vectorized call. */
3368 auto_vec
<vec
<tree
> > vec_defs (nargs
);
3369 vec
<tree
> vec_oprnds0
;
3371 for (i
= 0; i
< nargs
; i
++)
3372 vargs
[i
] = gimple_call_arg (stmt
, i
);
3373 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
);
3374 vec_oprnds0
= vec_defs
[0];
3376 /* Arguments are ready. Create the new vector stmt. */
3377 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
3380 for (k
= 0; k
< nargs
; k
++)
3382 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
3383 vargs
[k
] = vec_oprndsk
[i
];
3385 if (modifier
== NARROW
)
3387 /* We don't define any narrowing conditional functions
3389 gcc_assert (mask_opno
< 0);
3390 tree half_res
= make_ssa_name (vectype_in
);
3392 = gimple_build_call_internal_vec (ifn
, vargs
);
3393 gimple_call_set_lhs (call
, half_res
);
3394 gimple_call_set_nothrow (call
, true);
3396 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
3399 prev_res
= half_res
;
3402 new_temp
= make_ssa_name (vec_dest
);
3404 = gimple_build_assign (new_temp
, convert_code
,
3405 prev_res
, half_res
);
3407 = vect_finish_stmt_generation (stmt_info
, new_stmt
,
3412 if (mask_opno
>= 0 && masked_loop_p
)
3414 unsigned int vec_num
= vec_oprnds0
.length ();
3415 /* Always true for SLP. */
3416 gcc_assert (ncopies
== 1);
3417 tree mask
= vect_get_loop_mask (gsi
, masks
, vec_num
,
3419 vargs
[mask_opno
] = prepare_load_store_mask
3420 (TREE_TYPE (mask
), mask
, vargs
[mask_opno
], gsi
);
3424 if (ifn
!= IFN_LAST
)
3425 call
= gimple_build_call_internal_vec (ifn
, vargs
);
3427 call
= gimple_build_call_vec (fndecl
, vargs
);
3428 new_temp
= make_ssa_name (vec_dest
, call
);
3429 gimple_call_set_lhs (call
, new_temp
);
3430 gimple_call_set_nothrow (call
, true);
3432 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
3434 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
3437 for (i
= 0; i
< nargs
; i
++)
3439 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
3440 vec_oprndsi
.release ();
3445 for (i
= 0; i
< nargs
; i
++)
3447 op
= gimple_call_arg (stmt
, i
);
3450 = vect_get_vec_def_for_operand (op
, stmt_info
);
3453 = vect_get_vec_def_for_stmt_copy (vinfo
, orig_vargs
[i
]);
3455 orig_vargs
[i
] = vargs
[i
] = vec_oprnd0
;
3458 if (mask_opno
>= 0 && masked_loop_p
)
3460 tree mask
= vect_get_loop_mask (gsi
, masks
, ncopies
,
3463 = prepare_load_store_mask (TREE_TYPE (mask
), mask
,
3464 vargs
[mask_opno
], gsi
);
3467 if (cfn
== CFN_GOMP_SIMD_LANE
)
3469 tree cst
= build_index_vector (vectype_out
, j
* nunits_out
, 1);
3471 = vect_get_new_ssa_name (vectype_out
, vect_simple_var
, "cst_");
3472 gimple
*init_stmt
= gimple_build_assign (new_var
, cst
);
3473 vect_init_vector_1 (stmt_info
, init_stmt
, NULL
);
3474 new_temp
= make_ssa_name (vec_dest
);
3475 gimple
*new_stmt
= gimple_build_assign (new_temp
, new_var
);
3477 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3479 else if (modifier
== NARROW
)
3481 /* We don't define any narrowing conditional functions at
3483 gcc_assert (mask_opno
< 0);
3484 tree half_res
= make_ssa_name (vectype_in
);
3485 gcall
*call
= gimple_build_call_internal_vec (ifn
, vargs
);
3486 gimple_call_set_lhs (call
, half_res
);
3487 gimple_call_set_nothrow (call
, true);
3489 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
3492 prev_res
= half_res
;
3495 new_temp
= make_ssa_name (vec_dest
);
3496 gassign
*new_stmt
= gimple_build_assign (new_temp
, convert_code
,
3497 prev_res
, half_res
);
3499 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3504 if (ifn
!= IFN_LAST
)
3505 call
= gimple_build_call_internal_vec (ifn
, vargs
);
3507 call
= gimple_build_call_vec (fndecl
, vargs
);
3508 new_temp
= make_ssa_name (vec_dest
, call
);
3509 gimple_call_set_lhs (call
, new_temp
);
3510 gimple_call_set_nothrow (call
, true);
3512 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
3515 if (j
== (modifier
== NARROW
? 1 : 0))
3516 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
3518 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
3520 prev_stmt_info
= new_stmt_info
;
3523 else if (modifier
== NARROW
)
3525 /* We don't define any narrowing conditional functions at present. */
3526 gcc_assert (mask_opno
< 0);
3527 for (j
= 0; j
< ncopies
; ++j
)
3529 /* Build argument list for the vectorized call. */
3531 vargs
.create (nargs
* 2);
3537 auto_vec
<vec
<tree
> > vec_defs (nargs
);
3538 vec
<tree
> vec_oprnds0
;
3540 for (i
= 0; i
< nargs
; i
++)
3541 vargs
.quick_push (gimple_call_arg (stmt
, i
));
3542 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
);
3543 vec_oprnds0
= vec_defs
[0];
3545 /* Arguments are ready. Create the new vector stmt. */
3546 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
3550 for (k
= 0; k
< nargs
; k
++)
3552 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
3553 vargs
.quick_push (vec_oprndsk
[i
]);
3554 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
3557 if (ifn
!= IFN_LAST
)
3558 call
= gimple_build_call_internal_vec (ifn
, vargs
);
3560 call
= gimple_build_call_vec (fndecl
, vargs
);
3561 new_temp
= make_ssa_name (vec_dest
, call
);
3562 gimple_call_set_lhs (call
, new_temp
);
3563 gimple_call_set_nothrow (call
, true);
3565 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
3566 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
3569 for (i
= 0; i
< nargs
; i
++)
3571 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
3572 vec_oprndsi
.release ();
3577 for (i
= 0; i
< nargs
; i
++)
3579 op
= gimple_call_arg (stmt
, i
);
3583 = vect_get_vec_def_for_operand (op
, stmt_info
);
3585 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd0
);
3589 vec_oprnd1
= gimple_call_arg (new_stmt_info
->stmt
,
3592 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd1
);
3594 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd0
);
3597 vargs
.quick_push (vec_oprnd0
);
3598 vargs
.quick_push (vec_oprnd1
);
3601 gcall
*new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3602 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3603 gimple_call_set_lhs (new_stmt
, new_temp
);
3605 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
3608 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt_info
;
3610 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
3612 prev_stmt_info
= new_stmt_info
;
3615 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3618 /* No current target implements this case. */
3623 /* The call in STMT might prevent it from being removed in dce.
3624 We however cannot remove it here, due to the way the ssa name
3625 it defines is mapped to the new definition. So just replace
3626 rhs of the statement with something harmless. */
3631 stmt_info
= vect_orig_stmt (stmt_info
);
3632 lhs
= gimple_get_lhs (stmt_info
->stmt
);
3635 = gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
3636 vinfo
->replace_stmt (gsi
, stmt_info
, new_stmt
);
3642 struct simd_call_arg_info
3646 HOST_WIDE_INT linear_step
;
3647 enum vect_def_type dt
;
3649 bool simd_lane_linear
;
3652 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3653 is linear within simd lane (but not within whole loop), note it in
3657 vect_simd_lane_linear (tree op
, struct loop
*loop
,
3658 struct simd_call_arg_info
*arginfo
)
3660 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
3662 if (!is_gimple_assign (def_stmt
)
3663 || gimple_assign_rhs_code (def_stmt
) != POINTER_PLUS_EXPR
3664 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt
)))
3667 tree base
= gimple_assign_rhs1 (def_stmt
);
3668 HOST_WIDE_INT linear_step
= 0;
3669 tree v
= gimple_assign_rhs2 (def_stmt
);
3670 while (TREE_CODE (v
) == SSA_NAME
)
3673 def_stmt
= SSA_NAME_DEF_STMT (v
);
3674 if (is_gimple_assign (def_stmt
))
3675 switch (gimple_assign_rhs_code (def_stmt
))
3678 t
= gimple_assign_rhs2 (def_stmt
);
3679 if (linear_step
|| TREE_CODE (t
) != INTEGER_CST
)
3681 base
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (base
), base
, t
);
3682 v
= gimple_assign_rhs1 (def_stmt
);
3685 t
= gimple_assign_rhs2 (def_stmt
);
3686 if (linear_step
|| !tree_fits_shwi_p (t
) || integer_zerop (t
))
3688 linear_step
= tree_to_shwi (t
);
3689 v
= gimple_assign_rhs1 (def_stmt
);
3692 t
= gimple_assign_rhs1 (def_stmt
);
3693 if (TREE_CODE (TREE_TYPE (t
)) != INTEGER_TYPE
3694 || (TYPE_PRECISION (TREE_TYPE (v
))
3695 < TYPE_PRECISION (TREE_TYPE (t
))))
3704 else if (gimple_call_internal_p (def_stmt
, IFN_GOMP_SIMD_LANE
)
3706 && TREE_CODE (gimple_call_arg (def_stmt
, 0)) == SSA_NAME
3707 && (SSA_NAME_VAR (gimple_call_arg (def_stmt
, 0))
3712 arginfo
->linear_step
= linear_step
;
3714 arginfo
->simd_lane_linear
= true;
3720 /* Return the number of elements in vector type VECTYPE, which is associated
3721 with a SIMD clone. At present these vectors always have a constant
3724 static unsigned HOST_WIDE_INT
3725 simd_clone_subparts (tree vectype
)
3727 return TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
3730 /* Function vectorizable_simd_clone_call.
3732 Check if STMT_INFO performs a function call that can be vectorized
3733 by calling a simd clone of the function.
3734 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3735 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3736 Return true if STMT_INFO is vectorizable in this way. */
3739 vectorizable_simd_clone_call (stmt_vec_info stmt_info
,
3740 gimple_stmt_iterator
*gsi
,
3741 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
3742 stmt_vector_for_cost
*)
3747 tree vec_oprnd0
= NULL_TREE
;
3748 stmt_vec_info prev_stmt_info
;
3750 unsigned int nunits
;
3751 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3752 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3753 vec_info
*vinfo
= stmt_info
->vinfo
;
3754 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
3755 tree fndecl
, new_temp
;
3757 auto_vec
<simd_call_arg_info
> arginfo
;
3758 vec
<tree
> vargs
= vNULL
;
3760 tree lhs
, rtype
, ratype
;
3761 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
= NULL
;
3763 /* Is STMT a vectorizable call? */
3764 gcall
*stmt
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
3768 fndecl
= gimple_call_fndecl (stmt
);
3769 if (fndecl
== NULL_TREE
)
3772 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
3773 if (node
== NULL
|| node
->simd_clones
== NULL
)
3776 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3779 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
3783 if (gimple_call_lhs (stmt
)
3784 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
3787 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
3789 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3791 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt_info
))
3798 /* Process function arguments. */
3799 nargs
= gimple_call_num_args (stmt
);
3801 /* Bail out if the function has zero arguments. */
3805 arginfo
.reserve (nargs
, true);
3807 for (i
= 0; i
< nargs
; i
++)
3809 simd_call_arg_info thisarginfo
;
3812 thisarginfo
.linear_step
= 0;
3813 thisarginfo
.align
= 0;
3814 thisarginfo
.op
= NULL_TREE
;
3815 thisarginfo
.simd_lane_linear
= false;
3817 op
= gimple_call_arg (stmt
, i
);
3818 if (!vect_is_simple_use (op
, vinfo
, &thisarginfo
.dt
,
3819 &thisarginfo
.vectype
)
3820 || thisarginfo
.dt
== vect_uninitialized_def
)
3822 if (dump_enabled_p ())
3823 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3824 "use not simple.\n");
3828 if (thisarginfo
.dt
== vect_constant_def
3829 || thisarginfo
.dt
== vect_external_def
)
3830 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
3832 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
3834 /* For linear arguments, the analyze phase should have saved
3835 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
3836 if (i
* 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
3837 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2])
3839 gcc_assert (vec_stmt
);
3840 thisarginfo
.linear_step
3841 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2]);
3843 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 1];
3844 thisarginfo
.simd_lane_linear
3845 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 3]
3846 == boolean_true_node
);
3847 /* If loop has been peeled for alignment, we need to adjust it. */
3848 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
3849 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
3850 if (n1
!= n2
&& !thisarginfo
.simd_lane_linear
)
3852 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
3853 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2];
3854 tree opt
= TREE_TYPE (thisarginfo
.op
);
3855 bias
= fold_convert (TREE_TYPE (step
), bias
);
3856 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
3858 = fold_build2 (POINTER_TYPE_P (opt
)
3859 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
3860 thisarginfo
.op
, bias
);
3864 && thisarginfo
.dt
!= vect_constant_def
3865 && thisarginfo
.dt
!= vect_external_def
3867 && TREE_CODE (op
) == SSA_NAME
3868 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
3870 && tree_fits_shwi_p (iv
.step
))
3872 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
3873 thisarginfo
.op
= iv
.base
;
3875 else if ((thisarginfo
.dt
== vect_constant_def
3876 || thisarginfo
.dt
== vect_external_def
)
3877 && POINTER_TYPE_P (TREE_TYPE (op
)))
3878 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
3879 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
3881 if (POINTER_TYPE_P (TREE_TYPE (op
))
3882 && !thisarginfo
.linear_step
3884 && thisarginfo
.dt
!= vect_constant_def
3885 && thisarginfo
.dt
!= vect_external_def
3888 && TREE_CODE (op
) == SSA_NAME
)
3889 vect_simd_lane_linear (op
, loop
, &thisarginfo
);
3891 arginfo
.quick_push (thisarginfo
);
3894 unsigned HOST_WIDE_INT vf
;
3895 if (!LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant (&vf
))
3897 if (dump_enabled_p ())
3898 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3899 "not considering SIMD clones; not yet supported"
3900 " for variable-width vectors.\n");
3904 unsigned int badness
= 0;
3905 struct cgraph_node
*bestn
= NULL
;
3906 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
3907 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
3909 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
3910 n
= n
->simdclone
->next_clone
)
3912 unsigned int this_badness
= 0;
3913 if (n
->simdclone
->simdlen
> vf
3914 || n
->simdclone
->nargs
!= nargs
)
3916 if (n
->simdclone
->simdlen
< vf
)
3917 this_badness
+= (exact_log2 (vf
)
3918 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
3919 if (n
->simdclone
->inbranch
)
3920 this_badness
+= 2048;
3921 int target_badness
= targetm
.simd_clone
.usable (n
);
3922 if (target_badness
< 0)
3924 this_badness
+= target_badness
* 512;
3925 /* FORNOW: Have to add code to add the mask argument. */
3926 if (n
->simdclone
->inbranch
)
3928 for (i
= 0; i
< nargs
; i
++)
3930 switch (n
->simdclone
->args
[i
].arg_type
)
3932 case SIMD_CLONE_ARG_TYPE_VECTOR
:
3933 if (!useless_type_conversion_p
3934 (n
->simdclone
->args
[i
].orig_type
,
3935 TREE_TYPE (gimple_call_arg (stmt
, i
))))
3937 else if (arginfo
[i
].dt
== vect_constant_def
3938 || arginfo
[i
].dt
== vect_external_def
3939 || arginfo
[i
].linear_step
)
3942 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3943 if (arginfo
[i
].dt
!= vect_constant_def
3944 && arginfo
[i
].dt
!= vect_external_def
)
3947 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3948 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
3949 if (arginfo
[i
].dt
== vect_constant_def
3950 || arginfo
[i
].dt
== vect_external_def
3951 || (arginfo
[i
].linear_step
3952 != n
->simdclone
->args
[i
].linear_step
))
3955 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3956 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
3957 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
3958 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
3959 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
3960 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
3964 case SIMD_CLONE_ARG_TYPE_MASK
:
3967 if (i
== (size_t) -1)
3969 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
3974 if (arginfo
[i
].align
)
3975 this_badness
+= (exact_log2 (arginfo
[i
].align
)
3976 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
3978 if (i
== (size_t) -1)
3980 if (bestn
== NULL
|| this_badness
< badness
)
3983 badness
= this_badness
;
3990 for (i
= 0; i
< nargs
; i
++)
3991 if ((arginfo
[i
].dt
== vect_constant_def
3992 || arginfo
[i
].dt
== vect_external_def
)
3993 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
3996 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
3998 if (arginfo
[i
].vectype
== NULL
3999 || (simd_clone_subparts (arginfo
[i
].vectype
)
4000 > bestn
->simdclone
->simdlen
))
4004 fndecl
= bestn
->decl
;
4005 nunits
= bestn
->simdclone
->simdlen
;
4006 ncopies
= vf
/ nunits
;
4008 /* If the function isn't const, only allow it in simd loops where user
4009 has asserted that at least nunits consecutive iterations can be
4010 performed using SIMD instructions. */
4011 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
4012 && gimple_vuse (stmt
))
4015 /* Sanity check: make sure that at least one copy of the vectorized stmt
4016 needs to be generated. */
4017 gcc_assert (ncopies
>= 1);
4019 if (!vec_stmt
) /* transformation not required. */
4021 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
4022 for (i
= 0; i
< nargs
; i
++)
4023 if ((bestn
->simdclone
->args
[i
].arg_type
4024 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
4025 || (bestn
->simdclone
->args
[i
].arg_type
4026 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
))
4028 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 3
4030 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
4031 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
4032 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
4033 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
4034 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
4035 tree sll
= arginfo
[i
].simd_lane_linear
4036 ? boolean_true_node
: boolean_false_node
;
4037 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (sll
);
4039 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
4040 DUMP_VECT_SCOPE ("vectorizable_simd_clone_call");
4041 /* vect_model_simple_cost (stmt_info, ncopies, dt, slp_node, cost_vec); */
4047 if (dump_enabled_p ())
4048 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
4051 scalar_dest
= gimple_call_lhs (stmt
);
4052 vec_dest
= NULL_TREE
;
4057 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4058 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
4059 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
4062 rtype
= TREE_TYPE (ratype
);
4066 prev_stmt_info
= NULL
;
4067 for (j
= 0; j
< ncopies
; ++j
)
4069 /* Build argument list for the vectorized call. */
4071 vargs
.create (nargs
);
4075 for (i
= 0; i
< nargs
; i
++)
4077 unsigned int k
, l
, m
, o
;
4079 op
= gimple_call_arg (stmt
, i
);
4080 switch (bestn
->simdclone
->args
[i
].arg_type
)
4082 case SIMD_CLONE_ARG_TYPE_VECTOR
:
4083 atype
= bestn
->simdclone
->args
[i
].vector_type
;
4084 o
= nunits
/ simd_clone_subparts (atype
);
4085 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
4087 if (simd_clone_subparts (atype
)
4088 < simd_clone_subparts (arginfo
[i
].vectype
))
4090 poly_uint64 prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
4091 k
= (simd_clone_subparts (arginfo
[i
].vectype
)
4092 / simd_clone_subparts (atype
));
4093 gcc_assert ((k
& (k
- 1)) == 0);
4096 = vect_get_vec_def_for_operand (op
, stmt_info
);
4099 vec_oprnd0
= arginfo
[i
].op
;
4100 if ((m
& (k
- 1)) == 0)
4102 = vect_get_vec_def_for_stmt_copy (vinfo
,
4105 arginfo
[i
].op
= vec_oprnd0
;
4107 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
4109 bitsize_int ((m
& (k
- 1)) * prec
));
4111 = gimple_build_assign (make_ssa_name (atype
),
4113 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4114 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
4118 k
= (simd_clone_subparts (atype
)
4119 / simd_clone_subparts (arginfo
[i
].vectype
));
4120 gcc_assert ((k
& (k
- 1)) == 0);
4121 vec
<constructor_elt
, va_gc
> *ctor_elts
;
4123 vec_alloc (ctor_elts
, k
);
4126 for (l
= 0; l
< k
; l
++)
4128 if (m
== 0 && l
== 0)
4130 = vect_get_vec_def_for_operand (op
, stmt_info
);
4133 = vect_get_vec_def_for_stmt_copy (vinfo
,
4135 arginfo
[i
].op
= vec_oprnd0
;
4138 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
4142 vargs
.safe_push (vec_oprnd0
);
4145 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
4147 = gimple_build_assign (make_ssa_name (atype
),
4149 vect_finish_stmt_generation (stmt_info
, new_stmt
,
4151 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
4156 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
4157 vargs
.safe_push (op
);
4159 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
4160 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
4165 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
4170 edge pe
= loop_preheader_edge (loop
);
4171 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
4172 gcc_assert (!new_bb
);
4174 if (arginfo
[i
].simd_lane_linear
)
4176 vargs
.safe_push (arginfo
[i
].op
);
4179 tree phi_res
= copy_ssa_name (op
);
4180 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
4181 loop_vinfo
->add_stmt (new_phi
);
4182 add_phi_arg (new_phi
, arginfo
[i
].op
,
4183 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
4185 = POINTER_TYPE_P (TREE_TYPE (op
))
4186 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
4187 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
4188 ? sizetype
: TREE_TYPE (op
);
4190 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
4192 tree tcst
= wide_int_to_tree (type
, cst
);
4193 tree phi_arg
= copy_ssa_name (op
);
4195 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
4196 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
4197 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
4198 loop_vinfo
->add_stmt (new_stmt
);
4199 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
4201 arginfo
[i
].op
= phi_res
;
4202 vargs
.safe_push (phi_res
);
4207 = POINTER_TYPE_P (TREE_TYPE (op
))
4208 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
4209 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
4210 ? sizetype
: TREE_TYPE (op
);
4212 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
4214 tree tcst
= wide_int_to_tree (type
, cst
);
4215 new_temp
= make_ssa_name (TREE_TYPE (op
));
4217 = gimple_build_assign (new_temp
, code
,
4218 arginfo
[i
].op
, tcst
);
4219 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4220 vargs
.safe_push (new_temp
);
4223 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
4224 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
4225 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
4226 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
4227 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
4228 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
4234 gcall
*new_call
= gimple_build_call_vec (fndecl
, vargs
);
4237 gcc_assert (ratype
|| simd_clone_subparts (rtype
) == nunits
);
4239 new_temp
= create_tmp_var (ratype
);
4240 else if (simd_clone_subparts (vectype
)
4241 == simd_clone_subparts (rtype
))
4242 new_temp
= make_ssa_name (vec_dest
, new_call
);
4244 new_temp
= make_ssa_name (rtype
, new_call
);
4245 gimple_call_set_lhs (new_call
, new_temp
);
4247 stmt_vec_info new_stmt_info
4248 = vect_finish_stmt_generation (stmt_info
, new_call
, gsi
);
4252 if (simd_clone_subparts (vectype
) < nunits
)
4255 poly_uint64 prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
4256 poly_uint64 bytes
= GET_MODE_SIZE (TYPE_MODE (vectype
));
4257 k
= nunits
/ simd_clone_subparts (vectype
);
4258 gcc_assert ((k
& (k
- 1)) == 0);
4259 for (l
= 0; l
< k
; l
++)
4264 t
= build_fold_addr_expr (new_temp
);
4265 t
= build2 (MEM_REF
, vectype
, t
,
4266 build_int_cst (TREE_TYPE (t
), l
* bytes
));
4269 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
4270 bitsize_int (prec
), bitsize_int (l
* prec
));
4272 = gimple_build_assign (make_ssa_name (vectype
), t
);
4274 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4276 if (j
== 0 && l
== 0)
4277 STMT_VINFO_VEC_STMT (stmt_info
)
4278 = *vec_stmt
= new_stmt_info
;
4280 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
4282 prev_stmt_info
= new_stmt_info
;
4286 vect_clobber_variable (stmt_info
, gsi
, new_temp
);
4289 else if (simd_clone_subparts (vectype
) > nunits
)
4291 unsigned int k
= (simd_clone_subparts (vectype
)
4292 / simd_clone_subparts (rtype
));
4293 gcc_assert ((k
& (k
- 1)) == 0);
4294 if ((j
& (k
- 1)) == 0)
4295 vec_alloc (ret_ctor_elts
, k
);
4298 unsigned int m
, o
= nunits
/ simd_clone_subparts (rtype
);
4299 for (m
= 0; m
< o
; m
++)
4301 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
4302 size_int (m
), NULL_TREE
, NULL_TREE
);
4304 = gimple_build_assign (make_ssa_name (rtype
), tem
);
4306 = vect_finish_stmt_generation (stmt_info
, new_stmt
,
4308 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
4309 gimple_assign_lhs (new_stmt
));
4311 vect_clobber_variable (stmt_info
, gsi
, new_temp
);
4314 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
4315 if ((j
& (k
- 1)) != k
- 1)
4317 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
4319 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
4321 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4323 if ((unsigned) j
== k
- 1)
4324 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
4326 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
4328 prev_stmt_info
= new_stmt_info
;
4333 tree t
= build_fold_addr_expr (new_temp
);
4334 t
= build2 (MEM_REF
, vectype
, t
,
4335 build_int_cst (TREE_TYPE (t
), 0));
4337 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
4339 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4340 vect_clobber_variable (stmt_info
, gsi
, new_temp
);
4345 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
4347 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
4349 prev_stmt_info
= new_stmt_info
;
4354 /* The call in STMT might prevent it from being removed in dce.
4355 We however cannot remove it here, due to the way the ssa name
4356 it defines is mapped to the new definition. So just replace
4357 rhs of the statement with something harmless. */
4365 type
= TREE_TYPE (scalar_dest
);
4366 lhs
= gimple_call_lhs (vect_orig_stmt (stmt_info
)->stmt
);
4367 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
4370 new_stmt
= gimple_build_nop ();
4371 vinfo
->replace_stmt (gsi
, vect_orig_stmt (stmt_info
), new_stmt
);
4372 unlink_stmt_vdef (stmt
);
4378 /* Function vect_gen_widened_results_half
4380 Create a vector stmt whose code, type, number of arguments, and result
4381 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
4382 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
4383 In the case that CODE is a CALL_EXPR, this means that a call to DECL
4384 needs to be created (DECL is a function-decl of a target-builtin).
4385 STMT_INFO is the original scalar stmt that we are vectorizing. */
4388 vect_gen_widened_results_half (enum tree_code code
,
4390 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
4391 tree vec_dest
, gimple_stmt_iterator
*gsi
,
4392 stmt_vec_info stmt_info
)
4397 /* Generate half of the widened result: */
4398 if (code
== CALL_EXPR
)
4400 /* Target specific support */
4401 if (op_type
== binary_op
)
4402 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
4404 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
4405 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4406 gimple_call_set_lhs (new_stmt
, new_temp
);
4410 /* Generic support */
4411 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
4412 if (op_type
!= binary_op
)
4414 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
4415 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4416 gimple_assign_set_lhs (new_stmt
, new_temp
);
4418 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4424 /* Get vectorized definitions for loop-based vectorization of STMT_INFO.
4425 For the first operand we call vect_get_vec_def_for_operand (with OPRND
4426 containing scalar operand), and for the rest we get a copy with
4427 vect_get_vec_def_for_stmt_copy() using the previous vector definition
4428 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
4429 The vectors are collected into VEC_OPRNDS. */
4432 vect_get_loop_based_defs (tree
*oprnd
, stmt_vec_info stmt_info
,
4433 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
4435 vec_info
*vinfo
= stmt_info
->vinfo
;
4438 /* Get first vector operand. */
4439 /* All the vector operands except the very first one (that is scalar oprnd)
4441 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
4442 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt_info
);
4444 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
, *oprnd
);
4446 vec_oprnds
->quick_push (vec_oprnd
);
4448 /* Get second vector operand. */
4449 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd
);
4450 vec_oprnds
->quick_push (vec_oprnd
);
4454 /* For conversion in multiple steps, continue to get operands
4457 vect_get_loop_based_defs (oprnd
, stmt_info
, vec_oprnds
,
4458 multi_step_cvt
- 1);
4462 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
4463 For multi-step conversions store the resulting vectors and call the function
4467 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
4469 stmt_vec_info stmt_info
,
4471 gimple_stmt_iterator
*gsi
,
4472 slp_tree slp_node
, enum tree_code code
,
4473 stmt_vec_info
*prev_stmt_info
)
4476 tree vop0
, vop1
, new_tmp
, vec_dest
;
4478 vec_dest
= vec_dsts
.pop ();
4480 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
4482 /* Create demotion operation. */
4483 vop0
= (*vec_oprnds
)[i
];
4484 vop1
= (*vec_oprnds
)[i
+ 1];
4485 gassign
*new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4486 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
4487 gimple_assign_set_lhs (new_stmt
, new_tmp
);
4488 stmt_vec_info new_stmt_info
4489 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4492 /* Store the resulting vector for next recursive call. */
4493 (*vec_oprnds
)[i
/2] = new_tmp
;
4496 /* This is the last step of the conversion sequence. Store the
4497 vectors in SLP_NODE or in vector info of the scalar statement
4498 (or in STMT_VINFO_RELATED_STMT chain). */
4500 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
4503 if (!*prev_stmt_info
)
4504 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt_info
;
4506 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt_info
;
4508 *prev_stmt_info
= new_stmt_info
;
4513 /* For multi-step demotion operations we first generate demotion operations
4514 from the source type to the intermediate types, and then combine the
4515 results (stored in VEC_OPRNDS) in demotion operation to the destination
4519 /* At each level of recursion we have half of the operands we had at the
4521 vec_oprnds
->truncate ((i
+1)/2);
4522 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
4523 stmt_info
, vec_dsts
, gsi
,
4524 slp_node
, VEC_PACK_TRUNC_EXPR
,
4528 vec_dsts
.quick_push (vec_dest
);
4532 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
4533 and VEC_OPRNDS1, for a binary operation associated with scalar statement
4534 STMT_INFO. For multi-step conversions store the resulting vectors and
4535 call the function recursively. */
4538 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
4539 vec
<tree
> *vec_oprnds1
,
4540 stmt_vec_info stmt_info
, tree vec_dest
,
4541 gimple_stmt_iterator
*gsi
,
4542 enum tree_code code1
,
4543 enum tree_code code2
, tree decl1
,
4544 tree decl2
, int op_type
)
4547 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
4548 gimple
*new_stmt1
, *new_stmt2
;
4549 vec
<tree
> vec_tmp
= vNULL
;
4551 vec_tmp
.create (vec_oprnds0
->length () * 2);
4552 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
4554 if (op_type
== binary_op
)
4555 vop1
= (*vec_oprnds1
)[i
];
4559 /* Generate the two halves of promotion operation. */
4560 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
4561 op_type
, vec_dest
, gsi
,
4563 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
4564 op_type
, vec_dest
, gsi
,
4566 if (is_gimple_call (new_stmt1
))
4568 new_tmp1
= gimple_call_lhs (new_stmt1
);
4569 new_tmp2
= gimple_call_lhs (new_stmt2
);
4573 new_tmp1
= gimple_assign_lhs (new_stmt1
);
4574 new_tmp2
= gimple_assign_lhs (new_stmt2
);
4577 /* Store the results for the next step. */
4578 vec_tmp
.quick_push (new_tmp1
);
4579 vec_tmp
.quick_push (new_tmp2
);
4582 vec_oprnds0
->release ();
4583 *vec_oprnds0
= vec_tmp
;
4587 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
4588 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
4589 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
4590 Return true if STMT_INFO is vectorizable in this way. */
4593 vectorizable_conversion (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
4594 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
4595 stmt_vector_for_cost
*cost_vec
)
4599 tree op0
, op1
= NULL_TREE
;
4600 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
4601 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4602 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
4603 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
4604 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
4606 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4608 stmt_vec_info prev_stmt_info
;
4609 poly_uint64 nunits_in
;
4610 poly_uint64 nunits_out
;
4611 tree vectype_out
, vectype_in
;
4613 tree lhs_type
, rhs_type
;
4614 enum { NARROW
, NONE
, WIDEN
} modifier
;
4615 vec
<tree
> vec_oprnds0
= vNULL
;
4616 vec
<tree
> vec_oprnds1
= vNULL
;
4618 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4619 vec_info
*vinfo
= stmt_info
->vinfo
;
4620 int multi_step_cvt
= 0;
4621 vec
<tree
> interm_types
= vNULL
;
4622 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
4624 unsigned short fltsz
;
4626 /* Is STMT a vectorizable conversion? */
4628 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4631 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
4635 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
4639 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4642 code
= gimple_assign_rhs_code (stmt
);
4643 if (!CONVERT_EXPR_CODE_P (code
)
4644 && code
!= FIX_TRUNC_EXPR
4645 && code
!= FLOAT_EXPR
4646 && code
!= WIDEN_MULT_EXPR
4647 && code
!= WIDEN_LSHIFT_EXPR
)
4650 op_type
= TREE_CODE_LENGTH (code
);
4652 /* Check types of lhs and rhs. */
4653 scalar_dest
= gimple_assign_lhs (stmt
);
4654 lhs_type
= TREE_TYPE (scalar_dest
);
4655 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4657 op0
= gimple_assign_rhs1 (stmt
);
4658 rhs_type
= TREE_TYPE (op0
);
4660 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
4661 && !((INTEGRAL_TYPE_P (lhs_type
)
4662 && INTEGRAL_TYPE_P (rhs_type
))
4663 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
4664 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
4667 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4668 && ((INTEGRAL_TYPE_P (lhs_type
)
4669 && !type_has_mode_precision_p (lhs_type
))
4670 || (INTEGRAL_TYPE_P (rhs_type
)
4671 && !type_has_mode_precision_p (rhs_type
))))
4673 if (dump_enabled_p ())
4674 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4675 "type conversion to/from bit-precision unsupported."
4680 /* Check the operands of the operation. */
4681 if (!vect_is_simple_use (op0
, vinfo
, &dt
[0], &vectype_in
))
4683 if (dump_enabled_p ())
4684 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4685 "use not simple.\n");
4688 if (op_type
== binary_op
)
4692 op1
= gimple_assign_rhs2 (stmt
);
4693 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
4694 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
4696 if (CONSTANT_CLASS_P (op0
))
4697 ok
= vect_is_simple_use (op1
, vinfo
, &dt
[1], &vectype_in
);
4699 ok
= vect_is_simple_use (op1
, vinfo
, &dt
[1]);
4703 if (dump_enabled_p ())
4704 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4705 "use not simple.\n");
4710 /* If op0 is an external or constant defs use a vector type of
4711 the same size as the output vector type. */
4713 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
4715 gcc_assert (vectype_in
);
4718 if (dump_enabled_p ())
4720 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4721 "no vectype for scalar type ");
4722 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
4723 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4729 if (VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4730 && !VECTOR_BOOLEAN_TYPE_P (vectype_in
))
4732 if (dump_enabled_p ())
4734 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4735 "can't convert between boolean and non "
4737 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
4738 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4744 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
4745 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4746 if (known_eq (nunits_out
, nunits_in
))
4748 else if (multiple_p (nunits_out
, nunits_in
))
4752 gcc_checking_assert (multiple_p (nunits_in
, nunits_out
));
4756 /* Multiple types in SLP are handled by creating the appropriate number of
4757 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4761 else if (modifier
== NARROW
)
4762 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_out
);
4764 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
4766 /* Sanity check: make sure that at least one copy of the vectorized stmt
4767 needs to be generated. */
4768 gcc_assert (ncopies
>= 1);
4770 bool found_mode
= false;
4771 scalar_mode lhs_mode
= SCALAR_TYPE_MODE (lhs_type
);
4772 scalar_mode rhs_mode
= SCALAR_TYPE_MODE (rhs_type
);
4773 opt_scalar_mode rhs_mode_iter
;
4775 /* Supportable by target? */
4779 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
4781 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
4786 if (dump_enabled_p ())
4787 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4788 "conversion not supported by target.\n");
4792 if (supportable_widening_operation (code
, stmt_info
, vectype_out
,
4793 vectype_in
, &code1
, &code2
,
4794 &multi_step_cvt
, &interm_types
))
4796 /* Binary widening operation can only be supported directly by the
4798 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
4802 if (code
!= FLOAT_EXPR
4803 || GET_MODE_SIZE (lhs_mode
) <= GET_MODE_SIZE (rhs_mode
))
4806 fltsz
= GET_MODE_SIZE (lhs_mode
);
4807 FOR_EACH_2XWIDER_MODE (rhs_mode_iter
, rhs_mode
)
4809 rhs_mode
= rhs_mode_iter
.require ();
4810 if (GET_MODE_SIZE (rhs_mode
) > fltsz
)
4814 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
4815 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
4816 if (cvt_type
== NULL_TREE
)
4819 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
4821 if (!supportable_convert_operation (code
, vectype_out
,
4822 cvt_type
, &decl1
, &codecvt1
))
4825 else if (!supportable_widening_operation (code
, stmt_info
,
4826 vectype_out
, cvt_type
,
4827 &codecvt1
, &codecvt2
,
4832 gcc_assert (multi_step_cvt
== 0);
4834 if (supportable_widening_operation (NOP_EXPR
, stmt_info
, cvt_type
,
4835 vectype_in
, &code1
, &code2
,
4836 &multi_step_cvt
, &interm_types
))
4846 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
4847 codecvt2
= ERROR_MARK
;
4851 interm_types
.safe_push (cvt_type
);
4852 cvt_type
= NULL_TREE
;
4857 gcc_assert (op_type
== unary_op
);
4858 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
4859 &code1
, &multi_step_cvt
,
4863 if (code
!= FIX_TRUNC_EXPR
4864 || GET_MODE_SIZE (lhs_mode
) >= GET_MODE_SIZE (rhs_mode
))
4868 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
4869 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
4870 if (cvt_type
== NULL_TREE
)
4872 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
4875 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
4876 &code1
, &multi_step_cvt
,
4885 if (!vec_stmt
) /* transformation not required. */
4887 DUMP_VECT_SCOPE ("vectorizable_conversion");
4888 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
4890 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
4891 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, slp_node
,
4894 else if (modifier
== NARROW
)
4896 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
4897 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
,
4902 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
4903 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
,
4906 interm_types
.release ();
4911 if (dump_enabled_p ())
4912 dump_printf_loc (MSG_NOTE
, vect_location
,
4913 "transform conversion. ncopies = %d.\n", ncopies
);
4915 if (op_type
== binary_op
)
4917 if (CONSTANT_CLASS_P (op0
))
4918 op0
= fold_convert (TREE_TYPE (op1
), op0
);
4919 else if (CONSTANT_CLASS_P (op1
))
4920 op1
= fold_convert (TREE_TYPE (op0
), op1
);
4923 /* In case of multi-step conversion, we first generate conversion operations
4924 to the intermediate types, and then from that types to the final one.
4925 We create vector destinations for the intermediate type (TYPES) received
4926 from supportable_*_operation, and store them in the correct order
4927 for future use in vect_create_vectorized_*_stmts (). */
4928 auto_vec
<tree
> vec_dsts (multi_step_cvt
+ 1);
4929 vec_dest
= vect_create_destination_var (scalar_dest
,
4930 (cvt_type
&& modifier
== WIDEN
)
4931 ? cvt_type
: vectype_out
);
4932 vec_dsts
.quick_push (vec_dest
);
4936 for (i
= interm_types
.length () - 1;
4937 interm_types
.iterate (i
, &intermediate_type
); i
--)
4939 vec_dest
= vect_create_destination_var (scalar_dest
,
4941 vec_dsts
.quick_push (vec_dest
);
4946 vec_dest
= vect_create_destination_var (scalar_dest
,
4948 ? vectype_out
: cvt_type
);
4952 if (modifier
== WIDEN
)
4954 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
4955 if (op_type
== binary_op
)
4956 vec_oprnds1
.create (1);
4958 else if (modifier
== NARROW
)
4959 vec_oprnds0
.create (
4960 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
4962 else if (code
== WIDEN_LSHIFT_EXPR
)
4963 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
4966 prev_stmt_info
= NULL
;
4970 for (j
= 0; j
< ncopies
; j
++)
4973 vect_get_vec_defs (op0
, NULL
, stmt_info
, &vec_oprnds0
,
4976 vect_get_vec_defs_for_stmt_copy (vinfo
, &vec_oprnds0
, NULL
);
4978 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4980 stmt_vec_info new_stmt_info
;
4981 /* Arguments are ready, create the new vector stmt. */
4982 if (code1
== CALL_EXPR
)
4984 gcall
*new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4985 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4986 gimple_call_set_lhs (new_stmt
, new_temp
);
4988 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
4992 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
4994 = gimple_build_assign (vec_dest
, code1
, vop0
);
4995 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4996 gimple_assign_set_lhs (new_stmt
, new_temp
);
4998 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
5002 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
5005 if (!prev_stmt_info
)
5006 STMT_VINFO_VEC_STMT (stmt_info
)
5007 = *vec_stmt
= new_stmt_info
;
5009 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
5010 prev_stmt_info
= new_stmt_info
;
5017 /* In case the vectorization factor (VF) is bigger than the number
5018 of elements that we can fit in a vectype (nunits), we have to
5019 generate more than one vector stmt - i.e - we need to "unroll"
5020 the vector stmt by a factor VF/nunits. */
5021 for (j
= 0; j
< ncopies
; j
++)
5028 if (code
== WIDEN_LSHIFT_EXPR
)
5033 /* Store vec_oprnd1 for every vector stmt to be created
5034 for SLP_NODE. We check during the analysis that all
5035 the shift arguments are the same. */
5036 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
5037 vec_oprnds1
.quick_push (vec_oprnd1
);
5039 vect_get_vec_defs (op0
, NULL_TREE
, stmt_info
,
5040 &vec_oprnds0
, NULL
, slp_node
);
5043 vect_get_vec_defs (op0
, op1
, stmt_info
, &vec_oprnds0
,
5044 &vec_oprnds1
, slp_node
);
5048 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt_info
);
5049 vec_oprnds0
.quick_push (vec_oprnd0
);
5050 if (op_type
== binary_op
)
5052 if (code
== WIDEN_LSHIFT_EXPR
)
5056 = vect_get_vec_def_for_operand (op1
, stmt_info
);
5057 vec_oprnds1
.quick_push (vec_oprnd1
);
5063 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd0
);
5064 vec_oprnds0
.truncate (0);
5065 vec_oprnds0
.quick_push (vec_oprnd0
);
5066 if (op_type
== binary_op
)
5068 if (code
== WIDEN_LSHIFT_EXPR
)
5071 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (vinfo
,
5073 vec_oprnds1
.truncate (0);
5074 vec_oprnds1
.quick_push (vec_oprnd1
);
5078 /* Arguments are ready. Create the new vector stmts. */
5079 for (i
= multi_step_cvt
; i
>= 0; i
--)
5081 tree this_dest
= vec_dsts
[i
];
5082 enum tree_code c1
= code1
, c2
= code2
;
5083 if (i
== 0 && codecvt2
!= ERROR_MARK
)
5088 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
5089 &vec_oprnds1
, stmt_info
,
5091 c1
, c2
, decl1
, decl2
,
5095 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5097 stmt_vec_info new_stmt_info
;
5100 if (codecvt1
== CALL_EXPR
)
5102 gcall
*new_stmt
= gimple_build_call (decl1
, 1, vop0
);
5103 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5104 gimple_call_set_lhs (new_stmt
, new_temp
);
5106 = vect_finish_stmt_generation (stmt_info
, new_stmt
,
5111 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
5112 new_temp
= make_ssa_name (vec_dest
);
5114 = gimple_build_assign (new_temp
, codecvt1
, vop0
);
5116 = vect_finish_stmt_generation (stmt_info
, new_stmt
,
5121 new_stmt_info
= vinfo
->lookup_def (vop0
);
5124 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
5127 if (!prev_stmt_info
)
5128 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt_info
;
5130 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
5131 prev_stmt_info
= new_stmt_info
;
5136 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
5140 /* In case the vectorization factor (VF) is bigger than the number
5141 of elements that we can fit in a vectype (nunits), we have to
5142 generate more than one vector stmt - i.e - we need to "unroll"
5143 the vector stmt by a factor VF/nunits. */
5144 for (j
= 0; j
< ncopies
; j
++)
5148 vect_get_vec_defs (op0
, NULL_TREE
, stmt_info
, &vec_oprnds0
, NULL
,
5152 vec_oprnds0
.truncate (0);
5153 vect_get_loop_based_defs (&last_oprnd
, stmt_info
, &vec_oprnds0
,
5154 vect_pow2 (multi_step_cvt
) - 1);
5157 /* Arguments are ready. Create the new vector stmts. */
5159 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5161 if (codecvt1
== CALL_EXPR
)
5163 gcall
*new_stmt
= gimple_build_call (decl1
, 1, vop0
);
5164 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5165 gimple_call_set_lhs (new_stmt
, new_temp
);
5166 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
5170 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
5171 new_temp
= make_ssa_name (vec_dest
);
5173 = gimple_build_assign (new_temp
, codecvt1
, vop0
);
5174 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
5177 vec_oprnds0
[i
] = new_temp
;
5180 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
5181 stmt_info
, vec_dsts
, gsi
,
5186 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
5190 vec_oprnds0
.release ();
5191 vec_oprnds1
.release ();
5192 interm_types
.release ();
5198 /* Function vectorizable_assignment.
5200 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5201 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5202 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5203 Return true if STMT_INFO is vectorizable in this way. */
5206 vectorizable_assignment (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
5207 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
5208 stmt_vector_for_cost
*cost_vec
)
5213 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5215 enum vect_def_type dt
[1] = {vect_unknown_def_type
};
5219 vec
<tree
> vec_oprnds
= vNULL
;
5221 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5222 vec_info
*vinfo
= stmt_info
->vinfo
;
5223 stmt_vec_info prev_stmt_info
= NULL
;
5224 enum tree_code code
;
5227 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5230 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5234 /* Is vectorizable assignment? */
5235 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
5239 scalar_dest
= gimple_assign_lhs (stmt
);
5240 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
5243 code
= gimple_assign_rhs_code (stmt
);
5244 if (gimple_assign_single_p (stmt
)
5245 || code
== PAREN_EXPR
5246 || CONVERT_EXPR_CODE_P (code
))
5247 op
= gimple_assign_rhs1 (stmt
);
5251 if (code
== VIEW_CONVERT_EXPR
)
5252 op
= TREE_OPERAND (op
, 0);
5254 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5255 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5257 /* Multiple types in SLP are handled by creating the appropriate number of
5258 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5263 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
5265 gcc_assert (ncopies
>= 1);
5267 if (!vect_is_simple_use (op
, vinfo
, &dt
[0], &vectype_in
))
5269 if (dump_enabled_p ())
5270 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5271 "use not simple.\n");
5275 /* We can handle NOP_EXPR conversions that do not change the number
5276 of elements or the vector size. */
5277 if ((CONVERT_EXPR_CODE_P (code
)
5278 || code
== VIEW_CONVERT_EXPR
)
5280 || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in
), nunits
)
5281 || maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype
)),
5282 GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
5285 /* We do not handle bit-precision changes. */
5286 if ((CONVERT_EXPR_CODE_P (code
)
5287 || code
== VIEW_CONVERT_EXPR
)
5288 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
5289 && (!type_has_mode_precision_p (TREE_TYPE (scalar_dest
))
5290 || !type_has_mode_precision_p (TREE_TYPE (op
)))
5291 /* But a conversion that does not change the bit-pattern is ok. */
5292 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
5293 > TYPE_PRECISION (TREE_TYPE (op
)))
5294 && TYPE_UNSIGNED (TREE_TYPE (op
)))
5295 /* Conversion between boolean types of different sizes is
5296 a simple assignment in case their vectypes are same
5298 && (!VECTOR_BOOLEAN_TYPE_P (vectype
)
5299 || !VECTOR_BOOLEAN_TYPE_P (vectype_in
)))
5301 if (dump_enabled_p ())
5302 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5303 "type conversion to/from bit-precision "
5308 if (!vec_stmt
) /* transformation not required. */
5310 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
5311 DUMP_VECT_SCOPE ("vectorizable_assignment");
5312 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, slp_node
, cost_vec
);
5317 if (dump_enabled_p ())
5318 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
5321 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
5324 for (j
= 0; j
< ncopies
; j
++)
5328 vect_get_vec_defs (op
, NULL
, stmt_info
, &vec_oprnds
, NULL
, slp_node
);
5330 vect_get_vec_defs_for_stmt_copy (vinfo
, &vec_oprnds
, NULL
);
5332 /* Arguments are ready. create the new vector stmt. */
5333 stmt_vec_info new_stmt_info
= NULL
;
5334 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
5336 if (CONVERT_EXPR_CODE_P (code
)
5337 || code
== VIEW_CONVERT_EXPR
)
5338 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
5339 gassign
*new_stmt
= gimple_build_assign (vec_dest
, vop
);
5340 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5341 gimple_assign_set_lhs (new_stmt
, new_temp
);
5343 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
5345 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
5352 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
5354 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
5356 prev_stmt_info
= new_stmt_info
;
5359 vec_oprnds
.release ();
5364 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
5365 either as shift by a scalar or by a vector. */
5368 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
5371 machine_mode vec_mode
;
5376 vectype
= get_vectype_for_scalar_type (scalar_type
);
5380 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
5382 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
5384 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
5386 || (optab_handler (optab
, TYPE_MODE (vectype
))
5387 == CODE_FOR_nothing
))
5391 vec_mode
= TYPE_MODE (vectype
);
5392 icode
= (int) optab_handler (optab
, vec_mode
);
5393 if (icode
== CODE_FOR_nothing
)
5400 /* Function vectorizable_shift.
5402 Check if STMT_INFO performs a shift operation that can be vectorized.
5403 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5404 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5405 Return true if STMT_INFO is vectorizable in this way. */
5408 vectorizable_shift (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
5409 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
5410 stmt_vector_for_cost
*cost_vec
)
5414 tree op0
, op1
= NULL
;
5415 tree vec_oprnd1
= NULL_TREE
;
5417 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5418 enum tree_code code
;
5419 machine_mode vec_mode
;
5423 machine_mode optab_op2_mode
;
5424 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
5426 stmt_vec_info prev_stmt_info
;
5427 poly_uint64 nunits_in
;
5428 poly_uint64 nunits_out
;
5433 vec
<tree
> vec_oprnds0
= vNULL
;
5434 vec
<tree
> vec_oprnds1
= vNULL
;
5437 bool scalar_shift_arg
= true;
5438 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5439 vec_info
*vinfo
= stmt_info
->vinfo
;
5441 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5444 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5448 /* Is STMT a vectorizable binary/unary operation? */
5449 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
5453 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
5456 code
= gimple_assign_rhs_code (stmt
);
5458 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
5459 || code
== RROTATE_EXPR
))
5462 scalar_dest
= gimple_assign_lhs (stmt
);
5463 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
5464 if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest
)))
5466 if (dump_enabled_p ())
5467 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5468 "bit-precision shifts not supported.\n");
5472 op0
= gimple_assign_rhs1 (stmt
);
5473 if (!vect_is_simple_use (op0
, vinfo
, &dt
[0], &vectype
))
5475 if (dump_enabled_p ())
5476 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5477 "use not simple.\n");
5480 /* If op0 is an external or constant def use a vector type with
5481 the same size as the output vector type. */
5483 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
5485 gcc_assert (vectype
);
5488 if (dump_enabled_p ())
5489 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5490 "no vectype for scalar type\n");
5494 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
5495 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
5496 if (maybe_ne (nunits_out
, nunits_in
))
5499 op1
= gimple_assign_rhs2 (stmt
);
5500 stmt_vec_info op1_def_stmt_info
;
5501 if (!vect_is_simple_use (op1
, vinfo
, &dt
[1], &op1_vectype
,
5502 &op1_def_stmt_info
))
5504 if (dump_enabled_p ())
5505 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5506 "use not simple.\n");
5510 /* Multiple types in SLP are handled by creating the appropriate number of
5511 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5516 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
5518 gcc_assert (ncopies
>= 1);
5520 /* Determine whether the shift amount is a vector, or scalar. If the
5521 shift/rotate amount is a vector, use the vector/vector shift optabs. */
5523 if ((dt
[1] == vect_internal_def
5524 || dt
[1] == vect_induction_def
)
5526 scalar_shift_arg
= false;
5527 else if (dt
[1] == vect_constant_def
5528 || dt
[1] == vect_external_def
5529 || dt
[1] == vect_internal_def
)
5531 /* In SLP, need to check whether the shift count is the same,
5532 in loops if it is a constant or invariant, it is always
5536 vec
<stmt_vec_info
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
5537 stmt_vec_info slpstmt_info
;
5539 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt_info
)
5541 gassign
*slpstmt
= as_a
<gassign
*> (slpstmt_info
->stmt
);
5542 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
5543 scalar_shift_arg
= false;
5547 /* If the shift amount is computed by a pattern stmt we cannot
5548 use the scalar amount directly thus give up and use a vector
5550 if (op1_def_stmt_info
&& is_pattern_stmt_p (op1_def_stmt_info
))
5551 scalar_shift_arg
= false;
5555 if (dump_enabled_p ())
5556 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5557 "operand mode requires invariant argument.\n");
5561 /* Vector shifted by vector. */
5562 if (!scalar_shift_arg
)
5564 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
5565 if (dump_enabled_p ())
5566 dump_printf_loc (MSG_NOTE
, vect_location
,
5567 "vector/vector shift/rotate found.\n");
5570 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
5571 if (op1_vectype
== NULL_TREE
5572 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
5574 if (dump_enabled_p ())
5575 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5576 "unusable type for last operand in"
5577 " vector/vector shift/rotate.\n");
5581 /* See if the machine has a vector shifted by scalar insn and if not
5582 then see if it has a vector shifted by vector insn. */
5585 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
5587 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
5589 if (dump_enabled_p ())
5590 dump_printf_loc (MSG_NOTE
, vect_location
,
5591 "vector/scalar shift/rotate found.\n");
5595 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
5597 && (optab_handler (optab
, TYPE_MODE (vectype
))
5598 != CODE_FOR_nothing
))
5600 scalar_shift_arg
= false;
5602 if (dump_enabled_p ())
5603 dump_printf_loc (MSG_NOTE
, vect_location
,
5604 "vector/vector shift/rotate found.\n");
5606 /* Unlike the other binary operators, shifts/rotates have
5607 the rhs being int, instead of the same type as the lhs,
5608 so make sure the scalar is the right type if we are
5609 dealing with vectors of long long/long/short/char. */
5610 if (dt
[1] == vect_constant_def
)
5611 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
5612 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
5616 && TYPE_MODE (TREE_TYPE (vectype
))
5617 != TYPE_MODE (TREE_TYPE (op1
)))
5619 if (dump_enabled_p ())
5620 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5621 "unusable type for last operand in"
5622 " vector/vector shift/rotate.\n");
5625 if (vec_stmt
&& !slp_node
)
5627 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
5628 op1
= vect_init_vector (stmt_info
, op1
,
5629 TREE_TYPE (vectype
), NULL
);
5636 /* Supportable by target? */
5639 if (dump_enabled_p ())
5640 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5644 vec_mode
= TYPE_MODE (vectype
);
5645 icode
= (int) optab_handler (optab
, vec_mode
);
5646 if (icode
== CODE_FOR_nothing
)
5648 if (dump_enabled_p ())
5649 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5650 "op not supported by target.\n");
5651 /* Check only during analysis. */
5652 if (maybe_ne (GET_MODE_SIZE (vec_mode
), UNITS_PER_WORD
)
5654 && !vect_worthwhile_without_simd_p (vinfo
, code
)))
5656 if (dump_enabled_p ())
5657 dump_printf_loc (MSG_NOTE
, vect_location
,
5658 "proceeding using word mode.\n");
5661 /* Worthwhile without SIMD support? Check only during analysis. */
5663 && !VECTOR_MODE_P (TYPE_MODE (vectype
))
5664 && !vect_worthwhile_without_simd_p (vinfo
, code
))
5666 if (dump_enabled_p ())
5667 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5668 "not worthwhile without SIMD support.\n");
5672 if (!vec_stmt
) /* transformation not required. */
5674 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
5675 DUMP_VECT_SCOPE ("vectorizable_shift");
5676 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, slp_node
, cost_vec
);
5682 if (dump_enabled_p ())
5683 dump_printf_loc (MSG_NOTE
, vect_location
,
5684 "transform binary/unary operation.\n");
5687 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
5689 prev_stmt_info
= NULL
;
5690 for (j
= 0; j
< ncopies
; j
++)
5695 if (scalar_shift_arg
)
5697 /* Vector shl and shr insn patterns can be defined with scalar
5698 operand 2 (shift operand). In this case, use constant or loop
5699 invariant op1 directly, without extending it to vector mode
5701 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
5702 if (!VECTOR_MODE_P (optab_op2_mode
))
5704 if (dump_enabled_p ())
5705 dump_printf_loc (MSG_NOTE
, vect_location
,
5706 "operand 1 using scalar mode.\n");
5708 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
5709 vec_oprnds1
.quick_push (vec_oprnd1
);
5712 /* Store vec_oprnd1 for every vector stmt to be created
5713 for SLP_NODE. We check during the analysis that all
5714 the shift arguments are the same.
5715 TODO: Allow different constants for different vector
5716 stmts generated for an SLP instance. */
5717 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
5718 vec_oprnds1
.quick_push (vec_oprnd1
);
5723 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
5724 (a special case for certain kind of vector shifts); otherwise,
5725 operand 1 should be of a vector type (the usual case). */
5727 vect_get_vec_defs (op0
, NULL_TREE
, stmt_info
, &vec_oprnds0
, NULL
,
5730 vect_get_vec_defs (op0
, op1
, stmt_info
, &vec_oprnds0
, &vec_oprnds1
,
5734 vect_get_vec_defs_for_stmt_copy (vinfo
, &vec_oprnds0
, &vec_oprnds1
);
5736 /* Arguments are ready. Create the new vector stmt. */
5737 stmt_vec_info new_stmt_info
= NULL
;
5738 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
5740 vop1
= vec_oprnds1
[i
];
5741 gassign
*new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
5742 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
5743 gimple_assign_set_lhs (new_stmt
, new_temp
);
5745 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
5747 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
5754 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
5756 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
5757 prev_stmt_info
= new_stmt_info
;
5760 vec_oprnds0
.release ();
5761 vec_oprnds1
.release ();
5767 /* Function vectorizable_operation.
5769 Check if STMT_INFO performs a binary, unary or ternary operation that can
5771 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5772 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5773 Return true if STMT_INFO is vectorizable in this way. */
5776 vectorizable_operation (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
5777 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
5778 stmt_vector_for_cost
*cost_vec
)
5782 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
5784 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5785 enum tree_code code
, orig_code
;
5786 machine_mode vec_mode
;
5790 bool target_support_p
;
5791 enum vect_def_type dt
[3]
5792 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
5794 stmt_vec_info prev_stmt_info
;
5795 poly_uint64 nunits_in
;
5796 poly_uint64 nunits_out
;
5800 vec
<tree
> vec_oprnds0
= vNULL
;
5801 vec
<tree
> vec_oprnds1
= vNULL
;
5802 vec
<tree
> vec_oprnds2
= vNULL
;
5803 tree vop0
, vop1
, vop2
;
5804 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5805 vec_info
*vinfo
= stmt_info
->vinfo
;
5807 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5810 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
5814 /* Is STMT a vectorizable binary/unary operation? */
5815 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
5819 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
5822 orig_code
= code
= gimple_assign_rhs_code (stmt
);
5824 /* For pointer addition and subtraction, we should use the normal
5825 plus and minus for the vector operation. */
5826 if (code
== POINTER_PLUS_EXPR
)
5828 if (code
== POINTER_DIFF_EXPR
)
5831 /* Support only unary or binary operations. */
5832 op_type
= TREE_CODE_LENGTH (code
);
5833 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
5835 if (dump_enabled_p ())
5836 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5837 "num. args = %d (not unary/binary/ternary op).\n",
5842 scalar_dest
= gimple_assign_lhs (stmt
);
5843 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
5845 /* Most operations cannot handle bit-precision types without extra
5847 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
5848 && !type_has_mode_precision_p (TREE_TYPE (scalar_dest
))
5849 /* Exception are bitwise binary operations. */
5850 && code
!= BIT_IOR_EXPR
5851 && code
!= BIT_XOR_EXPR
5852 && code
!= BIT_AND_EXPR
)
5854 if (dump_enabled_p ())
5855 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5856 "bit-precision arithmetic not supported.\n");
5860 op0
= gimple_assign_rhs1 (stmt
);
5861 if (!vect_is_simple_use (op0
, vinfo
, &dt
[0], &vectype
))
5863 if (dump_enabled_p ())
5864 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5865 "use not simple.\n");
5868 /* If op0 is an external or constant def use a vector type with
5869 the same size as the output vector type. */
5872 /* For boolean type we cannot determine vectype by
5873 invariant value (don't know whether it is a vector
5874 of booleans or vector of integers). We use output
5875 vectype because operations on boolean don't change
5877 if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)))
5879 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest
)))
5881 if (dump_enabled_p ())
5882 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5883 "not supported operation on bool value.\n");
5886 vectype
= vectype_out
;
5889 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
5892 gcc_assert (vectype
);
5895 if (dump_enabled_p ())
5897 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5898 "no vectype for scalar type ");
5899 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
5901 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
5907 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
5908 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
5909 if (maybe_ne (nunits_out
, nunits_in
))
5912 if (op_type
== binary_op
|| op_type
== ternary_op
)
5914 op1
= gimple_assign_rhs2 (stmt
);
5915 if (!vect_is_simple_use (op1
, vinfo
, &dt
[1]))
5917 if (dump_enabled_p ())
5918 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5919 "use not simple.\n");
5923 if (op_type
== ternary_op
)
5925 op2
= gimple_assign_rhs3 (stmt
);
5926 if (!vect_is_simple_use (op2
, vinfo
, &dt
[2]))
5928 if (dump_enabled_p ())
5929 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5930 "use not simple.\n");
5935 /* Multiple types in SLP are handled by creating the appropriate number of
5936 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5941 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
5943 gcc_assert (ncopies
>= 1);
5945 /* Shifts are handled in vectorizable_shift (). */
5946 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
5947 || code
== RROTATE_EXPR
)
5950 /* Supportable by target? */
5952 vec_mode
= TYPE_MODE (vectype
);
5953 if (code
== MULT_HIGHPART_EXPR
)
5954 target_support_p
= can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
));
5957 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
5960 if (dump_enabled_p ())
5961 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5965 target_support_p
= (optab_handler (optab
, vec_mode
)
5966 != CODE_FOR_nothing
);
5969 if (!target_support_p
)
5971 if (dump_enabled_p ())
5972 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5973 "op not supported by target.\n");
5974 /* Check only during analysis. */
5975 if (maybe_ne (GET_MODE_SIZE (vec_mode
), UNITS_PER_WORD
)
5976 || (!vec_stmt
&& !vect_worthwhile_without_simd_p (vinfo
, code
)))
5978 if (dump_enabled_p ())
5979 dump_printf_loc (MSG_NOTE
, vect_location
,
5980 "proceeding using word mode.\n");
5983 /* Worthwhile without SIMD support? Check only during analysis. */
5984 if (!VECTOR_MODE_P (vec_mode
)
5986 && !vect_worthwhile_without_simd_p (vinfo
, code
))
5988 if (dump_enabled_p ())
5989 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5990 "not worthwhile without SIMD support.\n");
5994 if (!vec_stmt
) /* transformation not required. */
5996 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
5997 DUMP_VECT_SCOPE ("vectorizable_operation");
5998 vect_model_simple_cost (stmt_info
, ncopies
, dt
, ndts
, slp_node
, cost_vec
);
6004 if (dump_enabled_p ())
6005 dump_printf_loc (MSG_NOTE
, vect_location
,
6006 "transform binary/unary operation.\n");
6008 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6009 vectors with unsigned elements, but the result is signed. So, we
6010 need to compute the MINUS_EXPR into vectype temporary and
6011 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6012 tree vec_cvt_dest
= NULL_TREE
;
6013 if (orig_code
== POINTER_DIFF_EXPR
)
6015 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6016 vec_cvt_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
6020 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
6022 /* In case the vectorization factor (VF) is bigger than the number
6023 of elements that we can fit in a vectype (nunits), we have to generate
6024 more than one vector stmt - i.e - we need to "unroll" the
6025 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6026 from one copy of the vector stmt to the next, in the field
6027 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6028 stages to find the correct vector defs to be used when vectorizing
6029 stmts that use the defs of the current stmt. The example below
6030 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6031 we need to create 4 vectorized stmts):
6033 before vectorization:
6034 RELATED_STMT VEC_STMT
6038 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6040 RELATED_STMT VEC_STMT
6041 VS1_0: vx0 = memref0 VS1_1 -
6042 VS1_1: vx1 = memref1 VS1_2 -
6043 VS1_2: vx2 = memref2 VS1_3 -
6044 VS1_3: vx3 = memref3 - -
6045 S1: x = load - VS1_0
6048 step2: vectorize stmt S2 (done here):
6049 To vectorize stmt S2 we first need to find the relevant vector
6050 def for the first operand 'x'. This is, as usual, obtained from
6051 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6052 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6053 relevant vector def 'vx0'. Having found 'vx0' we can generate
6054 the vector stmt VS2_0, and as usual, record it in the
6055 STMT_VINFO_VEC_STMT of stmt S2.
6056 When creating the second copy (VS2_1), we obtain the relevant vector
6057 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6058 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6059 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6060 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6061 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6062 chain of stmts and pointers:
6063 RELATED_STMT VEC_STMT
6064 VS1_0: vx0 = memref0 VS1_1 -
6065 VS1_1: vx1 = memref1 VS1_2 -
6066 VS1_2: vx2 = memref2 VS1_3 -
6067 VS1_3: vx3 = memref3 - -
6068 S1: x = load - VS1_0
6069 VS2_0: vz0 = vx0 + v1 VS2_1 -
6070 VS2_1: vz1 = vx1 + v1 VS2_2 -
6071 VS2_2: vz2 = vx2 + v1 VS2_3 -
6072 VS2_3: vz3 = vx3 + v1 - -
6073 S2: z = x + 1 - VS2_0 */
6075 prev_stmt_info
= NULL
;
6076 for (j
= 0; j
< ncopies
; j
++)
6081 if (op_type
== binary_op
)
6082 vect_get_vec_defs (op0
, op1
, stmt_info
, &vec_oprnds0
, &vec_oprnds1
,
6084 else if (op_type
== ternary_op
)
6088 auto_vec
<tree
> ops(3);
6089 ops
.quick_push (op0
);
6090 ops
.quick_push (op1
);
6091 ops
.quick_push (op2
);
6092 auto_vec
<vec
<tree
> > vec_defs(3);
6093 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
6094 vec_oprnds0
= vec_defs
[0];
6095 vec_oprnds1
= vec_defs
[1];
6096 vec_oprnds2
= vec_defs
[2];
6100 vect_get_vec_defs (op0
, op1
, stmt_info
, &vec_oprnds0
,
6101 &vec_oprnds1
, NULL
);
6102 vect_get_vec_defs (op2
, NULL_TREE
, stmt_info
, &vec_oprnds2
,
6107 vect_get_vec_defs (op0
, NULL_TREE
, stmt_info
, &vec_oprnds0
, NULL
,
6112 vect_get_vec_defs_for_stmt_copy (vinfo
, &vec_oprnds0
, &vec_oprnds1
);
6113 if (op_type
== ternary_op
)
6115 tree vec_oprnd
= vec_oprnds2
.pop ();
6116 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (vinfo
,
6121 /* Arguments are ready. Create the new vector stmt. */
6122 stmt_vec_info new_stmt_info
= NULL
;
6123 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
6125 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
6126 ? vec_oprnds1
[i
] : NULL_TREE
);
6127 vop2
= ((op_type
== ternary_op
)
6128 ? vec_oprnds2
[i
] : NULL_TREE
);
6129 gassign
*new_stmt
= gimple_build_assign (vec_dest
, code
,
6131 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6132 gimple_assign_set_lhs (new_stmt
, new_temp
);
6134 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6137 new_temp
= build1 (VIEW_CONVERT_EXPR
, vectype_out
, new_temp
);
6139 = gimple_build_assign (vec_cvt_dest
, VIEW_CONVERT_EXPR
,
6141 new_temp
= make_ssa_name (vec_cvt_dest
, new_stmt
);
6142 gimple_assign_set_lhs (new_stmt
, new_temp
);
6144 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6147 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
6154 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
6156 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
6157 prev_stmt_info
= new_stmt_info
;
6160 vec_oprnds0
.release ();
6161 vec_oprnds1
.release ();
6162 vec_oprnds2
.release ();
6167 /* A helper function to ensure data reference DR_INFO's base alignment. */
6170 ensure_base_align (dr_vec_info
*dr_info
)
6172 if (dr_info
->misalignment
== DR_MISALIGNMENT_UNINITIALIZED
)
6175 if (dr_info
->base_misaligned
)
6177 tree base_decl
= dr_info
->base_decl
;
6179 unsigned int align_base_to
6180 = DR_TARGET_ALIGNMENT (dr_info
) * BITS_PER_UNIT
;
6182 if (decl_in_symtab_p (base_decl
))
6183 symtab_node::get (base_decl
)->increase_alignment (align_base_to
);
6186 SET_DECL_ALIGN (base_decl
, align_base_to
);
6187 DECL_USER_ALIGN (base_decl
) = 1;
6189 dr_info
->base_misaligned
= false;
6194 /* Function get_group_alias_ptr_type.
6196 Return the alias type for the group starting at FIRST_STMT_INFO. */
6199 get_group_alias_ptr_type (stmt_vec_info first_stmt_info
)
6201 struct data_reference
*first_dr
, *next_dr
;
6203 first_dr
= STMT_VINFO_DATA_REF (first_stmt_info
);
6204 stmt_vec_info next_stmt_info
= DR_GROUP_NEXT_ELEMENT (first_stmt_info
);
6205 while (next_stmt_info
)
6207 next_dr
= STMT_VINFO_DATA_REF (next_stmt_info
);
6208 if (get_alias_set (DR_REF (first_dr
))
6209 != get_alias_set (DR_REF (next_dr
)))
6211 if (dump_enabled_p ())
6212 dump_printf_loc (MSG_NOTE
, vect_location
,
6213 "conflicting alias set types.\n");
6214 return ptr_type_node
;
6216 next_stmt_info
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
6218 return reference_alias_ptr_type (DR_REF (first_dr
));
6222 /* Function vectorizable_store.
6224 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
6225 that can be vectorized.
6226 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6227 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6228 Return true if STMT_INFO is vectorizable in this way. */
6231 vectorizable_store (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
6232 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
6233 stmt_vector_for_cost
*cost_vec
)
6237 tree vec_oprnd
= NULL_TREE
;
6239 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6240 struct loop
*loop
= NULL
;
6241 machine_mode vec_mode
;
6243 enum dr_alignment_support alignment_support_scheme
;
6244 enum vect_def_type rhs_dt
= vect_unknown_def_type
;
6245 enum vect_def_type mask_dt
= vect_unknown_def_type
;
6246 stmt_vec_info prev_stmt_info
= NULL
;
6247 tree dataref_ptr
= NULL_TREE
;
6248 tree dataref_offset
= NULL_TREE
;
6249 gimple
*ptr_incr
= NULL
;
6252 stmt_vec_info first_stmt_info
;
6254 unsigned int group_size
, i
;
6255 vec
<tree
> oprnds
= vNULL
;
6256 vec
<tree
> result_chain
= vNULL
;
6257 tree offset
= NULL_TREE
;
6258 vec
<tree
> vec_oprnds
= vNULL
;
6259 bool slp
= (slp_node
!= NULL
);
6260 unsigned int vec_num
;
6261 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6262 vec_info
*vinfo
= stmt_info
->vinfo
;
6264 gather_scatter_info gs_info
;
6266 vec_load_store_type vls_type
;
6269 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6272 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6276 /* Is vectorizable store? */
6278 tree mask
= NULL_TREE
, mask_vectype
= NULL_TREE
;
6279 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt_info
->stmt
))
6281 tree scalar_dest
= gimple_assign_lhs (assign
);
6282 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
6283 && is_pattern_stmt_p (stmt_info
))
6284 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
6285 if (TREE_CODE (scalar_dest
) != ARRAY_REF
6286 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
6287 && TREE_CODE (scalar_dest
) != INDIRECT_REF
6288 && TREE_CODE (scalar_dest
) != COMPONENT_REF
6289 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
6290 && TREE_CODE (scalar_dest
) != REALPART_EXPR
6291 && TREE_CODE (scalar_dest
) != MEM_REF
)
6296 gcall
*call
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
6297 if (!call
|| !gimple_call_internal_p (call
))
6300 internal_fn ifn
= gimple_call_internal_fn (call
);
6301 if (!internal_store_fn_p (ifn
))
6304 if (slp_node
!= NULL
)
6306 if (dump_enabled_p ())
6307 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6308 "SLP of masked stores not supported.\n");
6312 int mask_index
= internal_fn_mask_index (ifn
);
6313 if (mask_index
>= 0)
6315 mask
= gimple_call_arg (call
, mask_index
);
6316 if (!vect_check_load_store_mask (stmt_info
, mask
, &mask_dt
,
6322 op
= vect_get_store_rhs (stmt_info
);
6324 /* Cannot have hybrid store SLP -- that would mean storing to the
6325 same location twice. */
6326 gcc_assert (slp
== PURE_SLP_STMT (stmt_info
));
6328 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
), rhs_vectype
= NULL_TREE
;
6329 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6333 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6334 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
6339 /* Multiple types in SLP are handled by creating the appropriate number of
6340 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6345 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
6347 gcc_assert (ncopies
>= 1);
6349 /* FORNOW. This restriction should be relaxed. */
6350 if (loop
&& nested_in_vect_loop_p (loop
, stmt_info
) && ncopies
> 1)
6352 if (dump_enabled_p ())
6353 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6354 "multiple types in nested loop.\n");
6358 if (!vect_check_store_rhs (stmt_info
, op
, &rhs_dt
, &rhs_vectype
, &vls_type
))
6361 elem_type
= TREE_TYPE (vectype
);
6362 vec_mode
= TYPE_MODE (vectype
);
6364 if (!STMT_VINFO_DATA_REF (stmt_info
))
6367 vect_memory_access_type memory_access_type
;
6368 if (!get_load_store_type (stmt_info
, vectype
, slp
, mask
, vls_type
, ncopies
,
6369 &memory_access_type
, &gs_info
))
6374 if (memory_access_type
== VMAT_CONTIGUOUS
)
6376 if (!VECTOR_MODE_P (vec_mode
)
6377 || !can_vec_mask_load_store_p (vec_mode
,
6378 TYPE_MODE (mask_vectype
), false))
6381 else if (memory_access_type
!= VMAT_LOAD_STORE_LANES
6382 && (memory_access_type
!= VMAT_GATHER_SCATTER
|| gs_info
.decl
))
6384 if (dump_enabled_p ())
6385 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6386 "unsupported access type for masked store.\n");
6392 /* FORNOW. In some cases can vectorize even if data-type not supported
6393 (e.g. - array initialization with 0). */
6394 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
6398 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
), *first_dr_info
= NULL
;
6399 grouped_store
= (STMT_VINFO_GROUPED_ACCESS (stmt_info
)
6400 && memory_access_type
!= VMAT_GATHER_SCATTER
6401 && (slp
|| memory_access_type
!= VMAT_CONTIGUOUS
));
6404 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
6405 first_dr_info
= STMT_VINFO_DR_INFO (first_stmt_info
);
6406 group_size
= DR_GROUP_SIZE (first_stmt_info
);
6410 first_stmt_info
= stmt_info
;
6411 first_dr_info
= dr_info
;
6412 group_size
= vec_num
= 1;
6415 if (!vec_stmt
) /* transformation not required. */
6417 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
) = memory_access_type
;
6420 && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
))
6421 check_load_store_masking (loop_vinfo
, vectype
, vls_type
, group_size
,
6422 memory_access_type
, &gs_info
);
6424 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
6425 vect_model_store_cost (stmt_info
, ncopies
, rhs_dt
, memory_access_type
,
6426 vls_type
, slp_node
, cost_vec
);
6429 gcc_assert (memory_access_type
== STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
));
6433 ensure_base_align (dr_info
);
6435 if (memory_access_type
== VMAT_GATHER_SCATTER
&& gs_info
.decl
)
6437 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
, src
;
6438 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
6439 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6440 tree ptr
, mask
, var
, scale
, perm_mask
= NULL_TREE
;
6441 edge pe
= loop_preheader_edge (loop
);
6444 enum { NARROW
, NONE
, WIDEN
} modifier
;
6445 poly_uint64 scatter_off_nunits
6446 = TYPE_VECTOR_SUBPARTS (gs_info
.offset_vectype
);
6448 if (known_eq (nunits
, scatter_off_nunits
))
6450 else if (known_eq (nunits
* 2, scatter_off_nunits
))
6454 /* Currently gathers and scatters are only supported for
6455 fixed-length vectors. */
6456 unsigned int count
= scatter_off_nunits
.to_constant ();
6457 vec_perm_builder
sel (count
, count
, 1);
6458 for (i
= 0; i
< (unsigned int) count
; ++i
)
6459 sel
.quick_push (i
| (count
/ 2));
6461 vec_perm_indices
indices (sel
, 1, count
);
6462 perm_mask
= vect_gen_perm_mask_checked (gs_info
.offset_vectype
,
6464 gcc_assert (perm_mask
!= NULL_TREE
);
6466 else if (known_eq (nunits
, scatter_off_nunits
* 2))
6470 /* Currently gathers and scatters are only supported for
6471 fixed-length vectors. */
6472 unsigned int count
= nunits
.to_constant ();
6473 vec_perm_builder
sel (count
, count
, 1);
6474 for (i
= 0; i
< (unsigned int) count
; ++i
)
6475 sel
.quick_push (i
| (count
/ 2));
6477 vec_perm_indices
indices (sel
, 2, count
);
6478 perm_mask
= vect_gen_perm_mask_checked (vectype
, indices
);
6479 gcc_assert (perm_mask
!= NULL_TREE
);
6485 rettype
= TREE_TYPE (TREE_TYPE (gs_info
.decl
));
6486 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6487 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6488 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6489 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6490 scaletype
= TREE_VALUE (arglist
);
6492 gcc_checking_assert (TREE_CODE (masktype
) == INTEGER_TYPE
6493 && TREE_CODE (rettype
) == VOID_TYPE
);
6495 ptr
= fold_convert (ptrtype
, gs_info
.base
);
6496 if (!is_gimple_min_invariant (ptr
))
6498 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6499 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6500 gcc_assert (!new_bb
);
6503 /* Currently we support only unconditional scatter stores,
6504 so mask should be all ones. */
6505 mask
= build_int_cst (masktype
, -1);
6506 mask
= vect_init_vector (stmt_info
, mask
, masktype
, NULL
);
6508 scale
= build_int_cst (scaletype
, gs_info
.scale
);
6510 prev_stmt_info
= NULL
;
6511 for (j
= 0; j
< ncopies
; ++j
)
6516 = vect_get_vec_def_for_operand (op
, stmt_info
);
6518 = vect_get_vec_def_for_operand (gs_info
.offset
, stmt_info
);
6520 else if (modifier
!= NONE
&& (j
& 1))
6522 if (modifier
== WIDEN
)
6525 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd1
);
6526 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
, perm_mask
,
6529 else if (modifier
== NARROW
)
6531 src
= permute_vec_elements (vec_oprnd1
, vec_oprnd1
, perm_mask
,
6534 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd0
);
6542 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd1
);
6544 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnd0
);
6547 if (!useless_type_conversion_p (srctype
, TREE_TYPE (src
)))
6549 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src
)),
6550 TYPE_VECTOR_SUBPARTS (srctype
)));
6551 var
= vect_get_new_ssa_name (srctype
, vect_simple_var
);
6552 src
= build1 (VIEW_CONVERT_EXPR
, srctype
, src
);
6554 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, src
);
6555 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6559 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6561 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
)),
6562 TYPE_VECTOR_SUBPARTS (idxtype
)));
6563 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
6564 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6566 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6567 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6572 = gimple_build_call (gs_info
.decl
, 5, ptr
, mask
, op
, src
, scale
);
6573 stmt_vec_info new_stmt_info
6574 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6576 if (prev_stmt_info
== NULL
)
6577 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
6579 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
6580 prev_stmt_info
= new_stmt_info
;
6585 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
6586 DR_GROUP_STORE_COUNT (DR_GROUP_FIRST_ELEMENT (stmt_info
))++;
6591 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt_info
));
6593 /* We vectorize all the stmts of the interleaving group when we
6594 reach the last stmt in the group. */
6595 if (DR_GROUP_STORE_COUNT (first_stmt_info
)
6596 < DR_GROUP_SIZE (first_stmt_info
)
6605 grouped_store
= false;
6606 /* VEC_NUM is the number of vect stmts to be created for this
6608 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6609 first_stmt_info
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6610 gcc_assert (DR_GROUP_FIRST_ELEMENT (first_stmt_info
)
6611 == first_stmt_info
);
6612 first_dr_info
= STMT_VINFO_DR_INFO (first_stmt_info
);
6613 op
= vect_get_store_rhs (first_stmt_info
);
6616 /* VEC_NUM is the number of vect stmts to be created for this
6618 vec_num
= group_size
;
6620 ref_type
= get_group_alias_ptr_type (first_stmt_info
);
6623 ref_type
= reference_alias_ptr_type (DR_REF (first_dr_info
->dr
));
6625 if (dump_enabled_p ())
6626 dump_printf_loc (MSG_NOTE
, vect_location
,
6627 "transform store. ncopies = %d\n", ncopies
);
6629 if (memory_access_type
== VMAT_ELEMENTWISE
6630 || memory_access_type
== VMAT_STRIDED_SLP
)
6632 gimple_stmt_iterator incr_gsi
;
6638 tree stride_base
, stride_step
, alias_off
;
6641 /* Checked by get_load_store_type. */
6642 unsigned int const_nunits
= nunits
.to_constant ();
6644 gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
));
6645 gcc_assert (!nested_in_vect_loop_p (loop
, stmt_info
));
6648 = fold_build_pointer_plus
6649 (DR_BASE_ADDRESS (first_dr_info
->dr
),
6650 size_binop (PLUS_EXPR
,
6651 convert_to_ptrofftype (DR_OFFSET (first_dr_info
->dr
)),
6652 convert_to_ptrofftype (DR_INIT (first_dr_info
->dr
))));
6653 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr_info
->dr
));
6655 /* For a store with loop-invariant (but other than power-of-2)
6656 stride (i.e. not a grouped access) like so:
6658 for (i = 0; i < n; i += stride)
6661 we generate a new induction variable and new stores from
6662 the components of the (vectorized) rhs:
6664 for (j = 0; ; j += VF*stride)
6669 array[j + stride] = tmp2;
6673 unsigned nstores
= const_nunits
;
6675 tree ltype
= elem_type
;
6676 tree lvectype
= vectype
;
6679 if (group_size
< const_nunits
6680 && const_nunits
% group_size
== 0)
6682 nstores
= const_nunits
/ group_size
;
6684 ltype
= build_vector_type (elem_type
, group_size
);
6687 /* First check if vec_extract optab doesn't support extraction
6688 of vector elts directly. */
6689 scalar_mode elmode
= SCALAR_TYPE_MODE (elem_type
);
6691 if (!mode_for_vector (elmode
, group_size
).exists (&vmode
)
6692 || !VECTOR_MODE_P (vmode
)
6693 || !targetm
.vector_mode_supported_p (vmode
)
6694 || (convert_optab_handler (vec_extract_optab
,
6695 TYPE_MODE (vectype
), vmode
)
6696 == CODE_FOR_nothing
))
6698 /* Try to avoid emitting an extract of vector elements
6699 by performing the extracts using an integer type of the
6700 same size, extracting from a vector of those and then
6701 re-interpreting it as the original vector type if
6704 = group_size
* GET_MODE_BITSIZE (elmode
);
6705 elmode
= int_mode_for_size (lsize
, 0).require ();
6706 unsigned int lnunits
= const_nunits
/ group_size
;
6707 /* If we can't construct such a vector fall back to
6708 element extracts from the original vector type and
6709 element size stores. */
6710 if (mode_for_vector (elmode
, lnunits
).exists (&vmode
)
6711 && VECTOR_MODE_P (vmode
)
6712 && targetm
.vector_mode_supported_p (vmode
)
6713 && (convert_optab_handler (vec_extract_optab
,
6715 != CODE_FOR_nothing
))
6719 ltype
= build_nonstandard_integer_type (lsize
, 1);
6720 lvectype
= build_vector_type (ltype
, nstores
);
6722 /* Else fall back to vector extraction anyway.
6723 Fewer stores are more important than avoiding spilling
6724 of the vector we extract from. Compared to the
6725 construction case in vectorizable_load no store-forwarding
6726 issue exists here for reasonable archs. */
6729 else if (group_size
>= const_nunits
6730 && group_size
% const_nunits
== 0)
6733 lnel
= const_nunits
;
6737 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
6738 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6741 ivstep
= stride_step
;
6742 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
6743 build_int_cst (TREE_TYPE (ivstep
), vf
));
6745 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6747 stride_base
= cse_and_gimplify_to_preheader (loop_vinfo
, stride_base
);
6748 ivstep
= cse_and_gimplify_to_preheader (loop_vinfo
, ivstep
);
6749 create_iv (stride_base
, ivstep
, NULL
,
6750 loop
, &incr_gsi
, insert_after
,
6752 incr
= gsi_stmt (incr_gsi
);
6753 loop_vinfo
->add_stmt (incr
);
6755 stride_step
= cse_and_gimplify_to_preheader (loop_vinfo
, stride_step
);
6757 prev_stmt_info
= NULL
;
6758 alias_off
= build_int_cst (ref_type
, 0);
6759 stmt_vec_info next_stmt_info
= first_stmt_info
;
6760 for (g
= 0; g
< group_size
; g
++)
6762 running_off
= offvar
;
6765 tree size
= TYPE_SIZE_UNIT (ltype
);
6766 tree pos
= fold_build2 (MULT_EXPR
, sizetype
, size_int (g
),
6768 tree newoff
= copy_ssa_name (running_off
, NULL
);
6769 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6771 vect_finish_stmt_generation (stmt_info
, incr
, gsi
);
6772 running_off
= newoff
;
6774 unsigned int group_el
= 0;
6775 unsigned HOST_WIDE_INT
6776 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
6777 for (j
= 0; j
< ncopies
; j
++)
6779 /* We've set op and dt above, from vect_get_store_rhs,
6780 and first_stmt_info == stmt_info. */
6785 vect_get_vec_defs (op
, NULL_TREE
, stmt_info
,
6786 &vec_oprnds
, NULL
, slp_node
);
6787 vec_oprnd
= vec_oprnds
[0];
6791 op
= vect_get_store_rhs (next_stmt_info
);
6792 vec_oprnd
= vect_get_vec_def_for_operand
6793 (op
, next_stmt_info
);
6799 vec_oprnd
= vec_oprnds
[j
];
6801 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
,
6804 /* Pun the vector to extract from if necessary. */
6805 if (lvectype
!= vectype
)
6807 tree tem
= make_ssa_name (lvectype
);
6809 = gimple_build_assign (tem
, build1 (VIEW_CONVERT_EXPR
,
6810 lvectype
, vec_oprnd
));
6811 vect_finish_stmt_generation (stmt_info
, pun
, gsi
);
6814 for (i
= 0; i
< nstores
; i
++)
6816 tree newref
, newoff
;
6817 gimple
*incr
, *assign
;
6818 tree size
= TYPE_SIZE (ltype
);
6819 /* Extract the i'th component. */
6820 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
,
6821 bitsize_int (i
), size
);
6822 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
6825 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
6829 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
6831 newref
= build2 (MEM_REF
, ltype
,
6832 running_off
, this_off
);
6833 vect_copy_ref_info (newref
, DR_REF (first_dr_info
->dr
));
6835 /* And store it to *running_off. */
6836 assign
= gimple_build_assign (newref
, elem
);
6837 stmt_vec_info assign_info
6838 = vect_finish_stmt_generation (stmt_info
, assign
, gsi
);
6842 || group_el
== group_size
)
6844 newoff
= copy_ssa_name (running_off
, NULL
);
6845 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6846 running_off
, stride_step
);
6847 vect_finish_stmt_generation (stmt_info
, incr
, gsi
);
6849 running_off
= newoff
;
6852 if (g
== group_size
- 1
6855 if (j
== 0 && i
== 0)
6856 STMT_VINFO_VEC_STMT (stmt_info
)
6857 = *vec_stmt
= assign_info
;
6859 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign_info
;
6860 prev_stmt_info
= assign_info
;
6864 next_stmt_info
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
6869 vec_oprnds
.release ();
6873 auto_vec
<tree
> dr_chain (group_size
);
6874 oprnds
.create (group_size
);
6876 alignment_support_scheme
6877 = vect_supportable_dr_alignment (first_dr_info
, false);
6878 gcc_assert (alignment_support_scheme
);
6879 vec_loop_masks
*loop_masks
6880 = (loop_vinfo
&& LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
6881 ? &LOOP_VINFO_MASKS (loop_vinfo
)
6883 /* Targets with store-lane instructions must not require explicit
6884 realignment. vect_supportable_dr_alignment always returns either
6885 dr_aligned or dr_unaligned_supported for masked operations. */
6886 gcc_assert ((memory_access_type
!= VMAT_LOAD_STORE_LANES
6889 || alignment_support_scheme
== dr_aligned
6890 || alignment_support_scheme
== dr_unaligned_supported
);
6892 if (memory_access_type
== VMAT_CONTIGUOUS_DOWN
6893 || memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
6894 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6897 tree vec_offset
= NULL_TREE
;
6898 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
6900 aggr_type
= NULL_TREE
;
6903 else if (memory_access_type
== VMAT_GATHER_SCATTER
)
6905 aggr_type
= elem_type
;
6906 vect_get_strided_load_store_ops (stmt_info
, loop_vinfo
, &gs_info
,
6907 &bump
, &vec_offset
);
6911 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
6912 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6914 aggr_type
= vectype
;
6915 bump
= vect_get_data_ptr_increment (dr_info
, aggr_type
,
6916 memory_access_type
);
6920 LOOP_VINFO_HAS_MASK_STORE (loop_vinfo
) = true;
6922 /* In case the vectorization factor (VF) is bigger than the number
6923 of elements that we can fit in a vectype (nunits), we have to generate
6924 more than one vector stmt - i.e - we need to "unroll" the
6925 vector stmt by a factor VF/nunits. For more details see documentation in
6926 vect_get_vec_def_for_copy_stmt. */
6928 /* In case of interleaving (non-unit grouped access):
6935 We create vectorized stores starting from base address (the access of the
6936 first stmt in the chain (S2 in the above example), when the last store stmt
6937 of the chain (S4) is reached:
6940 VS2: &base + vec_size*1 = vx0
6941 VS3: &base + vec_size*2 = vx1
6942 VS4: &base + vec_size*3 = vx3
6944 Then permutation statements are generated:
6946 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
6947 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
6950 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6951 (the order of the data-refs in the output of vect_permute_store_chain
6952 corresponds to the order of scalar stmts in the interleaving chain - see
6953 the documentation of vect_permute_store_chain()).
6955 In case of both multiple types and interleaving, above vector stores and
6956 permutation stmts are created for every copy. The result vector stmts are
6957 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
6958 STMT_VINFO_RELATED_STMT for the next copies.
6961 prev_stmt_info
= NULL
;
6962 tree vec_mask
= NULL_TREE
;
6963 for (j
= 0; j
< ncopies
; j
++)
6965 stmt_vec_info new_stmt_info
;
6970 /* Get vectorized arguments for SLP_NODE. */
6971 vect_get_vec_defs (op
, NULL_TREE
, stmt_info
, &vec_oprnds
,
6974 vec_oprnd
= vec_oprnds
[0];
6978 /* For interleaved stores we collect vectorized defs for all the
6979 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
6980 used as an input to vect_permute_store_chain(), and OPRNDS as
6981 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
6983 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and
6984 OPRNDS are of size 1. */
6985 stmt_vec_info next_stmt_info
= first_stmt_info
;
6986 for (i
= 0; i
< group_size
; i
++)
6988 /* Since gaps are not supported for interleaved stores,
6989 DR_GROUP_SIZE is the exact number of stmts in the chain.
6990 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
6991 that there is no interleaving, DR_GROUP_SIZE is 1,
6992 and only one iteration of the loop will be executed. */
6993 op
= vect_get_store_rhs (next_stmt_info
);
6994 vec_oprnd
= vect_get_vec_def_for_operand
6995 (op
, next_stmt_info
);
6996 dr_chain
.quick_push (vec_oprnd
);
6997 oprnds
.quick_push (vec_oprnd
);
6998 next_stmt_info
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
7001 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt_info
,
7005 /* We should have catched mismatched types earlier. */
7006 gcc_assert (useless_type_conversion_p (vectype
,
7007 TREE_TYPE (vec_oprnd
)));
7008 bool simd_lane_access_p
7009 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
7010 if (simd_lane_access_p
7011 && TREE_CODE (DR_BASE_ADDRESS (first_dr_info
->dr
)) == ADDR_EXPR
7012 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info
->dr
), 0))
7013 && integer_zerop (DR_OFFSET (first_dr_info
->dr
))
7014 && integer_zerop (DR_INIT (first_dr_info
->dr
))
7015 && alias_sets_conflict_p (get_alias_set (aggr_type
),
7016 get_alias_set (TREE_TYPE (ref_type
))))
7018 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr_info
->dr
));
7019 dataref_offset
= build_int_cst (ref_type
, 0);
7021 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
7022 vect_get_gather_scatter_ops (loop
, stmt_info
, &gs_info
,
7023 &dataref_ptr
, &vec_offset
);
7026 = vect_create_data_ref_ptr (first_stmt_info
, aggr_type
,
7027 simd_lane_access_p
? loop
: NULL
,
7028 offset
, &dummy
, gsi
, &ptr_incr
,
7029 simd_lane_access_p
, NULL_TREE
, bump
);
7033 /* For interleaved stores we created vectorized defs for all the
7034 defs stored in OPRNDS in the previous iteration (previous copy).
7035 DR_CHAIN is then used as an input to vect_permute_store_chain(),
7036 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
7038 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN and
7039 OPRNDS are of size 1. */
7040 for (i
= 0; i
< group_size
; i
++)
7043 vec_oprnd
= vect_get_vec_def_for_stmt_copy (vinfo
, op
);
7044 dr_chain
[i
] = vec_oprnd
;
7045 oprnds
[i
] = vec_oprnd
;
7048 vec_mask
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_mask
);
7051 = int_const_binop (PLUS_EXPR
, dataref_offset
, bump
);
7052 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
7053 vec_offset
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_offset
);
7055 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7059 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
7063 /* Get an array into which we can store the individual vectors. */
7064 vec_array
= create_vector_array (vectype
, vec_num
);
7066 /* Invalidate the current contents of VEC_ARRAY. This should
7067 become an RTL clobber too, which prevents the vector registers
7068 from being upward-exposed. */
7069 vect_clobber_variable (stmt_info
, gsi
, vec_array
);
7071 /* Store the individual vectors into the array. */
7072 for (i
= 0; i
< vec_num
; i
++)
7074 vec_oprnd
= dr_chain
[i
];
7075 write_vector_array (stmt_info
, gsi
, vec_oprnd
, vec_array
, i
);
7078 tree final_mask
= NULL
;
7080 final_mask
= vect_get_loop_mask (gsi
, loop_masks
, ncopies
,
7083 final_mask
= prepare_load_store_mask (mask_vectype
, final_mask
,
7090 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
7092 unsigned int align
= TYPE_ALIGN_UNIT (TREE_TYPE (vectype
));
7093 tree alias_ptr
= build_int_cst (ref_type
, align
);
7094 call
= gimple_build_call_internal (IFN_MASK_STORE_LANES
, 4,
7095 dataref_ptr
, alias_ptr
,
7096 final_mask
, vec_array
);
7101 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
7102 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, ref_type
);
7103 call
= gimple_build_call_internal (IFN_STORE_LANES
, 1,
7105 gimple_call_set_lhs (call
, data_ref
);
7107 gimple_call_set_nothrow (call
, true);
7108 new_stmt_info
= vect_finish_stmt_generation (stmt_info
, call
, gsi
);
7110 /* Record that VEC_ARRAY is now dead. */
7111 vect_clobber_variable (stmt_info
, gsi
, vec_array
);
7115 new_stmt_info
= NULL
;
7119 result_chain
.create (group_size
);
7121 vect_permute_store_chain (dr_chain
, group_size
, stmt_info
, gsi
,
7125 stmt_vec_info next_stmt_info
= first_stmt_info
;
7126 for (i
= 0; i
< vec_num
; i
++)
7128 unsigned align
, misalign
;
7130 tree final_mask
= NULL_TREE
;
7132 final_mask
= vect_get_loop_mask (gsi
, loop_masks
,
7134 vectype
, vec_num
* j
+ i
);
7136 final_mask
= prepare_load_store_mask (mask_vectype
, final_mask
,
7139 if (memory_access_type
== VMAT_GATHER_SCATTER
)
7141 tree scale
= size_int (gs_info
.scale
);
7144 call
= gimple_build_call_internal
7145 (IFN_MASK_SCATTER_STORE
, 5, dataref_ptr
, vec_offset
,
7146 scale
, vec_oprnd
, final_mask
);
7148 call
= gimple_build_call_internal
7149 (IFN_SCATTER_STORE
, 4, dataref_ptr
, vec_offset
,
7151 gimple_call_set_nothrow (call
, true);
7153 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
7158 /* Bump the vector pointer. */
7159 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7163 vec_oprnd
= vec_oprnds
[i
];
7164 else if (grouped_store
)
7165 /* For grouped stores vectorized defs are interleaved in
7166 vect_permute_store_chain(). */
7167 vec_oprnd
= result_chain
[i
];
7169 align
= DR_TARGET_ALIGNMENT (first_dr_info
);
7170 if (aligned_access_p (first_dr_info
))
7172 else if (DR_MISALIGNMENT (first_dr_info
) == -1)
7174 align
= dr_alignment (vect_dr_behavior (first_dr_info
));
7178 misalign
= DR_MISALIGNMENT (first_dr_info
);
7179 if (dataref_offset
== NULL_TREE
7180 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
7181 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
7184 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
7186 tree perm_mask
= perm_mask_for_reverse (vectype
);
7187 tree perm_dest
= vect_create_destination_var
7188 (vect_get_store_rhs (stmt_info
), vectype
);
7189 tree new_temp
= make_ssa_name (perm_dest
);
7191 /* Generate the permute statement. */
7193 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
7194 vec_oprnd
, perm_mask
);
7195 vect_finish_stmt_generation (stmt_info
, perm_stmt
, gsi
);
7197 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7198 vec_oprnd
= new_temp
;
7201 /* Arguments are ready. Create the new vector stmt. */
7204 align
= least_bit_hwi (misalign
| align
);
7205 tree ptr
= build_int_cst (ref_type
, align
);
7207 = gimple_build_call_internal (IFN_MASK_STORE
, 4,
7209 final_mask
, vec_oprnd
);
7210 gimple_call_set_nothrow (call
, true);
7212 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
7216 data_ref
= fold_build2 (MEM_REF
, vectype
,
7220 : build_int_cst (ref_type
, 0));
7221 if (aligned_access_p (first_dr_info
))
7223 else if (DR_MISALIGNMENT (first_dr_info
) == -1)
7224 TREE_TYPE (data_ref
)
7225 = build_aligned_type (TREE_TYPE (data_ref
),
7226 align
* BITS_PER_UNIT
);
7228 TREE_TYPE (data_ref
)
7229 = build_aligned_type (TREE_TYPE (data_ref
),
7230 TYPE_ALIGN (elem_type
));
7231 vect_copy_ref_info (data_ref
, DR_REF (first_dr_info
->dr
));
7233 = gimple_build_assign (data_ref
, vec_oprnd
);
7235 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
7241 next_stmt_info
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
7242 if (!next_stmt_info
)
7249 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
7251 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
7252 prev_stmt_info
= new_stmt_info
;
7257 result_chain
.release ();
7258 vec_oprnds
.release ();
7263 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
7264 VECTOR_CST mask. No checks are made that the target platform supports the
7265 mask, so callers may wish to test can_vec_perm_const_p separately, or use
7266 vect_gen_perm_mask_checked. */
7269 vect_gen_perm_mask_any (tree vectype
, const vec_perm_indices
&sel
)
7273 poly_uint64 nunits
= sel
.length ();
7274 gcc_assert (known_eq (nunits
, TYPE_VECTOR_SUBPARTS (vectype
)));
7276 mask_type
= build_vector_type (ssizetype
, nunits
);
7277 return vec_perm_indices_to_tree (mask_type
, sel
);
7280 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
7281 i.e. that the target supports the pattern _for arbitrary input vectors_. */
7284 vect_gen_perm_mask_checked (tree vectype
, const vec_perm_indices
&sel
)
7286 gcc_assert (can_vec_perm_const_p (TYPE_MODE (vectype
), sel
));
7287 return vect_gen_perm_mask_any (vectype
, sel
);
7290 /* Given a vector variable X and Y, that was generated for the scalar
7291 STMT_INFO, generate instructions to permute the vector elements of X and Y
7292 using permutation mask MASK_VEC, insert them at *GSI and return the
7293 permuted vector variable. */
7296 permute_vec_elements (tree x
, tree y
, tree mask_vec
, stmt_vec_info stmt_info
,
7297 gimple_stmt_iterator
*gsi
)
7299 tree vectype
= TREE_TYPE (x
);
7300 tree perm_dest
, data_ref
;
7303 tree scalar_dest
= gimple_get_lhs (stmt_info
->stmt
);
7304 if (TREE_CODE (scalar_dest
) == SSA_NAME
)
7305 perm_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7307 perm_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, NULL
);
7308 data_ref
= make_ssa_name (perm_dest
);
7310 /* Generate the permute statement. */
7311 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
7312 vect_finish_stmt_generation (stmt_info
, perm_stmt
, gsi
);
7317 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
7318 inserting them on the loops preheader edge. Returns true if we
7319 were successful in doing so (and thus STMT_INFO can be moved then),
7320 otherwise returns false. */
7323 hoist_defs_of_uses (stmt_vec_info stmt_info
, struct loop
*loop
)
7329 FOR_EACH_SSA_TREE_OPERAND (op
, stmt_info
->stmt
, i
, SSA_OP_USE
)
7331 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
7332 if (!gimple_nop_p (def_stmt
)
7333 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
7335 /* Make sure we don't need to recurse. While we could do
7336 so in simple cases when there are more complex use webs
7337 we don't have an easy way to preserve stmt order to fulfil
7338 dependencies within them. */
7341 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
7343 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
7345 gimple
*def_stmt2
= SSA_NAME_DEF_STMT (op2
);
7346 if (!gimple_nop_p (def_stmt2
)
7347 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
7357 FOR_EACH_SSA_TREE_OPERAND (op
, stmt_info
->stmt
, i
, SSA_OP_USE
)
7359 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
7360 if (!gimple_nop_p (def_stmt
)
7361 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
7363 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
7364 gsi_remove (&gsi
, false);
7365 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
7372 /* vectorizable_load.
7374 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
7375 that can be vectorized.
7376 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
7377 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
7378 Return true if STMT_INFO is vectorizable in this way. */
7381 vectorizable_load (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
7382 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
7383 slp_instance slp_node_instance
,
7384 stmt_vector_for_cost
*cost_vec
)
7387 tree vec_dest
= NULL
;
7388 tree data_ref
= NULL
;
7389 stmt_vec_info prev_stmt_info
;
7390 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7391 struct loop
*loop
= NULL
;
7392 struct loop
*containing_loop
= gimple_bb (stmt_info
->stmt
)->loop_father
;
7393 bool nested_in_vect_loop
= false;
7398 enum dr_alignment_support alignment_support_scheme
;
7399 tree dataref_ptr
= NULL_TREE
;
7400 tree dataref_offset
= NULL_TREE
;
7401 gimple
*ptr_incr
= NULL
;
7404 unsigned int group_size
;
7405 poly_uint64 group_gap_adj
;
7406 tree msq
= NULL_TREE
, lsq
;
7407 tree offset
= NULL_TREE
;
7408 tree byte_offset
= NULL_TREE
;
7409 tree realignment_token
= NULL_TREE
;
7411 vec
<tree
> dr_chain
= vNULL
;
7412 bool grouped_load
= false;
7413 stmt_vec_info first_stmt_info
;
7414 stmt_vec_info first_stmt_info_for_drptr
= NULL
;
7415 bool compute_in_loop
= false;
7416 struct loop
*at_loop
;
7418 bool slp
= (slp_node
!= NULL
);
7419 bool slp_perm
= false;
7420 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7423 gather_scatter_info gs_info
;
7424 vec_info
*vinfo
= stmt_info
->vinfo
;
7426 enum vect_def_type mask_dt
= vect_unknown_def_type
;
7428 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7431 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7435 tree mask
= NULL_TREE
, mask_vectype
= NULL_TREE
;
7436 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt_info
->stmt
))
7438 scalar_dest
= gimple_assign_lhs (assign
);
7439 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
7442 tree_code code
= gimple_assign_rhs_code (assign
);
7443 if (code
!= ARRAY_REF
7444 && code
!= BIT_FIELD_REF
7445 && code
!= INDIRECT_REF
7446 && code
!= COMPONENT_REF
7447 && code
!= IMAGPART_EXPR
7448 && code
!= REALPART_EXPR
7450 && TREE_CODE_CLASS (code
) != tcc_declaration
)
7455 gcall
*call
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
7456 if (!call
|| !gimple_call_internal_p (call
))
7459 internal_fn ifn
= gimple_call_internal_fn (call
);
7460 if (!internal_load_fn_p (ifn
))
7463 scalar_dest
= gimple_call_lhs (call
);
7467 if (slp_node
!= NULL
)
7469 if (dump_enabled_p ())
7470 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7471 "SLP of masked loads not supported.\n");
7475 int mask_index
= internal_fn_mask_index (ifn
);
7476 if (mask_index
>= 0)
7478 mask
= gimple_call_arg (call
, mask_index
);
7479 if (!vect_check_load_store_mask (stmt_info
, mask
, &mask_dt
,
7485 if (!STMT_VINFO_DATA_REF (stmt_info
))
7488 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7489 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7493 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7494 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt_info
);
7495 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
7500 /* Multiple types in SLP are handled by creating the appropriate number of
7501 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
7506 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
7508 gcc_assert (ncopies
>= 1);
7510 /* FORNOW. This restriction should be relaxed. */
7511 if (nested_in_vect_loop
&& ncopies
> 1)
7513 if (dump_enabled_p ())
7514 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7515 "multiple types in nested loop.\n");
7519 /* Invalidate assumptions made by dependence analysis when vectorization
7520 on the unrolled body effectively re-orders stmts. */
7522 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
7523 && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo
),
7524 STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
7526 if (dump_enabled_p ())
7527 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7528 "cannot perform implicit CSE when unrolling "
7529 "with negative dependence distance\n");
7533 elem_type
= TREE_TYPE (vectype
);
7534 mode
= TYPE_MODE (vectype
);
7536 /* FORNOW. In some cases can vectorize even if data-type not supported
7537 (e.g. - data copies). */
7538 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
7540 if (dump_enabled_p ())
7541 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7542 "Aligned load, but unsupported type.\n");
7546 /* Check if the load is a part of an interleaving chain. */
7547 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
7549 grouped_load
= true;
7551 gcc_assert (!nested_in_vect_loop
);
7552 gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info
));
7554 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
7555 group_size
= DR_GROUP_SIZE (first_stmt_info
);
7557 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
7560 /* Invalidate assumptions made by dependence analysis when vectorization
7561 on the unrolled body effectively re-orders stmts. */
7562 if (!PURE_SLP_STMT (stmt_info
)
7563 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
7564 && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo
),
7565 STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
7567 if (dump_enabled_p ())
7568 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7569 "cannot perform implicit CSE when performing "
7570 "group loads with negative dependence distance\n");
7574 /* Similarly when the stmt is a load that is both part of a SLP
7575 instance and a loop vectorized stmt via the same-dr mechanism
7576 we have to give up. */
7577 if (DR_GROUP_SAME_DR_STMT (stmt_info
)
7578 && (STMT_SLP_TYPE (stmt_info
)
7579 != STMT_SLP_TYPE (DR_GROUP_SAME_DR_STMT (stmt_info
))))
7581 if (dump_enabled_p ())
7582 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7583 "conflicting SLP types for CSEd load\n");
7590 vect_memory_access_type memory_access_type
;
7591 if (!get_load_store_type (stmt_info
, vectype
, slp
, mask
, VLS_LOAD
, ncopies
,
7592 &memory_access_type
, &gs_info
))
7597 if (memory_access_type
== VMAT_CONTIGUOUS
)
7599 machine_mode vec_mode
= TYPE_MODE (vectype
);
7600 if (!VECTOR_MODE_P (vec_mode
)
7601 || !can_vec_mask_load_store_p (vec_mode
,
7602 TYPE_MODE (mask_vectype
), true))
7605 else if (memory_access_type
== VMAT_GATHER_SCATTER
&& gs_info
.decl
)
7607 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gs_info
.decl
));
7609 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
7610 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
7612 if (dump_enabled_p ())
7613 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7614 "masked gather with integer mask not"
7619 else if (memory_access_type
!= VMAT_LOAD_STORE_LANES
7620 && memory_access_type
!= VMAT_GATHER_SCATTER
)
7622 if (dump_enabled_p ())
7623 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7624 "unsupported access type for masked load.\n");
7629 if (!vec_stmt
) /* transformation not required. */
7632 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
) = memory_access_type
;
7635 && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
))
7636 check_load_store_masking (loop_vinfo
, vectype
, VLS_LOAD
, group_size
,
7637 memory_access_type
, &gs_info
);
7639 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
7640 vect_model_load_cost (stmt_info
, ncopies
, memory_access_type
,
7641 slp_node_instance
, slp_node
, cost_vec
);
7646 gcc_assert (memory_access_type
7647 == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info
));
7649 if (dump_enabled_p ())
7650 dump_printf_loc (MSG_NOTE
, vect_location
,
7651 "transform load. ncopies = %d\n", ncopies
);
7655 dr_vec_info
*dr_info
= STMT_VINFO_DR_INFO (stmt_info
), *first_dr_info
= NULL
;
7656 ensure_base_align (dr_info
);
7658 if (memory_access_type
== VMAT_GATHER_SCATTER
&& gs_info
.decl
)
7660 vect_build_gather_load_calls (stmt_info
, gsi
, vec_stmt
, &gs_info
, mask
);
7664 if (memory_access_type
== VMAT_INVARIANT
)
7666 gcc_assert (!grouped_load
&& !mask
&& !bb_vinfo
);
7667 /* If we have versioned for aliasing or the loop doesn't
7668 have any data dependencies that would preclude this,
7669 then we are sure this is a loop invariant load and
7670 thus we can insert it on the preheader edge. */
7671 bool hoist_p
= (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
7672 && !nested_in_vect_loop
7673 && hoist_defs_of_uses (stmt_info
, loop
));
7676 gassign
*stmt
= as_a
<gassign
*> (stmt_info
->stmt
);
7677 if (dump_enabled_p ())
7679 dump_printf_loc (MSG_NOTE
, vect_location
,
7680 "hoisting out of the vectorized loop: ");
7681 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7683 scalar_dest
= copy_ssa_name (scalar_dest
);
7684 tree rhs
= unshare_expr (gimple_assign_rhs1 (stmt
));
7685 gsi_insert_on_edge_immediate
7686 (loop_preheader_edge (loop
),
7687 gimple_build_assign (scalar_dest
, rhs
));
7689 /* These copies are all equivalent, but currently the representation
7690 requires a separate STMT_VINFO_VEC_STMT for each one. */
7691 prev_stmt_info
= NULL
;
7692 gimple_stmt_iterator gsi2
= *gsi
;
7694 for (j
= 0; j
< ncopies
; j
++)
7696 stmt_vec_info new_stmt_info
;
7699 new_temp
= vect_init_vector (stmt_info
, scalar_dest
,
7701 gimple
*new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7702 new_stmt_info
= vinfo
->add_stmt (new_stmt
);
7706 new_temp
= vect_init_vector (stmt_info
, scalar_dest
,
7708 new_stmt_info
= vinfo
->lookup_def (new_temp
);
7711 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
7713 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
7715 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
7716 prev_stmt_info
= new_stmt_info
;
7721 if (memory_access_type
== VMAT_ELEMENTWISE
7722 || memory_access_type
== VMAT_STRIDED_SLP
)
7724 gimple_stmt_iterator incr_gsi
;
7730 vec
<constructor_elt
, va_gc
> *v
= NULL
;
7731 tree stride_base
, stride_step
, alias_off
;
7732 /* Checked by get_load_store_type. */
7733 unsigned int const_nunits
= nunits
.to_constant ();
7734 unsigned HOST_WIDE_INT cst_offset
= 0;
7736 gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
));
7737 gcc_assert (!nested_in_vect_loop
);
7741 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
7742 first_dr_info
= STMT_VINFO_DR_INFO (first_stmt_info
);
7746 first_stmt_info
= stmt_info
;
7747 first_dr_info
= dr_info
;
7749 if (slp
&& grouped_load
)
7751 group_size
= DR_GROUP_SIZE (first_stmt_info
);
7752 ref_type
= get_group_alias_ptr_type (first_stmt_info
);
7758 = (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)))
7759 * vect_get_place_in_interleaving_chain (stmt_info
,
7762 ref_type
= reference_alias_ptr_type (DR_REF (dr_info
->dr
));
7766 = fold_build_pointer_plus
7767 (DR_BASE_ADDRESS (first_dr_info
->dr
),
7768 size_binop (PLUS_EXPR
,
7769 convert_to_ptrofftype (DR_OFFSET (first_dr_info
->dr
)),
7770 convert_to_ptrofftype (DR_INIT (first_dr_info
->dr
))));
7771 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr_info
->dr
));
7773 /* For a load with loop-invariant (but other than power-of-2)
7774 stride (i.e. not a grouped access) like so:
7776 for (i = 0; i < n; i += stride)
7779 we generate a new induction variable and new accesses to
7780 form a new vector (or vectors, depending on ncopies):
7782 for (j = 0; ; j += VF*stride)
7784 tmp2 = array[j + stride];
7786 vectemp = {tmp1, tmp2, ...}
7789 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
7790 build_int_cst (TREE_TYPE (stride_step
), vf
));
7792 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
7794 stride_base
= cse_and_gimplify_to_preheader (loop_vinfo
, stride_base
);
7795 ivstep
= cse_and_gimplify_to_preheader (loop_vinfo
, ivstep
);
7796 create_iv (stride_base
, ivstep
, NULL
,
7797 loop
, &incr_gsi
, insert_after
,
7799 incr
= gsi_stmt (incr_gsi
);
7800 loop_vinfo
->add_stmt (incr
);
7802 stride_step
= cse_and_gimplify_to_preheader (loop_vinfo
, stride_step
);
7804 prev_stmt_info
= NULL
;
7805 running_off
= offvar
;
7806 alias_off
= build_int_cst (ref_type
, 0);
7807 int nloads
= const_nunits
;
7809 tree ltype
= TREE_TYPE (vectype
);
7810 tree lvectype
= vectype
;
7811 auto_vec
<tree
> dr_chain
;
7812 if (memory_access_type
== VMAT_STRIDED_SLP
)
7814 if (group_size
< const_nunits
)
7816 /* First check if vec_init optab supports construction from
7817 vector elts directly. */
7818 scalar_mode elmode
= SCALAR_TYPE_MODE (TREE_TYPE (vectype
));
7820 if (mode_for_vector (elmode
, group_size
).exists (&vmode
)
7821 && VECTOR_MODE_P (vmode
)
7822 && targetm
.vector_mode_supported_p (vmode
)
7823 && (convert_optab_handler (vec_init_optab
,
7824 TYPE_MODE (vectype
), vmode
)
7825 != CODE_FOR_nothing
))
7827 nloads
= const_nunits
/ group_size
;
7829 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
7833 /* Otherwise avoid emitting a constructor of vector elements
7834 by performing the loads using an integer type of the same
7835 size, constructing a vector of those and then
7836 re-interpreting it as the original vector type.
7837 This avoids a huge runtime penalty due to the general
7838 inability to perform store forwarding from smaller stores
7839 to a larger load. */
7841 = group_size
* TYPE_PRECISION (TREE_TYPE (vectype
));
7842 elmode
= int_mode_for_size (lsize
, 0).require ();
7843 unsigned int lnunits
= const_nunits
/ group_size
;
7844 /* If we can't construct such a vector fall back to
7845 element loads of the original vector type. */
7846 if (mode_for_vector (elmode
, lnunits
).exists (&vmode
)
7847 && VECTOR_MODE_P (vmode
)
7848 && targetm
.vector_mode_supported_p (vmode
)
7849 && (convert_optab_handler (vec_init_optab
, vmode
, elmode
)
7850 != CODE_FOR_nothing
))
7854 ltype
= build_nonstandard_integer_type (lsize
, 1);
7855 lvectype
= build_vector_type (ltype
, nloads
);
7862 lnel
= const_nunits
;
7865 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
7867 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
7868 else if (nloads
== 1)
7873 /* For SLP permutation support we need to load the whole group,
7874 not only the number of vector stmts the permutation result
7878 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
7880 unsigned int const_vf
= vf
.to_constant ();
7881 ncopies
= CEIL (group_size
* const_vf
, const_nunits
);
7882 dr_chain
.create (ncopies
);
7885 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
7887 unsigned int group_el
= 0;
7888 unsigned HOST_WIDE_INT
7889 elsz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype
)));
7890 for (j
= 0; j
< ncopies
; j
++)
7893 vec_alloc (v
, nloads
);
7894 stmt_vec_info new_stmt_info
= NULL
;
7895 for (i
= 0; i
< nloads
; i
++)
7897 tree this_off
= build_int_cst (TREE_TYPE (alias_off
),
7898 group_el
* elsz
+ cst_offset
);
7899 tree data_ref
= build2 (MEM_REF
, ltype
, running_off
, this_off
);
7900 vect_copy_ref_info (data_ref
, DR_REF (first_dr_info
->dr
));
7902 = gimple_build_assign (make_ssa_name (ltype
), data_ref
);
7904 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
7906 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
,
7907 gimple_assign_lhs (new_stmt
));
7911 || group_el
== group_size
)
7913 tree newoff
= copy_ssa_name (running_off
);
7914 gimple
*incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
7915 running_off
, stride_step
);
7916 vect_finish_stmt_generation (stmt_info
, incr
, gsi
);
7918 running_off
= newoff
;
7924 tree vec_inv
= build_constructor (lvectype
, v
);
7925 new_temp
= vect_init_vector (stmt_info
, vec_inv
, lvectype
, gsi
);
7926 new_stmt_info
= vinfo
->lookup_def (new_temp
);
7927 if (lvectype
!= vectype
)
7930 = gimple_build_assign (make_ssa_name (vectype
),
7932 build1 (VIEW_CONVERT_EXPR
,
7933 vectype
, new_temp
));
7935 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
7942 dr_chain
.quick_push (gimple_assign_lhs (new_stmt_info
->stmt
));
7944 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
7949 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
7951 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
7952 prev_stmt_info
= new_stmt_info
;
7958 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
7959 slp_node_instance
, false, &n_perms
);
7964 if (memory_access_type
== VMAT_GATHER_SCATTER
7965 || (!slp
&& memory_access_type
== VMAT_CONTIGUOUS
))
7966 grouped_load
= false;
7970 first_stmt_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
7971 group_size
= DR_GROUP_SIZE (first_stmt_info
);
7972 /* For SLP vectorization we directly vectorize a subchain
7973 without permutation. */
7974 if (slp
&& ! SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
7975 first_stmt_info
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
7976 /* For BB vectorization always use the first stmt to base
7977 the data ref pointer on. */
7979 first_stmt_info_for_drptr
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
7981 /* Check if the chain of loads is already vectorized. */
7982 if (STMT_VINFO_VEC_STMT (first_stmt_info
)
7983 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
7984 ??? But we can only do so if there is exactly one
7985 as we have no way to get at the rest. Leave the CSE
7987 ??? With the group load eventually participating
7988 in multiple different permutations (having multiple
7989 slp nodes which refer to the same group) the CSE
7990 is even wrong code. See PR56270. */
7993 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7996 first_dr_info
= STMT_VINFO_DR_INFO (first_stmt_info
);
7999 /* VEC_NUM is the number of vect stmts to be created for this group. */
8002 grouped_load
= false;
8003 /* For SLP permutation support we need to load the whole group,
8004 not only the number of vector stmts the permutation result
8008 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
8010 unsigned int const_vf
= vf
.to_constant ();
8011 unsigned int const_nunits
= nunits
.to_constant ();
8012 vec_num
= CEIL (group_size
* const_vf
, const_nunits
);
8013 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
8017 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
8019 = group_size
- SLP_INSTANCE_GROUP_SIZE (slp_node_instance
);
8023 vec_num
= group_size
;
8025 ref_type
= get_group_alias_ptr_type (first_stmt_info
);
8029 first_stmt_info
= stmt_info
;
8030 first_dr_info
= dr_info
;
8031 group_size
= vec_num
= 1;
8033 ref_type
= reference_alias_ptr_type (DR_REF (first_dr_info
->dr
));
8036 alignment_support_scheme
8037 = vect_supportable_dr_alignment (first_dr_info
, false);
8038 gcc_assert (alignment_support_scheme
);
8039 vec_loop_masks
*loop_masks
8040 = (loop_vinfo
&& LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
8041 ? &LOOP_VINFO_MASKS (loop_vinfo
)
8043 /* Targets with store-lane instructions must not require explicit
8044 realignment. vect_supportable_dr_alignment always returns either
8045 dr_aligned or dr_unaligned_supported for masked operations. */
8046 gcc_assert ((memory_access_type
!= VMAT_LOAD_STORE_LANES
8049 || alignment_support_scheme
== dr_aligned
8050 || alignment_support_scheme
== dr_unaligned_supported
);
8052 /* In case the vectorization factor (VF) is bigger than the number
8053 of elements that we can fit in a vectype (nunits), we have to generate
8054 more than one vector stmt - i.e - we need to "unroll" the
8055 vector stmt by a factor VF/nunits. In doing so, we record a pointer
8056 from one copy of the vector stmt to the next, in the field
8057 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
8058 stages to find the correct vector defs to be used when vectorizing
8059 stmts that use the defs of the current stmt. The example below
8060 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
8061 need to create 4 vectorized stmts):
8063 before vectorization:
8064 RELATED_STMT VEC_STMT
8068 step 1: vectorize stmt S1:
8069 We first create the vector stmt VS1_0, and, as usual, record a
8070 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
8071 Next, we create the vector stmt VS1_1, and record a pointer to
8072 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
8073 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
8075 RELATED_STMT VEC_STMT
8076 VS1_0: vx0 = memref0 VS1_1 -
8077 VS1_1: vx1 = memref1 VS1_2 -
8078 VS1_2: vx2 = memref2 VS1_3 -
8079 VS1_3: vx3 = memref3 - -
8080 S1: x = load - VS1_0
8083 See in documentation in vect_get_vec_def_for_stmt_copy for how the
8084 information we recorded in RELATED_STMT field is used to vectorize
8087 /* In case of interleaving (non-unit grouped access):
8094 Vectorized loads are created in the order of memory accesses
8095 starting from the access of the first stmt of the chain:
8098 VS2: vx1 = &base + vec_size*1
8099 VS3: vx3 = &base + vec_size*2
8100 VS4: vx4 = &base + vec_size*3
8102 Then permutation statements are generated:
8104 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
8105 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
8108 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8109 (the order of the data-refs in the output of vect_permute_load_chain
8110 corresponds to the order of scalar stmts in the interleaving chain - see
8111 the documentation of vect_permute_load_chain()).
8112 The generation of permutation stmts and recording them in
8113 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
8115 In case of both multiple types and interleaving, the vector loads and
8116 permutation stmts above are created for every copy. The result vector
8117 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
8118 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
8120 /* If the data reference is aligned (dr_aligned) or potentially unaligned
8121 on a target that supports unaligned accesses (dr_unaligned_supported)
8122 we generate the following code:
8126 p = p + indx * vectype_size;
8131 Otherwise, the data reference is potentially unaligned on a target that
8132 does not support unaligned accesses (dr_explicit_realign_optimized) -
8133 then generate the following code, in which the data in each iteration is
8134 obtained by two vector loads, one from the previous iteration, and one
8135 from the current iteration:
8137 msq_init = *(floor(p1))
8138 p2 = initial_addr + VS - 1;
8139 realignment_token = call target_builtin;
8142 p2 = p2 + indx * vectype_size
8144 vec_dest = realign_load (msq, lsq, realignment_token)
8149 /* If the misalignment remains the same throughout the execution of the
8150 loop, we can create the init_addr and permutation mask at the loop
8151 preheader. Otherwise, it needs to be created inside the loop.
8152 This can only occur when vectorizing memory accesses in the inner-loop
8153 nested within an outer-loop that is being vectorized. */
8155 if (nested_in_vect_loop
8156 && !multiple_p (DR_STEP_ALIGNMENT (dr_info
->dr
),
8157 GET_MODE_SIZE (TYPE_MODE (vectype
))))
8159 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
8160 compute_in_loop
= true;
8163 if ((alignment_support_scheme
== dr_explicit_realign_optimized
8164 || alignment_support_scheme
== dr_explicit_realign
)
8165 && !compute_in_loop
)
8167 msq
= vect_setup_realignment (first_stmt_info
, gsi
, &realignment_token
,
8168 alignment_support_scheme
, NULL_TREE
,
8170 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
8172 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
8173 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
8180 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
8181 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
8184 tree vec_offset
= NULL_TREE
;
8185 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
8187 aggr_type
= NULL_TREE
;
8190 else if (memory_access_type
== VMAT_GATHER_SCATTER
)
8192 aggr_type
= elem_type
;
8193 vect_get_strided_load_store_ops (stmt_info
, loop_vinfo
, &gs_info
,
8194 &bump
, &vec_offset
);
8198 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
8199 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
8201 aggr_type
= vectype
;
8202 bump
= vect_get_data_ptr_increment (dr_info
, aggr_type
,
8203 memory_access_type
);
8206 tree vec_mask
= NULL_TREE
;
8207 prev_stmt_info
= NULL
;
8208 poly_uint64 group_elt
= 0;
8209 for (j
= 0; j
< ncopies
; j
++)
8211 stmt_vec_info new_stmt_info
= NULL
;
8212 /* 1. Create the vector or array pointer update chain. */
8215 bool simd_lane_access_p
8216 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
8217 if (simd_lane_access_p
8218 && TREE_CODE (DR_BASE_ADDRESS (first_dr_info
->dr
)) == ADDR_EXPR
8219 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info
->dr
), 0))
8220 && integer_zerop (DR_OFFSET (first_dr_info
->dr
))
8221 && integer_zerop (DR_INIT (first_dr_info
->dr
))
8222 && alias_sets_conflict_p (get_alias_set (aggr_type
),
8223 get_alias_set (TREE_TYPE (ref_type
)))
8224 && (alignment_support_scheme
== dr_aligned
8225 || alignment_support_scheme
== dr_unaligned_supported
))
8227 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr_info
->dr
));
8228 dataref_offset
= build_int_cst (ref_type
, 0);
8230 else if (first_stmt_info_for_drptr
8231 && first_stmt_info
!= first_stmt_info_for_drptr
)
8234 = vect_create_data_ref_ptr (first_stmt_info_for_drptr
,
8235 aggr_type
, at_loop
, offset
, &dummy
,
8236 gsi
, &ptr_incr
, simd_lane_access_p
,
8238 /* Adjust the pointer by the difference to first_stmt. */
8239 data_reference_p ptrdr
8240 = STMT_VINFO_DATA_REF (first_stmt_info_for_drptr
);
8242 = fold_convert (sizetype
,
8243 size_binop (MINUS_EXPR
,
8244 DR_INIT (first_dr_info
->dr
),
8246 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
8249 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
8250 vect_get_gather_scatter_ops (loop
, stmt_info
, &gs_info
,
8251 &dataref_ptr
, &vec_offset
);
8254 = vect_create_data_ref_ptr (first_stmt_info
, aggr_type
, at_loop
,
8255 offset
, &dummy
, gsi
, &ptr_incr
,
8259 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt_info
,
8265 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
8267 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
8268 vec_offset
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_offset
);
8270 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
8273 vec_mask
= vect_get_vec_def_for_stmt_copy (vinfo
, vec_mask
);
8276 if (grouped_load
|| slp_perm
)
8277 dr_chain
.create (vec_num
);
8279 if (memory_access_type
== VMAT_LOAD_STORE_LANES
)
8283 vec_array
= create_vector_array (vectype
, vec_num
);
8285 tree final_mask
= NULL_TREE
;
8287 final_mask
= vect_get_loop_mask (gsi
, loop_masks
, ncopies
,
8290 final_mask
= prepare_load_store_mask (mask_vectype
, final_mask
,
8297 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
8299 unsigned int align
= TYPE_ALIGN_UNIT (TREE_TYPE (vectype
));
8300 tree alias_ptr
= build_int_cst (ref_type
, align
);
8301 call
= gimple_build_call_internal (IFN_MASK_LOAD_LANES
, 3,
8302 dataref_ptr
, alias_ptr
,
8308 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
8309 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, ref_type
);
8310 call
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
8312 gimple_call_set_lhs (call
, vec_array
);
8313 gimple_call_set_nothrow (call
, true);
8314 new_stmt_info
= vect_finish_stmt_generation (stmt_info
, call
, gsi
);
8316 /* Extract each vector into an SSA_NAME. */
8317 for (i
= 0; i
< vec_num
; i
++)
8319 new_temp
= read_vector_array (stmt_info
, gsi
, scalar_dest
,
8321 dr_chain
.quick_push (new_temp
);
8324 /* Record the mapping between SSA_NAMEs and statements. */
8325 vect_record_grouped_load_vectors (stmt_info
, dr_chain
);
8327 /* Record that VEC_ARRAY is now dead. */
8328 vect_clobber_variable (stmt_info
, gsi
, vec_array
);
8332 for (i
= 0; i
< vec_num
; i
++)
8334 tree final_mask
= NULL_TREE
;
8336 && memory_access_type
!= VMAT_INVARIANT
)
8337 final_mask
= vect_get_loop_mask (gsi
, loop_masks
,
8339 vectype
, vec_num
* j
+ i
);
8341 final_mask
= prepare_load_store_mask (mask_vectype
, final_mask
,
8345 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
8348 /* 2. Create the vector-load in the loop. */
8349 gimple
*new_stmt
= NULL
;
8350 switch (alignment_support_scheme
)
8353 case dr_unaligned_supported
:
8355 unsigned int align
, misalign
;
8357 if (memory_access_type
== VMAT_GATHER_SCATTER
)
8359 tree scale
= size_int (gs_info
.scale
);
8362 call
= gimple_build_call_internal
8363 (IFN_MASK_GATHER_LOAD
, 4, dataref_ptr
,
8364 vec_offset
, scale
, final_mask
);
8366 call
= gimple_build_call_internal
8367 (IFN_GATHER_LOAD
, 3, dataref_ptr
,
8369 gimple_call_set_nothrow (call
, true);
8371 data_ref
= NULL_TREE
;
8375 align
= DR_TARGET_ALIGNMENT (dr_info
);
8376 if (alignment_support_scheme
== dr_aligned
)
8378 gcc_assert (aligned_access_p (first_dr_info
));
8381 else if (DR_MISALIGNMENT (first_dr_info
) == -1)
8383 align
= dr_alignment
8384 (vect_dr_behavior (first_dr_info
));
8388 misalign
= DR_MISALIGNMENT (first_dr_info
);
8389 if (dataref_offset
== NULL_TREE
8390 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
8391 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
8396 align
= least_bit_hwi (misalign
| align
);
8397 tree ptr
= build_int_cst (ref_type
, align
);
8399 = gimple_build_call_internal (IFN_MASK_LOAD
, 3,
8402 gimple_call_set_nothrow (call
, true);
8404 data_ref
= NULL_TREE
;
8409 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
8412 : build_int_cst (ref_type
, 0));
8413 if (alignment_support_scheme
== dr_aligned
)
8415 else if (DR_MISALIGNMENT (first_dr_info
) == -1)
8416 TREE_TYPE (data_ref
)
8417 = build_aligned_type (TREE_TYPE (data_ref
),
8418 align
* BITS_PER_UNIT
);
8420 TREE_TYPE (data_ref
)
8421 = build_aligned_type (TREE_TYPE (data_ref
),
8422 TYPE_ALIGN (elem_type
));
8426 case dr_explicit_realign
:
8430 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
8432 if (compute_in_loop
)
8433 msq
= vect_setup_realignment (first_stmt_info
, gsi
,
8435 dr_explicit_realign
,
8438 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
8439 ptr
= copy_ssa_name (dataref_ptr
);
8441 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
8442 unsigned int align
= DR_TARGET_ALIGNMENT (first_dr_info
);
8443 new_stmt
= gimple_build_assign
8444 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
8446 (TREE_TYPE (dataref_ptr
),
8447 -(HOST_WIDE_INT
) align
));
8448 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8450 = build2 (MEM_REF
, vectype
, ptr
,
8451 build_int_cst (ref_type
, 0));
8452 vect_copy_ref_info (data_ref
, DR_REF (first_dr_info
->dr
));
8453 vec_dest
= vect_create_destination_var (scalar_dest
,
8455 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
8456 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
8457 gimple_assign_set_lhs (new_stmt
, new_temp
);
8458 gimple_set_vdef (new_stmt
, gimple_vdef (stmt_info
->stmt
));
8459 gimple_set_vuse (new_stmt
, gimple_vuse (stmt_info
->stmt
));
8460 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8463 bump
= size_binop (MULT_EXPR
, vs
,
8464 TYPE_SIZE_UNIT (elem_type
));
8465 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
8466 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
,
8468 new_stmt
= gimple_build_assign
8469 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
8471 (TREE_TYPE (ptr
), -(HOST_WIDE_INT
) align
));
8472 ptr
= copy_ssa_name (ptr
, new_stmt
);
8473 gimple_assign_set_lhs (new_stmt
, ptr
);
8474 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8476 = build2 (MEM_REF
, vectype
, ptr
,
8477 build_int_cst (ref_type
, 0));
8480 case dr_explicit_realign_optimized
:
8482 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
8483 new_temp
= copy_ssa_name (dataref_ptr
);
8485 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
8486 unsigned int align
= DR_TARGET_ALIGNMENT (first_dr_info
);
8487 new_stmt
= gimple_build_assign
8488 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
8489 build_int_cst (TREE_TYPE (dataref_ptr
),
8490 -(HOST_WIDE_INT
) align
));
8491 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8493 = build2 (MEM_REF
, vectype
, new_temp
,
8494 build_int_cst (ref_type
, 0));
8500 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
8501 /* DATA_REF is null if we've already built the statement. */
8504 vect_copy_ref_info (data_ref
, DR_REF (first_dr_info
->dr
));
8505 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
8507 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
8508 gimple_set_lhs (new_stmt
, new_temp
);
8510 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8512 /* 3. Handle explicit realignment if necessary/supported.
8514 vec_dest = realign_load (msq, lsq, realignment_token) */
8515 if (alignment_support_scheme
== dr_explicit_realign_optimized
8516 || alignment_support_scheme
== dr_explicit_realign
)
8518 lsq
= gimple_assign_lhs (new_stmt
);
8519 if (!realignment_token
)
8520 realignment_token
= dataref_ptr
;
8521 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
8522 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
8523 msq
, lsq
, realignment_token
);
8524 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
8525 gimple_assign_set_lhs (new_stmt
, new_temp
);
8527 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
8529 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
8532 if (i
== vec_num
- 1 && j
== ncopies
- 1)
8533 add_phi_arg (phi
, lsq
,
8534 loop_latch_edge (containing_loop
),
8540 if (memory_access_type
== VMAT_CONTIGUOUS_REVERSE
)
8542 tree perm_mask
= perm_mask_for_reverse (vectype
);
8543 new_temp
= permute_vec_elements (new_temp
, new_temp
,
8544 perm_mask
, stmt_info
, gsi
);
8545 new_stmt_info
= vinfo
->lookup_def (new_temp
);
8548 /* Collect vector loads and later create their permutation in
8549 vect_transform_grouped_load (). */
8550 if (grouped_load
|| slp_perm
)
8551 dr_chain
.quick_push (new_temp
);
8553 /* Store vector loads in the corresponding SLP_NODE. */
8554 if (slp
&& !slp_perm
)
8555 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
8557 /* With SLP permutation we load the gaps as well, without
8558 we need to skip the gaps after we manage to fully load
8559 all elements. group_gap_adj is DR_GROUP_SIZE here. */
8560 group_elt
+= nunits
;
8561 if (maybe_ne (group_gap_adj
, 0U)
8563 && known_eq (group_elt
, group_size
- group_gap_adj
))
8565 poly_wide_int bump_val
8566 = (wi::to_wide (TYPE_SIZE_UNIT (elem_type
))
8568 tree bump
= wide_int_to_tree (sizetype
, bump_val
);
8569 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
8574 /* Bump the vector pointer to account for a gap or for excess
8575 elements loaded for a permuted SLP load. */
8576 if (maybe_ne (group_gap_adj
, 0U) && slp_perm
)
8578 poly_wide_int bump_val
8579 = (wi::to_wide (TYPE_SIZE_UNIT (elem_type
))
8581 tree bump
= wide_int_to_tree (sizetype
, bump_val
);
8582 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
8587 if (slp
&& !slp_perm
)
8593 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
8594 slp_node_instance
, false,
8597 dr_chain
.release ();
8605 if (memory_access_type
!= VMAT_LOAD_STORE_LANES
)
8606 vect_transform_grouped_load (stmt_info
, dr_chain
,
8608 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
8613 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
8615 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
8616 prev_stmt_info
= new_stmt_info
;
8619 dr_chain
.release ();
8625 /* Function vect_is_simple_cond.
8628 LOOP - the loop that is being vectorized.
8629 COND - Condition that is checked for simple use.
8632 *COMP_VECTYPE - the vector type for the comparison.
8633 *DTS - The def types for the arguments of the comparison
8635 Returns whether a COND can be vectorized. Checks whether
8636 condition operands are supportable using vec_is_simple_use. */
8639 vect_is_simple_cond (tree cond
, vec_info
*vinfo
,
8640 tree
*comp_vectype
, enum vect_def_type
*dts
,
8644 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
8647 if (TREE_CODE (cond
) == SSA_NAME
8648 && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond
)))
8650 if (!vect_is_simple_use (cond
, vinfo
, &dts
[0], comp_vectype
)
8652 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype
))
8657 if (!COMPARISON_CLASS_P (cond
))
8660 lhs
= TREE_OPERAND (cond
, 0);
8661 rhs
= TREE_OPERAND (cond
, 1);
8663 if (TREE_CODE (lhs
) == SSA_NAME
)
8665 if (!vect_is_simple_use (lhs
, vinfo
, &dts
[0], &vectype1
))
8668 else if (TREE_CODE (lhs
) == INTEGER_CST
|| TREE_CODE (lhs
) == REAL_CST
8669 || TREE_CODE (lhs
) == FIXED_CST
)
8670 dts
[0] = vect_constant_def
;
8674 if (TREE_CODE (rhs
) == SSA_NAME
)
8676 if (!vect_is_simple_use (rhs
, vinfo
, &dts
[1], &vectype2
))
8679 else if (TREE_CODE (rhs
) == INTEGER_CST
|| TREE_CODE (rhs
) == REAL_CST
8680 || TREE_CODE (rhs
) == FIXED_CST
)
8681 dts
[1] = vect_constant_def
;
8685 if (vectype1
&& vectype2
8686 && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1
),
8687 TYPE_VECTOR_SUBPARTS (vectype2
)))
8690 *comp_vectype
= vectype1
? vectype1
: vectype2
;
8691 /* Invariant comparison. */
8692 if (! *comp_vectype
&& vectype
)
8694 tree scalar_type
= TREE_TYPE (lhs
);
8695 /* If we can widen the comparison to match vectype do so. */
8696 if (INTEGRAL_TYPE_P (scalar_type
)
8697 && tree_int_cst_lt (TYPE_SIZE (scalar_type
),
8698 TYPE_SIZE (TREE_TYPE (vectype
))))
8699 scalar_type
= build_nonstandard_integer_type
8700 (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype
))),
8701 TYPE_UNSIGNED (scalar_type
));
8702 *comp_vectype
= get_vectype_for_scalar_type (scalar_type
);
8708 /* vectorizable_condition.
8710 Check if STMT_INFO is conditional modify expression that can be vectorized.
8711 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8712 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
8715 When STMT_INFO is vectorized as a nested cycle, REDUC_DEF is the vector
8716 variable to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1,
8717 and in else clause if it is 2).
8719 Return true if STMT_INFO is vectorizable in this way. */
8722 vectorizable_condition (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
8723 stmt_vec_info
*vec_stmt
, tree reduc_def
,
8724 int reduc_index
, slp_tree slp_node
,
8725 stmt_vector_for_cost
*cost_vec
)
8727 vec_info
*vinfo
= stmt_info
->vinfo
;
8728 tree scalar_dest
= NULL_TREE
;
8729 tree vec_dest
= NULL_TREE
;
8730 tree cond_expr
, cond_expr0
= NULL_TREE
, cond_expr1
= NULL_TREE
;
8731 tree then_clause
, else_clause
;
8732 tree comp_vectype
= NULL_TREE
;
8733 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
8734 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
8737 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8738 enum vect_def_type dts
[4]
8739 = {vect_unknown_def_type
, vect_unknown_def_type
,
8740 vect_unknown_def_type
, vect_unknown_def_type
};
8743 enum tree_code code
, cond_code
, bitop1
= NOP_EXPR
, bitop2
= NOP_EXPR
;
8744 stmt_vec_info prev_stmt_info
= NULL
;
8746 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
8747 vec
<tree
> vec_oprnds0
= vNULL
;
8748 vec
<tree
> vec_oprnds1
= vNULL
;
8749 vec
<tree
> vec_oprnds2
= vNULL
;
8750 vec
<tree
> vec_oprnds3
= vNULL
;
8752 bool masked
= false;
8754 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
8757 vect_reduction_type reduction_type
8758 = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info
);
8759 if (reduction_type
== TREE_CODE_REDUCTION
)
8761 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
8764 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
8765 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
8769 /* FORNOW: not yet supported. */
8770 if (STMT_VINFO_LIVE_P (stmt_info
))
8772 if (dump_enabled_p ())
8773 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8774 "value used after loop.\n");
8779 /* Is vectorizable conditional operation? */
8780 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
8784 code
= gimple_assign_rhs_code (stmt
);
8786 if (code
!= COND_EXPR
)
8789 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8790 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
8795 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
8797 gcc_assert (ncopies
>= 1);
8798 if (reduc_index
&& ncopies
> 1)
8799 return false; /* FORNOW */
8801 cond_expr
= gimple_assign_rhs1 (stmt
);
8802 then_clause
= gimple_assign_rhs2 (stmt
);
8803 else_clause
= gimple_assign_rhs3 (stmt
);
8805 if (!vect_is_simple_cond (cond_expr
, stmt_info
->vinfo
,
8806 &comp_vectype
, &dts
[0], slp_node
? NULL
: vectype
)
8810 if (!vect_is_simple_use (then_clause
, stmt_info
->vinfo
, &dts
[2], &vectype1
))
8812 if (!vect_is_simple_use (else_clause
, stmt_info
->vinfo
, &dts
[3], &vectype2
))
8815 if (vectype1
&& !useless_type_conversion_p (vectype
, vectype1
))
8818 if (vectype2
&& !useless_type_conversion_p (vectype
, vectype2
))
8821 masked
= !COMPARISON_CLASS_P (cond_expr
);
8822 vec_cmp_type
= build_same_sized_truth_vector_type (comp_vectype
);
8824 if (vec_cmp_type
== NULL_TREE
)
8827 cond_code
= TREE_CODE (cond_expr
);
8830 cond_expr0
= TREE_OPERAND (cond_expr
, 0);
8831 cond_expr1
= TREE_OPERAND (cond_expr
, 1);
8834 if (!masked
&& VECTOR_BOOLEAN_TYPE_P (comp_vectype
))
8836 /* Boolean values may have another representation in vectors
8837 and therefore we prefer bit operations over comparison for
8838 them (which also works for scalar masks). We store opcodes
8839 to use in bitop1 and bitop2. Statement is vectorized as
8840 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
8841 depending on bitop1 and bitop2 arity. */
8845 bitop1
= BIT_NOT_EXPR
;
8846 bitop2
= BIT_AND_EXPR
;
8849 bitop1
= BIT_NOT_EXPR
;
8850 bitop2
= BIT_IOR_EXPR
;
8853 bitop1
= BIT_NOT_EXPR
;
8854 bitop2
= BIT_AND_EXPR
;
8855 std::swap (cond_expr0
, cond_expr1
);
8858 bitop1
= BIT_NOT_EXPR
;
8859 bitop2
= BIT_IOR_EXPR
;
8860 std::swap (cond_expr0
, cond_expr1
);
8863 bitop1
= BIT_XOR_EXPR
;
8866 bitop1
= BIT_XOR_EXPR
;
8867 bitop2
= BIT_NOT_EXPR
;
8872 cond_code
= SSA_NAME
;
8877 if (bitop1
!= NOP_EXPR
)
8879 machine_mode mode
= TYPE_MODE (comp_vectype
);
8882 optab
= optab_for_tree_code (bitop1
, comp_vectype
, optab_default
);
8883 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8886 if (bitop2
!= NOP_EXPR
)
8888 optab
= optab_for_tree_code (bitop2
, comp_vectype
,
8890 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
8894 if (expand_vec_cond_expr_p (vectype
, comp_vectype
,
8897 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
8898 vect_model_simple_cost (stmt_info
, ncopies
, dts
, ndts
, slp_node
,
8909 vec_oprnds0
.create (1);
8910 vec_oprnds1
.create (1);
8911 vec_oprnds2
.create (1);
8912 vec_oprnds3
.create (1);
8916 scalar_dest
= gimple_assign_lhs (stmt
);
8917 if (reduction_type
!= EXTRACT_LAST_REDUCTION
)
8918 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
8920 /* Handle cond expr. */
8921 for (j
= 0; j
< ncopies
; j
++)
8923 stmt_vec_info new_stmt_info
= NULL
;
8928 auto_vec
<tree
, 4> ops
;
8929 auto_vec
<vec
<tree
>, 4> vec_defs
;
8932 ops
.safe_push (cond_expr
);
8935 ops
.safe_push (cond_expr0
);
8936 ops
.safe_push (cond_expr1
);
8938 ops
.safe_push (then_clause
);
8939 ops
.safe_push (else_clause
);
8940 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
8941 vec_oprnds3
= vec_defs
.pop ();
8942 vec_oprnds2
= vec_defs
.pop ();
8944 vec_oprnds1
= vec_defs
.pop ();
8945 vec_oprnds0
= vec_defs
.pop ();
8952 = vect_get_vec_def_for_operand (cond_expr
, stmt_info
,
8954 vect_is_simple_use (cond_expr
, stmt_info
->vinfo
, &dts
[0]);
8959 = vect_get_vec_def_for_operand (cond_expr0
,
8960 stmt_info
, comp_vectype
);
8961 vect_is_simple_use (cond_expr0
, loop_vinfo
, &dts
[0]);
8964 = vect_get_vec_def_for_operand (cond_expr1
,
8965 stmt_info
, comp_vectype
);
8966 vect_is_simple_use (cond_expr1
, loop_vinfo
, &dts
[1]);
8968 if (reduc_index
== 1)
8969 vec_then_clause
= reduc_def
;
8972 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
8974 vect_is_simple_use (then_clause
, loop_vinfo
, &dts
[2]);
8976 if (reduc_index
== 2)
8977 vec_else_clause
= reduc_def
;
8980 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
8982 vect_is_simple_use (else_clause
, loop_vinfo
, &dts
[3]);
8989 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnds0
.pop ());
8992 = vect_get_vec_def_for_stmt_copy (vinfo
, vec_oprnds1
.pop ());
8994 vec_then_clause
= vect_get_vec_def_for_stmt_copy (vinfo
,
8995 vec_oprnds2
.pop ());
8996 vec_else_clause
= vect_get_vec_def_for_stmt_copy (vinfo
,
8997 vec_oprnds3
.pop ());
9002 vec_oprnds0
.quick_push (vec_cond_lhs
);
9004 vec_oprnds1
.quick_push (vec_cond_rhs
);
9005 vec_oprnds2
.quick_push (vec_then_clause
);
9006 vec_oprnds3
.quick_push (vec_else_clause
);
9009 /* Arguments are ready. Create the new vector stmt. */
9010 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
9012 vec_then_clause
= vec_oprnds2
[i
];
9013 vec_else_clause
= vec_oprnds3
[i
];
9016 vec_compare
= vec_cond_lhs
;
9019 vec_cond_rhs
= vec_oprnds1
[i
];
9020 if (bitop1
== NOP_EXPR
)
9021 vec_compare
= build2 (cond_code
, vec_cmp_type
,
9022 vec_cond_lhs
, vec_cond_rhs
);
9025 new_temp
= make_ssa_name (vec_cmp_type
);
9027 if (bitop1
== BIT_NOT_EXPR
)
9028 new_stmt
= gimple_build_assign (new_temp
, bitop1
,
9032 = gimple_build_assign (new_temp
, bitop1
, vec_cond_lhs
,
9034 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9035 if (bitop2
== NOP_EXPR
)
9036 vec_compare
= new_temp
;
9037 else if (bitop2
== BIT_NOT_EXPR
)
9039 /* Instead of doing ~x ? y : z do x ? z : y. */
9040 vec_compare
= new_temp
;
9041 std::swap (vec_then_clause
, vec_else_clause
);
9045 vec_compare
= make_ssa_name (vec_cmp_type
);
9047 = gimple_build_assign (vec_compare
, bitop2
,
9048 vec_cond_lhs
, new_temp
);
9049 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9053 if (reduction_type
== EXTRACT_LAST_REDUCTION
)
9055 if (!is_gimple_val (vec_compare
))
9057 tree vec_compare_name
= make_ssa_name (vec_cmp_type
);
9058 gassign
*new_stmt
= gimple_build_assign (vec_compare_name
,
9060 vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9061 vec_compare
= vec_compare_name
;
9063 gcc_assert (reduc_index
== 2);
9064 gcall
*new_stmt
= gimple_build_call_internal
9065 (IFN_FOLD_EXTRACT_LAST
, 3, else_clause
, vec_compare
,
9067 gimple_call_set_lhs (new_stmt
, scalar_dest
);
9068 SSA_NAME_DEF_STMT (scalar_dest
) = new_stmt
;
9069 if (stmt_info
->stmt
== gsi_stmt (*gsi
))
9070 new_stmt_info
= vect_finish_replace_stmt (stmt_info
, new_stmt
);
9073 /* In this case we're moving the definition to later in the
9074 block. That doesn't matter because the only uses of the
9075 lhs are in phi statements. */
9076 gimple_stmt_iterator old_gsi
9077 = gsi_for_stmt (stmt_info
->stmt
);
9078 gsi_remove (&old_gsi
, true);
9080 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9085 new_temp
= make_ssa_name (vec_dest
);
9087 = gimple_build_assign (new_temp
, VEC_COND_EXPR
, vec_compare
,
9088 vec_then_clause
, vec_else_clause
);
9090 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9093 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
9100 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
9102 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
9104 prev_stmt_info
= new_stmt_info
;
9107 vec_oprnds0
.release ();
9108 vec_oprnds1
.release ();
9109 vec_oprnds2
.release ();
9110 vec_oprnds3
.release ();
9115 /* vectorizable_comparison.
9117 Check if STMT_INFO is comparison expression that can be vectorized.
9118 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9119 comparison, put it in VEC_STMT, and insert it at GSI.
9121 Return true if STMT_INFO is vectorizable in this way. */
9124 vectorizable_comparison (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
9125 stmt_vec_info
*vec_stmt
, tree reduc_def
,
9126 slp_tree slp_node
, stmt_vector_for_cost
*cost_vec
)
9128 vec_info
*vinfo
= stmt_info
->vinfo
;
9129 tree lhs
, rhs1
, rhs2
;
9130 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
9131 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
9132 tree vec_rhs1
= NULL_TREE
, vec_rhs2
= NULL_TREE
;
9134 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
9135 enum vect_def_type dts
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
9139 enum tree_code code
, bitop1
= NOP_EXPR
, bitop2
= NOP_EXPR
;
9140 stmt_vec_info prev_stmt_info
= NULL
;
9142 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
9143 vec
<tree
> vec_oprnds0
= vNULL
;
9144 vec
<tree
> vec_oprnds1
= vNULL
;
9148 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
9151 if (!vectype
|| !VECTOR_BOOLEAN_TYPE_P (vectype
))
9154 mask_type
= vectype
;
9155 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
9160 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
9162 gcc_assert (ncopies
>= 1);
9163 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
9164 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
9168 if (STMT_VINFO_LIVE_P (stmt_info
))
9170 if (dump_enabled_p ())
9171 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9172 "value used after loop.\n");
9176 gassign
*stmt
= dyn_cast
<gassign
*> (stmt_info
->stmt
);
9180 code
= gimple_assign_rhs_code (stmt
);
9182 if (TREE_CODE_CLASS (code
) != tcc_comparison
)
9185 rhs1
= gimple_assign_rhs1 (stmt
);
9186 rhs2
= gimple_assign_rhs2 (stmt
);
9188 if (!vect_is_simple_use (rhs1
, stmt_info
->vinfo
, &dts
[0], &vectype1
))
9191 if (!vect_is_simple_use (rhs2
, stmt_info
->vinfo
, &dts
[1], &vectype2
))
9194 if (vectype1
&& vectype2
9195 && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1
),
9196 TYPE_VECTOR_SUBPARTS (vectype2
)))
9199 vectype
= vectype1
? vectype1
: vectype2
;
9201 /* Invariant comparison. */
9204 vectype
= get_vectype_for_scalar_type (TREE_TYPE (rhs1
));
9205 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype
), nunits
))
9208 else if (maybe_ne (nunits
, TYPE_VECTOR_SUBPARTS (vectype
)))
9211 /* Can't compare mask and non-mask types. */
9212 if (vectype1
&& vectype2
9213 && (VECTOR_BOOLEAN_TYPE_P (vectype1
) ^ VECTOR_BOOLEAN_TYPE_P (vectype2
)))
9216 /* Boolean values may have another representation in vectors
9217 and therefore we prefer bit operations over comparison for
9218 them (which also works for scalar masks). We store opcodes
9219 to use in bitop1 and bitop2. Statement is vectorized as
9220 BITOP2 (rhs1 BITOP1 rhs2) or
9221 rhs1 BITOP2 (BITOP1 rhs2)
9222 depending on bitop1 and bitop2 arity. */
9223 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
9225 if (code
== GT_EXPR
)
9227 bitop1
= BIT_NOT_EXPR
;
9228 bitop2
= BIT_AND_EXPR
;
9230 else if (code
== GE_EXPR
)
9232 bitop1
= BIT_NOT_EXPR
;
9233 bitop2
= BIT_IOR_EXPR
;
9235 else if (code
== LT_EXPR
)
9237 bitop1
= BIT_NOT_EXPR
;
9238 bitop2
= BIT_AND_EXPR
;
9239 std::swap (rhs1
, rhs2
);
9240 std::swap (dts
[0], dts
[1]);
9242 else if (code
== LE_EXPR
)
9244 bitop1
= BIT_NOT_EXPR
;
9245 bitop2
= BIT_IOR_EXPR
;
9246 std::swap (rhs1
, rhs2
);
9247 std::swap (dts
[0], dts
[1]);
9251 bitop1
= BIT_XOR_EXPR
;
9252 if (code
== EQ_EXPR
)
9253 bitop2
= BIT_NOT_EXPR
;
9259 if (bitop1
== NOP_EXPR
)
9261 if (!expand_vec_cmp_expr_p (vectype
, mask_type
, code
))
9266 machine_mode mode
= TYPE_MODE (vectype
);
9269 optab
= optab_for_tree_code (bitop1
, vectype
, optab_default
);
9270 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
9273 if (bitop2
!= NOP_EXPR
)
9275 optab
= optab_for_tree_code (bitop2
, vectype
, optab_default
);
9276 if (!optab
|| optab_handler (optab
, mode
) == CODE_FOR_nothing
)
9281 STMT_VINFO_TYPE (stmt_info
) = comparison_vec_info_type
;
9282 vect_model_simple_cost (stmt_info
, ncopies
* (1 + (bitop2
!= NOP_EXPR
)),
9283 dts
, ndts
, slp_node
, cost_vec
);
9290 vec_oprnds0
.create (1);
9291 vec_oprnds1
.create (1);
9295 lhs
= gimple_assign_lhs (stmt
);
9296 mask
= vect_create_destination_var (lhs
, mask_type
);
9298 /* Handle cmp expr. */
9299 for (j
= 0; j
< ncopies
; j
++)
9301 stmt_vec_info new_stmt_info
= NULL
;
9306 auto_vec
<tree
, 2> ops
;
9307 auto_vec
<vec
<tree
>, 2> vec_defs
;
9309 ops
.safe_push (rhs1
);
9310 ops
.safe_push (rhs2
);
9311 vect_get_slp_defs (ops
, slp_node
, &vec_defs
);
9312 vec_oprnds1
= vec_defs
.pop ();
9313 vec_oprnds0
= vec_defs
.pop ();
9317 vec_rhs1
= vect_get_vec_def_for_operand (rhs1
, stmt_info
,
9319 vec_rhs2
= vect_get_vec_def_for_operand (rhs2
, stmt_info
,
9325 vec_rhs1
= vect_get_vec_def_for_stmt_copy (vinfo
,
9326 vec_oprnds0
.pop ());
9327 vec_rhs2
= vect_get_vec_def_for_stmt_copy (vinfo
,
9328 vec_oprnds1
.pop ());
9333 vec_oprnds0
.quick_push (vec_rhs1
);
9334 vec_oprnds1
.quick_push (vec_rhs2
);
9337 /* Arguments are ready. Create the new vector stmt. */
9338 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_rhs1
)
9340 vec_rhs2
= vec_oprnds1
[i
];
9342 new_temp
= make_ssa_name (mask
);
9343 if (bitop1
== NOP_EXPR
)
9345 gassign
*new_stmt
= gimple_build_assign (new_temp
, code
,
9346 vec_rhs1
, vec_rhs2
);
9348 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9353 if (bitop1
== BIT_NOT_EXPR
)
9354 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs2
);
9356 new_stmt
= gimple_build_assign (new_temp
, bitop1
, vec_rhs1
,
9359 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9360 if (bitop2
!= NOP_EXPR
)
9362 tree res
= make_ssa_name (mask
);
9363 if (bitop2
== BIT_NOT_EXPR
)
9364 new_stmt
= gimple_build_assign (res
, bitop2
, new_temp
);
9366 new_stmt
= gimple_build_assign (res
, bitop2
, vec_rhs1
,
9369 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
9373 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
9380 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
9382 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
9384 prev_stmt_info
= new_stmt_info
;
9387 vec_oprnds0
.release ();
9388 vec_oprnds1
.release ();
9393 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
9394 can handle all live statements in the node. Otherwise return true
9395 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
9396 GSI and VEC_STMT are as for vectorizable_live_operation. */
9399 can_vectorize_live_stmts (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
9400 slp_tree slp_node
, stmt_vec_info
*vec_stmt
,
9401 stmt_vector_for_cost
*cost_vec
)
9405 stmt_vec_info slp_stmt_info
;
9407 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node
), i
, slp_stmt_info
)
9409 if (STMT_VINFO_LIVE_P (slp_stmt_info
)
9410 && !vectorizable_live_operation (slp_stmt_info
, gsi
, slp_node
, i
,
9411 vec_stmt
, cost_vec
))
9415 else if (STMT_VINFO_LIVE_P (stmt_info
)
9416 && !vectorizable_live_operation (stmt_info
, gsi
, slp_node
, -1,
9417 vec_stmt
, cost_vec
))
9423 /* Make sure the statement is vectorizable. */
9426 vect_analyze_stmt (stmt_vec_info stmt_info
, bool *need_to_vectorize
,
9427 slp_tree node
, slp_instance node_instance
,
9428 stmt_vector_for_cost
*cost_vec
)
9430 vec_info
*vinfo
= stmt_info
->vinfo
;
9431 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
9432 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
9434 gimple_seq pattern_def_seq
;
9436 if (dump_enabled_p ())
9438 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
9439 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt_info
->stmt
, 0);
9442 if (gimple_has_volatile_ops (stmt_info
->stmt
))
9444 if (dump_enabled_p ())
9445 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9446 "not vectorized: stmt has volatile operands\n");
9451 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
9453 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
9455 gimple_stmt_iterator si
;
9457 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
9459 stmt_vec_info pattern_def_stmt_info
9460 = vinfo
->lookup_stmt (gsi_stmt (si
));
9461 if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info
)
9462 || STMT_VINFO_LIVE_P (pattern_def_stmt_info
))
9464 /* Analyze def stmt of STMT if it's a pattern stmt. */
9465 if (dump_enabled_p ())
9467 dump_printf_loc (MSG_NOTE
, vect_location
,
9468 "==> examining pattern def statement: ");
9469 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
,
9470 pattern_def_stmt_info
->stmt
, 0);
9473 if (!vect_analyze_stmt (pattern_def_stmt_info
,
9474 need_to_vectorize
, node
, node_instance
,
9481 /* Skip stmts that do not need to be vectorized. In loops this is expected
9483 - the COND_EXPR which is the loop exit condition
9484 - any LABEL_EXPRs in the loop
9485 - computations that are used only for array indexing or loop control.
9486 In basic blocks we only analyze statements that are a part of some SLP
9487 instance, therefore, all the statements are relevant.
9489 Pattern statement needs to be analyzed instead of the original statement
9490 if the original statement is not relevant. Otherwise, we analyze both
9491 statements. In basic blocks we are called from some SLP instance
9492 traversal, don't analyze pattern stmts instead, the pattern stmts
9493 already will be part of SLP instance. */
9495 stmt_vec_info pattern_stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
9496 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
9497 && !STMT_VINFO_LIVE_P (stmt_info
))
9499 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
9500 && pattern_stmt_info
9501 && (STMT_VINFO_RELEVANT_P (pattern_stmt_info
)
9502 || STMT_VINFO_LIVE_P (pattern_stmt_info
)))
9504 /* Analyze PATTERN_STMT instead of the original stmt. */
9505 stmt_info
= pattern_stmt_info
;
9506 if (dump_enabled_p ())
9508 dump_printf_loc (MSG_NOTE
, vect_location
,
9509 "==> examining pattern statement: ");
9510 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt_info
->stmt
, 0);
9515 if (dump_enabled_p ())
9516 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
9521 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
9523 && pattern_stmt_info
9524 && (STMT_VINFO_RELEVANT_P (pattern_stmt_info
)
9525 || STMT_VINFO_LIVE_P (pattern_stmt_info
)))
9527 /* Analyze PATTERN_STMT too. */
9528 if (dump_enabled_p ())
9530 dump_printf_loc (MSG_NOTE
, vect_location
,
9531 "==> examining pattern statement: ");
9532 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_stmt_info
->stmt
, 0);
9535 if (!vect_analyze_stmt (pattern_stmt_info
, need_to_vectorize
, node
,
9536 node_instance
, cost_vec
))
9540 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
9542 case vect_internal_def
:
9545 case vect_reduction_def
:
9546 case vect_nested_cycle
:
9547 gcc_assert (!bb_vinfo
9548 && (relevance
== vect_used_in_outer
9549 || relevance
== vect_used_in_outer_by_reduction
9550 || relevance
== vect_used_by_reduction
9551 || relevance
== vect_unused_in_scope
9552 || relevance
== vect_used_only_live
));
9555 case vect_induction_def
:
9556 gcc_assert (!bb_vinfo
);
9559 case vect_constant_def
:
9560 case vect_external_def
:
9561 case vect_unknown_def_type
:
9566 if (STMT_VINFO_RELEVANT_P (stmt_info
))
9568 tree type
= gimple_expr_type (stmt_info
->stmt
);
9569 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (type
)));
9570 gcall
*call
= dyn_cast
<gcall
*> (stmt_info
->stmt
);
9571 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
9572 || (call
&& gimple_call_lhs (call
) == NULL_TREE
));
9573 *need_to_vectorize
= true;
9576 if (PURE_SLP_STMT (stmt_info
) && !node
)
9578 dump_printf_loc (MSG_NOTE
, vect_location
,
9579 "handled only by SLP analysis\n");
9585 && (STMT_VINFO_RELEVANT_P (stmt_info
)
9586 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
9587 ok
= (vectorizable_simd_clone_call (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9588 || vectorizable_conversion (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9589 || vectorizable_shift (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9590 || vectorizable_operation (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9591 || vectorizable_assignment (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9592 || vectorizable_load (stmt_info
, NULL
, NULL
, node
, node_instance
,
9594 || vectorizable_call (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9595 || vectorizable_store (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9596 || vectorizable_reduction (stmt_info
, NULL
, NULL
, node
,
9597 node_instance
, cost_vec
)
9598 || vectorizable_induction (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9599 || vectorizable_condition (stmt_info
, NULL
, NULL
, NULL
, 0, node
,
9601 || vectorizable_comparison (stmt_info
, NULL
, NULL
, NULL
, node
,
9606 ok
= (vectorizable_simd_clone_call (stmt_info
, NULL
, NULL
, node
,
9608 || vectorizable_conversion (stmt_info
, NULL
, NULL
, node
,
9610 || vectorizable_shift (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9611 || vectorizable_operation (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9612 || vectorizable_assignment (stmt_info
, NULL
, NULL
, node
,
9614 || vectorizable_load (stmt_info
, NULL
, NULL
, node
, node_instance
,
9616 || vectorizable_call (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9617 || vectorizable_store (stmt_info
, NULL
, NULL
, node
, cost_vec
)
9618 || vectorizable_condition (stmt_info
, NULL
, NULL
, NULL
, 0, node
,
9620 || vectorizable_comparison (stmt_info
, NULL
, NULL
, NULL
, node
,
9626 if (dump_enabled_p ())
9628 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9629 "not vectorized: relevant stmt not ");
9630 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
9631 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
9632 stmt_info
->stmt
, 0);
9638 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
9639 need extra handling, except for vectorizable reductions. */
9641 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
9642 && !can_vectorize_live_stmts (stmt_info
, NULL
, node
, NULL
, cost_vec
))
9644 if (dump_enabled_p ())
9646 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9647 "not vectorized: live stmt not supported: ");
9648 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
9649 stmt_info
->stmt
, 0);
9659 /* Function vect_transform_stmt.
9661 Create a vectorized stmt to replace STMT_INFO, and insert it at BSI. */
9664 vect_transform_stmt (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
9665 slp_tree slp_node
, slp_instance slp_node_instance
)
9667 vec_info
*vinfo
= stmt_info
->vinfo
;
9668 bool is_store
= false;
9669 stmt_vec_info vec_stmt
= NULL
;
9672 gcc_assert (slp_node
|| !PURE_SLP_STMT (stmt_info
));
9673 stmt_vec_info old_vec_stmt_info
= STMT_VINFO_VEC_STMT (stmt_info
);
9675 bool nested_p
= (STMT_VINFO_LOOP_VINFO (stmt_info
)
9676 && nested_in_vect_loop_p
9677 (LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info
)),
9680 gimple
*stmt
= stmt_info
->stmt
;
9681 switch (STMT_VINFO_TYPE (stmt_info
))
9683 case type_demotion_vec_info_type
:
9684 case type_promotion_vec_info_type
:
9685 case type_conversion_vec_info_type
:
9686 done
= vectorizable_conversion (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9691 case induc_vec_info_type
:
9692 done
= vectorizable_induction (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9697 case shift_vec_info_type
:
9698 done
= vectorizable_shift (stmt_info
, gsi
, &vec_stmt
, slp_node
, NULL
);
9702 case op_vec_info_type
:
9703 done
= vectorizable_operation (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9708 case assignment_vec_info_type
:
9709 done
= vectorizable_assignment (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9714 case load_vec_info_type
:
9715 done
= vectorizable_load (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9716 slp_node_instance
, NULL
);
9720 case store_vec_info_type
:
9721 done
= vectorizable_store (stmt_info
, gsi
, &vec_stmt
, slp_node
, NULL
);
9723 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
9725 /* In case of interleaving, the whole chain is vectorized when the
9726 last store in the chain is reached. Store stmts before the last
9727 one are skipped, and there vec_stmt_info shouldn't be freed
9729 stmt_vec_info group_info
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
9730 if (DR_GROUP_STORE_COUNT (group_info
) == DR_GROUP_SIZE (group_info
))
9737 case condition_vec_info_type
:
9738 done
= vectorizable_condition (stmt_info
, gsi
, &vec_stmt
, NULL
, 0,
9743 case comparison_vec_info_type
:
9744 done
= vectorizable_comparison (stmt_info
, gsi
, &vec_stmt
, NULL
,
9749 case call_vec_info_type
:
9750 done
= vectorizable_call (stmt_info
, gsi
, &vec_stmt
, slp_node
, NULL
);
9751 stmt
= gsi_stmt (*gsi
);
9754 case call_simd_clone_vec_info_type
:
9755 done
= vectorizable_simd_clone_call (stmt_info
, gsi
, &vec_stmt
,
9757 stmt
= gsi_stmt (*gsi
);
9760 case reduc_vec_info_type
:
9761 done
= vectorizable_reduction (stmt_info
, gsi
, &vec_stmt
, slp_node
,
9762 slp_node_instance
, NULL
);
9767 if (!STMT_VINFO_LIVE_P (stmt_info
))
9769 if (dump_enabled_p ())
9770 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
9771 "stmt not supported.\n");
9776 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
9777 This would break hybrid SLP vectorization. */
9779 gcc_assert (!vec_stmt
9780 && STMT_VINFO_VEC_STMT (stmt_info
) == old_vec_stmt_info
);
9782 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
9783 is being vectorized, but outside the immediately enclosing loop. */
9786 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
9787 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
9788 || STMT_VINFO_RELEVANT (stmt_info
) ==
9789 vect_used_in_outer_by_reduction
))
9791 struct loop
*innerloop
= LOOP_VINFO_LOOP (
9792 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
9793 imm_use_iterator imm_iter
;
9794 use_operand_p use_p
;
9797 if (dump_enabled_p ())
9798 dump_printf_loc (MSG_NOTE
, vect_location
,
9799 "Record the vdef for outer-loop vectorization.\n");
9801 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
9802 (to be used when vectorizing outer-loop stmts that use the DEF of
9804 if (gimple_code (stmt
) == GIMPLE_PHI
)
9805 scalar_dest
= PHI_RESULT (stmt
);
9807 scalar_dest
= gimple_assign_lhs (stmt
);
9809 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
9810 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
9812 stmt_vec_info exit_phi_info
9813 = vinfo
->lookup_stmt (USE_STMT (use_p
));
9814 STMT_VINFO_VEC_STMT (exit_phi_info
) = vec_stmt
;
9818 /* Handle stmts whose DEF is used outside the loop-nest that is
9819 being vectorized. */
9820 if (STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
9822 done
= can_vectorize_live_stmts (stmt_info
, gsi
, slp_node
, &vec_stmt
,
9828 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
9834 /* Remove a group of stores (for SLP or interleaving), free their
9838 vect_remove_stores (stmt_vec_info first_stmt_info
)
9840 vec_info
*vinfo
= first_stmt_info
->vinfo
;
9841 stmt_vec_info next_stmt_info
= first_stmt_info
;
9843 while (next_stmt_info
)
9845 stmt_vec_info tmp
= DR_GROUP_NEXT_ELEMENT (next_stmt_info
);
9846 next_stmt_info
= vect_orig_stmt (next_stmt_info
);
9847 /* Free the attached stmt_vec_info and remove the stmt. */
9848 vinfo
->remove_stmt (next_stmt_info
);
9849 next_stmt_info
= tmp
;
9853 /* Function get_vectype_for_scalar_type_and_size.
9855 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
9859 get_vectype_for_scalar_type_and_size (tree scalar_type
, poly_uint64 size
)
9861 tree orig_scalar_type
= scalar_type
;
9862 scalar_mode inner_mode
;
9863 machine_mode simd_mode
;
9867 if (!is_int_mode (TYPE_MODE (scalar_type
), &inner_mode
)
9868 && !is_float_mode (TYPE_MODE (scalar_type
), &inner_mode
))
9871 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
9873 /* For vector types of elements whose mode precision doesn't
9874 match their types precision we use a element type of mode
9875 precision. The vectorization routines will have to make sure
9876 they support the proper result truncation/extension.
9877 We also make sure to build vector types with INTEGER_TYPE
9878 component type only. */
9879 if (INTEGRAL_TYPE_P (scalar_type
)
9880 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
9881 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
9882 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
9883 TYPE_UNSIGNED (scalar_type
));
9885 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
9886 When the component mode passes the above test simply use a type
9887 corresponding to that mode. The theory is that any use that
9888 would cause problems with this will disable vectorization anyway. */
9889 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
9890 && !INTEGRAL_TYPE_P (scalar_type
))
9891 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
9893 /* We can't build a vector type of elements with alignment bigger than
9895 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
9896 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
9897 TYPE_UNSIGNED (scalar_type
));
9899 /* If we felt back to using the mode fail if there was
9900 no scalar type for it. */
9901 if (scalar_type
== NULL_TREE
)
9904 /* If no size was supplied use the mode the target prefers. Otherwise
9905 lookup a vector mode of the specified size. */
9906 if (known_eq (size
, 0U))
9907 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
9908 else if (!multiple_p (size
, nbytes
, &nunits
)
9909 || !mode_for_vector (inner_mode
, nunits
).exists (&simd_mode
))
9911 /* NOTE: nunits == 1 is allowed to support single element vector types. */
9912 if (!multiple_p (GET_MODE_SIZE (simd_mode
), nbytes
, &nunits
))
9915 vectype
= build_vector_type (scalar_type
, nunits
);
9917 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
9918 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
9921 /* Re-attach the address-space qualifier if we canonicalized the scalar
9923 if (TYPE_ADDR_SPACE (orig_scalar_type
) != TYPE_ADDR_SPACE (vectype
))
9924 return build_qualified_type
9925 (vectype
, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type
)));
9930 poly_uint64 current_vector_size
;
9932 /* Function get_vectype_for_scalar_type.
9934 Returns the vector type corresponding to SCALAR_TYPE as supported
9938 get_vectype_for_scalar_type (tree scalar_type
)
9941 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
9942 current_vector_size
);
9944 && known_eq (current_vector_size
, 0U))
9945 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
9949 /* Function get_mask_type_for_scalar_type.
9951 Returns the mask type corresponding to a result of comparison
9952 of vectors of specified SCALAR_TYPE as supported by target. */
9955 get_mask_type_for_scalar_type (tree scalar_type
)
9957 tree vectype
= get_vectype_for_scalar_type (scalar_type
);
9962 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype
),
9963 current_vector_size
);
9966 /* Function get_same_sized_vectype
9968 Returns a vector type corresponding to SCALAR_TYPE of size
9969 VECTOR_TYPE if supported by the target. */
9972 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
9974 if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type
))
9975 return build_same_sized_truth_vector_type (vector_type
);
9977 return get_vectype_for_scalar_type_and_size
9978 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
9981 /* Function vect_is_simple_use.
9984 VINFO - the vect info of the loop or basic block that is being vectorized.
9985 OPERAND - operand in the loop or bb.
9987 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
9988 case OPERAND is an SSA_NAME that is defined in the vectorizable region
9989 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
9990 the definition could be anywhere in the function
9991 DT - the type of definition
9993 Returns whether a stmt with OPERAND can be vectorized.
9994 For loops, supportable operands are constants, loop invariants, and operands
9995 that are defined by the current iteration of the loop. Unsupportable
9996 operands are those that are defined by a previous iteration of the loop (as
9997 is the case in reduction/induction computations).
9998 For basic blocks, supportable operands are constants and bb invariants.
9999 For now, operands defined outside the basic block are not supported. */
10002 vect_is_simple_use (tree operand
, vec_info
*vinfo
, enum vect_def_type
*dt
,
10003 stmt_vec_info
*def_stmt_info_out
, gimple
**def_stmt_out
)
10005 if (def_stmt_info_out
)
10006 *def_stmt_info_out
= NULL
;
10008 *def_stmt_out
= NULL
;
10009 *dt
= vect_unknown_def_type
;
10011 if (dump_enabled_p ())
10013 dump_printf_loc (MSG_NOTE
, vect_location
,
10014 "vect_is_simple_use: operand ");
10015 if (TREE_CODE (operand
) == SSA_NAME
10016 && !SSA_NAME_IS_DEFAULT_DEF (operand
))
10017 dump_gimple_expr (MSG_NOTE
, TDF_SLIM
, SSA_NAME_DEF_STMT (operand
), 0);
10019 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
10022 if (CONSTANT_CLASS_P (operand
))
10023 *dt
= vect_constant_def
;
10024 else if (is_gimple_min_invariant (operand
))
10025 *dt
= vect_external_def
;
10026 else if (TREE_CODE (operand
) != SSA_NAME
)
10027 *dt
= vect_unknown_def_type
;
10028 else if (SSA_NAME_IS_DEFAULT_DEF (operand
))
10029 *dt
= vect_external_def
;
10032 gimple
*def_stmt
= SSA_NAME_DEF_STMT (operand
);
10033 stmt_vec_info stmt_vinfo
= vinfo
->lookup_def (operand
);
10035 *dt
= vect_external_def
;
10038 stmt_vinfo
= vect_stmt_to_vectorize (stmt_vinfo
);
10039 def_stmt
= stmt_vinfo
->stmt
;
10040 switch (gimple_code (def_stmt
))
10043 case GIMPLE_ASSIGN
:
10045 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
10048 *dt
= vect_unknown_def_type
;
10051 if (def_stmt_info_out
)
10052 *def_stmt_info_out
= stmt_vinfo
;
10055 *def_stmt_out
= def_stmt
;
10058 if (dump_enabled_p ())
10060 dump_printf (MSG_NOTE
, ", type of def: ");
10063 case vect_uninitialized_def
:
10064 dump_printf (MSG_NOTE
, "uninitialized\n");
10066 case vect_constant_def
:
10067 dump_printf (MSG_NOTE
, "constant\n");
10069 case vect_external_def
:
10070 dump_printf (MSG_NOTE
, "external\n");
10072 case vect_internal_def
:
10073 dump_printf (MSG_NOTE
, "internal\n");
10075 case vect_induction_def
:
10076 dump_printf (MSG_NOTE
, "induction\n");
10078 case vect_reduction_def
:
10079 dump_printf (MSG_NOTE
, "reduction\n");
10081 case vect_double_reduction_def
:
10082 dump_printf (MSG_NOTE
, "double reduction\n");
10084 case vect_nested_cycle
:
10085 dump_printf (MSG_NOTE
, "nested cycle\n");
10087 case vect_unknown_def_type
:
10088 dump_printf (MSG_NOTE
, "unknown\n");
10093 if (*dt
== vect_unknown_def_type
)
10095 if (dump_enabled_p ())
10096 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10097 "Unsupported pattern.\n");
10104 /* Function vect_is_simple_use.
10106 Same as vect_is_simple_use but also determines the vector operand
10107 type of OPERAND and stores it to *VECTYPE. If the definition of
10108 OPERAND is vect_uninitialized_def, vect_constant_def or
10109 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
10110 is responsible to compute the best suited vector type for the
10114 vect_is_simple_use (tree operand
, vec_info
*vinfo
, enum vect_def_type
*dt
,
10115 tree
*vectype
, stmt_vec_info
*def_stmt_info_out
,
10116 gimple
**def_stmt_out
)
10118 stmt_vec_info def_stmt_info
;
10120 if (!vect_is_simple_use (operand
, vinfo
, dt
, &def_stmt_info
, &def_stmt
))
10124 *def_stmt_out
= def_stmt
;
10125 if (def_stmt_info_out
)
10126 *def_stmt_info_out
= def_stmt_info
;
10128 /* Now get a vector type if the def is internal, otherwise supply
10129 NULL_TREE and leave it up to the caller to figure out a proper
10130 type for the use stmt. */
10131 if (*dt
== vect_internal_def
10132 || *dt
== vect_induction_def
10133 || *dt
== vect_reduction_def
10134 || *dt
== vect_double_reduction_def
10135 || *dt
== vect_nested_cycle
)
10137 *vectype
= STMT_VINFO_VECTYPE (def_stmt_info
);
10138 gcc_assert (*vectype
!= NULL_TREE
);
10139 if (dump_enabled_p ())
10141 dump_printf_loc (MSG_NOTE
, vect_location
,
10142 "vect_is_simple_use: vectype ");
10143 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, *vectype
);
10144 dump_printf (MSG_NOTE
, "\n");
10147 else if (*dt
== vect_uninitialized_def
10148 || *dt
== vect_constant_def
10149 || *dt
== vect_external_def
)
10150 *vectype
= NULL_TREE
;
10152 gcc_unreachable ();
10158 /* Function supportable_widening_operation
10160 Check whether an operation represented by the code CODE is a
10161 widening operation that is supported by the target platform in
10162 vector form (i.e., when operating on arguments of type VECTYPE_IN
10163 producing a result of type VECTYPE_OUT).
10165 Widening operations we currently support are NOP (CONVERT), FLOAT,
10166 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
10167 are supported by the target platform either directly (via vector
10168 tree-codes), or via target builtins.
10171 - CODE1 and CODE2 are codes of vector operations to be used when
10172 vectorizing the operation, if available.
10173 - MULTI_STEP_CVT determines the number of required intermediate steps in
10174 case of multi-step conversion (like char->short->int - in that case
10175 MULTI_STEP_CVT will be 1).
10176 - INTERM_TYPES contains the intermediate type required to perform the
10177 widening operation (short in the above example). */
10180 supportable_widening_operation (enum tree_code code
, stmt_vec_info stmt_info
,
10181 tree vectype_out
, tree vectype_in
,
10182 enum tree_code
*code1
, enum tree_code
*code2
,
10183 int *multi_step_cvt
,
10184 vec
<tree
> *interm_types
)
10186 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
10187 struct loop
*vect_loop
= NULL
;
10188 machine_mode vec_mode
;
10189 enum insn_code icode1
, icode2
;
10190 optab optab1
, optab2
;
10191 tree vectype
= vectype_in
;
10192 tree wide_vectype
= vectype_out
;
10193 enum tree_code c1
, c2
;
10195 tree prev_type
, intermediate_type
;
10196 machine_mode intermediate_mode
, prev_mode
;
10197 optab optab3
, optab4
;
10199 *multi_step_cvt
= 0;
10201 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
10205 case WIDEN_MULT_EXPR
:
10206 /* The result of a vectorized widening operation usually requires
10207 two vectors (because the widened results do not fit into one vector).
10208 The generated vector results would normally be expected to be
10209 generated in the same order as in the original scalar computation,
10210 i.e. if 8 results are generated in each vector iteration, they are
10211 to be organized as follows:
10212 vect1: [res1,res2,res3,res4],
10213 vect2: [res5,res6,res7,res8].
10215 However, in the special case that the result of the widening
10216 operation is used in a reduction computation only, the order doesn't
10217 matter (because when vectorizing a reduction we change the order of
10218 the computation). Some targets can take advantage of this and
10219 generate more efficient code. For example, targets like Altivec,
10220 that support widen_mult using a sequence of {mult_even,mult_odd}
10221 generate the following vectors:
10222 vect1: [res1,res3,res5,res7],
10223 vect2: [res2,res4,res6,res8].
10225 When vectorizing outer-loops, we execute the inner-loop sequentially
10226 (each vectorized inner-loop iteration contributes to VF outer-loop
10227 iterations in parallel). We therefore don't allow to change the
10228 order of the computation in the inner-loop during outer-loop
10230 /* TODO: Another case in which order doesn't *really* matter is when we
10231 widen and then contract again, e.g. (short)((int)x * y >> 8).
10232 Normally, pack_trunc performs an even/odd permute, whereas the
10233 repack from an even/odd expansion would be an interleave, which
10234 would be significantly simpler for e.g. AVX2. */
10235 /* In any case, in order to avoid duplicating the code below, recurse
10236 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
10237 are properly set up for the caller. If we fail, we'll continue with
10238 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
10240 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
10241 && !nested_in_vect_loop_p (vect_loop
, stmt_info
)
10242 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
10243 stmt_info
, vectype_out
,
10244 vectype_in
, code1
, code2
,
10245 multi_step_cvt
, interm_types
))
10247 /* Elements in a vector with vect_used_by_reduction property cannot
10248 be reordered if the use chain with this property does not have the
10249 same operation. One such an example is s += a * b, where elements
10250 in a and b cannot be reordered. Here we check if the vector defined
10251 by STMT is only directly used in the reduction statement. */
10252 tree lhs
= gimple_assign_lhs (stmt_info
->stmt
);
10253 stmt_vec_info use_stmt_info
= loop_info
->lookup_single_use (lhs
);
10255 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
10258 c1
= VEC_WIDEN_MULT_LO_EXPR
;
10259 c2
= VEC_WIDEN_MULT_HI_EXPR
;
10262 case DOT_PROD_EXPR
:
10263 c1
= DOT_PROD_EXPR
;
10264 c2
= DOT_PROD_EXPR
;
10272 case VEC_WIDEN_MULT_EVEN_EXPR
:
10273 /* Support the recursion induced just above. */
10274 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
10275 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
10278 case WIDEN_LSHIFT_EXPR
:
10279 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
10280 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
10284 c1
= VEC_UNPACK_LO_EXPR
;
10285 c2
= VEC_UNPACK_HI_EXPR
;
10289 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
10290 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
10293 case FIX_TRUNC_EXPR
:
10294 c1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
10295 c2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
10299 gcc_unreachable ();
10302 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
10303 std::swap (c1
, c2
);
10305 if (code
== FIX_TRUNC_EXPR
)
10307 /* The signedness is determined from output operand. */
10308 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
10309 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
10313 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
10314 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
10317 if (!optab1
|| !optab2
)
10320 vec_mode
= TYPE_MODE (vectype
);
10321 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
10322 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
10328 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
10329 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
10330 /* For scalar masks we may have different boolean
10331 vector types having the same QImode. Thus we
10332 add additional check for elements number. */
10333 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
10334 || known_eq (TYPE_VECTOR_SUBPARTS (vectype
),
10335 TYPE_VECTOR_SUBPARTS (wide_vectype
) * 2));
10337 /* Check if it's a multi-step conversion that can be done using intermediate
10340 prev_type
= vectype
;
10341 prev_mode
= vec_mode
;
10343 if (!CONVERT_EXPR_CODE_P (code
))
10346 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
10347 intermediate steps in promotion sequence. We try
10348 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
10350 interm_types
->create (MAX_INTERM_CVT_STEPS
);
10351 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
10353 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
10354 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
10356 intermediate_type
= vect_halve_mask_nunits (prev_type
);
10357 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
10362 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
10363 TYPE_UNSIGNED (prev_type
));
10365 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
10366 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
10368 if (!optab3
|| !optab4
10369 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
10370 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
10371 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
10372 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
10373 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
10374 == CODE_FOR_nothing
)
10375 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
10376 == CODE_FOR_nothing
))
10379 interm_types
->quick_push (intermediate_type
);
10380 (*multi_step_cvt
)++;
10382 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
10383 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
10384 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
10385 || known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type
),
10386 TYPE_VECTOR_SUBPARTS (wide_vectype
) * 2));
10388 prev_type
= intermediate_type
;
10389 prev_mode
= intermediate_mode
;
10392 interm_types
->release ();
10397 /* Function supportable_narrowing_operation
10399 Check whether an operation represented by the code CODE is a
10400 narrowing operation that is supported by the target platform in
10401 vector form (i.e., when operating on arguments of type VECTYPE_IN
10402 and producing a result of type VECTYPE_OUT).
10404 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
10405 and FLOAT. This function checks if these operations are supported by
10406 the target platform directly via vector tree-codes.
10409 - CODE1 is the code of a vector operation to be used when
10410 vectorizing the operation, if available.
10411 - MULTI_STEP_CVT determines the number of required intermediate steps in
10412 case of multi-step conversion (like int->short->char - in that case
10413 MULTI_STEP_CVT will be 1).
10414 - INTERM_TYPES contains the intermediate type required to perform the
10415 narrowing operation (short in the above example). */
10418 supportable_narrowing_operation (enum tree_code code
,
10419 tree vectype_out
, tree vectype_in
,
10420 enum tree_code
*code1
, int *multi_step_cvt
,
10421 vec
<tree
> *interm_types
)
10423 machine_mode vec_mode
;
10424 enum insn_code icode1
;
10425 optab optab1
, interm_optab
;
10426 tree vectype
= vectype_in
;
10427 tree narrow_vectype
= vectype_out
;
10429 tree intermediate_type
, prev_type
;
10430 machine_mode intermediate_mode
, prev_mode
;
10434 *multi_step_cvt
= 0;
10438 c1
= VEC_PACK_TRUNC_EXPR
;
10441 case FIX_TRUNC_EXPR
:
10442 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
10446 c1
= VEC_PACK_FLOAT_EXPR
;
10450 gcc_unreachable ();
10453 if (code
== FIX_TRUNC_EXPR
)
10454 /* The signedness is determined from output operand. */
10455 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
10457 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
10462 vec_mode
= TYPE_MODE (vectype
);
10463 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
10468 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
10469 /* For scalar masks we may have different boolean
10470 vector types having the same QImode. Thus we
10471 add additional check for elements number. */
10472 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
10473 || known_eq (TYPE_VECTOR_SUBPARTS (vectype
) * 2,
10474 TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
10476 if (code
== FLOAT_EXPR
)
10479 /* Check if it's a multi-step conversion that can be done using intermediate
10481 prev_mode
= vec_mode
;
10482 prev_type
= vectype
;
10483 if (code
== FIX_TRUNC_EXPR
)
10484 uns
= TYPE_UNSIGNED (vectype_out
);
10486 uns
= TYPE_UNSIGNED (vectype
);
10488 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
10489 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
10490 costly than signed. */
10491 if (code
== FIX_TRUNC_EXPR
&& uns
)
10493 enum insn_code icode2
;
10496 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
10498 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
10499 if (interm_optab
!= unknown_optab
10500 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
10501 && insn_data
[icode1
].operand
[0].mode
10502 == insn_data
[icode2
].operand
[0].mode
)
10505 optab1
= interm_optab
;
10510 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
10511 intermediate steps in promotion sequence. We try
10512 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
10513 interm_types
->create (MAX_INTERM_CVT_STEPS
);
10514 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
10516 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
10517 if (VECTOR_BOOLEAN_TYPE_P (prev_type
))
10519 intermediate_type
= vect_double_mask_nunits (prev_type
);
10520 if (intermediate_mode
!= TYPE_MODE (intermediate_type
))
10525 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
10527 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
10530 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
10531 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
10532 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
10533 == CODE_FOR_nothing
))
10536 interm_types
->quick_push (intermediate_type
);
10537 (*multi_step_cvt
)++;
10539 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
10540 return (!VECTOR_BOOLEAN_TYPE_P (vectype
)
10541 || known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type
) * 2,
10542 TYPE_VECTOR_SUBPARTS (narrow_vectype
)));
10544 prev_mode
= intermediate_mode
;
10545 prev_type
= intermediate_type
;
10546 optab1
= interm_optab
;
10549 interm_types
->release ();
10553 /* Generate and return a statement that sets vector mask MASK such that
10554 MASK[I] is true iff J + START_INDEX < END_INDEX for all J <= I. */
10557 vect_gen_while (tree mask
, tree start_index
, tree end_index
)
10559 tree cmp_type
= TREE_TYPE (start_index
);
10560 tree mask_type
= TREE_TYPE (mask
);
10561 gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT
,
10562 cmp_type
, mask_type
,
10563 OPTIMIZE_FOR_SPEED
));
10564 gcall
*call
= gimple_build_call_internal (IFN_WHILE_ULT
, 3,
10565 start_index
, end_index
,
10566 build_zero_cst (mask_type
));
10567 gimple_call_set_lhs (call
, mask
);
10571 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
10572 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
10575 vect_gen_while_not (gimple_seq
*seq
, tree mask_type
, tree start_index
,
10578 tree tmp
= make_ssa_name (mask_type
);
10579 gcall
*call
= vect_gen_while (tmp
, start_index
, end_index
);
10580 gimple_seq_add_stmt (seq
, call
);
10581 return gimple_build (seq
, BIT_NOT_EXPR
, mask_type
, tmp
);
10584 /* Try to compute the vector types required to vectorize STMT_INFO,
10585 returning true on success and false if vectorization isn't possible.
10589 - Set *STMT_VECTYPE_OUT to:
10590 - NULL_TREE if the statement doesn't need to be vectorized;
10591 - boolean_type_node if the statement is a boolean operation whose
10592 vector type can only be determined once all the other vector types
10594 - the equivalent of STMT_VINFO_VECTYPE otherwise.
10596 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
10597 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
10598 statement does not help to determine the overall number of units. */
10601 vect_get_vector_types_for_stmt (stmt_vec_info stmt_info
,
10602 tree
*stmt_vectype_out
,
10603 tree
*nunits_vectype_out
)
10605 gimple
*stmt
= stmt_info
->stmt
;
10607 *stmt_vectype_out
= NULL_TREE
;
10608 *nunits_vectype_out
= NULL_TREE
;
10610 if (gimple_get_lhs (stmt
) == NULL_TREE
10611 /* MASK_STORE has no lhs, but is ok. */
10612 && !gimple_call_internal_p (stmt
, IFN_MASK_STORE
))
10614 if (is_a
<gcall
*> (stmt
))
10616 /* Ignore calls with no lhs. These must be calls to
10617 #pragma omp simd functions, and what vectorization factor
10618 it really needs can't be determined until
10619 vectorizable_simd_clone_call. */
10620 if (dump_enabled_p ())
10621 dump_printf_loc (MSG_NOTE
, vect_location
,
10622 "defer to SIMD clone analysis.\n");
10626 if (dump_enabled_p ())
10628 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10629 "not vectorized: irregular stmt.");
10630 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
10635 if (VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))))
10637 if (dump_enabled_p ())
10639 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10640 "not vectorized: vector stmt in loop:");
10641 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
10647 tree scalar_type
= NULL_TREE
;
10648 if (STMT_VINFO_VECTYPE (stmt_info
))
10649 *stmt_vectype_out
= vectype
= STMT_VINFO_VECTYPE (stmt_info
);
10652 gcc_assert (!STMT_VINFO_DATA_REF (stmt_info
));
10653 if (gimple_call_internal_p (stmt
, IFN_MASK_STORE
))
10654 scalar_type
= TREE_TYPE (gimple_call_arg (stmt
, 3));
10656 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
10658 /* Pure bool ops don't participate in number-of-units computation.
10659 For comparisons use the types being compared. */
10660 if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type
)
10661 && is_gimple_assign (stmt
)
10662 && gimple_assign_rhs_code (stmt
) != COND_EXPR
)
10664 *stmt_vectype_out
= boolean_type_node
;
10666 tree rhs1
= gimple_assign_rhs1 (stmt
);
10667 if (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) == tcc_comparison
10668 && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
)))
10669 scalar_type
= TREE_TYPE (rhs1
);
10672 if (dump_enabled_p ())
10673 dump_printf_loc (MSG_NOTE
, vect_location
,
10674 "pure bool operation.\n");
10679 if (dump_enabled_p ())
10681 dump_printf_loc (MSG_NOTE
, vect_location
,
10682 "get vectype for scalar type: ");
10683 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
10684 dump_printf (MSG_NOTE
, "\n");
10686 vectype
= get_vectype_for_scalar_type (scalar_type
);
10689 if (dump_enabled_p ())
10691 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10692 "not vectorized: unsupported data-type ");
10693 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
10695 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
10700 if (!*stmt_vectype_out
)
10701 *stmt_vectype_out
= vectype
;
10703 if (dump_enabled_p ())
10705 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
10706 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
10707 dump_printf (MSG_NOTE
, "\n");
10711 /* Don't try to compute scalar types if the stmt produces a boolean
10712 vector; use the existing vector type instead. */
10713 tree nunits_vectype
;
10714 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
10715 nunits_vectype
= vectype
;
10718 /* The number of units is set according to the smallest scalar
10719 type (or the largest vector size, but we only support one
10720 vector size per vectorization). */
10721 if (*stmt_vectype_out
!= boolean_type_node
)
10723 HOST_WIDE_INT dummy
;
10724 scalar_type
= vect_get_smallest_scalar_type (stmt_info
,
10727 if (dump_enabled_p ())
10729 dump_printf_loc (MSG_NOTE
, vect_location
,
10730 "get vectype for scalar type: ");
10731 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
10732 dump_printf (MSG_NOTE
, "\n");
10734 nunits_vectype
= get_vectype_for_scalar_type (scalar_type
);
10736 if (!nunits_vectype
)
10738 if (dump_enabled_p ())
10740 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10741 "not vectorized: unsupported data-type ");
10742 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, scalar_type
);
10743 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
10748 if (maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype
)),
10749 GET_MODE_SIZE (TYPE_MODE (nunits_vectype
))))
10751 if (dump_enabled_p ())
10753 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10754 "not vectorized: different sized vector "
10755 "types in statement, ");
10756 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, vectype
);
10757 dump_printf (MSG_MISSED_OPTIMIZATION
, " and ");
10758 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, nunits_vectype
);
10759 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
10764 if (dump_enabled_p ())
10766 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
10767 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, nunits_vectype
);
10768 dump_printf (MSG_NOTE
, "\n");
10770 dump_printf_loc (MSG_NOTE
, vect_location
, "nunits = ");
10771 dump_dec (MSG_NOTE
, TYPE_VECTOR_SUBPARTS (nunits_vectype
));
10772 dump_printf (MSG_NOTE
, "\n");
10775 *nunits_vectype_out
= nunits_vectype
;
10779 /* Try to determine the correct vector type for STMT_INFO, which is a
10780 statement that produces a scalar boolean result. Return the vector
10781 type on success, otherwise return NULL_TREE. */
10784 vect_get_mask_type_for_stmt (stmt_vec_info stmt_info
)
10786 gimple
*stmt
= stmt_info
->stmt
;
10787 tree mask_type
= NULL
;
10788 tree vectype
, scalar_type
;
10790 if (is_gimple_assign (stmt
)
10791 && TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) == tcc_comparison
10792 && !VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (gimple_assign_rhs1 (stmt
))))
10794 scalar_type
= TREE_TYPE (gimple_assign_rhs1 (stmt
));
10795 mask_type
= get_mask_type_for_scalar_type (scalar_type
);
10799 if (dump_enabled_p ())
10800 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10801 "not vectorized: unsupported mask\n");
10809 enum vect_def_type dt
;
10811 FOR_EACH_SSA_TREE_OPERAND (rhs
, stmt
, iter
, SSA_OP_USE
)
10813 if (!vect_is_simple_use (rhs
, stmt_info
->vinfo
, &dt
, &vectype
))
10815 if (dump_enabled_p ())
10817 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10818 "not vectorized: can't compute mask type "
10819 "for statement, ");
10820 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
,
10826 /* No vectype probably means external definition.
10827 Allow it in case there is another operand which
10828 allows to determine mask type. */
10833 mask_type
= vectype
;
10834 else if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type
),
10835 TYPE_VECTOR_SUBPARTS (vectype
)))
10837 if (dump_enabled_p ())
10839 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10840 "not vectorized: different sized masks "
10841 "types in statement, ");
10842 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
10844 dump_printf (MSG_MISSED_OPTIMIZATION
, " and ");
10845 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
10847 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
10851 else if (VECTOR_BOOLEAN_TYPE_P (mask_type
)
10852 != VECTOR_BOOLEAN_TYPE_P (vectype
))
10854 if (dump_enabled_p ())
10856 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10857 "not vectorized: mixed mask and "
10858 "nonmask vector types in statement, ");
10859 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
10861 dump_printf (MSG_MISSED_OPTIMIZATION
, " and ");
10862 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
10864 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
10870 /* We may compare boolean value loaded as vector of integers.
10871 Fix mask_type in such case. */
10873 && !VECTOR_BOOLEAN_TYPE_P (mask_type
)
10874 && gimple_code (stmt
) == GIMPLE_ASSIGN
10875 && TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) == tcc_comparison
)
10876 mask_type
= build_same_sized_truth_vector_type (mask_type
);
10879 /* No mask_type should mean loop invariant predicate.
10880 This is probably a subject for optimization in if-conversion. */
10881 if (!mask_type
&& dump_enabled_p ())
10883 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
10884 "not vectorized: can't compute mask type "
10885 "for statement, ");
10886 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);