i386: Use const reference of struct ix86_frame to avoid copy
[official-gcc.git] / gcc / tree-vect-stmts.c
blobda76572ce45d9ddd5434e2dd4e886f4f15e0dc61
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2018 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #include "config.h"
23 #include "system.h"
24 #include "coretypes.h"
25 #include "backend.h"
26 #include "target.h"
27 #include "rtl.h"
28 #include "tree.h"
29 #include "gimple.h"
30 #include "ssa.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
34 #include "cgraph.h"
35 #include "dumpfile.h"
36 #include "alias.h"
37 #include "fold-const.h"
38 #include "stor-layout.h"
39 #include "tree-eh.h"
40 #include "gimplify.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "tree-cfg.h"
44 #include "tree-ssa-loop-manip.h"
45 #include "cfgloop.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
49 #include "builtins.h"
50 #include "internal-fn.h"
51 #include "tree-vector-builder.h"
52 #include "vec-perm-indices.h"
53 #include "tree-ssa-loop-niter.h"
54 #include "gimple-fold.h"
56 /* For lang_hooks.types.type_for_mode. */
57 #include "langhooks.h"
59 /* Return the vectorized type for the given statement. */
61 tree
62 stmt_vectype (struct _stmt_vec_info *stmt_info)
64 return STMT_VINFO_VECTYPE (stmt_info);
67 /* Return TRUE iff the given statement is in an inner loop relative to
68 the loop being vectorized. */
69 bool
70 stmt_in_inner_loop_p (struct _stmt_vec_info *stmt_info)
72 gimple *stmt = STMT_VINFO_STMT (stmt_info);
73 basic_block bb = gimple_bb (stmt);
74 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
75 struct loop* loop;
77 if (!loop_vinfo)
78 return false;
80 loop = LOOP_VINFO_LOOP (loop_vinfo);
82 return (bb->loop_father == loop->inner);
85 /* Record the cost of a statement, either by directly informing the
86 target model or by saving it in a vector for later processing.
87 Return a preliminary estimate of the statement's cost. */
89 unsigned
90 record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
91 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
92 int misalign, enum vect_cost_model_location where)
94 if ((kind == vector_load || kind == unaligned_load)
95 && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
96 kind = vector_gather_load;
97 if ((kind == vector_store || kind == unaligned_store)
98 && STMT_VINFO_GATHER_SCATTER_P (stmt_info))
99 kind = vector_scatter_store;
100 if (body_cost_vec)
102 tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
103 stmt_info_for_cost si = { count, kind,
104 stmt_info ? STMT_VINFO_STMT (stmt_info) : NULL,
105 misalign };
106 body_cost_vec->safe_push (si);
107 return (unsigned)
108 (builtin_vectorization_cost (kind, vectype, misalign) * count);
110 else
111 return add_stmt_cost (stmt_info->vinfo->target_cost_data,
112 count, kind, stmt_info, misalign, where);
115 /* Return a variable of type ELEM_TYPE[NELEMS]. */
117 static tree
118 create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
120 return create_tmp_var (build_array_type_nelts (elem_type, nelems),
121 "vect_array");
124 /* ARRAY is an array of vectors created by create_vector_array.
125 Return an SSA_NAME for the vector in index N. The reference
126 is part of the vectorization of STMT and the vector is associated
127 with scalar destination SCALAR_DEST. */
129 static tree
130 read_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree scalar_dest,
131 tree array, unsigned HOST_WIDE_INT n)
133 tree vect_type, vect, vect_name, array_ref;
134 gimple *new_stmt;
136 gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
137 vect_type = TREE_TYPE (TREE_TYPE (array));
138 vect = vect_create_destination_var (scalar_dest, vect_type);
139 array_ref = build4 (ARRAY_REF, vect_type, array,
140 build_int_cst (size_type_node, n),
141 NULL_TREE, NULL_TREE);
143 new_stmt = gimple_build_assign (vect, array_ref);
144 vect_name = make_ssa_name (vect, new_stmt);
145 gimple_assign_set_lhs (new_stmt, vect_name);
146 vect_finish_stmt_generation (stmt, new_stmt, gsi);
148 return vect_name;
151 /* ARRAY is an array of vectors created by create_vector_array.
152 Emit code to store SSA_NAME VECT in index N of the array.
153 The store is part of the vectorization of STMT. */
155 static void
156 write_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree vect,
157 tree array, unsigned HOST_WIDE_INT n)
159 tree array_ref;
160 gimple *new_stmt;
162 array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
163 build_int_cst (size_type_node, n),
164 NULL_TREE, NULL_TREE);
166 new_stmt = gimple_build_assign (array_ref, vect);
167 vect_finish_stmt_generation (stmt, new_stmt, gsi);
170 /* PTR is a pointer to an array of type TYPE. Return a representation
171 of *PTR. The memory reference replaces those in FIRST_DR
172 (and its group). */
174 static tree
175 create_array_ref (tree type, tree ptr, tree alias_ptr_type)
177 tree mem_ref;
179 mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
180 /* Arrays have the same alignment as their type. */
181 set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
182 return mem_ref;
185 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
187 /* Function vect_mark_relevant.
189 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
191 static void
192 vect_mark_relevant (vec<gimple *> *worklist, gimple *stmt,
193 enum vect_relevant relevant, bool live_p)
195 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
196 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
197 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
198 gimple *pattern_stmt;
200 if (dump_enabled_p ())
202 dump_printf_loc (MSG_NOTE, vect_location,
203 "mark relevant %d, live %d: ", relevant, live_p);
204 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
207 /* If this stmt is an original stmt in a pattern, we might need to mark its
208 related pattern stmt instead of the original stmt. However, such stmts
209 may have their own uses that are not in any pattern, in such cases the
210 stmt itself should be marked. */
211 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
213 /* This is the last stmt in a sequence that was detected as a
214 pattern that can potentially be vectorized. Don't mark the stmt
215 as relevant/live because it's not going to be vectorized.
216 Instead mark the pattern-stmt that replaces it. */
218 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
220 if (dump_enabled_p ())
221 dump_printf_loc (MSG_NOTE, vect_location,
222 "last stmt in pattern. don't mark"
223 " relevant/live.\n");
224 stmt_info = vinfo_for_stmt (pattern_stmt);
225 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
226 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
227 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
228 stmt = pattern_stmt;
231 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
232 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
233 STMT_VINFO_RELEVANT (stmt_info) = relevant;
235 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
236 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
238 if (dump_enabled_p ())
239 dump_printf_loc (MSG_NOTE, vect_location,
240 "already marked relevant/live.\n");
241 return;
244 worklist->safe_push (stmt);
248 /* Function is_simple_and_all_uses_invariant
250 Return true if STMT is simple and all uses of it are invariant. */
252 bool
253 is_simple_and_all_uses_invariant (gimple *stmt, loop_vec_info loop_vinfo)
255 tree op;
256 gimple *def_stmt;
257 ssa_op_iter iter;
259 if (!is_gimple_assign (stmt))
260 return false;
262 FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
264 enum vect_def_type dt = vect_uninitialized_def;
266 if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt))
268 if (dump_enabled_p ())
269 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
270 "use not simple.\n");
271 return false;
274 if (dt != vect_external_def && dt != vect_constant_def)
275 return false;
277 return true;
280 /* Function vect_stmt_relevant_p.
282 Return true if STMT in loop that is represented by LOOP_VINFO is
283 "relevant for vectorization".
285 A stmt is considered "relevant for vectorization" if:
286 - it has uses outside the loop.
287 - it has vdefs (it alters memory).
288 - control stmts in the loop (except for the exit condition).
290 CHECKME: what other side effects would the vectorizer allow? */
292 static bool
293 vect_stmt_relevant_p (gimple *stmt, loop_vec_info loop_vinfo,
294 enum vect_relevant *relevant, bool *live_p)
296 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
297 ssa_op_iter op_iter;
298 imm_use_iterator imm_iter;
299 use_operand_p use_p;
300 def_operand_p def_p;
302 *relevant = vect_unused_in_scope;
303 *live_p = false;
305 /* cond stmt other than loop exit cond. */
306 if (is_ctrl_stmt (stmt)
307 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
308 != loop_exit_ctrl_vec_info_type)
309 *relevant = vect_used_in_scope;
311 /* changing memory. */
312 if (gimple_code (stmt) != GIMPLE_PHI)
313 if (gimple_vdef (stmt)
314 && !gimple_clobber_p (stmt))
316 if (dump_enabled_p ())
317 dump_printf_loc (MSG_NOTE, vect_location,
318 "vec_stmt_relevant_p: stmt has vdefs.\n");
319 *relevant = vect_used_in_scope;
322 /* uses outside the loop. */
323 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
325 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
327 basic_block bb = gimple_bb (USE_STMT (use_p));
328 if (!flow_bb_inside_loop_p (loop, bb))
330 if (dump_enabled_p ())
331 dump_printf_loc (MSG_NOTE, vect_location,
332 "vec_stmt_relevant_p: used out of loop.\n");
334 if (is_gimple_debug (USE_STMT (use_p)))
335 continue;
337 /* We expect all such uses to be in the loop exit phis
338 (because of loop closed form) */
339 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
340 gcc_assert (bb == single_exit (loop)->dest);
342 *live_p = true;
347 if (*live_p && *relevant == vect_unused_in_scope
348 && !is_simple_and_all_uses_invariant (stmt, loop_vinfo))
350 if (dump_enabled_p ())
351 dump_printf_loc (MSG_NOTE, vect_location,
352 "vec_stmt_relevant_p: stmt live but not relevant.\n");
353 *relevant = vect_used_only_live;
356 return (*live_p || *relevant);
360 /* Function exist_non_indexing_operands_for_use_p
362 USE is one of the uses attached to STMT. Check if USE is
363 used in STMT for anything other than indexing an array. */
365 static bool
366 exist_non_indexing_operands_for_use_p (tree use, gimple *stmt)
368 tree operand;
369 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
371 /* USE corresponds to some operand in STMT. If there is no data
372 reference in STMT, then any operand that corresponds to USE
373 is not indexing an array. */
374 if (!STMT_VINFO_DATA_REF (stmt_info))
375 return true;
377 /* STMT has a data_ref. FORNOW this means that its of one of
378 the following forms:
379 -1- ARRAY_REF = var
380 -2- var = ARRAY_REF
381 (This should have been verified in analyze_data_refs).
383 'var' in the second case corresponds to a def, not a use,
384 so USE cannot correspond to any operands that are not used
385 for array indexing.
387 Therefore, all we need to check is if STMT falls into the
388 first case, and whether var corresponds to USE. */
390 if (!gimple_assign_copy_p (stmt))
392 if (is_gimple_call (stmt)
393 && gimple_call_internal_p (stmt))
395 internal_fn ifn = gimple_call_internal_fn (stmt);
396 int mask_index = internal_fn_mask_index (ifn);
397 if (mask_index >= 0
398 && use == gimple_call_arg (stmt, mask_index))
399 return true;
400 int stored_value_index = internal_fn_stored_value_index (ifn);
401 if (stored_value_index >= 0
402 && use == gimple_call_arg (stmt, stored_value_index))
403 return true;
404 if (internal_gather_scatter_fn_p (ifn)
405 && use == gimple_call_arg (stmt, 1))
406 return true;
408 return false;
411 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
412 return false;
413 operand = gimple_assign_rhs1 (stmt);
414 if (TREE_CODE (operand) != SSA_NAME)
415 return false;
417 if (operand == use)
418 return true;
420 return false;
425 Function process_use.
427 Inputs:
428 - a USE in STMT in a loop represented by LOOP_VINFO
429 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
430 that defined USE. This is done by calling mark_relevant and passing it
431 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
432 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
433 be performed.
435 Outputs:
436 Generally, LIVE_P and RELEVANT are used to define the liveness and
437 relevance info of the DEF_STMT of this USE:
438 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
439 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
440 Exceptions:
441 - case 1: If USE is used only for address computations (e.g. array indexing),
442 which does not need to be directly vectorized, then the liveness/relevance
443 of the respective DEF_STMT is left unchanged.
444 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
445 skip DEF_STMT cause it had already been processed.
446 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
447 be modified accordingly.
449 Return true if everything is as expected. Return false otherwise. */
451 static bool
452 process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo,
453 enum vect_relevant relevant, vec<gimple *> *worklist,
454 bool force)
456 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
457 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
458 stmt_vec_info dstmt_vinfo;
459 basic_block bb, def_bb;
460 gimple *def_stmt;
461 enum vect_def_type dt;
463 /* case 1: we are only interested in uses that need to be vectorized. Uses
464 that are used for address computation are not considered relevant. */
465 if (!force && !exist_non_indexing_operands_for_use_p (use, stmt))
466 return true;
468 if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &dt))
470 if (dump_enabled_p ())
471 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
472 "not vectorized: unsupported use in stmt.\n");
473 return false;
476 if (!def_stmt || gimple_nop_p (def_stmt))
477 return true;
479 def_bb = gimple_bb (def_stmt);
480 if (!flow_bb_inside_loop_p (loop, def_bb))
482 if (dump_enabled_p ())
483 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt is out of loop.\n");
484 return true;
487 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
488 DEF_STMT must have already been processed, because this should be the
489 only way that STMT, which is a reduction-phi, was put in the worklist,
490 as there should be no other uses for DEF_STMT in the loop. So we just
491 check that everything is as expected, and we are done. */
492 dstmt_vinfo = vinfo_for_stmt (def_stmt);
493 bb = gimple_bb (stmt);
494 if (gimple_code (stmt) == GIMPLE_PHI
495 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
496 && gimple_code (def_stmt) != GIMPLE_PHI
497 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
498 && bb->loop_father == def_bb->loop_father)
500 if (dump_enabled_p ())
501 dump_printf_loc (MSG_NOTE, vect_location,
502 "reduc-stmt defining reduc-phi in the same nest.\n");
503 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
504 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
505 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
506 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
507 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
508 return true;
511 /* case 3a: outer-loop stmt defining an inner-loop stmt:
512 outer-loop-header-bb:
513 d = def_stmt
514 inner-loop:
515 stmt # use (d)
516 outer-loop-tail-bb:
517 ... */
518 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
520 if (dump_enabled_p ())
521 dump_printf_loc (MSG_NOTE, vect_location,
522 "outer-loop def-stmt defining inner-loop stmt.\n");
524 switch (relevant)
526 case vect_unused_in_scope:
527 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
528 vect_used_in_scope : vect_unused_in_scope;
529 break;
531 case vect_used_in_outer_by_reduction:
532 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
533 relevant = vect_used_by_reduction;
534 break;
536 case vect_used_in_outer:
537 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
538 relevant = vect_used_in_scope;
539 break;
541 case vect_used_in_scope:
542 break;
544 default:
545 gcc_unreachable ();
549 /* case 3b: inner-loop stmt defining an outer-loop stmt:
550 outer-loop-header-bb:
552 inner-loop:
553 d = def_stmt
554 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
555 stmt # use (d) */
556 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
558 if (dump_enabled_p ())
559 dump_printf_loc (MSG_NOTE, vect_location,
560 "inner-loop def-stmt defining outer-loop stmt.\n");
562 switch (relevant)
564 case vect_unused_in_scope:
565 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
566 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
567 vect_used_in_outer_by_reduction : vect_unused_in_scope;
568 break;
570 case vect_used_by_reduction:
571 case vect_used_only_live:
572 relevant = vect_used_in_outer_by_reduction;
573 break;
575 case vect_used_in_scope:
576 relevant = vect_used_in_outer;
577 break;
579 default:
580 gcc_unreachable ();
583 /* We are also not interested in uses on loop PHI backedges that are
584 inductions. Otherwise we'll needlessly vectorize the IV increment
585 and cause hybrid SLP for SLP inductions. Unless the PHI is live
586 of course. */
587 else if (gimple_code (stmt) == GIMPLE_PHI
588 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_induction_def
589 && ! STMT_VINFO_LIVE_P (stmt_vinfo)
590 && (PHI_ARG_DEF_FROM_EDGE (stmt, loop_latch_edge (bb->loop_father))
591 == use))
593 if (dump_enabled_p ())
594 dump_printf_loc (MSG_NOTE, vect_location,
595 "induction value on backedge.\n");
596 return true;
600 vect_mark_relevant (worklist, def_stmt, relevant, false);
601 return true;
605 /* Function vect_mark_stmts_to_be_vectorized.
607 Not all stmts in the loop need to be vectorized. For example:
609 for i...
610 for j...
611 1. T0 = i + j
612 2. T1 = a[T0]
614 3. j = j + 1
616 Stmt 1 and 3 do not need to be vectorized, because loop control and
617 addressing of vectorized data-refs are handled differently.
619 This pass detects such stmts. */
621 bool
622 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
624 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
625 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
626 unsigned int nbbs = loop->num_nodes;
627 gimple_stmt_iterator si;
628 gimple *stmt;
629 unsigned int i;
630 stmt_vec_info stmt_vinfo;
631 basic_block bb;
632 gimple *phi;
633 bool live_p;
634 enum vect_relevant relevant;
636 if (dump_enabled_p ())
637 dump_printf_loc (MSG_NOTE, vect_location,
638 "=== vect_mark_stmts_to_be_vectorized ===\n");
640 auto_vec<gimple *, 64> worklist;
642 /* 1. Init worklist. */
643 for (i = 0; i < nbbs; i++)
645 bb = bbs[i];
646 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
648 phi = gsi_stmt (si);
649 if (dump_enabled_p ())
651 dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? ");
652 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
655 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
656 vect_mark_relevant (&worklist, phi, relevant, live_p);
658 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
660 stmt = gsi_stmt (si);
661 if (dump_enabled_p ())
663 dump_printf_loc (MSG_NOTE, vect_location, "init: stmt relevant? ");
664 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
667 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
668 vect_mark_relevant (&worklist, stmt, relevant, live_p);
672 /* 2. Process_worklist */
673 while (worklist.length () > 0)
675 use_operand_p use_p;
676 ssa_op_iter iter;
678 stmt = worklist.pop ();
679 if (dump_enabled_p ())
681 dump_printf_loc (MSG_NOTE, vect_location, "worklist: examine stmt: ");
682 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
685 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
686 (DEF_STMT) as relevant/irrelevant according to the relevance property
687 of STMT. */
688 stmt_vinfo = vinfo_for_stmt (stmt);
689 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
691 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
692 propagated as is to the DEF_STMTs of its USEs.
694 One exception is when STMT has been identified as defining a reduction
695 variable; in this case we set the relevance to vect_used_by_reduction.
696 This is because we distinguish between two kinds of relevant stmts -
697 those that are used by a reduction computation, and those that are
698 (also) used by a regular computation. This allows us later on to
699 identify stmts that are used solely by a reduction, and therefore the
700 order of the results that they produce does not have to be kept. */
702 switch (STMT_VINFO_DEF_TYPE (stmt_vinfo))
704 case vect_reduction_def:
705 gcc_assert (relevant != vect_unused_in_scope);
706 if (relevant != vect_unused_in_scope
707 && relevant != vect_used_in_scope
708 && relevant != vect_used_by_reduction
709 && relevant != vect_used_only_live)
711 if (dump_enabled_p ())
712 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
713 "unsupported use of reduction.\n");
714 return false;
716 break;
718 case vect_nested_cycle:
719 if (relevant != vect_unused_in_scope
720 && relevant != vect_used_in_outer_by_reduction
721 && relevant != vect_used_in_outer)
723 if (dump_enabled_p ())
724 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
725 "unsupported use of nested cycle.\n");
727 return false;
729 break;
731 case vect_double_reduction_def:
732 if (relevant != vect_unused_in_scope
733 && relevant != vect_used_by_reduction
734 && relevant != vect_used_only_live)
736 if (dump_enabled_p ())
737 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
738 "unsupported use of double reduction.\n");
740 return false;
742 break;
744 default:
745 break;
748 if (is_pattern_stmt_p (stmt_vinfo))
750 /* Pattern statements are not inserted into the code, so
751 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
752 have to scan the RHS or function arguments instead. */
753 if (is_gimple_assign (stmt))
755 enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
756 tree op = gimple_assign_rhs1 (stmt);
758 i = 1;
759 if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
761 if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo,
762 relevant, &worklist, false)
763 || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo,
764 relevant, &worklist, false))
765 return false;
766 i = 2;
768 for (; i < gimple_num_ops (stmt); i++)
770 op = gimple_op (stmt, i);
771 if (TREE_CODE (op) == SSA_NAME
772 && !process_use (stmt, op, loop_vinfo, relevant,
773 &worklist, false))
774 return false;
777 else if (is_gimple_call (stmt))
779 for (i = 0; i < gimple_call_num_args (stmt); i++)
781 tree arg = gimple_call_arg (stmt, i);
782 if (!process_use (stmt, arg, loop_vinfo, relevant,
783 &worklist, false))
784 return false;
788 else
789 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
791 tree op = USE_FROM_PTR (use_p);
792 if (!process_use (stmt, op, loop_vinfo, relevant,
793 &worklist, false))
794 return false;
797 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
799 gather_scatter_info gs_info;
800 if (!vect_check_gather_scatter (stmt, loop_vinfo, &gs_info))
801 gcc_unreachable ();
802 if (!process_use (stmt, gs_info.offset, loop_vinfo, relevant,
803 &worklist, true))
804 return false;
806 } /* while worklist */
808 return true;
812 /* Function vect_model_simple_cost.
814 Models cost for simple operations, i.e. those that only emit ncopies of a
815 single op. Right now, this does not account for multiple insns that could
816 be generated for the single vector op. We will handle that shortly. */
818 void
819 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
820 enum vect_def_type *dt,
821 int ndts,
822 stmt_vector_for_cost *prologue_cost_vec,
823 stmt_vector_for_cost *body_cost_vec)
825 int i;
826 int inside_cost = 0, prologue_cost = 0;
828 /* The SLP costs were already calculated during SLP tree build. */
829 if (PURE_SLP_STMT (stmt_info))
830 return;
832 /* Cost the "broadcast" of a scalar operand in to a vector operand.
833 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
834 cost model. */
835 for (i = 0; i < ndts; i++)
836 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
837 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, scalar_to_vec,
838 stmt_info, 0, vect_prologue);
840 /* Pass the inside-of-loop statements to the target-specific cost model. */
841 inside_cost = record_stmt_cost (body_cost_vec, ncopies, vector_stmt,
842 stmt_info, 0, vect_body);
844 if (dump_enabled_p ())
845 dump_printf_loc (MSG_NOTE, vect_location,
846 "vect_model_simple_cost: inside_cost = %d, "
847 "prologue_cost = %d .\n", inside_cost, prologue_cost);
851 /* Model cost for type demotion and promotion operations. PWR is normally
852 zero for single-step promotions and demotions. It will be one if
853 two-step promotion/demotion is required, and so on. Each additional
854 step doubles the number of instructions required. */
856 static void
857 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
858 enum vect_def_type *dt, int pwr)
860 int i, tmp;
861 int inside_cost = 0, prologue_cost = 0;
862 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
863 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
864 void *target_cost_data;
866 /* The SLP costs were already calculated during SLP tree build. */
867 if (PURE_SLP_STMT (stmt_info))
868 return;
870 if (loop_vinfo)
871 target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
872 else
873 target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
875 for (i = 0; i < pwr + 1; i++)
877 tmp = (STMT_VINFO_TYPE (stmt_info) == type_promotion_vec_info_type) ?
878 (i + 1) : i;
879 inside_cost += add_stmt_cost (target_cost_data, vect_pow2 (tmp),
880 vec_promote_demote, stmt_info, 0,
881 vect_body);
884 /* FORNOW: Assuming maximum 2 args per stmts. */
885 for (i = 0; i < 2; i++)
886 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
887 prologue_cost += add_stmt_cost (target_cost_data, 1, vector_stmt,
888 stmt_info, 0, vect_prologue);
890 if (dump_enabled_p ())
891 dump_printf_loc (MSG_NOTE, vect_location,
892 "vect_model_promotion_demotion_cost: inside_cost = %d, "
893 "prologue_cost = %d .\n", inside_cost, prologue_cost);
896 /* Function vect_model_store_cost
898 Models cost for stores. In the case of grouped accesses, one access
899 has the overhead of the grouped access attributed to it. */
901 void
902 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
903 vect_memory_access_type memory_access_type,
904 vec_load_store_type vls_type, slp_tree slp_node,
905 stmt_vector_for_cost *prologue_cost_vec,
906 stmt_vector_for_cost *body_cost_vec)
908 unsigned int inside_cost = 0, prologue_cost = 0;
909 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
910 gimple *first_stmt = STMT_VINFO_STMT (stmt_info);
911 bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
913 if (vls_type == VLS_STORE_INVARIANT)
914 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, scalar_to_vec,
915 stmt_info, 0, vect_prologue);
917 /* Grouped stores update all elements in the group at once,
918 so we want the DR for the first statement. */
919 if (!slp_node && grouped_access_p)
921 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
922 dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
925 /* True if we should include any once-per-group costs as well as
926 the cost of the statement itself. For SLP we only get called
927 once per group anyhow. */
928 bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info));
930 /* We assume that the cost of a single store-lanes instruction is
931 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
932 access is instead being provided by a permute-and-store operation,
933 include the cost of the permutes. */
934 if (first_stmt_p
935 && memory_access_type == VMAT_CONTIGUOUS_PERMUTE)
937 /* Uses a high and low interleave or shuffle operations for each
938 needed permute. */
939 int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
940 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
941 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
942 stmt_info, 0, vect_body);
944 if (dump_enabled_p ())
945 dump_printf_loc (MSG_NOTE, vect_location,
946 "vect_model_store_cost: strided group_size = %d .\n",
947 group_size);
950 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
951 /* Costs of the stores. */
952 if (memory_access_type == VMAT_ELEMENTWISE
953 || memory_access_type == VMAT_GATHER_SCATTER)
955 /* N scalar stores plus extracting the elements. */
956 unsigned int assumed_nunits = vect_nunits_for_cost (vectype);
957 inside_cost += record_stmt_cost (body_cost_vec,
958 ncopies * assumed_nunits,
959 scalar_store, stmt_info, 0, vect_body);
961 else
962 vect_get_store_cost (dr, ncopies, &inside_cost, body_cost_vec);
964 if (memory_access_type == VMAT_ELEMENTWISE
965 || memory_access_type == VMAT_STRIDED_SLP)
967 /* N scalar stores plus extracting the elements. */
968 unsigned int assumed_nunits = vect_nunits_for_cost (vectype);
969 inside_cost += record_stmt_cost (body_cost_vec,
970 ncopies * assumed_nunits,
971 vec_to_scalar, stmt_info, 0, vect_body);
974 if (dump_enabled_p ())
975 dump_printf_loc (MSG_NOTE, vect_location,
976 "vect_model_store_cost: inside_cost = %d, "
977 "prologue_cost = %d .\n", inside_cost, prologue_cost);
981 /* Calculate cost of DR's memory access. */
982 void
983 vect_get_store_cost (struct data_reference *dr, int ncopies,
984 unsigned int *inside_cost,
985 stmt_vector_for_cost *body_cost_vec)
987 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
988 gimple *stmt = DR_STMT (dr);
989 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
991 switch (alignment_support_scheme)
993 case dr_aligned:
995 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
996 vector_store, stmt_info, 0,
997 vect_body);
999 if (dump_enabled_p ())
1000 dump_printf_loc (MSG_NOTE, vect_location,
1001 "vect_model_store_cost: aligned.\n");
1002 break;
1005 case dr_unaligned_supported:
1007 /* Here, we assign an additional cost for the unaligned store. */
1008 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1009 unaligned_store, stmt_info,
1010 DR_MISALIGNMENT (dr), vect_body);
1011 if (dump_enabled_p ())
1012 dump_printf_loc (MSG_NOTE, vect_location,
1013 "vect_model_store_cost: unaligned supported by "
1014 "hardware.\n");
1015 break;
1018 case dr_unaligned_unsupported:
1020 *inside_cost = VECT_MAX_COST;
1022 if (dump_enabled_p ())
1023 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1024 "vect_model_store_cost: unsupported access.\n");
1025 break;
1028 default:
1029 gcc_unreachable ();
1034 /* Function vect_model_load_cost
1036 Models cost for loads. In the case of grouped accesses, one access has
1037 the overhead of the grouped access attributed to it. Since unaligned
1038 accesses are supported for loads, we also account for the costs of the
1039 access scheme chosen. */
1041 void
1042 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
1043 vect_memory_access_type memory_access_type,
1044 slp_tree slp_node,
1045 stmt_vector_for_cost *prologue_cost_vec,
1046 stmt_vector_for_cost *body_cost_vec)
1048 gimple *first_stmt = STMT_VINFO_STMT (stmt_info);
1049 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
1050 unsigned int inside_cost = 0, prologue_cost = 0;
1051 bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
1053 /* Grouped loads read all elements in the group at once,
1054 so we want the DR for the first statement. */
1055 if (!slp_node && grouped_access_p)
1057 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
1058 dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
1061 /* True if we should include any once-per-group costs as well as
1062 the cost of the statement itself. For SLP we only get called
1063 once per group anyhow. */
1064 bool first_stmt_p = (first_stmt == STMT_VINFO_STMT (stmt_info));
1066 /* We assume that the cost of a single load-lanes instruction is
1067 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1068 access is instead being provided by a load-and-permute operation,
1069 include the cost of the permutes. */
1070 if (first_stmt_p
1071 && memory_access_type == VMAT_CONTIGUOUS_PERMUTE)
1073 /* Uses an even and odd extract operations or shuffle operations
1074 for each needed permute. */
1075 int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
1076 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
1077 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
1078 stmt_info, 0, vect_body);
1080 if (dump_enabled_p ())
1081 dump_printf_loc (MSG_NOTE, vect_location,
1082 "vect_model_load_cost: strided group_size = %d .\n",
1083 group_size);
1086 /* The loads themselves. */
1087 if (memory_access_type == VMAT_ELEMENTWISE
1088 || memory_access_type == VMAT_GATHER_SCATTER)
1090 /* N scalar loads plus gathering them into a vector. */
1091 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1092 unsigned int assumed_nunits = vect_nunits_for_cost (vectype);
1093 inside_cost += record_stmt_cost (body_cost_vec,
1094 ncopies * assumed_nunits,
1095 scalar_load, stmt_info, 0, vect_body);
1097 else
1098 vect_get_load_cost (dr, ncopies, first_stmt_p,
1099 &inside_cost, &prologue_cost,
1100 prologue_cost_vec, body_cost_vec, true);
1101 if (memory_access_type == VMAT_ELEMENTWISE
1102 || memory_access_type == VMAT_STRIDED_SLP)
1103 inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_construct,
1104 stmt_info, 0, vect_body);
1106 if (dump_enabled_p ())
1107 dump_printf_loc (MSG_NOTE, vect_location,
1108 "vect_model_load_cost: inside_cost = %d, "
1109 "prologue_cost = %d .\n", inside_cost, prologue_cost);
1113 /* Calculate cost of DR's memory access. */
1114 void
1115 vect_get_load_cost (struct data_reference *dr, int ncopies,
1116 bool add_realign_cost, unsigned int *inside_cost,
1117 unsigned int *prologue_cost,
1118 stmt_vector_for_cost *prologue_cost_vec,
1119 stmt_vector_for_cost *body_cost_vec,
1120 bool record_prologue_costs)
1122 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
1123 gimple *stmt = DR_STMT (dr);
1124 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1126 switch (alignment_support_scheme)
1128 case dr_aligned:
1130 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1131 stmt_info, 0, vect_body);
1133 if (dump_enabled_p ())
1134 dump_printf_loc (MSG_NOTE, vect_location,
1135 "vect_model_load_cost: aligned.\n");
1137 break;
1139 case dr_unaligned_supported:
1141 /* Here, we assign an additional cost for the unaligned load. */
1142 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1143 unaligned_load, stmt_info,
1144 DR_MISALIGNMENT (dr), vect_body);
1146 if (dump_enabled_p ())
1147 dump_printf_loc (MSG_NOTE, vect_location,
1148 "vect_model_load_cost: unaligned supported by "
1149 "hardware.\n");
1151 break;
1153 case dr_explicit_realign:
1155 *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1156 vector_load, stmt_info, 0, vect_body);
1157 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1158 vec_perm, stmt_info, 0, vect_body);
1160 /* FIXME: If the misalignment remains fixed across the iterations of
1161 the containing loop, the following cost should be added to the
1162 prologue costs. */
1163 if (targetm.vectorize.builtin_mask_for_load)
1164 *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1165 stmt_info, 0, vect_body);
1167 if (dump_enabled_p ())
1168 dump_printf_loc (MSG_NOTE, vect_location,
1169 "vect_model_load_cost: explicit realign\n");
1171 break;
1173 case dr_explicit_realign_optimized:
1175 if (dump_enabled_p ())
1176 dump_printf_loc (MSG_NOTE, vect_location,
1177 "vect_model_load_cost: unaligned software "
1178 "pipelined.\n");
1180 /* Unaligned software pipeline has a load of an address, an initial
1181 load, and possibly a mask operation to "prime" the loop. However,
1182 if this is an access in a group of loads, which provide grouped
1183 access, then the above cost should only be considered for one
1184 access in the group. Inside the loop, there is a load op
1185 and a realignment op. */
1187 if (add_realign_cost && record_prologue_costs)
1189 *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1190 vector_stmt, stmt_info,
1191 0, vect_prologue);
1192 if (targetm.vectorize.builtin_mask_for_load)
1193 *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1194 vector_stmt, stmt_info,
1195 0, vect_prologue);
1198 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1199 stmt_info, 0, vect_body);
1200 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1201 stmt_info, 0, vect_body);
1203 if (dump_enabled_p ())
1204 dump_printf_loc (MSG_NOTE, vect_location,
1205 "vect_model_load_cost: explicit realign optimized"
1206 "\n");
1208 break;
1211 case dr_unaligned_unsupported:
1213 *inside_cost = VECT_MAX_COST;
1215 if (dump_enabled_p ())
1216 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1217 "vect_model_load_cost: unsupported access.\n");
1218 break;
1221 default:
1222 gcc_unreachable ();
1226 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1227 the loop preheader for the vectorized stmt STMT. */
1229 static void
1230 vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi)
1232 if (gsi)
1233 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1234 else
1236 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1237 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1239 if (loop_vinfo)
1241 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1242 basic_block new_bb;
1243 edge pe;
1245 if (nested_in_vect_loop_p (loop, stmt))
1246 loop = loop->inner;
1248 pe = loop_preheader_edge (loop);
1249 new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
1250 gcc_assert (!new_bb);
1252 else
1254 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
1255 basic_block bb;
1256 gimple_stmt_iterator gsi_bb_start;
1258 gcc_assert (bb_vinfo);
1259 bb = BB_VINFO_BB (bb_vinfo);
1260 gsi_bb_start = gsi_after_labels (bb);
1261 gsi_insert_before (&gsi_bb_start, new_stmt, GSI_SAME_STMT);
1265 if (dump_enabled_p ())
1267 dump_printf_loc (MSG_NOTE, vect_location,
1268 "created new init_stmt: ");
1269 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, new_stmt, 0);
1273 /* Function vect_init_vector.
1275 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1276 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1277 vector type a vector with all elements equal to VAL is created first.
1278 Place the initialization at BSI if it is not NULL. Otherwise, place the
1279 initialization at the loop preheader.
1280 Return the DEF of INIT_STMT.
1281 It will be used in the vectorization of STMT. */
1283 tree
1284 vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi)
1286 gimple *init_stmt;
1287 tree new_temp;
1289 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1290 if (! useless_type_conversion_p (type, TREE_TYPE (val)))
1292 gcc_assert (TREE_CODE (type) == VECTOR_TYPE);
1293 if (! types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1295 /* Scalar boolean value should be transformed into
1296 all zeros or all ones value before building a vector. */
1297 if (VECTOR_BOOLEAN_TYPE_P (type))
1299 tree true_val = build_all_ones_cst (TREE_TYPE (type));
1300 tree false_val = build_zero_cst (TREE_TYPE (type));
1302 if (CONSTANT_CLASS_P (val))
1303 val = integer_zerop (val) ? false_val : true_val;
1304 else
1306 new_temp = make_ssa_name (TREE_TYPE (type));
1307 init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1308 val, true_val, false_val);
1309 vect_init_vector_1 (stmt, init_stmt, gsi);
1310 val = new_temp;
1313 else if (CONSTANT_CLASS_P (val))
1314 val = fold_convert (TREE_TYPE (type), val);
1315 else
1317 new_temp = make_ssa_name (TREE_TYPE (type));
1318 if (! INTEGRAL_TYPE_P (TREE_TYPE (val)))
1319 init_stmt = gimple_build_assign (new_temp,
1320 fold_build1 (VIEW_CONVERT_EXPR,
1321 TREE_TYPE (type),
1322 val));
1323 else
1324 init_stmt = gimple_build_assign (new_temp, NOP_EXPR, val);
1325 vect_init_vector_1 (stmt, init_stmt, gsi);
1326 val = new_temp;
1329 val = build_vector_from_val (type, val);
1332 new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1333 init_stmt = gimple_build_assign (new_temp, val);
1334 vect_init_vector_1 (stmt, init_stmt, gsi);
1335 return new_temp;
1338 /* Function vect_get_vec_def_for_operand_1.
1340 For a defining stmt DEF_STMT of a scalar stmt, return a vector def with type
1341 DT that will be used in the vectorized stmt. */
1343 tree
1344 vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt)
1346 tree vec_oprnd;
1347 gimple *vec_stmt;
1348 stmt_vec_info def_stmt_info = NULL;
1350 switch (dt)
1352 /* operand is a constant or a loop invariant. */
1353 case vect_constant_def:
1354 case vect_external_def:
1355 /* Code should use vect_get_vec_def_for_operand. */
1356 gcc_unreachable ();
1358 /* operand is defined inside the loop. */
1359 case vect_internal_def:
1361 /* Get the def from the vectorized stmt. */
1362 def_stmt_info = vinfo_for_stmt (def_stmt);
1364 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1365 /* Get vectorized pattern statement. */
1366 if (!vec_stmt
1367 && STMT_VINFO_IN_PATTERN_P (def_stmt_info)
1368 && !STMT_VINFO_RELEVANT (def_stmt_info))
1369 vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1370 STMT_VINFO_RELATED_STMT (def_stmt_info)));
1371 gcc_assert (vec_stmt);
1372 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1373 vec_oprnd = PHI_RESULT (vec_stmt);
1374 else if (is_gimple_call (vec_stmt))
1375 vec_oprnd = gimple_call_lhs (vec_stmt);
1376 else
1377 vec_oprnd = gimple_assign_lhs (vec_stmt);
1378 return vec_oprnd;
1381 /* operand is defined by a loop header phi. */
1382 case vect_reduction_def:
1383 case vect_double_reduction_def:
1384 case vect_nested_cycle:
1385 case vect_induction_def:
1387 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1389 /* Get the def from the vectorized stmt. */
1390 def_stmt_info = vinfo_for_stmt (def_stmt);
1391 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1392 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1393 vec_oprnd = PHI_RESULT (vec_stmt);
1394 else
1395 vec_oprnd = gimple_get_lhs (vec_stmt);
1396 return vec_oprnd;
1399 default:
1400 gcc_unreachable ();
1405 /* Function vect_get_vec_def_for_operand.
1407 OP is an operand in STMT. This function returns a (vector) def that will be
1408 used in the vectorized stmt for STMT.
1410 In the case that OP is an SSA_NAME which is defined in the loop, then
1411 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1413 In case OP is an invariant or constant, a new stmt that creates a vector def
1414 needs to be introduced. VECTYPE may be used to specify a required type for
1415 vector invariant. */
1417 tree
1418 vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype)
1420 gimple *def_stmt;
1421 enum vect_def_type dt;
1422 bool is_simple_use;
1423 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1424 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1426 if (dump_enabled_p ())
1428 dump_printf_loc (MSG_NOTE, vect_location,
1429 "vect_get_vec_def_for_operand: ");
1430 dump_generic_expr (MSG_NOTE, TDF_SLIM, op);
1431 dump_printf (MSG_NOTE, "\n");
1434 is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt);
1435 gcc_assert (is_simple_use);
1436 if (def_stmt && dump_enabled_p ())
1438 dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = ");
1439 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0);
1442 if (dt == vect_constant_def || dt == vect_external_def)
1444 tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
1445 tree vector_type;
1447 if (vectype)
1448 vector_type = vectype;
1449 else if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op))
1450 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype))
1451 vector_type = build_same_sized_truth_vector_type (stmt_vectype);
1452 else
1453 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
1455 gcc_assert (vector_type);
1456 return vect_init_vector (stmt, op, vector_type, NULL);
1458 else
1459 return vect_get_vec_def_for_operand_1 (def_stmt, dt);
1463 /* Function vect_get_vec_def_for_stmt_copy
1465 Return a vector-def for an operand. This function is used when the
1466 vectorized stmt to be created (by the caller to this function) is a "copy"
1467 created in case the vectorized result cannot fit in one vector, and several
1468 copies of the vector-stmt are required. In this case the vector-def is
1469 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1470 of the stmt that defines VEC_OPRND.
1471 DT is the type of the vector def VEC_OPRND.
1473 Context:
1474 In case the vectorization factor (VF) is bigger than the number
1475 of elements that can fit in a vectype (nunits), we have to generate
1476 more than one vector stmt to vectorize the scalar stmt. This situation
1477 arises when there are multiple data-types operated upon in the loop; the
1478 smallest data-type determines the VF, and as a result, when vectorizing
1479 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1480 vector stmt (each computing a vector of 'nunits' results, and together
1481 computing 'VF' results in each iteration). This function is called when
1482 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1483 which VF=16 and nunits=4, so the number of copies required is 4):
1485 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1487 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1488 VS1.1: vx.1 = memref1 VS1.2
1489 VS1.2: vx.2 = memref2 VS1.3
1490 VS1.3: vx.3 = memref3
1492 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1493 VSnew.1: vz1 = vx.1 + ... VSnew.2
1494 VSnew.2: vz2 = vx.2 + ... VSnew.3
1495 VSnew.3: vz3 = vx.3 + ...
1497 The vectorization of S1 is explained in vectorizable_load.
1498 The vectorization of S2:
1499 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1500 the function 'vect_get_vec_def_for_operand' is called to
1501 get the relevant vector-def for each operand of S2. For operand x it
1502 returns the vector-def 'vx.0'.
1504 To create the remaining copies of the vector-stmt (VSnew.j), this
1505 function is called to get the relevant vector-def for each operand. It is
1506 obtained from the respective VS1.j stmt, which is recorded in the
1507 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1509 For example, to obtain the vector-def 'vx.1' in order to create the
1510 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1511 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1512 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1513 and return its def ('vx.1').
1514 Overall, to create the above sequence this function will be called 3 times:
1515 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1516 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1517 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1519 tree
1520 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1522 gimple *vec_stmt_for_operand;
1523 stmt_vec_info def_stmt_info;
1525 /* Do nothing; can reuse same def. */
1526 if (dt == vect_external_def || dt == vect_constant_def )
1527 return vec_oprnd;
1529 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1530 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1531 gcc_assert (def_stmt_info);
1532 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1533 gcc_assert (vec_stmt_for_operand);
1534 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1535 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1536 else
1537 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1538 return vec_oprnd;
1542 /* Get vectorized definitions for the operands to create a copy of an original
1543 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1545 void
1546 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1547 vec<tree> *vec_oprnds0,
1548 vec<tree> *vec_oprnds1)
1550 tree vec_oprnd = vec_oprnds0->pop ();
1552 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1553 vec_oprnds0->quick_push (vec_oprnd);
1555 if (vec_oprnds1 && vec_oprnds1->length ())
1557 vec_oprnd = vec_oprnds1->pop ();
1558 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1559 vec_oprnds1->quick_push (vec_oprnd);
1564 /* Get vectorized definitions for OP0 and OP1. */
1566 void
1567 vect_get_vec_defs (tree op0, tree op1, gimple *stmt,
1568 vec<tree> *vec_oprnds0,
1569 vec<tree> *vec_oprnds1,
1570 slp_tree slp_node)
1572 if (slp_node)
1574 int nops = (op1 == NULL_TREE) ? 1 : 2;
1575 auto_vec<tree> ops (nops);
1576 auto_vec<vec<tree> > vec_defs (nops);
1578 ops.quick_push (op0);
1579 if (op1)
1580 ops.quick_push (op1);
1582 vect_get_slp_defs (ops, slp_node, &vec_defs);
1584 *vec_oprnds0 = vec_defs[0];
1585 if (op1)
1586 *vec_oprnds1 = vec_defs[1];
1588 else
1590 tree vec_oprnd;
1592 vec_oprnds0->create (1);
1593 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt);
1594 vec_oprnds0->quick_push (vec_oprnd);
1596 if (op1)
1598 vec_oprnds1->create (1);
1599 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt);
1600 vec_oprnds1->quick_push (vec_oprnd);
1605 /* Helper function called by vect_finish_replace_stmt and
1606 vect_finish_stmt_generation. Set the location of the new
1607 statement and create a stmt_vec_info for it. */
1609 static void
1610 vect_finish_stmt_generation_1 (gimple *stmt, gimple *vec_stmt)
1612 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1613 vec_info *vinfo = stmt_info->vinfo;
1615 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, vinfo));
1617 if (dump_enabled_p ())
1619 dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: ");
1620 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, vec_stmt, 0);
1623 gimple_set_location (vec_stmt, gimple_location (stmt));
1625 /* While EH edges will generally prevent vectorization, stmt might
1626 e.g. be in a must-not-throw region. Ensure newly created stmts
1627 that could throw are part of the same region. */
1628 int lp_nr = lookup_stmt_eh_lp (stmt);
1629 if (lp_nr != 0 && stmt_could_throw_p (vec_stmt))
1630 add_stmt_to_eh_lp (vec_stmt, lp_nr);
1633 /* Replace the scalar statement STMT with a new vector statement VEC_STMT,
1634 which sets the same scalar result as STMT did. */
1636 void
1637 vect_finish_replace_stmt (gimple *stmt, gimple *vec_stmt)
1639 gcc_assert (gimple_get_lhs (stmt) == gimple_get_lhs (vec_stmt));
1641 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1642 gsi_replace (&gsi, vec_stmt, false);
1644 vect_finish_stmt_generation_1 (stmt, vec_stmt);
1647 /* Function vect_finish_stmt_generation.
1649 Insert a new stmt. */
1651 void
1652 vect_finish_stmt_generation (gimple *stmt, gimple *vec_stmt,
1653 gimple_stmt_iterator *gsi)
1655 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1657 if (!gsi_end_p (*gsi)
1658 && gimple_has_mem_ops (vec_stmt))
1660 gimple *at_stmt = gsi_stmt (*gsi);
1661 tree vuse = gimple_vuse (at_stmt);
1662 if (vuse && TREE_CODE (vuse) == SSA_NAME)
1664 tree vdef = gimple_vdef (at_stmt);
1665 gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1666 /* If we have an SSA vuse and insert a store, update virtual
1667 SSA form to avoid triggering the renamer. Do so only
1668 if we can easily see all uses - which is what almost always
1669 happens with the way vectorized stmts are inserted. */
1670 if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1671 && ((is_gimple_assign (vec_stmt)
1672 && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1673 || (is_gimple_call (vec_stmt)
1674 && !(gimple_call_flags (vec_stmt)
1675 & (ECF_CONST|ECF_PURE|ECF_NOVOPS)))))
1677 tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1678 gimple_set_vdef (vec_stmt, new_vdef);
1679 SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1683 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1684 vect_finish_stmt_generation_1 (stmt, vec_stmt);
1687 /* We want to vectorize a call to combined function CFN with function
1688 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1689 as the types of all inputs. Check whether this is possible using
1690 an internal function, returning its code if so or IFN_LAST if not. */
1692 static internal_fn
1693 vectorizable_internal_function (combined_fn cfn, tree fndecl,
1694 tree vectype_out, tree vectype_in)
1696 internal_fn ifn;
1697 if (internal_fn_p (cfn))
1698 ifn = as_internal_fn (cfn);
1699 else
1700 ifn = associated_internal_fn (fndecl);
1701 if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1703 const direct_internal_fn_info &info = direct_internal_fn (ifn);
1704 if (info.vectorizable)
1706 tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1707 tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1708 if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1709 OPTIMIZE_FOR_SPEED))
1710 return ifn;
1713 return IFN_LAST;
1717 static tree permute_vec_elements (tree, tree, tree, gimple *,
1718 gimple_stmt_iterator *);
1720 /* Check whether a load or store statement in the loop described by
1721 LOOP_VINFO is possible in a fully-masked loop. This is testing
1722 whether the vectorizer pass has the appropriate support, as well as
1723 whether the target does.
1725 VLS_TYPE says whether the statement is a load or store and VECTYPE
1726 is the type of the vector being loaded or stored. MEMORY_ACCESS_TYPE
1727 says how the load or store is going to be implemented and GROUP_SIZE
1728 is the number of load or store statements in the containing group.
1729 If the access is a gather load or scatter store, GS_INFO describes
1730 its arguments.
1732 Clear LOOP_VINFO_CAN_FULLY_MASK_P if a fully-masked loop is not
1733 supported, otherwise record the required mask types. */
1735 static void
1736 check_load_store_masking (loop_vec_info loop_vinfo, tree vectype,
1737 vec_load_store_type vls_type, int group_size,
1738 vect_memory_access_type memory_access_type,
1739 gather_scatter_info *gs_info)
1741 /* Invariant loads need no special support. */
1742 if (memory_access_type == VMAT_INVARIANT)
1743 return;
1745 vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
1746 machine_mode vecmode = TYPE_MODE (vectype);
1747 bool is_load = (vls_type == VLS_LOAD);
1748 if (memory_access_type == VMAT_LOAD_STORE_LANES)
1750 if (is_load
1751 ? !vect_load_lanes_supported (vectype, group_size, true)
1752 : !vect_store_lanes_supported (vectype, group_size, true))
1754 if (dump_enabled_p ())
1755 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1756 "can't use a fully-masked loop because the"
1757 " target doesn't have an appropriate masked"
1758 " load/store-lanes instruction.\n");
1759 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo) = false;
1760 return;
1762 unsigned int ncopies = vect_get_num_copies (loop_vinfo, vectype);
1763 vect_record_loop_mask (loop_vinfo, masks, ncopies, vectype);
1764 return;
1767 if (memory_access_type == VMAT_GATHER_SCATTER)
1769 internal_fn ifn = (is_load
1770 ? IFN_MASK_GATHER_LOAD
1771 : IFN_MASK_SCATTER_STORE);
1772 tree offset_type = TREE_TYPE (gs_info->offset);
1773 if (!internal_gather_scatter_fn_supported_p (ifn, vectype,
1774 gs_info->memory_type,
1775 TYPE_SIGN (offset_type),
1776 gs_info->scale))
1778 if (dump_enabled_p ())
1779 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1780 "can't use a fully-masked loop because the"
1781 " target doesn't have an appropriate masked"
1782 " gather load or scatter store instruction.\n");
1783 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo) = false;
1784 return;
1786 unsigned int ncopies = vect_get_num_copies (loop_vinfo, vectype);
1787 vect_record_loop_mask (loop_vinfo, masks, ncopies, vectype);
1788 return;
1791 if (memory_access_type != VMAT_CONTIGUOUS
1792 && memory_access_type != VMAT_CONTIGUOUS_PERMUTE)
1794 /* Element X of the data must come from iteration i * VF + X of the
1795 scalar loop. We need more work to support other mappings. */
1796 if (dump_enabled_p ())
1797 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1798 "can't use a fully-masked loop because an access"
1799 " isn't contiguous.\n");
1800 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo) = false;
1801 return;
1804 machine_mode mask_mode;
1805 if (!(targetm.vectorize.get_mask_mode
1806 (GET_MODE_NUNITS (vecmode),
1807 GET_MODE_SIZE (vecmode)).exists (&mask_mode))
1808 || !can_vec_mask_load_store_p (vecmode, mask_mode, is_load))
1810 if (dump_enabled_p ())
1811 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1812 "can't use a fully-masked loop because the target"
1813 " doesn't have the appropriate masked load or"
1814 " store.\n");
1815 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo) = false;
1816 return;
1818 /* We might load more scalars than we need for permuting SLP loads.
1819 We checked in get_group_load_store_type that the extra elements
1820 don't leak into a new vector. */
1821 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
1822 poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1823 unsigned int nvectors;
1824 if (can_div_away_from_zero_p (group_size * vf, nunits, &nvectors))
1825 vect_record_loop_mask (loop_vinfo, masks, nvectors, vectype);
1826 else
1827 gcc_unreachable ();
1830 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1831 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1832 that needs to be applied to all loads and stores in a vectorized loop.
1833 Return VEC_MASK if LOOP_MASK is null, otherwise return VEC_MASK & LOOP_MASK.
1835 MASK_TYPE is the type of both masks. If new statements are needed,
1836 insert them before GSI. */
1838 static tree
1839 prepare_load_store_mask (tree mask_type, tree loop_mask, tree vec_mask,
1840 gimple_stmt_iterator *gsi)
1842 gcc_assert (useless_type_conversion_p (mask_type, TREE_TYPE (vec_mask)));
1843 if (!loop_mask)
1844 return vec_mask;
1846 gcc_assert (TREE_TYPE (loop_mask) == mask_type);
1847 tree and_res = make_temp_ssa_name (mask_type, NULL, "vec_mask_and");
1848 gimple *and_stmt = gimple_build_assign (and_res, BIT_AND_EXPR,
1849 vec_mask, loop_mask);
1850 gsi_insert_before (gsi, and_stmt, GSI_SAME_STMT);
1851 return and_res;
1854 /* Determine whether we can use a gather load or scatter store to vectorize
1855 strided load or store STMT by truncating the current offset to a smaller
1856 width. We need to be able to construct an offset vector:
1858 { 0, X, X*2, X*3, ... }
1860 without loss of precision, where X is STMT's DR_STEP.
1862 Return true if this is possible, describing the gather load or scatter
1863 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1865 static bool
1866 vect_truncate_gather_scatter_offset (gimple *stmt, loop_vec_info loop_vinfo,
1867 bool masked_p,
1868 gather_scatter_info *gs_info)
1870 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1871 data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
1872 tree step = DR_STEP (dr);
1873 if (TREE_CODE (step) != INTEGER_CST)
1875 /* ??? Perhaps we could use range information here? */
1876 if (dump_enabled_p ())
1877 dump_printf_loc (MSG_NOTE, vect_location,
1878 "cannot truncate variable step.\n");
1879 return false;
1882 /* Get the number of bits in an element. */
1883 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1884 scalar_mode element_mode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
1885 unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1887 /* Set COUNT to the upper limit on the number of elements - 1.
1888 Start with the maximum vectorization factor. */
1889 unsigned HOST_WIDE_INT count = vect_max_vf (loop_vinfo) - 1;
1891 /* Try lowering COUNT to the number of scalar latch iterations. */
1892 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1893 widest_int max_iters;
1894 if (max_loop_iterations (loop, &max_iters)
1895 && max_iters < count)
1896 count = max_iters.to_shwi ();
1898 /* Try scales of 1 and the element size. */
1899 int scales[] = { 1, vect_get_scalar_dr_size (dr) };
1900 bool overflow_p = false;
1901 for (int i = 0; i < 2; ++i)
1903 int scale = scales[i];
1904 widest_int factor;
1905 if (!wi::multiple_of_p (wi::to_widest (step), scale, SIGNED, &factor))
1906 continue;
1908 /* See whether we can calculate (COUNT - 1) * STEP / SCALE
1909 in OFFSET_BITS bits. */
1910 widest_int range = wi::mul (count, factor, SIGNED, &overflow_p);
1911 if (overflow_p)
1912 continue;
1913 signop sign = range >= 0 ? UNSIGNED : SIGNED;
1914 if (wi::min_precision (range, sign) > element_bits)
1916 overflow_p = true;
1917 continue;
1920 /* See whether the target supports the operation. */
1921 tree memory_type = TREE_TYPE (DR_REF (dr));
1922 if (!vect_gather_scatter_fn_p (DR_IS_READ (dr), masked_p, vectype,
1923 memory_type, element_bits, sign, scale,
1924 &gs_info->ifn, &gs_info->element_type))
1925 continue;
1927 tree offset_type = build_nonstandard_integer_type (element_bits,
1928 sign == UNSIGNED);
1930 gs_info->decl = NULL_TREE;
1931 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1932 but we don't need to store that here. */
1933 gs_info->base = NULL_TREE;
1934 gs_info->offset = fold_convert (offset_type, step);
1935 gs_info->offset_dt = vect_constant_def;
1936 gs_info->offset_vectype = NULL_TREE;
1937 gs_info->scale = scale;
1938 gs_info->memory_type = memory_type;
1939 return true;
1942 if (overflow_p && dump_enabled_p ())
1943 dump_printf_loc (MSG_NOTE, vect_location,
1944 "truncating gather/scatter offset to %d bits"
1945 " might change its value.\n", element_bits);
1947 return false;
1950 /* Return true if we can use gather/scatter internal functions to
1951 vectorize STMT, which is a grouped or strided load or store.
1952 MASKED_P is true if load or store is conditional. When returning
1953 true, fill in GS_INFO with the information required to perform the
1954 operation. */
1956 static bool
1957 vect_use_strided_gather_scatters_p (gimple *stmt, loop_vec_info loop_vinfo,
1958 bool masked_p,
1959 gather_scatter_info *gs_info)
1961 if (!vect_check_gather_scatter (stmt, loop_vinfo, gs_info)
1962 || gs_info->decl)
1963 return vect_truncate_gather_scatter_offset (stmt, loop_vinfo,
1964 masked_p, gs_info);
1966 scalar_mode element_mode = SCALAR_TYPE_MODE (gs_info->element_type);
1967 unsigned int element_bits = GET_MODE_BITSIZE (element_mode);
1968 tree offset_type = TREE_TYPE (gs_info->offset);
1969 unsigned int offset_bits = TYPE_PRECISION (offset_type);
1971 /* Enforced by vect_check_gather_scatter. */
1972 gcc_assert (element_bits >= offset_bits);
1974 /* If the elements are wider than the offset, convert the offset to the
1975 same width, without changing its sign. */
1976 if (element_bits > offset_bits)
1978 bool unsigned_p = TYPE_UNSIGNED (offset_type);
1979 offset_type = build_nonstandard_integer_type (element_bits, unsigned_p);
1980 gs_info->offset = fold_convert (offset_type, gs_info->offset);
1983 if (dump_enabled_p ())
1984 dump_printf_loc (MSG_NOTE, vect_location,
1985 "using gather/scatter for strided/grouped access,"
1986 " scale = %d\n", gs_info->scale);
1988 return true;
1991 /* STMT is a non-strided load or store, meaning that it accesses
1992 elements with a known constant step. Return -1 if that step
1993 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1995 static int
1996 compare_step_with_zero (gimple *stmt)
1998 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1999 data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
2000 return tree_int_cst_compare (vect_dr_behavior (dr)->step,
2001 size_zero_node);
2004 /* If the target supports a permute mask that reverses the elements in
2005 a vector of type VECTYPE, return that mask, otherwise return null. */
2007 static tree
2008 perm_mask_for_reverse (tree vectype)
2010 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2012 /* The encoding has a single stepped pattern. */
2013 vec_perm_builder sel (nunits, 1, 3);
2014 for (int i = 0; i < 3; ++i)
2015 sel.quick_push (nunits - 1 - i);
2017 vec_perm_indices indices (sel, 1, nunits);
2018 if (!can_vec_perm_const_p (TYPE_MODE (vectype), indices))
2019 return NULL_TREE;
2020 return vect_gen_perm_mask_checked (vectype, indices);
2023 /* STMT is either a masked or unconditional store. Return the value
2024 being stored. */
2026 tree
2027 vect_get_store_rhs (gimple *stmt)
2029 if (gassign *assign = dyn_cast <gassign *> (stmt))
2031 gcc_assert (gimple_assign_single_p (assign));
2032 return gimple_assign_rhs1 (assign);
2034 if (gcall *call = dyn_cast <gcall *> (stmt))
2036 internal_fn ifn = gimple_call_internal_fn (call);
2037 int index = internal_fn_stored_value_index (ifn);
2038 gcc_assert (index >= 0);
2039 return gimple_call_arg (stmt, index);
2041 gcc_unreachable ();
2044 /* A subroutine of get_load_store_type, with a subset of the same
2045 arguments. Handle the case where STMT is part of a grouped load
2046 or store.
2048 For stores, the statements in the group are all consecutive
2049 and there is no gap at the end. For loads, the statements in the
2050 group might not be consecutive; there can be gaps between statements
2051 as well as at the end. */
2053 static bool
2054 get_group_load_store_type (gimple *stmt, tree vectype, bool slp,
2055 bool masked_p, vec_load_store_type vls_type,
2056 vect_memory_access_type *memory_access_type,
2057 gather_scatter_info *gs_info)
2059 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2060 vec_info *vinfo = stmt_info->vinfo;
2061 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2062 struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2063 gimple *first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
2064 data_reference *first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
2065 unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
2066 bool single_element_p = (stmt == first_stmt
2067 && !GROUP_NEXT_ELEMENT (stmt_info));
2068 unsigned HOST_WIDE_INT gap = GROUP_GAP (vinfo_for_stmt (first_stmt));
2069 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2071 /* True if the vectorized statements would access beyond the last
2072 statement in the group. */
2073 bool overrun_p = false;
2075 /* True if we can cope with such overrun by peeling for gaps, so that
2076 there is at least one final scalar iteration after the vector loop. */
2077 bool can_overrun_p = (!masked_p
2078 && vls_type == VLS_LOAD
2079 && loop_vinfo
2080 && !loop->inner);
2082 /* There can only be a gap at the end of the group if the stride is
2083 known at compile time. */
2084 gcc_assert (!STMT_VINFO_STRIDED_P (stmt_info) || gap == 0);
2086 /* Stores can't yet have gaps. */
2087 gcc_assert (slp || vls_type == VLS_LOAD || gap == 0);
2089 if (slp)
2091 if (STMT_VINFO_STRIDED_P (stmt_info))
2093 /* Try to use consecutive accesses of GROUP_SIZE elements,
2094 separated by the stride, until we have a complete vector.
2095 Fall back to scalar accesses if that isn't possible. */
2096 if (multiple_p (nunits, group_size))
2097 *memory_access_type = VMAT_STRIDED_SLP;
2098 else
2099 *memory_access_type = VMAT_ELEMENTWISE;
2101 else
2103 overrun_p = loop_vinfo && gap != 0;
2104 if (overrun_p && vls_type != VLS_LOAD)
2106 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2107 "Grouped store with gaps requires"
2108 " non-consecutive accesses\n");
2109 return false;
2111 /* An overrun is fine if the trailing elements are smaller
2112 than the alignment boundary B. Every vector access will
2113 be a multiple of B and so we are guaranteed to access a
2114 non-gap element in the same B-sized block. */
2115 if (overrun_p
2116 && gap < (vect_known_alignment_in_bytes (first_dr)
2117 / vect_get_scalar_dr_size (first_dr)))
2118 overrun_p = false;
2119 if (overrun_p && !can_overrun_p)
2121 if (dump_enabled_p ())
2122 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2123 "Peeling for outer loop is not supported\n");
2124 return false;
2126 *memory_access_type = VMAT_CONTIGUOUS;
2129 else
2131 /* We can always handle this case using elementwise accesses,
2132 but see if something more efficient is available. */
2133 *memory_access_type = VMAT_ELEMENTWISE;
2135 /* If there is a gap at the end of the group then these optimizations
2136 would access excess elements in the last iteration. */
2137 bool would_overrun_p = (gap != 0);
2138 /* An overrun is fine if the trailing elements are smaller than the
2139 alignment boundary B. Every vector access will be a multiple of B
2140 and so we are guaranteed to access a non-gap element in the
2141 same B-sized block. */
2142 if (would_overrun_p
2143 && !masked_p
2144 && gap < (vect_known_alignment_in_bytes (first_dr)
2145 / vect_get_scalar_dr_size (first_dr)))
2146 would_overrun_p = false;
2148 if (!STMT_VINFO_STRIDED_P (stmt_info)
2149 && (can_overrun_p || !would_overrun_p)
2150 && compare_step_with_zero (stmt) > 0)
2152 /* First cope with the degenerate case of a single-element
2153 vector. */
2154 if (known_eq (TYPE_VECTOR_SUBPARTS (vectype), 1U))
2155 *memory_access_type = VMAT_CONTIGUOUS;
2157 /* Otherwise try using LOAD/STORE_LANES. */
2158 if (*memory_access_type == VMAT_ELEMENTWISE
2159 && (vls_type == VLS_LOAD
2160 ? vect_load_lanes_supported (vectype, group_size, masked_p)
2161 : vect_store_lanes_supported (vectype, group_size,
2162 masked_p)))
2164 *memory_access_type = VMAT_LOAD_STORE_LANES;
2165 overrun_p = would_overrun_p;
2168 /* If that fails, try using permuting loads. */
2169 if (*memory_access_type == VMAT_ELEMENTWISE
2170 && (vls_type == VLS_LOAD
2171 ? vect_grouped_load_supported (vectype, single_element_p,
2172 group_size)
2173 : vect_grouped_store_supported (vectype, group_size)))
2175 *memory_access_type = VMAT_CONTIGUOUS_PERMUTE;
2176 overrun_p = would_overrun_p;
2180 /* As a last resort, trying using a gather load or scatter store.
2182 ??? Although the code can handle all group sizes correctly,
2183 it probably isn't a win to use separate strided accesses based
2184 on nearby locations. Or, even if it's a win over scalar code,
2185 it might not be a win over vectorizing at a lower VF, if that
2186 allows us to use contiguous accesses. */
2187 if (*memory_access_type == VMAT_ELEMENTWISE
2188 && single_element_p
2189 && loop_vinfo
2190 && vect_use_strided_gather_scatters_p (stmt, loop_vinfo,
2191 masked_p, gs_info))
2192 *memory_access_type = VMAT_GATHER_SCATTER;
2195 if (vls_type != VLS_LOAD && first_stmt == stmt)
2197 /* STMT is the leader of the group. Check the operands of all the
2198 stmts of the group. */
2199 gimple *next_stmt = GROUP_NEXT_ELEMENT (stmt_info);
2200 while (next_stmt)
2202 tree op = vect_get_store_rhs (next_stmt);
2203 gimple *def_stmt;
2204 enum vect_def_type dt;
2205 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
2207 if (dump_enabled_p ())
2208 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2209 "use not simple.\n");
2210 return false;
2212 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
2216 if (overrun_p)
2218 gcc_assert (can_overrun_p);
2219 if (dump_enabled_p ())
2220 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2221 "Data access with gaps requires scalar "
2222 "epilogue loop\n");
2223 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
2226 return true;
2229 /* A subroutine of get_load_store_type, with a subset of the same
2230 arguments. Handle the case where STMT is a load or store that
2231 accesses consecutive elements with a negative step. */
2233 static vect_memory_access_type
2234 get_negative_load_store_type (gimple *stmt, tree vectype,
2235 vec_load_store_type vls_type,
2236 unsigned int ncopies)
2238 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2239 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
2240 dr_alignment_support alignment_support_scheme;
2242 if (ncopies > 1)
2244 if (dump_enabled_p ())
2245 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2246 "multiple types with negative step.\n");
2247 return VMAT_ELEMENTWISE;
2250 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
2251 if (alignment_support_scheme != dr_aligned
2252 && alignment_support_scheme != dr_unaligned_supported)
2254 if (dump_enabled_p ())
2255 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2256 "negative step but alignment required.\n");
2257 return VMAT_ELEMENTWISE;
2260 if (vls_type == VLS_STORE_INVARIANT)
2262 if (dump_enabled_p ())
2263 dump_printf_loc (MSG_NOTE, vect_location,
2264 "negative step with invariant source;"
2265 " no permute needed.\n");
2266 return VMAT_CONTIGUOUS_DOWN;
2269 if (!perm_mask_for_reverse (vectype))
2271 if (dump_enabled_p ())
2272 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2273 "negative step and reversing not supported.\n");
2274 return VMAT_ELEMENTWISE;
2277 return VMAT_CONTIGUOUS_REVERSE;
2280 /* Analyze load or store statement STMT of type VLS_TYPE. Return true
2281 if there is a memory access type that the vectorized form can use,
2282 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2283 or scatters, fill in GS_INFO accordingly.
2285 SLP says whether we're performing SLP rather than loop vectorization.
2286 MASKED_P is true if the statement is conditional on a vectorized mask.
2287 VECTYPE is the vector type that the vectorized statements will use.
2288 NCOPIES is the number of vector statements that will be needed. */
2290 static bool
2291 get_load_store_type (gimple *stmt, tree vectype, bool slp, bool masked_p,
2292 vec_load_store_type vls_type, unsigned int ncopies,
2293 vect_memory_access_type *memory_access_type,
2294 gather_scatter_info *gs_info)
2296 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2297 vec_info *vinfo = stmt_info->vinfo;
2298 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2299 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2300 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2302 *memory_access_type = VMAT_GATHER_SCATTER;
2303 gimple *def_stmt;
2304 if (!vect_check_gather_scatter (stmt, loop_vinfo, gs_info))
2305 gcc_unreachable ();
2306 else if (!vect_is_simple_use (gs_info->offset, vinfo, &def_stmt,
2307 &gs_info->offset_dt,
2308 &gs_info->offset_vectype))
2310 if (dump_enabled_p ())
2311 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2312 "%s index use not simple.\n",
2313 vls_type == VLS_LOAD ? "gather" : "scatter");
2314 return false;
2317 else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2319 if (!get_group_load_store_type (stmt, vectype, slp, masked_p, vls_type,
2320 memory_access_type, gs_info))
2321 return false;
2323 else if (STMT_VINFO_STRIDED_P (stmt_info))
2325 gcc_assert (!slp);
2326 if (loop_vinfo
2327 && vect_use_strided_gather_scatters_p (stmt, loop_vinfo,
2328 masked_p, gs_info))
2329 *memory_access_type = VMAT_GATHER_SCATTER;
2330 else
2331 *memory_access_type = VMAT_ELEMENTWISE;
2333 else
2335 int cmp = compare_step_with_zero (stmt);
2336 if (cmp < 0)
2337 *memory_access_type = get_negative_load_store_type
2338 (stmt, vectype, vls_type, ncopies);
2339 else if (cmp == 0)
2341 gcc_assert (vls_type == VLS_LOAD);
2342 *memory_access_type = VMAT_INVARIANT;
2344 else
2345 *memory_access_type = VMAT_CONTIGUOUS;
2348 if ((*memory_access_type == VMAT_ELEMENTWISE
2349 || *memory_access_type == VMAT_STRIDED_SLP)
2350 && !nunits.is_constant ())
2352 if (dump_enabled_p ())
2353 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2354 "Not using elementwise accesses due to variable "
2355 "vectorization factor.\n");
2356 return false;
2359 /* FIXME: At the moment the cost model seems to underestimate the
2360 cost of using elementwise accesses. This check preserves the
2361 traditional behavior until that can be fixed. */
2362 if (*memory_access_type == VMAT_ELEMENTWISE
2363 && !STMT_VINFO_STRIDED_P (stmt_info)
2364 && !(stmt == GROUP_FIRST_ELEMENT (stmt_info)
2365 && !GROUP_NEXT_ELEMENT (stmt_info)
2366 && !pow2p_hwi (GROUP_SIZE (stmt_info))))
2368 if (dump_enabled_p ())
2369 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2370 "not falling back to elementwise accesses\n");
2371 return false;
2373 return true;
2376 /* Return true if boolean argument MASK is suitable for vectorizing
2377 conditional load or store STMT. When returning true, store the type
2378 of the definition in *MASK_DT_OUT and the type of the vectorized mask
2379 in *MASK_VECTYPE_OUT. */
2381 static bool
2382 vect_check_load_store_mask (gimple *stmt, tree mask,
2383 vect_def_type *mask_dt_out,
2384 tree *mask_vectype_out)
2386 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (mask)))
2388 if (dump_enabled_p ())
2389 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2390 "mask argument is not a boolean.\n");
2391 return false;
2394 if (TREE_CODE (mask) != SSA_NAME)
2396 if (dump_enabled_p ())
2397 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2398 "mask argument is not an SSA name.\n");
2399 return false;
2402 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2403 gimple *def_stmt;
2404 enum vect_def_type mask_dt;
2405 tree mask_vectype;
2406 if (!vect_is_simple_use (mask, stmt_info->vinfo, &def_stmt, &mask_dt,
2407 &mask_vectype))
2409 if (dump_enabled_p ())
2410 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2411 "mask use not simple.\n");
2412 return false;
2415 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2416 if (!mask_vectype)
2417 mask_vectype = get_mask_type_for_scalar_type (TREE_TYPE (vectype));
2419 if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype))
2421 if (dump_enabled_p ())
2422 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2423 "could not find an appropriate vector mask type.\n");
2424 return false;
2427 if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_vectype),
2428 TYPE_VECTOR_SUBPARTS (vectype)))
2430 if (dump_enabled_p ())
2432 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2433 "vector mask type ");
2434 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, mask_vectype);
2435 dump_printf (MSG_MISSED_OPTIMIZATION,
2436 " does not match vector data type ");
2437 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, vectype);
2438 dump_printf (MSG_MISSED_OPTIMIZATION, ".\n");
2440 return false;
2443 *mask_dt_out = mask_dt;
2444 *mask_vectype_out = mask_vectype;
2445 return true;
2448 /* Return true if stored value RHS is suitable for vectorizing store
2449 statement STMT. When returning true, store the type of the
2450 definition in *RHS_DT_OUT, the type of the vectorized store value in
2451 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2453 static bool
2454 vect_check_store_rhs (gimple *stmt, tree rhs, vect_def_type *rhs_dt_out,
2455 tree *rhs_vectype_out, vec_load_store_type *vls_type_out)
2457 /* In the case this is a store from a constant make sure
2458 native_encode_expr can handle it. */
2459 if (CONSTANT_CLASS_P (rhs) && native_encode_expr (rhs, NULL, 64) == 0)
2461 if (dump_enabled_p ())
2462 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2463 "cannot encode constant as a byte sequence.\n");
2464 return false;
2467 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2468 gimple *def_stmt;
2469 enum vect_def_type rhs_dt;
2470 tree rhs_vectype;
2471 if (!vect_is_simple_use (rhs, stmt_info->vinfo, &def_stmt, &rhs_dt,
2472 &rhs_vectype))
2474 if (dump_enabled_p ())
2475 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2476 "use not simple.\n");
2477 return false;
2480 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2481 if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
2483 if (dump_enabled_p ())
2484 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2485 "incompatible vector types.\n");
2486 return false;
2489 *rhs_dt_out = rhs_dt;
2490 *rhs_vectype_out = rhs_vectype;
2491 if (rhs_dt == vect_constant_def || rhs_dt == vect_external_def)
2492 *vls_type_out = VLS_STORE_INVARIANT;
2493 else
2494 *vls_type_out = VLS_STORE;
2495 return true;
2498 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT.
2499 Note that we support masks with floating-point type, in which case the
2500 floats are interpreted as a bitmask. */
2502 static tree
2503 vect_build_all_ones_mask (gimple *stmt, tree masktype)
2505 if (TREE_CODE (masktype) == INTEGER_TYPE)
2506 return build_int_cst (masktype, -1);
2507 else if (TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
2509 tree mask = build_int_cst (TREE_TYPE (masktype), -1);
2510 mask = build_vector_from_val (masktype, mask);
2511 return vect_init_vector (stmt, mask, masktype, NULL);
2513 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
2515 REAL_VALUE_TYPE r;
2516 long tmp[6];
2517 for (int j = 0; j < 6; ++j)
2518 tmp[j] = -1;
2519 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
2520 tree mask = build_real (TREE_TYPE (masktype), r);
2521 mask = build_vector_from_val (masktype, mask);
2522 return vect_init_vector (stmt, mask, masktype, NULL);
2524 gcc_unreachable ();
2527 /* Build an all-zero merge value of type VECTYPE while vectorizing
2528 STMT as a gather load. */
2530 static tree
2531 vect_build_zero_merge_argument (gimple *stmt, tree vectype)
2533 tree merge;
2534 if (TREE_CODE (TREE_TYPE (vectype)) == INTEGER_TYPE)
2535 merge = build_int_cst (TREE_TYPE (vectype), 0);
2536 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (vectype)))
2538 REAL_VALUE_TYPE r;
2539 long tmp[6];
2540 for (int j = 0; j < 6; ++j)
2541 tmp[j] = 0;
2542 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (vectype)));
2543 merge = build_real (TREE_TYPE (vectype), r);
2545 else
2546 gcc_unreachable ();
2547 merge = build_vector_from_val (vectype, merge);
2548 return vect_init_vector (stmt, merge, vectype, NULL);
2551 /* Build a gather load call while vectorizing STMT. Insert new instructions
2552 before GSI and add them to VEC_STMT. GS_INFO describes the gather load
2553 operation. If the load is conditional, MASK is the unvectorized
2554 condition and MASK_DT is its definition type, otherwise MASK is null. */
2556 static void
2557 vect_build_gather_load_calls (gimple *stmt, gimple_stmt_iterator *gsi,
2558 gimple **vec_stmt, gather_scatter_info *gs_info,
2559 tree mask, vect_def_type mask_dt)
2561 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2562 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2563 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2564 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2565 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
2566 int ncopies = vect_get_num_copies (loop_vinfo, vectype);
2567 edge pe = loop_preheader_edge (loop);
2568 enum { NARROW, NONE, WIDEN } modifier;
2569 poly_uint64 gather_off_nunits
2570 = TYPE_VECTOR_SUBPARTS (gs_info->offset_vectype);
2572 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info->decl));
2573 tree rettype = TREE_TYPE (TREE_TYPE (gs_info->decl));
2574 tree srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2575 tree ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2576 tree idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2577 tree masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
2578 tree scaletype = TREE_VALUE (arglist);
2579 gcc_checking_assert (types_compatible_p (srctype, rettype)
2580 && (!mask || types_compatible_p (srctype, masktype)));
2582 tree perm_mask = NULL_TREE;
2583 tree mask_perm_mask = NULL_TREE;
2584 if (known_eq (nunits, gather_off_nunits))
2585 modifier = NONE;
2586 else if (known_eq (nunits * 2, gather_off_nunits))
2588 modifier = WIDEN;
2590 /* Currently widening gathers and scatters are only supported for
2591 fixed-length vectors. */
2592 int count = gather_off_nunits.to_constant ();
2593 vec_perm_builder sel (count, count, 1);
2594 for (int i = 0; i < count; ++i)
2595 sel.quick_push (i | (count / 2));
2597 vec_perm_indices indices (sel, 1, count);
2598 perm_mask = vect_gen_perm_mask_checked (gs_info->offset_vectype,
2599 indices);
2601 else if (known_eq (nunits, gather_off_nunits * 2))
2603 modifier = NARROW;
2605 /* Currently narrowing gathers and scatters are only supported for
2606 fixed-length vectors. */
2607 int count = nunits.to_constant ();
2608 vec_perm_builder sel (count, count, 1);
2609 sel.quick_grow (count);
2610 for (int i = 0; i < count; ++i)
2611 sel[i] = i < count / 2 ? i : i + count / 2;
2612 vec_perm_indices indices (sel, 2, count);
2613 perm_mask = vect_gen_perm_mask_checked (vectype, indices);
2615 ncopies *= 2;
2617 if (mask)
2619 for (int i = 0; i < count; ++i)
2620 sel[i] = i | (count / 2);
2621 indices.new_vector (sel, 2, count);
2622 mask_perm_mask = vect_gen_perm_mask_checked (masktype, indices);
2625 else
2626 gcc_unreachable ();
2628 tree vec_dest = vect_create_destination_var (gimple_get_lhs (stmt),
2629 vectype);
2631 tree ptr = fold_convert (ptrtype, gs_info->base);
2632 if (!is_gimple_min_invariant (ptr))
2634 gimple_seq seq;
2635 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
2636 basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
2637 gcc_assert (!new_bb);
2640 tree scale = build_int_cst (scaletype, gs_info->scale);
2642 tree vec_oprnd0 = NULL_TREE;
2643 tree vec_mask = NULL_TREE;
2644 tree src_op = NULL_TREE;
2645 tree mask_op = NULL_TREE;
2646 tree prev_res = NULL_TREE;
2647 stmt_vec_info prev_stmt_info = NULL;
2649 if (!mask)
2651 src_op = vect_build_zero_merge_argument (stmt, rettype);
2652 mask_op = vect_build_all_ones_mask (stmt, masktype);
2655 for (int j = 0; j < ncopies; ++j)
2657 tree op, var;
2658 gimple *new_stmt;
2659 if (modifier == WIDEN && (j & 1))
2660 op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
2661 perm_mask, stmt, gsi);
2662 else if (j == 0)
2663 op = vec_oprnd0
2664 = vect_get_vec_def_for_operand (gs_info->offset, stmt);
2665 else
2666 op = vec_oprnd0
2667 = vect_get_vec_def_for_stmt_copy (gs_info->offset_dt, vec_oprnd0);
2669 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
2671 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
2672 TYPE_VECTOR_SUBPARTS (idxtype)));
2673 var = vect_get_new_ssa_name (idxtype, vect_simple_var);
2674 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
2675 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
2676 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2677 op = var;
2680 if (mask)
2682 if (mask_perm_mask && (j & 1))
2683 mask_op = permute_vec_elements (mask_op, mask_op,
2684 mask_perm_mask, stmt, gsi);
2685 else
2687 if (j == 0)
2688 vec_mask = vect_get_vec_def_for_operand (mask, stmt);
2689 else
2690 vec_mask = vect_get_vec_def_for_stmt_copy (mask_dt, vec_mask);
2692 mask_op = vec_mask;
2693 if (!useless_type_conversion_p (masktype, TREE_TYPE (vec_mask)))
2695 gcc_assert
2696 (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op)),
2697 TYPE_VECTOR_SUBPARTS (masktype)));
2698 var = vect_get_new_ssa_name (masktype, vect_simple_var);
2699 mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op);
2700 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR,
2701 mask_op);
2702 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2703 mask_op = var;
2706 src_op = mask_op;
2709 new_stmt = gimple_build_call (gs_info->decl, 5, src_op, ptr, op,
2710 mask_op, scale);
2712 if (!useless_type_conversion_p (vectype, rettype))
2714 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype),
2715 TYPE_VECTOR_SUBPARTS (rettype)));
2716 op = vect_get_new_ssa_name (rettype, vect_simple_var);
2717 gimple_call_set_lhs (new_stmt, op);
2718 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2719 var = make_ssa_name (vec_dest);
2720 op = build1 (VIEW_CONVERT_EXPR, vectype, op);
2721 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
2723 else
2725 var = make_ssa_name (vec_dest, new_stmt);
2726 gimple_call_set_lhs (new_stmt, var);
2729 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2731 if (modifier == NARROW)
2733 if ((j & 1) == 0)
2735 prev_res = var;
2736 continue;
2738 var = permute_vec_elements (prev_res, var, perm_mask, stmt, gsi);
2739 new_stmt = SSA_NAME_DEF_STMT (var);
2742 if (prev_stmt_info == NULL)
2743 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2744 else
2745 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2746 prev_stmt_info = vinfo_for_stmt (new_stmt);
2750 /* Prepare the base and offset in GS_INFO for vectorization.
2751 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2752 to the vectorized offset argument for the first copy of STMT. STMT
2753 is the statement described by GS_INFO and LOOP is the containing loop. */
2755 static void
2756 vect_get_gather_scatter_ops (struct loop *loop, gimple *stmt,
2757 gather_scatter_info *gs_info,
2758 tree *dataref_ptr, tree *vec_offset)
2760 gimple_seq stmts = NULL;
2761 *dataref_ptr = force_gimple_operand (gs_info->base, &stmts, true, NULL_TREE);
2762 if (stmts != NULL)
2764 basic_block new_bb;
2765 edge pe = loop_preheader_edge (loop);
2766 new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
2767 gcc_assert (!new_bb);
2769 tree offset_type = TREE_TYPE (gs_info->offset);
2770 tree offset_vectype = get_vectype_for_scalar_type (offset_type);
2771 *vec_offset = vect_get_vec_def_for_operand (gs_info->offset, stmt,
2772 offset_vectype);
2775 /* Prepare to implement a grouped or strided load or store using
2776 the gather load or scatter store operation described by GS_INFO.
2777 STMT is the load or store statement.
2779 Set *DATAREF_BUMP to the amount that should be added to the base
2780 address after each copy of the vectorized statement. Set *VEC_OFFSET
2781 to an invariant offset vector in which element I has the value
2782 I * DR_STEP / SCALE. */
2784 static void
2785 vect_get_strided_load_store_ops (gimple *stmt, loop_vec_info loop_vinfo,
2786 gather_scatter_info *gs_info,
2787 tree *dataref_bump, tree *vec_offset)
2789 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2790 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
2791 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2792 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2793 gimple_seq stmts;
2795 tree bump = size_binop (MULT_EXPR,
2796 fold_convert (sizetype, DR_STEP (dr)),
2797 size_int (TYPE_VECTOR_SUBPARTS (vectype)));
2798 *dataref_bump = force_gimple_operand (bump, &stmts, true, NULL_TREE);
2799 if (stmts)
2800 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
2802 /* The offset given in GS_INFO can have pointer type, so use the element
2803 type of the vector instead. */
2804 tree offset_type = TREE_TYPE (gs_info->offset);
2805 tree offset_vectype = get_vectype_for_scalar_type (offset_type);
2806 offset_type = TREE_TYPE (offset_vectype);
2808 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2809 tree step = size_binop (EXACT_DIV_EXPR, DR_STEP (dr),
2810 ssize_int (gs_info->scale));
2811 step = fold_convert (offset_type, step);
2812 step = force_gimple_operand (step, &stmts, true, NULL_TREE);
2814 /* Create {0, X, X*2, X*3, ...}. */
2815 *vec_offset = gimple_build (&stmts, VEC_SERIES_EXPR, offset_vectype,
2816 build_zero_cst (offset_type), step);
2817 if (stmts)
2818 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
2821 /* Return the amount that should be added to a vector pointer to move
2822 to the next or previous copy of AGGR_TYPE. DR is the data reference
2823 being vectorized and MEMORY_ACCESS_TYPE describes the type of
2824 vectorization. */
2826 static tree
2827 vect_get_data_ptr_increment (data_reference *dr, tree aggr_type,
2828 vect_memory_access_type memory_access_type)
2830 if (memory_access_type == VMAT_INVARIANT)
2831 return size_zero_node;
2833 tree iv_step = TYPE_SIZE_UNIT (aggr_type);
2834 tree step = vect_dr_behavior (dr)->step;
2835 if (tree_int_cst_sgn (step) == -1)
2836 iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
2837 return iv_step;
2840 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64}. */
2842 static bool
2843 vectorizable_bswap (gimple *stmt, gimple_stmt_iterator *gsi,
2844 gimple **vec_stmt, slp_tree slp_node,
2845 tree vectype_in, enum vect_def_type *dt)
2847 tree op, vectype;
2848 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
2849 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2850 unsigned ncopies;
2851 unsigned HOST_WIDE_INT nunits, num_bytes;
2853 op = gimple_call_arg (stmt, 0);
2854 vectype = STMT_VINFO_VECTYPE (stmt_info);
2856 if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nunits))
2857 return false;
2859 /* Multiple types in SLP are handled by creating the appropriate number of
2860 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2861 case of SLP. */
2862 if (slp_node)
2863 ncopies = 1;
2864 else
2865 ncopies = vect_get_num_copies (loop_vinfo, vectype);
2867 gcc_assert (ncopies >= 1);
2869 tree char_vectype = get_same_sized_vectype (char_type_node, vectype_in);
2870 if (! char_vectype)
2871 return false;
2873 if (!TYPE_VECTOR_SUBPARTS (char_vectype).is_constant (&num_bytes))
2874 return false;
2876 unsigned word_bytes = num_bytes / nunits;
2878 /* The encoding uses one stepped pattern for each byte in the word. */
2879 vec_perm_builder elts (num_bytes, word_bytes, 3);
2880 for (unsigned i = 0; i < 3; ++i)
2881 for (unsigned j = 0; j < word_bytes; ++j)
2882 elts.quick_push ((i + 1) * word_bytes - j - 1);
2884 vec_perm_indices indices (elts, 1, num_bytes);
2885 if (!can_vec_perm_const_p (TYPE_MODE (char_vectype), indices))
2886 return false;
2888 if (! vec_stmt)
2890 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
2891 if (dump_enabled_p ())
2892 dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_bswap ==="
2893 "\n");
2894 if (! PURE_SLP_STMT (stmt_info))
2896 add_stmt_cost (stmt_info->vinfo->target_cost_data,
2897 1, vector_stmt, stmt_info, 0, vect_prologue);
2898 add_stmt_cost (stmt_info->vinfo->target_cost_data,
2899 ncopies, vec_perm, stmt_info, 0, vect_body);
2901 return true;
2904 tree bswap_vconst = vec_perm_indices_to_tree (char_vectype, indices);
2906 /* Transform. */
2907 vec<tree> vec_oprnds = vNULL;
2908 gimple *new_stmt = NULL;
2909 stmt_vec_info prev_stmt_info = NULL;
2910 for (unsigned j = 0; j < ncopies; j++)
2912 /* Handle uses. */
2913 if (j == 0)
2914 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node);
2915 else
2916 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
2918 /* Arguments are ready. create the new vector stmt. */
2919 unsigned i;
2920 tree vop;
2921 FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
2923 tree tem = make_ssa_name (char_vectype);
2924 new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
2925 char_vectype, vop));
2926 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2927 tree tem2 = make_ssa_name (char_vectype);
2928 new_stmt = gimple_build_assign (tem2, VEC_PERM_EXPR,
2929 tem, tem, bswap_vconst);
2930 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2931 tem = make_ssa_name (vectype);
2932 new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
2933 vectype, tem2));
2934 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2935 if (slp_node)
2936 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
2939 if (slp_node)
2940 continue;
2942 if (j == 0)
2943 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2944 else
2945 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2947 prev_stmt_info = vinfo_for_stmt (new_stmt);
2950 vec_oprnds.release ();
2951 return true;
2954 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
2955 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
2956 in a single step. On success, store the binary pack code in
2957 *CONVERT_CODE. */
2959 static bool
2960 simple_integer_narrowing (tree vectype_out, tree vectype_in,
2961 tree_code *convert_code)
2963 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
2964 || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
2965 return false;
2967 tree_code code;
2968 int multi_step_cvt = 0;
2969 auto_vec <tree, 8> interm_types;
2970 if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
2971 &code, &multi_step_cvt,
2972 &interm_types)
2973 || multi_step_cvt)
2974 return false;
2976 *convert_code = code;
2977 return true;
2980 /* Function vectorizable_call.
2982 Check if GS performs a function call that can be vectorized.
2983 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2984 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2985 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2987 static bool
2988 vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, gimple **vec_stmt,
2989 slp_tree slp_node)
2991 gcall *stmt;
2992 tree vec_dest;
2993 tree scalar_dest;
2994 tree op, type;
2995 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
2996 stmt_vec_info stmt_info = vinfo_for_stmt (gs), prev_stmt_info;
2997 tree vectype_out, vectype_in;
2998 poly_uint64 nunits_in;
2999 poly_uint64 nunits_out;
3000 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3001 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3002 vec_info *vinfo = stmt_info->vinfo;
3003 tree fndecl, new_temp, rhs_type;
3004 gimple *def_stmt;
3005 enum vect_def_type dt[3]
3006 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
3007 int ndts = 3;
3008 gimple *new_stmt = NULL;
3009 int ncopies, j;
3010 vec<tree> vargs = vNULL;
3011 enum { NARROW, NONE, WIDEN } modifier;
3012 size_t i, nargs;
3013 tree lhs;
3015 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3016 return false;
3018 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3019 && ! vec_stmt)
3020 return false;
3022 /* Is GS a vectorizable call? */
3023 stmt = dyn_cast <gcall *> (gs);
3024 if (!stmt)
3025 return false;
3027 if (gimple_call_internal_p (stmt)
3028 && (internal_load_fn_p (gimple_call_internal_fn (stmt))
3029 || internal_store_fn_p (gimple_call_internal_fn (stmt))))
3030 /* Handled by vectorizable_load and vectorizable_store. */
3031 return false;
3033 if (gimple_call_lhs (stmt) == NULL_TREE
3034 || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3035 return false;
3037 gcc_checking_assert (!stmt_can_throw_internal (stmt));
3039 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
3041 /* Process function arguments. */
3042 rhs_type = NULL_TREE;
3043 vectype_in = NULL_TREE;
3044 nargs = gimple_call_num_args (stmt);
3046 /* Bail out if the function has more than three arguments, we do not have
3047 interesting builtin functions to vectorize with more than two arguments
3048 except for fma. No arguments is also not good. */
3049 if (nargs == 0 || nargs > 3)
3050 return false;
3052 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
3053 if (gimple_call_internal_p (stmt)
3054 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
3056 nargs = 0;
3057 rhs_type = unsigned_type_node;
3060 for (i = 0; i < nargs; i++)
3062 tree opvectype;
3064 op = gimple_call_arg (stmt, i);
3066 /* We can only handle calls with arguments of the same type. */
3067 if (rhs_type
3068 && !types_compatible_p (rhs_type, TREE_TYPE (op)))
3070 if (dump_enabled_p ())
3071 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3072 "argument types differ.\n");
3073 return false;
3075 if (!rhs_type)
3076 rhs_type = TREE_TYPE (op);
3078 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[i], &opvectype))
3080 if (dump_enabled_p ())
3081 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3082 "use not simple.\n");
3083 return false;
3086 if (!vectype_in)
3087 vectype_in = opvectype;
3088 else if (opvectype
3089 && opvectype != vectype_in)
3091 if (dump_enabled_p ())
3092 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3093 "argument vector types differ.\n");
3094 return false;
3097 /* If all arguments are external or constant defs use a vector type with
3098 the same size as the output vector type. */
3099 if (!vectype_in)
3100 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
3101 if (vec_stmt)
3102 gcc_assert (vectype_in);
3103 if (!vectype_in)
3105 if (dump_enabled_p ())
3107 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3108 "no vectype for scalar type ");
3109 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
3110 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
3113 return false;
3116 /* FORNOW */
3117 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3118 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3119 if (known_eq (nunits_in * 2, nunits_out))
3120 modifier = NARROW;
3121 else if (known_eq (nunits_out, nunits_in))
3122 modifier = NONE;
3123 else if (known_eq (nunits_out * 2, nunits_in))
3124 modifier = WIDEN;
3125 else
3126 return false;
3128 /* We only handle functions that do not read or clobber memory. */
3129 if (gimple_vuse (stmt))
3131 if (dump_enabled_p ())
3132 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3133 "function reads from or writes to memory.\n");
3134 return false;
3137 /* For now, we only vectorize functions if a target specific builtin
3138 is available. TODO -- in some cases, it might be profitable to
3139 insert the calls for pieces of the vector, in order to be able
3140 to vectorize other operations in the loop. */
3141 fndecl = NULL_TREE;
3142 internal_fn ifn = IFN_LAST;
3143 combined_fn cfn = gimple_call_combined_fn (stmt);
3144 tree callee = gimple_call_fndecl (stmt);
3146 /* First try using an internal function. */
3147 tree_code convert_code = ERROR_MARK;
3148 if (cfn != CFN_LAST
3149 && (modifier == NONE
3150 || (modifier == NARROW
3151 && simple_integer_narrowing (vectype_out, vectype_in,
3152 &convert_code))))
3153 ifn = vectorizable_internal_function (cfn, callee, vectype_out,
3154 vectype_in);
3156 /* If that fails, try asking for a target-specific built-in function. */
3157 if (ifn == IFN_LAST)
3159 if (cfn != CFN_LAST)
3160 fndecl = targetm.vectorize.builtin_vectorized_function
3161 (cfn, vectype_out, vectype_in);
3162 else
3163 fndecl = targetm.vectorize.builtin_md_vectorized_function
3164 (callee, vectype_out, vectype_in);
3167 if (ifn == IFN_LAST && !fndecl)
3169 if (cfn == CFN_GOMP_SIMD_LANE
3170 && !slp_node
3171 && loop_vinfo
3172 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3173 && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
3174 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
3175 == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
3177 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3178 { 0, 1, 2, ... vf - 1 } vector. */
3179 gcc_assert (nargs == 0);
3181 else if (modifier == NONE
3182 && (gimple_call_builtin_p (stmt, BUILT_IN_BSWAP16)
3183 || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP32)
3184 || gimple_call_builtin_p (stmt, BUILT_IN_BSWAP64)))
3185 return vectorizable_bswap (stmt, gsi, vec_stmt, slp_node,
3186 vectype_in, dt);
3187 else
3189 if (dump_enabled_p ())
3190 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3191 "function is not vectorizable.\n");
3192 return false;
3196 if (slp_node)
3197 ncopies = 1;
3198 else if (modifier == NARROW && ifn == IFN_LAST)
3199 ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
3200 else
3201 ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
3203 /* Sanity check: make sure that at least one copy of the vectorized stmt
3204 needs to be generated. */
3205 gcc_assert (ncopies >= 1);
3207 if (!vec_stmt) /* transformation not required. */
3209 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
3210 if (dump_enabled_p ())
3211 dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_call ==="
3212 "\n");
3213 vect_model_simple_cost (stmt_info, ncopies, dt, ndts, NULL, NULL);
3214 if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
3215 add_stmt_cost (stmt_info->vinfo->target_cost_data, ncopies / 2,
3216 vec_promote_demote, stmt_info, 0, vect_body);
3218 return true;
3221 /* Transform. */
3223 if (dump_enabled_p ())
3224 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3226 /* Handle def. */
3227 scalar_dest = gimple_call_lhs (stmt);
3228 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
3230 prev_stmt_info = NULL;
3231 if (modifier == NONE || ifn != IFN_LAST)
3233 tree prev_res = NULL_TREE;
3234 for (j = 0; j < ncopies; ++j)
3236 /* Build argument list for the vectorized call. */
3237 if (j == 0)
3238 vargs.create (nargs);
3239 else
3240 vargs.truncate (0);
3242 if (slp_node)
3244 auto_vec<vec<tree> > vec_defs (nargs);
3245 vec<tree> vec_oprnds0;
3247 for (i = 0; i < nargs; i++)
3248 vargs.quick_push (gimple_call_arg (stmt, i));
3249 vect_get_slp_defs (vargs, slp_node, &vec_defs);
3250 vec_oprnds0 = vec_defs[0];
3252 /* Arguments are ready. Create the new vector stmt. */
3253 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
3255 size_t k;
3256 for (k = 0; k < nargs; k++)
3258 vec<tree> vec_oprndsk = vec_defs[k];
3259 vargs[k] = vec_oprndsk[i];
3261 if (modifier == NARROW)
3263 tree half_res = make_ssa_name (vectype_in);
3264 gcall *call
3265 = gimple_build_call_internal_vec (ifn, vargs);
3266 gimple_call_set_lhs (call, half_res);
3267 gimple_call_set_nothrow (call, true);
3268 new_stmt = call;
3269 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3270 if ((i & 1) == 0)
3272 prev_res = half_res;
3273 continue;
3275 new_temp = make_ssa_name (vec_dest);
3276 new_stmt = gimple_build_assign (new_temp, convert_code,
3277 prev_res, half_res);
3279 else
3281 gcall *call;
3282 if (ifn != IFN_LAST)
3283 call = gimple_build_call_internal_vec (ifn, vargs);
3284 else
3285 call = gimple_build_call_vec (fndecl, vargs);
3286 new_temp = make_ssa_name (vec_dest, call);
3287 gimple_call_set_lhs (call, new_temp);
3288 gimple_call_set_nothrow (call, true);
3289 new_stmt = call;
3291 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3292 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3295 for (i = 0; i < nargs; i++)
3297 vec<tree> vec_oprndsi = vec_defs[i];
3298 vec_oprndsi.release ();
3300 continue;
3303 for (i = 0; i < nargs; i++)
3305 op = gimple_call_arg (stmt, i);
3306 if (j == 0)
3307 vec_oprnd0
3308 = vect_get_vec_def_for_operand (op, stmt);
3309 else
3311 vec_oprnd0 = gimple_call_arg (new_stmt, i);
3312 vec_oprnd0
3313 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
3316 vargs.quick_push (vec_oprnd0);
3319 if (gimple_call_internal_p (stmt)
3320 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
3322 tree cst = build_index_vector (vectype_out, j * nunits_out, 1);
3323 tree new_var
3324 = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
3325 gimple *init_stmt = gimple_build_assign (new_var, cst);
3326 vect_init_vector_1 (stmt, init_stmt, NULL);
3327 new_temp = make_ssa_name (vec_dest);
3328 new_stmt = gimple_build_assign (new_temp, new_var);
3330 else if (modifier == NARROW)
3332 tree half_res = make_ssa_name (vectype_in);
3333 gcall *call = gimple_build_call_internal_vec (ifn, vargs);
3334 gimple_call_set_lhs (call, half_res);
3335 gimple_call_set_nothrow (call, true);
3336 new_stmt = call;
3337 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3338 if ((j & 1) == 0)
3340 prev_res = half_res;
3341 continue;
3343 new_temp = make_ssa_name (vec_dest);
3344 new_stmt = gimple_build_assign (new_temp, convert_code,
3345 prev_res, half_res);
3347 else
3349 gcall *call;
3350 if (ifn != IFN_LAST)
3351 call = gimple_build_call_internal_vec (ifn, vargs);
3352 else
3353 call = gimple_build_call_vec (fndecl, vargs);
3354 new_temp = make_ssa_name (vec_dest, new_stmt);
3355 gimple_call_set_lhs (call, new_temp);
3356 gimple_call_set_nothrow (call, true);
3357 new_stmt = call;
3359 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3361 if (j == (modifier == NARROW ? 1 : 0))
3362 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3363 else
3364 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3366 prev_stmt_info = vinfo_for_stmt (new_stmt);
3369 else if (modifier == NARROW)
3371 for (j = 0; j < ncopies; ++j)
3373 /* Build argument list for the vectorized call. */
3374 if (j == 0)
3375 vargs.create (nargs * 2);
3376 else
3377 vargs.truncate (0);
3379 if (slp_node)
3381 auto_vec<vec<tree> > vec_defs (nargs);
3382 vec<tree> vec_oprnds0;
3384 for (i = 0; i < nargs; i++)
3385 vargs.quick_push (gimple_call_arg (stmt, i));
3386 vect_get_slp_defs (vargs, slp_node, &vec_defs);
3387 vec_oprnds0 = vec_defs[0];
3389 /* Arguments are ready. Create the new vector stmt. */
3390 for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
3392 size_t k;
3393 vargs.truncate (0);
3394 for (k = 0; k < nargs; k++)
3396 vec<tree> vec_oprndsk = vec_defs[k];
3397 vargs.quick_push (vec_oprndsk[i]);
3398 vargs.quick_push (vec_oprndsk[i + 1]);
3400 gcall *call;
3401 if (ifn != IFN_LAST)
3402 call = gimple_build_call_internal_vec (ifn, vargs);
3403 else
3404 call = gimple_build_call_vec (fndecl, vargs);
3405 new_temp = make_ssa_name (vec_dest, call);
3406 gimple_call_set_lhs (call, new_temp);
3407 gimple_call_set_nothrow (call, true);
3408 new_stmt = call;
3409 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3410 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3413 for (i = 0; i < nargs; i++)
3415 vec<tree> vec_oprndsi = vec_defs[i];
3416 vec_oprndsi.release ();
3418 continue;
3421 for (i = 0; i < nargs; i++)
3423 op = gimple_call_arg (stmt, i);
3424 if (j == 0)
3426 vec_oprnd0
3427 = vect_get_vec_def_for_operand (op, stmt);
3428 vec_oprnd1
3429 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
3431 else
3433 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1);
3434 vec_oprnd0
3435 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
3436 vec_oprnd1
3437 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
3440 vargs.quick_push (vec_oprnd0);
3441 vargs.quick_push (vec_oprnd1);
3444 new_stmt = gimple_build_call_vec (fndecl, vargs);
3445 new_temp = make_ssa_name (vec_dest, new_stmt);
3446 gimple_call_set_lhs (new_stmt, new_temp);
3447 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3449 if (j == 0)
3450 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
3451 else
3452 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3454 prev_stmt_info = vinfo_for_stmt (new_stmt);
3457 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3459 else
3460 /* No current target implements this case. */
3461 return false;
3463 vargs.release ();
3465 /* The call in STMT might prevent it from being removed in dce.
3466 We however cannot remove it here, due to the way the ssa name
3467 it defines is mapped to the new definition. So just replace
3468 rhs of the statement with something harmless. */
3470 if (slp_node)
3471 return true;
3473 type = TREE_TYPE (scalar_dest);
3474 if (is_pattern_stmt_p (stmt_info))
3475 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
3476 else
3477 lhs = gimple_call_lhs (stmt);
3479 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
3480 set_vinfo_for_stmt (new_stmt, stmt_info);
3481 set_vinfo_for_stmt (stmt, NULL);
3482 STMT_VINFO_STMT (stmt_info) = new_stmt;
3483 gsi_replace (gsi, new_stmt, false);
3485 return true;
3489 struct simd_call_arg_info
3491 tree vectype;
3492 tree op;
3493 HOST_WIDE_INT linear_step;
3494 enum vect_def_type dt;
3495 unsigned int align;
3496 bool simd_lane_linear;
3499 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3500 is linear within simd lane (but not within whole loop), note it in
3501 *ARGINFO. */
3503 static void
3504 vect_simd_lane_linear (tree op, struct loop *loop,
3505 struct simd_call_arg_info *arginfo)
3507 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
3509 if (!is_gimple_assign (def_stmt)
3510 || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
3511 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
3512 return;
3514 tree base = gimple_assign_rhs1 (def_stmt);
3515 HOST_WIDE_INT linear_step = 0;
3516 tree v = gimple_assign_rhs2 (def_stmt);
3517 while (TREE_CODE (v) == SSA_NAME)
3519 tree t;
3520 def_stmt = SSA_NAME_DEF_STMT (v);
3521 if (is_gimple_assign (def_stmt))
3522 switch (gimple_assign_rhs_code (def_stmt))
3524 case PLUS_EXPR:
3525 t = gimple_assign_rhs2 (def_stmt);
3526 if (linear_step || TREE_CODE (t) != INTEGER_CST)
3527 return;
3528 base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
3529 v = gimple_assign_rhs1 (def_stmt);
3530 continue;
3531 case MULT_EXPR:
3532 t = gimple_assign_rhs2 (def_stmt);
3533 if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
3534 return;
3535 linear_step = tree_to_shwi (t);
3536 v = gimple_assign_rhs1 (def_stmt);
3537 continue;
3538 CASE_CONVERT:
3539 t = gimple_assign_rhs1 (def_stmt);
3540 if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
3541 || (TYPE_PRECISION (TREE_TYPE (v))
3542 < TYPE_PRECISION (TREE_TYPE (t))))
3543 return;
3544 if (!linear_step)
3545 linear_step = 1;
3546 v = t;
3547 continue;
3548 default:
3549 return;
3551 else if (gimple_call_internal_p (def_stmt, IFN_GOMP_SIMD_LANE)
3552 && loop->simduid
3553 && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
3554 && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
3555 == loop->simduid))
3557 if (!linear_step)
3558 linear_step = 1;
3559 arginfo->linear_step = linear_step;
3560 arginfo->op = base;
3561 arginfo->simd_lane_linear = true;
3562 return;
3567 /* Return the number of elements in vector type VECTYPE, which is associated
3568 with a SIMD clone. At present these vectors always have a constant
3569 length. */
3571 static unsigned HOST_WIDE_INT
3572 simd_clone_subparts (tree vectype)
3574 return TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
3577 /* Function vectorizable_simd_clone_call.
3579 Check if STMT performs a function call that can be vectorized
3580 by calling a simd clone of the function.
3581 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3582 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3583 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3585 static bool
3586 vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi,
3587 gimple **vec_stmt, slp_tree slp_node)
3589 tree vec_dest;
3590 tree scalar_dest;
3591 tree op, type;
3592 tree vec_oprnd0 = NULL_TREE;
3593 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
3594 tree vectype;
3595 unsigned int nunits;
3596 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3597 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3598 vec_info *vinfo = stmt_info->vinfo;
3599 struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
3600 tree fndecl, new_temp;
3601 gimple *def_stmt;
3602 gimple *new_stmt = NULL;
3603 int ncopies, j;
3604 auto_vec<simd_call_arg_info> arginfo;
3605 vec<tree> vargs = vNULL;
3606 size_t i, nargs;
3607 tree lhs, rtype, ratype;
3608 vec<constructor_elt, va_gc> *ret_ctor_elts = NULL;
3610 /* Is STMT a vectorizable call? */
3611 if (!is_gimple_call (stmt))
3612 return false;
3614 fndecl = gimple_call_fndecl (stmt);
3615 if (fndecl == NULL_TREE)
3616 return false;
3618 struct cgraph_node *node = cgraph_node::get (fndecl);
3619 if (node == NULL || node->simd_clones == NULL)
3620 return false;
3622 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3623 return false;
3625 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3626 && ! vec_stmt)
3627 return false;
3629 if (gimple_call_lhs (stmt)
3630 && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
3631 return false;
3633 gcc_checking_assert (!stmt_can_throw_internal (stmt));
3635 vectype = STMT_VINFO_VECTYPE (stmt_info);
3637 if (loop_vinfo && nested_in_vect_loop_p (loop, stmt))
3638 return false;
3640 /* FORNOW */
3641 if (slp_node)
3642 return false;
3644 /* Process function arguments. */
3645 nargs = gimple_call_num_args (stmt);
3647 /* Bail out if the function has zero arguments. */
3648 if (nargs == 0)
3649 return false;
3651 arginfo.reserve (nargs, true);
3653 for (i = 0; i < nargs; i++)
3655 simd_call_arg_info thisarginfo;
3656 affine_iv iv;
3658 thisarginfo.linear_step = 0;
3659 thisarginfo.align = 0;
3660 thisarginfo.op = NULL_TREE;
3661 thisarginfo.simd_lane_linear = false;
3663 op = gimple_call_arg (stmt, i);
3664 if (!vect_is_simple_use (op, vinfo, &def_stmt, &thisarginfo.dt,
3665 &thisarginfo.vectype)
3666 || thisarginfo.dt == vect_uninitialized_def)
3668 if (dump_enabled_p ())
3669 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3670 "use not simple.\n");
3671 return false;
3674 if (thisarginfo.dt == vect_constant_def
3675 || thisarginfo.dt == vect_external_def)
3676 gcc_assert (thisarginfo.vectype == NULL_TREE);
3677 else
3678 gcc_assert (thisarginfo.vectype != NULL_TREE);
3680 /* For linear arguments, the analyze phase should have saved
3681 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
3682 if (i * 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info).length ()
3683 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2])
3685 gcc_assert (vec_stmt);
3686 thisarginfo.linear_step
3687 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2]);
3688 thisarginfo.op
3689 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 1];
3690 thisarginfo.simd_lane_linear
3691 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 3]
3692 == boolean_true_node);
3693 /* If loop has been peeled for alignment, we need to adjust it. */
3694 tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
3695 tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
3696 if (n1 != n2 && !thisarginfo.simd_lane_linear)
3698 tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
3699 tree step = STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2];
3700 tree opt = TREE_TYPE (thisarginfo.op);
3701 bias = fold_convert (TREE_TYPE (step), bias);
3702 bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
3703 thisarginfo.op
3704 = fold_build2 (POINTER_TYPE_P (opt)
3705 ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
3706 thisarginfo.op, bias);
3709 else if (!vec_stmt
3710 && thisarginfo.dt != vect_constant_def
3711 && thisarginfo.dt != vect_external_def
3712 && loop_vinfo
3713 && TREE_CODE (op) == SSA_NAME
3714 && simple_iv (loop, loop_containing_stmt (stmt), op,
3715 &iv, false)
3716 && tree_fits_shwi_p (iv.step))
3718 thisarginfo.linear_step = tree_to_shwi (iv.step);
3719 thisarginfo.op = iv.base;
3721 else if ((thisarginfo.dt == vect_constant_def
3722 || thisarginfo.dt == vect_external_def)
3723 && POINTER_TYPE_P (TREE_TYPE (op)))
3724 thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
3725 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
3726 linear too. */
3727 if (POINTER_TYPE_P (TREE_TYPE (op))
3728 && !thisarginfo.linear_step
3729 && !vec_stmt
3730 && thisarginfo.dt != vect_constant_def
3731 && thisarginfo.dt != vect_external_def
3732 && loop_vinfo
3733 && !slp_node
3734 && TREE_CODE (op) == SSA_NAME)
3735 vect_simd_lane_linear (op, loop, &thisarginfo);
3737 arginfo.quick_push (thisarginfo);
3740 unsigned HOST_WIDE_INT vf;
3741 if (!LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
3743 if (dump_enabled_p ())
3744 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3745 "not considering SIMD clones; not yet supported"
3746 " for variable-width vectors.\n");
3747 return NULL;
3750 unsigned int badness = 0;
3751 struct cgraph_node *bestn = NULL;
3752 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info).exists ())
3753 bestn = cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[0]);
3754 else
3755 for (struct cgraph_node *n = node->simd_clones; n != NULL;
3756 n = n->simdclone->next_clone)
3758 unsigned int this_badness = 0;
3759 if (n->simdclone->simdlen > vf
3760 || n->simdclone->nargs != nargs)
3761 continue;
3762 if (n->simdclone->simdlen < vf)
3763 this_badness += (exact_log2 (vf)
3764 - exact_log2 (n->simdclone->simdlen)) * 1024;
3765 if (n->simdclone->inbranch)
3766 this_badness += 2048;
3767 int target_badness = targetm.simd_clone.usable (n);
3768 if (target_badness < 0)
3769 continue;
3770 this_badness += target_badness * 512;
3771 /* FORNOW: Have to add code to add the mask argument. */
3772 if (n->simdclone->inbranch)
3773 continue;
3774 for (i = 0; i < nargs; i++)
3776 switch (n->simdclone->args[i].arg_type)
3778 case SIMD_CLONE_ARG_TYPE_VECTOR:
3779 if (!useless_type_conversion_p
3780 (n->simdclone->args[i].orig_type,
3781 TREE_TYPE (gimple_call_arg (stmt, i))))
3782 i = -1;
3783 else if (arginfo[i].dt == vect_constant_def
3784 || arginfo[i].dt == vect_external_def
3785 || arginfo[i].linear_step)
3786 this_badness += 64;
3787 break;
3788 case SIMD_CLONE_ARG_TYPE_UNIFORM:
3789 if (arginfo[i].dt != vect_constant_def
3790 && arginfo[i].dt != vect_external_def)
3791 i = -1;
3792 break;
3793 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
3794 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
3795 if (arginfo[i].dt == vect_constant_def
3796 || arginfo[i].dt == vect_external_def
3797 || (arginfo[i].linear_step
3798 != n->simdclone->args[i].linear_step))
3799 i = -1;
3800 break;
3801 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
3802 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
3803 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
3804 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
3805 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
3806 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
3807 /* FORNOW */
3808 i = -1;
3809 break;
3810 case SIMD_CLONE_ARG_TYPE_MASK:
3811 gcc_unreachable ();
3813 if (i == (size_t) -1)
3814 break;
3815 if (n->simdclone->args[i].alignment > arginfo[i].align)
3817 i = -1;
3818 break;
3820 if (arginfo[i].align)
3821 this_badness += (exact_log2 (arginfo[i].align)
3822 - exact_log2 (n->simdclone->args[i].alignment));
3824 if (i == (size_t) -1)
3825 continue;
3826 if (bestn == NULL || this_badness < badness)
3828 bestn = n;
3829 badness = this_badness;
3833 if (bestn == NULL)
3834 return false;
3836 for (i = 0; i < nargs; i++)
3837 if ((arginfo[i].dt == vect_constant_def
3838 || arginfo[i].dt == vect_external_def)
3839 && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
3841 arginfo[i].vectype
3842 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt,
3843 i)));
3844 if (arginfo[i].vectype == NULL
3845 || (simd_clone_subparts (arginfo[i].vectype)
3846 > bestn->simdclone->simdlen))
3847 return false;
3850 fndecl = bestn->decl;
3851 nunits = bestn->simdclone->simdlen;
3852 ncopies = vf / nunits;
3854 /* If the function isn't const, only allow it in simd loops where user
3855 has asserted that at least nunits consecutive iterations can be
3856 performed using SIMD instructions. */
3857 if ((loop == NULL || (unsigned) loop->safelen < nunits)
3858 && gimple_vuse (stmt))
3859 return false;
3861 /* Sanity check: make sure that at least one copy of the vectorized stmt
3862 needs to be generated. */
3863 gcc_assert (ncopies >= 1);
3865 if (!vec_stmt) /* transformation not required. */
3867 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (bestn->decl);
3868 for (i = 0; i < nargs; i++)
3869 if ((bestn->simdclone->args[i].arg_type
3870 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
3871 || (bestn->simdclone->args[i].arg_type
3872 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP))
3874 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_grow_cleared (i * 3
3875 + 1);
3876 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (arginfo[i].op);
3877 tree lst = POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
3878 ? size_type_node : TREE_TYPE (arginfo[i].op);
3879 tree ls = build_int_cst (lst, arginfo[i].linear_step);
3880 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (ls);
3881 tree sll = arginfo[i].simd_lane_linear
3882 ? boolean_true_node : boolean_false_node;
3883 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (sll);
3885 STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
3886 if (dump_enabled_p ())
3887 dump_printf_loc (MSG_NOTE, vect_location,
3888 "=== vectorizable_simd_clone_call ===\n");
3889 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
3890 return true;
3893 /* Transform. */
3895 if (dump_enabled_p ())
3896 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3898 /* Handle def. */
3899 scalar_dest = gimple_call_lhs (stmt);
3900 vec_dest = NULL_TREE;
3901 rtype = NULL_TREE;
3902 ratype = NULL_TREE;
3903 if (scalar_dest)
3905 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3906 rtype = TREE_TYPE (TREE_TYPE (fndecl));
3907 if (TREE_CODE (rtype) == ARRAY_TYPE)
3909 ratype = rtype;
3910 rtype = TREE_TYPE (ratype);
3914 prev_stmt_info = NULL;
3915 for (j = 0; j < ncopies; ++j)
3917 /* Build argument list for the vectorized call. */
3918 if (j == 0)
3919 vargs.create (nargs);
3920 else
3921 vargs.truncate (0);
3923 for (i = 0; i < nargs; i++)
3925 unsigned int k, l, m, o;
3926 tree atype;
3927 op = gimple_call_arg (stmt, i);
3928 switch (bestn->simdclone->args[i].arg_type)
3930 case SIMD_CLONE_ARG_TYPE_VECTOR:
3931 atype = bestn->simdclone->args[i].vector_type;
3932 o = nunits / simd_clone_subparts (atype);
3933 for (m = j * o; m < (j + 1) * o; m++)
3935 if (simd_clone_subparts (atype)
3936 < simd_clone_subparts (arginfo[i].vectype))
3938 poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
3939 k = (simd_clone_subparts (arginfo[i].vectype)
3940 / simd_clone_subparts (atype));
3941 gcc_assert ((k & (k - 1)) == 0);
3942 if (m == 0)
3943 vec_oprnd0
3944 = vect_get_vec_def_for_operand (op, stmt);
3945 else
3947 vec_oprnd0 = arginfo[i].op;
3948 if ((m & (k - 1)) == 0)
3949 vec_oprnd0
3950 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
3951 vec_oprnd0);
3953 arginfo[i].op = vec_oprnd0;
3954 vec_oprnd0
3955 = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
3956 bitsize_int (prec),
3957 bitsize_int ((m & (k - 1)) * prec));
3958 new_stmt
3959 = gimple_build_assign (make_ssa_name (atype),
3960 vec_oprnd0);
3961 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3962 vargs.safe_push (gimple_assign_lhs (new_stmt));
3964 else
3966 k = (simd_clone_subparts (atype)
3967 / simd_clone_subparts (arginfo[i].vectype));
3968 gcc_assert ((k & (k - 1)) == 0);
3969 vec<constructor_elt, va_gc> *ctor_elts;
3970 if (k != 1)
3971 vec_alloc (ctor_elts, k);
3972 else
3973 ctor_elts = NULL;
3974 for (l = 0; l < k; l++)
3976 if (m == 0 && l == 0)
3977 vec_oprnd0
3978 = vect_get_vec_def_for_operand (op, stmt);
3979 else
3980 vec_oprnd0
3981 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
3982 arginfo[i].op);
3983 arginfo[i].op = vec_oprnd0;
3984 if (k == 1)
3985 break;
3986 CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
3987 vec_oprnd0);
3989 if (k == 1)
3990 vargs.safe_push (vec_oprnd0);
3991 else
3993 vec_oprnd0 = build_constructor (atype, ctor_elts);
3994 new_stmt
3995 = gimple_build_assign (make_ssa_name (atype),
3996 vec_oprnd0);
3997 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3998 vargs.safe_push (gimple_assign_lhs (new_stmt));
4002 break;
4003 case SIMD_CLONE_ARG_TYPE_UNIFORM:
4004 vargs.safe_push (op);
4005 break;
4006 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
4007 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
4008 if (j == 0)
4010 gimple_seq stmts;
4011 arginfo[i].op
4012 = force_gimple_operand (arginfo[i].op, &stmts, true,
4013 NULL_TREE);
4014 if (stmts != NULL)
4016 basic_block new_bb;
4017 edge pe = loop_preheader_edge (loop);
4018 new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
4019 gcc_assert (!new_bb);
4021 if (arginfo[i].simd_lane_linear)
4023 vargs.safe_push (arginfo[i].op);
4024 break;
4026 tree phi_res = copy_ssa_name (op);
4027 gphi *new_phi = create_phi_node (phi_res, loop->header);
4028 set_vinfo_for_stmt (new_phi,
4029 new_stmt_vec_info (new_phi, loop_vinfo));
4030 add_phi_arg (new_phi, arginfo[i].op,
4031 loop_preheader_edge (loop), UNKNOWN_LOCATION);
4032 enum tree_code code
4033 = POINTER_TYPE_P (TREE_TYPE (op))
4034 ? POINTER_PLUS_EXPR : PLUS_EXPR;
4035 tree type = POINTER_TYPE_P (TREE_TYPE (op))
4036 ? sizetype : TREE_TYPE (op);
4037 widest_int cst
4038 = wi::mul (bestn->simdclone->args[i].linear_step,
4039 ncopies * nunits);
4040 tree tcst = wide_int_to_tree (type, cst);
4041 tree phi_arg = copy_ssa_name (op);
4042 new_stmt
4043 = gimple_build_assign (phi_arg, code, phi_res, tcst);
4044 gimple_stmt_iterator si = gsi_after_labels (loop->header);
4045 gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
4046 set_vinfo_for_stmt (new_stmt,
4047 new_stmt_vec_info (new_stmt, loop_vinfo));
4048 add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
4049 UNKNOWN_LOCATION);
4050 arginfo[i].op = phi_res;
4051 vargs.safe_push (phi_res);
4053 else
4055 enum tree_code code
4056 = POINTER_TYPE_P (TREE_TYPE (op))
4057 ? POINTER_PLUS_EXPR : PLUS_EXPR;
4058 tree type = POINTER_TYPE_P (TREE_TYPE (op))
4059 ? sizetype : TREE_TYPE (op);
4060 widest_int cst
4061 = wi::mul (bestn->simdclone->args[i].linear_step,
4062 j * nunits);
4063 tree tcst = wide_int_to_tree (type, cst);
4064 new_temp = make_ssa_name (TREE_TYPE (op));
4065 new_stmt = gimple_build_assign (new_temp, code,
4066 arginfo[i].op, tcst);
4067 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4068 vargs.safe_push (new_temp);
4070 break;
4071 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
4072 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
4073 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
4074 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
4075 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
4076 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
4077 default:
4078 gcc_unreachable ();
4082 new_stmt = gimple_build_call_vec (fndecl, vargs);
4083 if (vec_dest)
4085 gcc_assert (ratype || simd_clone_subparts (rtype) == nunits);
4086 if (ratype)
4087 new_temp = create_tmp_var (ratype);
4088 else if (simd_clone_subparts (vectype)
4089 == simd_clone_subparts (rtype))
4090 new_temp = make_ssa_name (vec_dest, new_stmt);
4091 else
4092 new_temp = make_ssa_name (rtype, new_stmt);
4093 gimple_call_set_lhs (new_stmt, new_temp);
4095 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4097 if (vec_dest)
4099 if (simd_clone_subparts (vectype) < nunits)
4101 unsigned int k, l;
4102 poly_uint64 prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
4103 poly_uint64 bytes = GET_MODE_SIZE (TYPE_MODE (vectype));
4104 k = nunits / simd_clone_subparts (vectype);
4105 gcc_assert ((k & (k - 1)) == 0);
4106 for (l = 0; l < k; l++)
4108 tree t;
4109 if (ratype)
4111 t = build_fold_addr_expr (new_temp);
4112 t = build2 (MEM_REF, vectype, t,
4113 build_int_cst (TREE_TYPE (t), l * bytes));
4115 else
4116 t = build3 (BIT_FIELD_REF, vectype, new_temp,
4117 bitsize_int (prec), bitsize_int (l * prec));
4118 new_stmt
4119 = gimple_build_assign (make_ssa_name (vectype), t);
4120 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4121 if (j == 0 && l == 0)
4122 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4123 else
4124 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4126 prev_stmt_info = vinfo_for_stmt (new_stmt);
4129 if (ratype)
4131 tree clobber = build_constructor (ratype, NULL);
4132 TREE_THIS_VOLATILE (clobber) = 1;
4133 new_stmt = gimple_build_assign (new_temp, clobber);
4134 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4136 continue;
4138 else if (simd_clone_subparts (vectype) > nunits)
4140 unsigned int k = (simd_clone_subparts (vectype)
4141 / simd_clone_subparts (rtype));
4142 gcc_assert ((k & (k - 1)) == 0);
4143 if ((j & (k - 1)) == 0)
4144 vec_alloc (ret_ctor_elts, k);
4145 if (ratype)
4147 unsigned int m, o = nunits / simd_clone_subparts (rtype);
4148 for (m = 0; m < o; m++)
4150 tree tem = build4 (ARRAY_REF, rtype, new_temp,
4151 size_int (m), NULL_TREE, NULL_TREE);
4152 new_stmt
4153 = gimple_build_assign (make_ssa_name (rtype), tem);
4154 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4155 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
4156 gimple_assign_lhs (new_stmt));
4158 tree clobber = build_constructor (ratype, NULL);
4159 TREE_THIS_VOLATILE (clobber) = 1;
4160 new_stmt = gimple_build_assign (new_temp, clobber);
4161 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4163 else
4164 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
4165 if ((j & (k - 1)) != k - 1)
4166 continue;
4167 vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
4168 new_stmt
4169 = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
4170 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4172 if ((unsigned) j == k - 1)
4173 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4174 else
4175 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4177 prev_stmt_info = vinfo_for_stmt (new_stmt);
4178 continue;
4180 else if (ratype)
4182 tree t = build_fold_addr_expr (new_temp);
4183 t = build2 (MEM_REF, vectype, t,
4184 build_int_cst (TREE_TYPE (t), 0));
4185 new_stmt
4186 = gimple_build_assign (make_ssa_name (vec_dest), t);
4187 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4188 tree clobber = build_constructor (ratype, NULL);
4189 TREE_THIS_VOLATILE (clobber) = 1;
4190 vect_finish_stmt_generation (stmt,
4191 gimple_build_assign (new_temp,
4192 clobber), gsi);
4196 if (j == 0)
4197 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4198 else
4199 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4201 prev_stmt_info = vinfo_for_stmt (new_stmt);
4204 vargs.release ();
4206 /* The call in STMT might prevent it from being removed in dce.
4207 We however cannot remove it here, due to the way the ssa name
4208 it defines is mapped to the new definition. So just replace
4209 rhs of the statement with something harmless. */
4211 if (slp_node)
4212 return true;
4214 if (scalar_dest)
4216 type = TREE_TYPE (scalar_dest);
4217 if (is_pattern_stmt_p (stmt_info))
4218 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
4219 else
4220 lhs = gimple_call_lhs (stmt);
4221 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
4223 else
4224 new_stmt = gimple_build_nop ();
4225 set_vinfo_for_stmt (new_stmt, stmt_info);
4226 set_vinfo_for_stmt (stmt, NULL);
4227 STMT_VINFO_STMT (stmt_info) = new_stmt;
4228 gsi_replace (gsi, new_stmt, true);
4229 unlink_stmt_vdef (stmt);
4231 return true;
4235 /* Function vect_gen_widened_results_half
4237 Create a vector stmt whose code, type, number of arguments, and result
4238 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
4239 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
4240 In the case that CODE is a CALL_EXPR, this means that a call to DECL
4241 needs to be created (DECL is a function-decl of a target-builtin).
4242 STMT is the original scalar stmt that we are vectorizing. */
4244 static gimple *
4245 vect_gen_widened_results_half (enum tree_code code,
4246 tree decl,
4247 tree vec_oprnd0, tree vec_oprnd1, int op_type,
4248 tree vec_dest, gimple_stmt_iterator *gsi,
4249 gimple *stmt)
4251 gimple *new_stmt;
4252 tree new_temp;
4254 /* Generate half of the widened result: */
4255 if (code == CALL_EXPR)
4257 /* Target specific support */
4258 if (op_type == binary_op)
4259 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
4260 else
4261 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
4262 new_temp = make_ssa_name (vec_dest, new_stmt);
4263 gimple_call_set_lhs (new_stmt, new_temp);
4265 else
4267 /* Generic support */
4268 gcc_assert (op_type == TREE_CODE_LENGTH (code));
4269 if (op_type != binary_op)
4270 vec_oprnd1 = NULL;
4271 new_stmt = gimple_build_assign (vec_dest, code, vec_oprnd0, vec_oprnd1);
4272 new_temp = make_ssa_name (vec_dest, new_stmt);
4273 gimple_assign_set_lhs (new_stmt, new_temp);
4275 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4277 return new_stmt;
4281 /* Get vectorized definitions for loop-based vectorization. For the first
4282 operand we call vect_get_vec_def_for_operand() (with OPRND containing
4283 scalar operand), and for the rest we get a copy with
4284 vect_get_vec_def_for_stmt_copy() using the previous vector definition
4285 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
4286 The vectors are collected into VEC_OPRNDS. */
4288 static void
4289 vect_get_loop_based_defs (tree *oprnd, gimple *stmt, enum vect_def_type dt,
4290 vec<tree> *vec_oprnds, int multi_step_cvt)
4292 tree vec_oprnd;
4294 /* Get first vector operand. */
4295 /* All the vector operands except the very first one (that is scalar oprnd)
4296 are stmt copies. */
4297 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
4298 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt);
4299 else
4300 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
4302 vec_oprnds->quick_push (vec_oprnd);
4304 /* Get second vector operand. */
4305 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
4306 vec_oprnds->quick_push (vec_oprnd);
4308 *oprnd = vec_oprnd;
4310 /* For conversion in multiple steps, continue to get operands
4311 recursively. */
4312 if (multi_step_cvt)
4313 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
4317 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
4318 For multi-step conversions store the resulting vectors and call the function
4319 recursively. */
4321 static void
4322 vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
4323 int multi_step_cvt, gimple *stmt,
4324 vec<tree> vec_dsts,
4325 gimple_stmt_iterator *gsi,
4326 slp_tree slp_node, enum tree_code code,
4327 stmt_vec_info *prev_stmt_info)
4329 unsigned int i;
4330 tree vop0, vop1, new_tmp, vec_dest;
4331 gimple *new_stmt;
4332 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4334 vec_dest = vec_dsts.pop ();
4336 for (i = 0; i < vec_oprnds->length (); i += 2)
4338 /* Create demotion operation. */
4339 vop0 = (*vec_oprnds)[i];
4340 vop1 = (*vec_oprnds)[i + 1];
4341 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
4342 new_tmp = make_ssa_name (vec_dest, new_stmt);
4343 gimple_assign_set_lhs (new_stmt, new_tmp);
4344 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4346 if (multi_step_cvt)
4347 /* Store the resulting vector for next recursive call. */
4348 (*vec_oprnds)[i/2] = new_tmp;
4349 else
4351 /* This is the last step of the conversion sequence. Store the
4352 vectors in SLP_NODE or in vector info of the scalar statement
4353 (or in STMT_VINFO_RELATED_STMT chain). */
4354 if (slp_node)
4355 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4356 else
4358 if (!*prev_stmt_info)
4359 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
4360 else
4361 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
4363 *prev_stmt_info = vinfo_for_stmt (new_stmt);
4368 /* For multi-step demotion operations we first generate demotion operations
4369 from the source type to the intermediate types, and then combine the
4370 results (stored in VEC_OPRNDS) in demotion operation to the destination
4371 type. */
4372 if (multi_step_cvt)
4374 /* At each level of recursion we have half of the operands we had at the
4375 previous level. */
4376 vec_oprnds->truncate ((i+1)/2);
4377 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
4378 stmt, vec_dsts, gsi, slp_node,
4379 VEC_PACK_TRUNC_EXPR,
4380 prev_stmt_info);
4383 vec_dsts.quick_push (vec_dest);
4387 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
4388 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
4389 the resulting vectors and call the function recursively. */
4391 static void
4392 vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
4393 vec<tree> *vec_oprnds1,
4394 gimple *stmt, tree vec_dest,
4395 gimple_stmt_iterator *gsi,
4396 enum tree_code code1,
4397 enum tree_code code2, tree decl1,
4398 tree decl2, int op_type)
4400 int i;
4401 tree vop0, vop1, new_tmp1, new_tmp2;
4402 gimple *new_stmt1, *new_stmt2;
4403 vec<tree> vec_tmp = vNULL;
4405 vec_tmp.create (vec_oprnds0->length () * 2);
4406 FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
4408 if (op_type == binary_op)
4409 vop1 = (*vec_oprnds1)[i];
4410 else
4411 vop1 = NULL_TREE;
4413 /* Generate the two halves of promotion operation. */
4414 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
4415 op_type, vec_dest, gsi, stmt);
4416 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
4417 op_type, vec_dest, gsi, stmt);
4418 if (is_gimple_call (new_stmt1))
4420 new_tmp1 = gimple_call_lhs (new_stmt1);
4421 new_tmp2 = gimple_call_lhs (new_stmt2);
4423 else
4425 new_tmp1 = gimple_assign_lhs (new_stmt1);
4426 new_tmp2 = gimple_assign_lhs (new_stmt2);
4429 /* Store the results for the next step. */
4430 vec_tmp.quick_push (new_tmp1);
4431 vec_tmp.quick_push (new_tmp2);
4434 vec_oprnds0->release ();
4435 *vec_oprnds0 = vec_tmp;
4439 /* Check if STMT performs a conversion operation, that can be vectorized.
4440 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4441 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
4442 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4444 static bool
4445 vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
4446 gimple **vec_stmt, slp_tree slp_node)
4448 tree vec_dest;
4449 tree scalar_dest;
4450 tree op0, op1 = NULL_TREE;
4451 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
4452 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4453 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4454 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
4455 enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
4456 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
4457 tree new_temp;
4458 gimple *def_stmt;
4459 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
4460 int ndts = 2;
4461 gimple *new_stmt = NULL;
4462 stmt_vec_info prev_stmt_info;
4463 poly_uint64 nunits_in;
4464 poly_uint64 nunits_out;
4465 tree vectype_out, vectype_in;
4466 int ncopies, i, j;
4467 tree lhs_type, rhs_type;
4468 enum { NARROW, NONE, WIDEN } modifier;
4469 vec<tree> vec_oprnds0 = vNULL;
4470 vec<tree> vec_oprnds1 = vNULL;
4471 tree vop0;
4472 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4473 vec_info *vinfo = stmt_info->vinfo;
4474 int multi_step_cvt = 0;
4475 vec<tree> interm_types = vNULL;
4476 tree last_oprnd, intermediate_type, cvt_type = NULL_TREE;
4477 int op_type;
4478 unsigned short fltsz;
4480 /* Is STMT a vectorizable conversion? */
4482 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4483 return false;
4485 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4486 && ! vec_stmt)
4487 return false;
4489 if (!is_gimple_assign (stmt))
4490 return false;
4492 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
4493 return false;
4495 code = gimple_assign_rhs_code (stmt);
4496 if (!CONVERT_EXPR_CODE_P (code)
4497 && code != FIX_TRUNC_EXPR
4498 && code != FLOAT_EXPR
4499 && code != WIDEN_MULT_EXPR
4500 && code != WIDEN_LSHIFT_EXPR)
4501 return false;
4503 op_type = TREE_CODE_LENGTH (code);
4505 /* Check types of lhs and rhs. */
4506 scalar_dest = gimple_assign_lhs (stmt);
4507 lhs_type = TREE_TYPE (scalar_dest);
4508 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
4510 op0 = gimple_assign_rhs1 (stmt);
4511 rhs_type = TREE_TYPE (op0);
4513 if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
4514 && !((INTEGRAL_TYPE_P (lhs_type)
4515 && INTEGRAL_TYPE_P (rhs_type))
4516 || (SCALAR_FLOAT_TYPE_P (lhs_type)
4517 && SCALAR_FLOAT_TYPE_P (rhs_type))))
4518 return false;
4520 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
4521 && ((INTEGRAL_TYPE_P (lhs_type)
4522 && !type_has_mode_precision_p (lhs_type))
4523 || (INTEGRAL_TYPE_P (rhs_type)
4524 && !type_has_mode_precision_p (rhs_type))))
4526 if (dump_enabled_p ())
4527 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4528 "type conversion to/from bit-precision unsupported."
4529 "\n");
4530 return false;
4533 /* Check the operands of the operation. */
4534 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype_in))
4536 if (dump_enabled_p ())
4537 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4538 "use not simple.\n");
4539 return false;
4541 if (op_type == binary_op)
4543 bool ok;
4545 op1 = gimple_assign_rhs2 (stmt);
4546 gcc_assert (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR);
4547 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
4548 OP1. */
4549 if (CONSTANT_CLASS_P (op0))
4550 ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &vectype_in);
4551 else
4552 ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]);
4554 if (!ok)
4556 if (dump_enabled_p ())
4557 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4558 "use not simple.\n");
4559 return false;
4563 /* If op0 is an external or constant defs use a vector type of
4564 the same size as the output vector type. */
4565 if (!vectype_in)
4566 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
4567 if (vec_stmt)
4568 gcc_assert (vectype_in);
4569 if (!vectype_in)
4571 if (dump_enabled_p ())
4573 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4574 "no vectype for scalar type ");
4575 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
4576 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
4579 return false;
4582 if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
4583 && !VECTOR_BOOLEAN_TYPE_P (vectype_in))
4585 if (dump_enabled_p ())
4587 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4588 "can't convert between boolean and non "
4589 "boolean vectors");
4590 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
4591 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
4594 return false;
4597 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
4598 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
4599 if (known_eq (nunits_out, nunits_in))
4600 modifier = NONE;
4601 else if (multiple_p (nunits_out, nunits_in))
4602 modifier = NARROW;
4603 else
4605 gcc_checking_assert (multiple_p (nunits_in, nunits_out));
4606 modifier = WIDEN;
4609 /* Multiple types in SLP are handled by creating the appropriate number of
4610 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4611 case of SLP. */
4612 if (slp_node)
4613 ncopies = 1;
4614 else if (modifier == NARROW)
4615 ncopies = vect_get_num_copies (loop_vinfo, vectype_out);
4616 else
4617 ncopies = vect_get_num_copies (loop_vinfo, vectype_in);
4619 /* Sanity check: make sure that at least one copy of the vectorized stmt
4620 needs to be generated. */
4621 gcc_assert (ncopies >= 1);
4623 bool found_mode = false;
4624 scalar_mode lhs_mode = SCALAR_TYPE_MODE (lhs_type);
4625 scalar_mode rhs_mode = SCALAR_TYPE_MODE (rhs_type);
4626 opt_scalar_mode rhs_mode_iter;
4628 /* Supportable by target? */
4629 switch (modifier)
4631 case NONE:
4632 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
4633 return false;
4634 if (supportable_convert_operation (code, vectype_out, vectype_in,
4635 &decl1, &code1))
4636 break;
4637 /* FALLTHRU */
4638 unsupported:
4639 if (dump_enabled_p ())
4640 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4641 "conversion not supported by target.\n");
4642 return false;
4644 case WIDEN:
4645 if (supportable_widening_operation (code, stmt, vectype_out, vectype_in,
4646 &code1, &code2, &multi_step_cvt,
4647 &interm_types))
4649 /* Binary widening operation can only be supported directly by the
4650 architecture. */
4651 gcc_assert (!(multi_step_cvt && op_type == binary_op));
4652 break;
4655 if (code != FLOAT_EXPR
4656 || GET_MODE_SIZE (lhs_mode) <= GET_MODE_SIZE (rhs_mode))
4657 goto unsupported;
4659 fltsz = GET_MODE_SIZE (lhs_mode);
4660 FOR_EACH_2XWIDER_MODE (rhs_mode_iter, rhs_mode)
4662 rhs_mode = rhs_mode_iter.require ();
4663 if (GET_MODE_SIZE (rhs_mode) > fltsz)
4664 break;
4666 cvt_type
4667 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
4668 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
4669 if (cvt_type == NULL_TREE)
4670 goto unsupported;
4672 if (GET_MODE_SIZE (rhs_mode) == fltsz)
4674 if (!supportable_convert_operation (code, vectype_out,
4675 cvt_type, &decl1, &codecvt1))
4676 goto unsupported;
4678 else if (!supportable_widening_operation (code, stmt, vectype_out,
4679 cvt_type, &codecvt1,
4680 &codecvt2, &multi_step_cvt,
4681 &interm_types))
4682 continue;
4683 else
4684 gcc_assert (multi_step_cvt == 0);
4686 if (supportable_widening_operation (NOP_EXPR, stmt, cvt_type,
4687 vectype_in, &code1, &code2,
4688 &multi_step_cvt, &interm_types))
4690 found_mode = true;
4691 break;
4695 if (!found_mode)
4696 goto unsupported;
4698 if (GET_MODE_SIZE (rhs_mode) == fltsz)
4699 codecvt2 = ERROR_MARK;
4700 else
4702 multi_step_cvt++;
4703 interm_types.safe_push (cvt_type);
4704 cvt_type = NULL_TREE;
4706 break;
4708 case NARROW:
4709 gcc_assert (op_type == unary_op);
4710 if (supportable_narrowing_operation (code, vectype_out, vectype_in,
4711 &code1, &multi_step_cvt,
4712 &interm_types))
4713 break;
4715 if (code != FIX_TRUNC_EXPR
4716 || GET_MODE_SIZE (lhs_mode) >= GET_MODE_SIZE (rhs_mode))
4717 goto unsupported;
4719 cvt_type
4720 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
4721 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
4722 if (cvt_type == NULL_TREE)
4723 goto unsupported;
4724 if (!supportable_convert_operation (code, cvt_type, vectype_in,
4725 &decl1, &codecvt1))
4726 goto unsupported;
4727 if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
4728 &code1, &multi_step_cvt,
4729 &interm_types))
4730 break;
4731 goto unsupported;
4733 default:
4734 gcc_unreachable ();
4737 if (!vec_stmt) /* transformation not required. */
4739 if (dump_enabled_p ())
4740 dump_printf_loc (MSG_NOTE, vect_location,
4741 "=== vectorizable_conversion ===\n");
4742 if (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR)
4744 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
4745 vect_model_simple_cost (stmt_info, ncopies, dt, ndts, NULL, NULL);
4747 else if (modifier == NARROW)
4749 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
4750 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
4752 else
4754 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
4755 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
4757 interm_types.release ();
4758 return true;
4761 /* Transform. */
4762 if (dump_enabled_p ())
4763 dump_printf_loc (MSG_NOTE, vect_location,
4764 "transform conversion. ncopies = %d.\n", ncopies);
4766 if (op_type == binary_op)
4768 if (CONSTANT_CLASS_P (op0))
4769 op0 = fold_convert (TREE_TYPE (op1), op0);
4770 else if (CONSTANT_CLASS_P (op1))
4771 op1 = fold_convert (TREE_TYPE (op0), op1);
4774 /* In case of multi-step conversion, we first generate conversion operations
4775 to the intermediate types, and then from that types to the final one.
4776 We create vector destinations for the intermediate type (TYPES) received
4777 from supportable_*_operation, and store them in the correct order
4778 for future use in vect_create_vectorized_*_stmts (). */
4779 auto_vec<tree> vec_dsts (multi_step_cvt + 1);
4780 vec_dest = vect_create_destination_var (scalar_dest,
4781 (cvt_type && modifier == WIDEN)
4782 ? cvt_type : vectype_out);
4783 vec_dsts.quick_push (vec_dest);
4785 if (multi_step_cvt)
4787 for (i = interm_types.length () - 1;
4788 interm_types.iterate (i, &intermediate_type); i--)
4790 vec_dest = vect_create_destination_var (scalar_dest,
4791 intermediate_type);
4792 vec_dsts.quick_push (vec_dest);
4796 if (cvt_type)
4797 vec_dest = vect_create_destination_var (scalar_dest,
4798 modifier == WIDEN
4799 ? vectype_out : cvt_type);
4801 if (!slp_node)
4803 if (modifier == WIDEN)
4805 vec_oprnds0.create (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1);
4806 if (op_type == binary_op)
4807 vec_oprnds1.create (1);
4809 else if (modifier == NARROW)
4810 vec_oprnds0.create (
4811 2 * (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
4813 else if (code == WIDEN_LSHIFT_EXPR)
4814 vec_oprnds1.create (slp_node->vec_stmts_size);
4816 last_oprnd = op0;
4817 prev_stmt_info = NULL;
4818 switch (modifier)
4820 case NONE:
4821 for (j = 0; j < ncopies; j++)
4823 if (j == 0)
4824 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node);
4825 else
4826 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
4828 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4830 /* Arguments are ready, create the new vector stmt. */
4831 if (code1 == CALL_EXPR)
4833 new_stmt = gimple_build_call (decl1, 1, vop0);
4834 new_temp = make_ssa_name (vec_dest, new_stmt);
4835 gimple_call_set_lhs (new_stmt, new_temp);
4837 else
4839 gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
4840 new_stmt = gimple_build_assign (vec_dest, code1, vop0);
4841 new_temp = make_ssa_name (vec_dest, new_stmt);
4842 gimple_assign_set_lhs (new_stmt, new_temp);
4845 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4846 if (slp_node)
4847 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4848 else
4850 if (!prev_stmt_info)
4851 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4852 else
4853 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4854 prev_stmt_info = vinfo_for_stmt (new_stmt);
4858 break;
4860 case WIDEN:
4861 /* In case the vectorization factor (VF) is bigger than the number
4862 of elements that we can fit in a vectype (nunits), we have to
4863 generate more than one vector stmt - i.e - we need to "unroll"
4864 the vector stmt by a factor VF/nunits. */
4865 for (j = 0; j < ncopies; j++)
4867 /* Handle uses. */
4868 if (j == 0)
4870 if (slp_node)
4872 if (code == WIDEN_LSHIFT_EXPR)
4874 unsigned int k;
4876 vec_oprnd1 = op1;
4877 /* Store vec_oprnd1 for every vector stmt to be created
4878 for SLP_NODE. We check during the analysis that all
4879 the shift arguments are the same. */
4880 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
4881 vec_oprnds1.quick_push (vec_oprnd1);
4883 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4884 slp_node);
4886 else
4887 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0,
4888 &vec_oprnds1, slp_node);
4890 else
4892 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt);
4893 vec_oprnds0.quick_push (vec_oprnd0);
4894 if (op_type == binary_op)
4896 if (code == WIDEN_LSHIFT_EXPR)
4897 vec_oprnd1 = op1;
4898 else
4899 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt);
4900 vec_oprnds1.quick_push (vec_oprnd1);
4904 else
4906 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
4907 vec_oprnds0.truncate (0);
4908 vec_oprnds0.quick_push (vec_oprnd0);
4909 if (op_type == binary_op)
4911 if (code == WIDEN_LSHIFT_EXPR)
4912 vec_oprnd1 = op1;
4913 else
4914 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1],
4915 vec_oprnd1);
4916 vec_oprnds1.truncate (0);
4917 vec_oprnds1.quick_push (vec_oprnd1);
4921 /* Arguments are ready. Create the new vector stmts. */
4922 for (i = multi_step_cvt; i >= 0; i--)
4924 tree this_dest = vec_dsts[i];
4925 enum tree_code c1 = code1, c2 = code2;
4926 if (i == 0 && codecvt2 != ERROR_MARK)
4928 c1 = codecvt1;
4929 c2 = codecvt2;
4931 vect_create_vectorized_promotion_stmts (&vec_oprnds0,
4932 &vec_oprnds1,
4933 stmt, this_dest, gsi,
4934 c1, c2, decl1, decl2,
4935 op_type);
4938 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4940 if (cvt_type)
4942 if (codecvt1 == CALL_EXPR)
4944 new_stmt = gimple_build_call (decl1, 1, vop0);
4945 new_temp = make_ssa_name (vec_dest, new_stmt);
4946 gimple_call_set_lhs (new_stmt, new_temp);
4948 else
4950 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
4951 new_temp = make_ssa_name (vec_dest);
4952 new_stmt = gimple_build_assign (new_temp, codecvt1,
4953 vop0);
4956 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4958 else
4959 new_stmt = SSA_NAME_DEF_STMT (vop0);
4961 if (slp_node)
4962 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4963 else
4965 if (!prev_stmt_info)
4966 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
4967 else
4968 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4969 prev_stmt_info = vinfo_for_stmt (new_stmt);
4974 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
4975 break;
4977 case NARROW:
4978 /* In case the vectorization factor (VF) is bigger than the number
4979 of elements that we can fit in a vectype (nunits), we have to
4980 generate more than one vector stmt - i.e - we need to "unroll"
4981 the vector stmt by a factor VF/nunits. */
4982 for (j = 0; j < ncopies; j++)
4984 /* Handle uses. */
4985 if (slp_node)
4986 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4987 slp_node);
4988 else
4990 vec_oprnds0.truncate (0);
4991 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
4992 vect_pow2 (multi_step_cvt) - 1);
4995 /* Arguments are ready. Create the new vector stmts. */
4996 if (cvt_type)
4997 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4999 if (codecvt1 == CALL_EXPR)
5001 new_stmt = gimple_build_call (decl1, 1, vop0);
5002 new_temp = make_ssa_name (vec_dest, new_stmt);
5003 gimple_call_set_lhs (new_stmt, new_temp);
5005 else
5007 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
5008 new_temp = make_ssa_name (vec_dest);
5009 new_stmt = gimple_build_assign (new_temp, codecvt1,
5010 vop0);
5013 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5014 vec_oprnds0[i] = new_temp;
5017 vect_create_vectorized_demotion_stmts (&vec_oprnds0, multi_step_cvt,
5018 stmt, vec_dsts, gsi,
5019 slp_node, code1,
5020 &prev_stmt_info);
5023 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
5024 break;
5027 vec_oprnds0.release ();
5028 vec_oprnds1.release ();
5029 interm_types.release ();
5031 return true;
5035 /* Function vectorizable_assignment.
5037 Check if STMT performs an assignment (copy) that can be vectorized.
5038 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5039 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5040 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5042 static bool
5043 vectorizable_assignment (gimple *stmt, gimple_stmt_iterator *gsi,
5044 gimple **vec_stmt, slp_tree slp_node)
5046 tree vec_dest;
5047 tree scalar_dest;
5048 tree op;
5049 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5050 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
5051 tree new_temp;
5052 gimple *def_stmt;
5053 enum vect_def_type dt[1] = {vect_unknown_def_type};
5054 int ndts = 1;
5055 int ncopies;
5056 int i, j;
5057 vec<tree> vec_oprnds = vNULL;
5058 tree vop;
5059 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
5060 vec_info *vinfo = stmt_info->vinfo;
5061 gimple *new_stmt = NULL;
5062 stmt_vec_info prev_stmt_info = NULL;
5063 enum tree_code code;
5064 tree vectype_in;
5066 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5067 return false;
5069 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5070 && ! vec_stmt)
5071 return false;
5073 /* Is vectorizable assignment? */
5074 if (!is_gimple_assign (stmt))
5075 return false;
5077 scalar_dest = gimple_assign_lhs (stmt);
5078 if (TREE_CODE (scalar_dest) != SSA_NAME)
5079 return false;
5081 code = gimple_assign_rhs_code (stmt);
5082 if (gimple_assign_single_p (stmt)
5083 || code == PAREN_EXPR
5084 || CONVERT_EXPR_CODE_P (code))
5085 op = gimple_assign_rhs1 (stmt);
5086 else
5087 return false;
5089 if (code == VIEW_CONVERT_EXPR)
5090 op = TREE_OPERAND (op, 0);
5092 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
5093 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
5095 /* Multiple types in SLP are handled by creating the appropriate number of
5096 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5097 case of SLP. */
5098 if (slp_node)
5099 ncopies = 1;
5100 else
5101 ncopies = vect_get_num_copies (loop_vinfo, vectype);
5103 gcc_assert (ncopies >= 1);
5105 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[0], &vectype_in))
5107 if (dump_enabled_p ())
5108 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5109 "use not simple.\n");
5110 return false;
5113 /* We can handle NOP_EXPR conversions that do not change the number
5114 of elements or the vector size. */
5115 if ((CONVERT_EXPR_CODE_P (code)
5116 || code == VIEW_CONVERT_EXPR)
5117 && (!vectype_in
5118 || maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in), nunits)
5119 || maybe_ne (GET_MODE_SIZE (TYPE_MODE (vectype)),
5120 GET_MODE_SIZE (TYPE_MODE (vectype_in)))))
5121 return false;
5123 /* We do not handle bit-precision changes. */
5124 if ((CONVERT_EXPR_CODE_P (code)
5125 || code == VIEW_CONVERT_EXPR)
5126 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
5127 && (!type_has_mode_precision_p (TREE_TYPE (scalar_dest))
5128 || !type_has_mode_precision_p (TREE_TYPE (op)))
5129 /* But a conversion that does not change the bit-pattern is ok. */
5130 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest))
5131 > TYPE_PRECISION (TREE_TYPE (op)))
5132 && TYPE_UNSIGNED (TREE_TYPE (op)))
5133 /* Conversion between boolean types of different sizes is
5134 a simple assignment in case their vectypes are same
5135 boolean vectors. */
5136 && (!VECTOR_BOOLEAN_TYPE_P (vectype)
5137 || !VECTOR_BOOLEAN_TYPE_P (vectype_in)))
5139 if (dump_enabled_p ())
5140 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5141 "type conversion to/from bit-precision "
5142 "unsupported.\n");
5143 return false;
5146 if (!vec_stmt) /* transformation not required. */
5148 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
5149 if (dump_enabled_p ())
5150 dump_printf_loc (MSG_NOTE, vect_location,
5151 "=== vectorizable_assignment ===\n");
5152 vect_model_simple_cost (stmt_info, ncopies, dt, ndts, NULL, NULL);
5153 return true;
5156 /* Transform. */
5157 if (dump_enabled_p ())
5158 dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
5160 /* Handle def. */
5161 vec_dest = vect_create_destination_var (scalar_dest, vectype);
5163 /* Handle use. */
5164 for (j = 0; j < ncopies; j++)
5166 /* Handle uses. */
5167 if (j == 0)
5168 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node);
5169 else
5170 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
5172 /* Arguments are ready. create the new vector stmt. */
5173 FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
5175 if (CONVERT_EXPR_CODE_P (code)
5176 || code == VIEW_CONVERT_EXPR)
5177 vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
5178 new_stmt = gimple_build_assign (vec_dest, vop);
5179 new_temp = make_ssa_name (vec_dest, new_stmt);
5180 gimple_assign_set_lhs (new_stmt, new_temp);
5181 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5182 if (slp_node)
5183 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
5186 if (slp_node)
5187 continue;
5189 if (j == 0)
5190 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5191 else
5192 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5194 prev_stmt_info = vinfo_for_stmt (new_stmt);
5197 vec_oprnds.release ();
5198 return true;
5202 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
5203 either as shift by a scalar or by a vector. */
5205 bool
5206 vect_supportable_shift (enum tree_code code, tree scalar_type)
5209 machine_mode vec_mode;
5210 optab optab;
5211 int icode;
5212 tree vectype;
5214 vectype = get_vectype_for_scalar_type (scalar_type);
5215 if (!vectype)
5216 return false;
5218 optab = optab_for_tree_code (code, vectype, optab_scalar);
5219 if (!optab
5220 || optab_handler (optab, TYPE_MODE (vectype)) == CODE_FOR_nothing)
5222 optab = optab_for_tree_code (code, vectype, optab_vector);
5223 if (!optab
5224 || (optab_handler (optab, TYPE_MODE (vectype))
5225 == CODE_FOR_nothing))
5226 return false;
5229 vec_mode = TYPE_MODE (vectype);
5230 icode = (int) optab_handler (optab, vec_mode);
5231 if (icode == CODE_FOR_nothing)
5232 return false;
5234 return true;
5238 /* Function vectorizable_shift.
5240 Check if STMT performs a shift operation that can be vectorized.
5241 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5242 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5243 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5245 static bool
5246 vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi,
5247 gimple **vec_stmt, slp_tree slp_node)
5249 tree vec_dest;
5250 tree scalar_dest;
5251 tree op0, op1 = NULL;
5252 tree vec_oprnd1 = NULL_TREE;
5253 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5254 tree vectype;
5255 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
5256 enum tree_code code;
5257 machine_mode vec_mode;
5258 tree new_temp;
5259 optab optab;
5260 int icode;
5261 machine_mode optab_op2_mode;
5262 gimple *def_stmt;
5263 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
5264 int ndts = 2;
5265 gimple *new_stmt = NULL;
5266 stmt_vec_info prev_stmt_info;
5267 poly_uint64 nunits_in;
5268 poly_uint64 nunits_out;
5269 tree vectype_out;
5270 tree op1_vectype;
5271 int ncopies;
5272 int j, i;
5273 vec<tree> vec_oprnds0 = vNULL;
5274 vec<tree> vec_oprnds1 = vNULL;
5275 tree vop0, vop1;
5276 unsigned int k;
5277 bool scalar_shift_arg = true;
5278 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
5279 vec_info *vinfo = stmt_info->vinfo;
5281 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5282 return false;
5284 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5285 && ! vec_stmt)
5286 return false;
5288 /* Is STMT a vectorizable binary/unary operation? */
5289 if (!is_gimple_assign (stmt))
5290 return false;
5292 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
5293 return false;
5295 code = gimple_assign_rhs_code (stmt);
5297 if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
5298 || code == RROTATE_EXPR))
5299 return false;
5301 scalar_dest = gimple_assign_lhs (stmt);
5302 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
5303 if (!type_has_mode_precision_p (TREE_TYPE (scalar_dest)))
5305 if (dump_enabled_p ())
5306 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5307 "bit-precision shifts not supported.\n");
5308 return false;
5311 op0 = gimple_assign_rhs1 (stmt);
5312 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
5314 if (dump_enabled_p ())
5315 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5316 "use not simple.\n");
5317 return false;
5319 /* If op0 is an external or constant def use a vector type with
5320 the same size as the output vector type. */
5321 if (!vectype)
5322 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
5323 if (vec_stmt)
5324 gcc_assert (vectype);
5325 if (!vectype)
5327 if (dump_enabled_p ())
5328 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5329 "no vectype for scalar type\n");
5330 return false;
5333 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5334 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
5335 if (maybe_ne (nunits_out, nunits_in))
5336 return false;
5338 op1 = gimple_assign_rhs2 (stmt);
5339 if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &op1_vectype))
5341 if (dump_enabled_p ())
5342 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5343 "use not simple.\n");
5344 return false;
5347 /* Multiple types in SLP are handled by creating the appropriate number of
5348 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5349 case of SLP. */
5350 if (slp_node)
5351 ncopies = 1;
5352 else
5353 ncopies = vect_get_num_copies (loop_vinfo, vectype);
5355 gcc_assert (ncopies >= 1);
5357 /* Determine whether the shift amount is a vector, or scalar. If the
5358 shift/rotate amount is a vector, use the vector/vector shift optabs. */
5360 if ((dt[1] == vect_internal_def
5361 || dt[1] == vect_induction_def)
5362 && !slp_node)
5363 scalar_shift_arg = false;
5364 else if (dt[1] == vect_constant_def
5365 || dt[1] == vect_external_def
5366 || dt[1] == vect_internal_def)
5368 /* In SLP, need to check whether the shift count is the same,
5369 in loops if it is a constant or invariant, it is always
5370 a scalar shift. */
5371 if (slp_node)
5373 vec<gimple *> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
5374 gimple *slpstmt;
5376 FOR_EACH_VEC_ELT (stmts, k, slpstmt)
5377 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
5378 scalar_shift_arg = false;
5381 /* If the shift amount is computed by a pattern stmt we cannot
5382 use the scalar amount directly thus give up and use a vector
5383 shift. */
5384 if (dt[1] == vect_internal_def)
5386 gimple *def = SSA_NAME_DEF_STMT (op1);
5387 if (is_pattern_stmt_p (vinfo_for_stmt (def)))
5388 scalar_shift_arg = false;
5391 else
5393 if (dump_enabled_p ())
5394 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5395 "operand mode requires invariant argument.\n");
5396 return false;
5399 /* Vector shifted by vector. */
5400 if (!scalar_shift_arg)
5402 optab = optab_for_tree_code (code, vectype, optab_vector);
5403 if (dump_enabled_p ())
5404 dump_printf_loc (MSG_NOTE, vect_location,
5405 "vector/vector shift/rotate found.\n");
5407 if (!op1_vectype)
5408 op1_vectype = get_same_sized_vectype (TREE_TYPE (op1), vectype_out);
5409 if (op1_vectype == NULL_TREE
5410 || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype))
5412 if (dump_enabled_p ())
5413 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5414 "unusable type for last operand in"
5415 " vector/vector shift/rotate.\n");
5416 return false;
5419 /* See if the machine has a vector shifted by scalar insn and if not
5420 then see if it has a vector shifted by vector insn. */
5421 else
5423 optab = optab_for_tree_code (code, vectype, optab_scalar);
5424 if (optab
5425 && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
5427 if (dump_enabled_p ())
5428 dump_printf_loc (MSG_NOTE, vect_location,
5429 "vector/scalar shift/rotate found.\n");
5431 else
5433 optab = optab_for_tree_code (code, vectype, optab_vector);
5434 if (optab
5435 && (optab_handler (optab, TYPE_MODE (vectype))
5436 != CODE_FOR_nothing))
5438 scalar_shift_arg = false;
5440 if (dump_enabled_p ())
5441 dump_printf_loc (MSG_NOTE, vect_location,
5442 "vector/vector shift/rotate found.\n");
5444 /* Unlike the other binary operators, shifts/rotates have
5445 the rhs being int, instead of the same type as the lhs,
5446 so make sure the scalar is the right type if we are
5447 dealing with vectors of long long/long/short/char. */
5448 if (dt[1] == vect_constant_def)
5449 op1 = fold_convert (TREE_TYPE (vectype), op1);
5450 else if (!useless_type_conversion_p (TREE_TYPE (vectype),
5451 TREE_TYPE (op1)))
5453 if (slp_node
5454 && TYPE_MODE (TREE_TYPE (vectype))
5455 != TYPE_MODE (TREE_TYPE (op1)))
5457 if (dump_enabled_p ())
5458 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5459 "unusable type for last operand in"
5460 " vector/vector shift/rotate.\n");
5461 return false;
5463 if (vec_stmt && !slp_node)
5465 op1 = fold_convert (TREE_TYPE (vectype), op1);
5466 op1 = vect_init_vector (stmt, op1,
5467 TREE_TYPE (vectype), NULL);
5474 /* Supportable by target? */
5475 if (!optab)
5477 if (dump_enabled_p ())
5478 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5479 "no optab.\n");
5480 return false;
5482 vec_mode = TYPE_MODE (vectype);
5483 icode = (int) optab_handler (optab, vec_mode);
5484 if (icode == CODE_FOR_nothing)
5486 if (dump_enabled_p ())
5487 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5488 "op not supported by target.\n");
5489 /* Check only during analysis. */
5490 if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
5491 || (!vec_stmt
5492 && !vect_worthwhile_without_simd_p (vinfo, code)))
5493 return false;
5494 if (dump_enabled_p ())
5495 dump_printf_loc (MSG_NOTE, vect_location,
5496 "proceeding using word mode.\n");
5499 /* Worthwhile without SIMD support? Check only during analysis. */
5500 if (!vec_stmt
5501 && !VECTOR_MODE_P (TYPE_MODE (vectype))
5502 && !vect_worthwhile_without_simd_p (vinfo, code))
5504 if (dump_enabled_p ())
5505 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5506 "not worthwhile without SIMD support.\n");
5507 return false;
5510 if (!vec_stmt) /* transformation not required. */
5512 STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
5513 if (dump_enabled_p ())
5514 dump_printf_loc (MSG_NOTE, vect_location,
5515 "=== vectorizable_shift ===\n");
5516 vect_model_simple_cost (stmt_info, ncopies, dt, ndts, NULL, NULL);
5517 return true;
5520 /* Transform. */
5522 if (dump_enabled_p ())
5523 dump_printf_loc (MSG_NOTE, vect_location,
5524 "transform binary/unary operation.\n");
5526 /* Handle def. */
5527 vec_dest = vect_create_destination_var (scalar_dest, vectype);
5529 prev_stmt_info = NULL;
5530 for (j = 0; j < ncopies; j++)
5532 /* Handle uses. */
5533 if (j == 0)
5535 if (scalar_shift_arg)
5537 /* Vector shl and shr insn patterns can be defined with scalar
5538 operand 2 (shift operand). In this case, use constant or loop
5539 invariant op1 directly, without extending it to vector mode
5540 first. */
5541 optab_op2_mode = insn_data[icode].operand[2].mode;
5542 if (!VECTOR_MODE_P (optab_op2_mode))
5544 if (dump_enabled_p ())
5545 dump_printf_loc (MSG_NOTE, vect_location,
5546 "operand 1 using scalar mode.\n");
5547 vec_oprnd1 = op1;
5548 vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : 1);
5549 vec_oprnds1.quick_push (vec_oprnd1);
5550 if (slp_node)
5552 /* Store vec_oprnd1 for every vector stmt to be created
5553 for SLP_NODE. We check during the analysis that all
5554 the shift arguments are the same.
5555 TODO: Allow different constants for different vector
5556 stmts generated for an SLP instance. */
5557 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
5558 vec_oprnds1.quick_push (vec_oprnd1);
5563 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
5564 (a special case for certain kind of vector shifts); otherwise,
5565 operand 1 should be of a vector type (the usual case). */
5566 if (vec_oprnd1)
5567 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
5568 slp_node);
5569 else
5570 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
5571 slp_node);
5573 else
5574 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
5576 /* Arguments are ready. Create the new vector stmt. */
5577 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5579 vop1 = vec_oprnds1[i];
5580 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
5581 new_temp = make_ssa_name (vec_dest, new_stmt);
5582 gimple_assign_set_lhs (new_stmt, new_temp);
5583 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5584 if (slp_node)
5585 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
5588 if (slp_node)
5589 continue;
5591 if (j == 0)
5592 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5593 else
5594 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5595 prev_stmt_info = vinfo_for_stmt (new_stmt);
5598 vec_oprnds0.release ();
5599 vec_oprnds1.release ();
5601 return true;
5605 /* Function vectorizable_operation.
5607 Check if STMT performs a binary, unary or ternary operation that can
5608 be vectorized.
5609 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5610 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5611 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5613 static bool
5614 vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi,
5615 gimple **vec_stmt, slp_tree slp_node)
5617 tree vec_dest;
5618 tree scalar_dest;
5619 tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
5620 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5621 tree vectype;
5622 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
5623 enum tree_code code, orig_code;
5624 machine_mode vec_mode;
5625 tree new_temp;
5626 int op_type;
5627 optab optab;
5628 bool target_support_p;
5629 gimple *def_stmt;
5630 enum vect_def_type dt[3]
5631 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
5632 int ndts = 3;
5633 gimple *new_stmt = NULL;
5634 stmt_vec_info prev_stmt_info;
5635 poly_uint64 nunits_in;
5636 poly_uint64 nunits_out;
5637 tree vectype_out;
5638 int ncopies;
5639 int j, i;
5640 vec<tree> vec_oprnds0 = vNULL;
5641 vec<tree> vec_oprnds1 = vNULL;
5642 vec<tree> vec_oprnds2 = vNULL;
5643 tree vop0, vop1, vop2;
5644 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
5645 vec_info *vinfo = stmt_info->vinfo;
5647 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5648 return false;
5650 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5651 && ! vec_stmt)
5652 return false;
5654 /* Is STMT a vectorizable binary/unary operation? */
5655 if (!is_gimple_assign (stmt))
5656 return false;
5658 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
5659 return false;
5661 orig_code = code = gimple_assign_rhs_code (stmt);
5663 /* For pointer addition and subtraction, we should use the normal
5664 plus and minus for the vector operation. */
5665 if (code == POINTER_PLUS_EXPR)
5666 code = PLUS_EXPR;
5667 if (code == POINTER_DIFF_EXPR)
5668 code = MINUS_EXPR;
5670 /* Support only unary or binary operations. */
5671 op_type = TREE_CODE_LENGTH (code);
5672 if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
5674 if (dump_enabled_p ())
5675 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5676 "num. args = %d (not unary/binary/ternary op).\n",
5677 op_type);
5678 return false;
5681 scalar_dest = gimple_assign_lhs (stmt);
5682 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
5684 /* Most operations cannot handle bit-precision types without extra
5685 truncations. */
5686 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
5687 && !type_has_mode_precision_p (TREE_TYPE (scalar_dest))
5688 /* Exception are bitwise binary operations. */
5689 && code != BIT_IOR_EXPR
5690 && code != BIT_XOR_EXPR
5691 && code != BIT_AND_EXPR)
5693 if (dump_enabled_p ())
5694 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5695 "bit-precision arithmetic not supported.\n");
5696 return false;
5699 op0 = gimple_assign_rhs1 (stmt);
5700 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
5702 if (dump_enabled_p ())
5703 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5704 "use not simple.\n");
5705 return false;
5707 /* If op0 is an external or constant def use a vector type with
5708 the same size as the output vector type. */
5709 if (!vectype)
5711 /* For boolean type we cannot determine vectype by
5712 invariant value (don't know whether it is a vector
5713 of booleans or vector of integers). We use output
5714 vectype because operations on boolean don't change
5715 type. */
5716 if (VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (op0)))
5718 if (!VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (scalar_dest)))
5720 if (dump_enabled_p ())
5721 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5722 "not supported operation on bool value.\n");
5723 return false;
5725 vectype = vectype_out;
5727 else
5728 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
5730 if (vec_stmt)
5731 gcc_assert (vectype);
5732 if (!vectype)
5734 if (dump_enabled_p ())
5736 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5737 "no vectype for scalar type ");
5738 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
5739 TREE_TYPE (op0));
5740 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
5743 return false;
5746 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
5747 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
5748 if (maybe_ne (nunits_out, nunits_in))
5749 return false;
5751 if (op_type == binary_op || op_type == ternary_op)
5753 op1 = gimple_assign_rhs2 (stmt);
5754 if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]))
5756 if (dump_enabled_p ())
5757 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5758 "use not simple.\n");
5759 return false;
5762 if (op_type == ternary_op)
5764 op2 = gimple_assign_rhs3 (stmt);
5765 if (!vect_is_simple_use (op2, vinfo, &def_stmt, &dt[2]))
5767 if (dump_enabled_p ())
5768 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5769 "use not simple.\n");
5770 return false;
5774 /* Multiple types in SLP are handled by creating the appropriate number of
5775 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5776 case of SLP. */
5777 if (slp_node)
5778 ncopies = 1;
5779 else
5780 ncopies = vect_get_num_copies (loop_vinfo, vectype);
5782 gcc_assert (ncopies >= 1);
5784 /* Shifts are handled in vectorizable_shift (). */
5785 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
5786 || code == RROTATE_EXPR)
5787 return false;
5789 /* Supportable by target? */
5791 vec_mode = TYPE_MODE (vectype);
5792 if (code == MULT_HIGHPART_EXPR)
5793 target_support_p = can_mult_highpart_p (vec_mode, TYPE_UNSIGNED (vectype));
5794 else
5796 optab = optab_for_tree_code (code, vectype, optab_default);
5797 if (!optab)
5799 if (dump_enabled_p ())
5800 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5801 "no optab.\n");
5802 return false;
5804 target_support_p = (optab_handler (optab, vec_mode)
5805 != CODE_FOR_nothing);
5808 if (!target_support_p)
5810 if (dump_enabled_p ())
5811 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5812 "op not supported by target.\n");
5813 /* Check only during analysis. */
5814 if (maybe_ne (GET_MODE_SIZE (vec_mode), UNITS_PER_WORD)
5815 || (!vec_stmt && !vect_worthwhile_without_simd_p (vinfo, code)))
5816 return false;
5817 if (dump_enabled_p ())
5818 dump_printf_loc (MSG_NOTE, vect_location,
5819 "proceeding using word mode.\n");
5822 /* Worthwhile without SIMD support? Check only during analysis. */
5823 if (!VECTOR_MODE_P (vec_mode)
5824 && !vec_stmt
5825 && !vect_worthwhile_without_simd_p (vinfo, code))
5827 if (dump_enabled_p ())
5828 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5829 "not worthwhile without SIMD support.\n");
5830 return false;
5833 if (!vec_stmt) /* transformation not required. */
5835 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
5836 if (dump_enabled_p ())
5837 dump_printf_loc (MSG_NOTE, vect_location,
5838 "=== vectorizable_operation ===\n");
5839 vect_model_simple_cost (stmt_info, ncopies, dt, ndts, NULL, NULL);
5840 return true;
5843 /* Transform. */
5845 if (dump_enabled_p ())
5846 dump_printf_loc (MSG_NOTE, vect_location,
5847 "transform binary/unary operation.\n");
5849 /* Handle def. */
5850 vec_dest = vect_create_destination_var (scalar_dest, vectype);
5852 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
5853 vectors with unsigned elements, but the result is signed. So, we
5854 need to compute the MINUS_EXPR into vectype temporary and
5855 VIEW_CONVERT_EXPR it into the final vectype_out result. */
5856 tree vec_cvt_dest = NULL_TREE;
5857 if (orig_code == POINTER_DIFF_EXPR)
5858 vec_cvt_dest = vect_create_destination_var (scalar_dest, vectype_out);
5860 /* In case the vectorization factor (VF) is bigger than the number
5861 of elements that we can fit in a vectype (nunits), we have to generate
5862 more than one vector stmt - i.e - we need to "unroll" the
5863 vector stmt by a factor VF/nunits. In doing so, we record a pointer
5864 from one copy of the vector stmt to the next, in the field
5865 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
5866 stages to find the correct vector defs to be used when vectorizing
5867 stmts that use the defs of the current stmt. The example below
5868 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
5869 we need to create 4 vectorized stmts):
5871 before vectorization:
5872 RELATED_STMT VEC_STMT
5873 S1: x = memref - -
5874 S2: z = x + 1 - -
5876 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
5877 there):
5878 RELATED_STMT VEC_STMT
5879 VS1_0: vx0 = memref0 VS1_1 -
5880 VS1_1: vx1 = memref1 VS1_2 -
5881 VS1_2: vx2 = memref2 VS1_3 -
5882 VS1_3: vx3 = memref3 - -
5883 S1: x = load - VS1_0
5884 S2: z = x + 1 - -
5886 step2: vectorize stmt S2 (done here):
5887 To vectorize stmt S2 we first need to find the relevant vector
5888 def for the first operand 'x'. This is, as usual, obtained from
5889 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
5890 that defines 'x' (S1). This way we find the stmt VS1_0, and the
5891 relevant vector def 'vx0'. Having found 'vx0' we can generate
5892 the vector stmt VS2_0, and as usual, record it in the
5893 STMT_VINFO_VEC_STMT of stmt S2.
5894 When creating the second copy (VS2_1), we obtain the relevant vector
5895 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
5896 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
5897 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
5898 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
5899 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
5900 chain of stmts and pointers:
5901 RELATED_STMT VEC_STMT
5902 VS1_0: vx0 = memref0 VS1_1 -
5903 VS1_1: vx1 = memref1 VS1_2 -
5904 VS1_2: vx2 = memref2 VS1_3 -
5905 VS1_3: vx3 = memref3 - -
5906 S1: x = load - VS1_0
5907 VS2_0: vz0 = vx0 + v1 VS2_1 -
5908 VS2_1: vz1 = vx1 + v1 VS2_2 -
5909 VS2_2: vz2 = vx2 + v1 VS2_3 -
5910 VS2_3: vz3 = vx3 + v1 - -
5911 S2: z = x + 1 - VS2_0 */
5913 prev_stmt_info = NULL;
5914 for (j = 0; j < ncopies; j++)
5916 /* Handle uses. */
5917 if (j == 0)
5919 if (op_type == binary_op || op_type == ternary_op)
5920 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
5921 slp_node);
5922 else
5923 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
5924 slp_node);
5925 if (op_type == ternary_op)
5926 vect_get_vec_defs (op2, NULL_TREE, stmt, &vec_oprnds2, NULL,
5927 slp_node);
5929 else
5931 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
5932 if (op_type == ternary_op)
5934 tree vec_oprnd = vec_oprnds2.pop ();
5935 vec_oprnds2.quick_push (vect_get_vec_def_for_stmt_copy (dt[2],
5936 vec_oprnd));
5940 /* Arguments are ready. Create the new vector stmt. */
5941 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5943 vop1 = ((op_type == binary_op || op_type == ternary_op)
5944 ? vec_oprnds1[i] : NULL_TREE);
5945 vop2 = ((op_type == ternary_op)
5946 ? vec_oprnds2[i] : NULL_TREE);
5947 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
5948 new_temp = make_ssa_name (vec_dest, new_stmt);
5949 gimple_assign_set_lhs (new_stmt, new_temp);
5950 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5951 if (vec_cvt_dest)
5953 new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp);
5954 new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR,
5955 new_temp);
5956 new_temp = make_ssa_name (vec_cvt_dest, new_stmt);
5957 gimple_assign_set_lhs (new_stmt, new_temp);
5958 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5960 if (slp_node)
5961 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
5964 if (slp_node)
5965 continue;
5967 if (j == 0)
5968 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5969 else
5970 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5971 prev_stmt_info = vinfo_for_stmt (new_stmt);
5974 vec_oprnds0.release ();
5975 vec_oprnds1.release ();
5976 vec_oprnds2.release ();
5978 return true;
5981 /* A helper function to ensure data reference DR's base alignment. */
5983 static void
5984 ensure_base_align (struct data_reference *dr)
5986 if (!dr->aux)
5987 return;
5989 if (DR_VECT_AUX (dr)->base_misaligned)
5991 tree base_decl = DR_VECT_AUX (dr)->base_decl;
5993 unsigned int align_base_to = DR_TARGET_ALIGNMENT (dr) * BITS_PER_UNIT;
5995 if (decl_in_symtab_p (base_decl))
5996 symtab_node::get (base_decl)->increase_alignment (align_base_to);
5997 else
5999 SET_DECL_ALIGN (base_decl, align_base_to);
6000 DECL_USER_ALIGN (base_decl) = 1;
6002 DR_VECT_AUX (dr)->base_misaligned = false;
6007 /* Function get_group_alias_ptr_type.
6009 Return the alias type for the group starting at FIRST_STMT. */
6011 static tree
6012 get_group_alias_ptr_type (gimple *first_stmt)
6014 struct data_reference *first_dr, *next_dr;
6015 gimple *next_stmt;
6017 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
6018 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (first_stmt));
6019 while (next_stmt)
6021 next_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (next_stmt));
6022 if (get_alias_set (DR_REF (first_dr))
6023 != get_alias_set (DR_REF (next_dr)))
6025 if (dump_enabled_p ())
6026 dump_printf_loc (MSG_NOTE, vect_location,
6027 "conflicting alias set types.\n");
6028 return ptr_type_node;
6030 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
6032 return reference_alias_ptr_type (DR_REF (first_dr));
6036 /* Function vectorizable_store.
6038 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
6039 can be vectorized.
6040 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6041 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
6042 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6044 static bool
6045 vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
6046 slp_tree slp_node)
6048 tree data_ref;
6049 tree op;
6050 tree vec_oprnd = NULL_TREE;
6051 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
6052 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
6053 tree elem_type;
6054 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
6055 struct loop *loop = NULL;
6056 machine_mode vec_mode;
6057 tree dummy;
6058 enum dr_alignment_support alignment_support_scheme;
6059 gimple *def_stmt;
6060 enum vect_def_type rhs_dt = vect_unknown_def_type;
6061 enum vect_def_type mask_dt = vect_unknown_def_type;
6062 stmt_vec_info prev_stmt_info = NULL;
6063 tree dataref_ptr = NULL_TREE;
6064 tree dataref_offset = NULL_TREE;
6065 gimple *ptr_incr = NULL;
6066 int ncopies;
6067 int j;
6068 gimple *next_stmt, *first_stmt;
6069 bool grouped_store;
6070 unsigned int group_size, i;
6071 vec<tree> oprnds = vNULL;
6072 vec<tree> result_chain = vNULL;
6073 bool inv_p;
6074 tree offset = NULL_TREE;
6075 vec<tree> vec_oprnds = vNULL;
6076 bool slp = (slp_node != NULL);
6077 unsigned int vec_num;
6078 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
6079 vec_info *vinfo = stmt_info->vinfo;
6080 tree aggr_type;
6081 gather_scatter_info gs_info;
6082 gimple *new_stmt;
6083 poly_uint64 vf;
6084 vec_load_store_type vls_type;
6085 tree ref_type;
6087 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6088 return false;
6090 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6091 && ! vec_stmt)
6092 return false;
6094 /* Is vectorizable store? */
6096 tree mask = NULL_TREE, mask_vectype = NULL_TREE;
6097 if (is_gimple_assign (stmt))
6099 tree scalar_dest = gimple_assign_lhs (stmt);
6100 if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
6101 && is_pattern_stmt_p (stmt_info))
6102 scalar_dest = TREE_OPERAND (scalar_dest, 0);
6103 if (TREE_CODE (scalar_dest) != ARRAY_REF
6104 && TREE_CODE (scalar_dest) != BIT_FIELD_REF
6105 && TREE_CODE (scalar_dest) != INDIRECT_REF
6106 && TREE_CODE (scalar_dest) != COMPONENT_REF
6107 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
6108 && TREE_CODE (scalar_dest) != REALPART_EXPR
6109 && TREE_CODE (scalar_dest) != MEM_REF)
6110 return false;
6112 else
6114 gcall *call = dyn_cast <gcall *> (stmt);
6115 if (!call || !gimple_call_internal_p (call))
6116 return false;
6118 internal_fn ifn = gimple_call_internal_fn (call);
6119 if (!internal_store_fn_p (ifn))
6120 return false;
6122 if (slp_node != NULL)
6124 if (dump_enabled_p ())
6125 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6126 "SLP of masked stores not supported.\n");
6127 return false;
6130 int mask_index = internal_fn_mask_index (ifn);
6131 if (mask_index >= 0)
6133 mask = gimple_call_arg (call, mask_index);
6134 if (!vect_check_load_store_mask (stmt, mask, &mask_dt,
6135 &mask_vectype))
6136 return false;
6140 op = vect_get_store_rhs (stmt);
6142 /* Cannot have hybrid store SLP -- that would mean storing to the
6143 same location twice. */
6144 gcc_assert (slp == PURE_SLP_STMT (stmt_info));
6146 tree vectype = STMT_VINFO_VECTYPE (stmt_info), rhs_vectype = NULL_TREE;
6147 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6149 if (loop_vinfo)
6151 loop = LOOP_VINFO_LOOP (loop_vinfo);
6152 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
6154 else
6155 vf = 1;
6157 /* Multiple types in SLP are handled by creating the appropriate number of
6158 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6159 case of SLP. */
6160 if (slp)
6161 ncopies = 1;
6162 else
6163 ncopies = vect_get_num_copies (loop_vinfo, vectype);
6165 gcc_assert (ncopies >= 1);
6167 /* FORNOW. This restriction should be relaxed. */
6168 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
6170 if (dump_enabled_p ())
6171 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6172 "multiple types in nested loop.\n");
6173 return false;
6176 if (!vect_check_store_rhs (stmt, op, &rhs_dt, &rhs_vectype, &vls_type))
6177 return false;
6179 elem_type = TREE_TYPE (vectype);
6180 vec_mode = TYPE_MODE (vectype);
6182 if (!STMT_VINFO_DATA_REF (stmt_info))
6183 return false;
6185 vect_memory_access_type memory_access_type;
6186 if (!get_load_store_type (stmt, vectype, slp, mask, vls_type, ncopies,
6187 &memory_access_type, &gs_info))
6188 return false;
6190 if (mask)
6192 if (memory_access_type == VMAT_CONTIGUOUS)
6194 if (!VECTOR_MODE_P (vec_mode)
6195 || !can_vec_mask_load_store_p (vec_mode,
6196 TYPE_MODE (mask_vectype), false))
6197 return false;
6199 else if (memory_access_type != VMAT_LOAD_STORE_LANES
6200 && (memory_access_type != VMAT_GATHER_SCATTER || gs_info.decl))
6202 if (dump_enabled_p ())
6203 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6204 "unsupported access type for masked store.\n");
6205 return false;
6208 else
6210 /* FORNOW. In some cases can vectorize even if data-type not supported
6211 (e.g. - array initialization with 0). */
6212 if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing)
6213 return false;
6216 grouped_store = (STMT_VINFO_GROUPED_ACCESS (stmt_info)
6217 && memory_access_type != VMAT_GATHER_SCATTER);
6218 if (grouped_store)
6220 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
6221 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
6222 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
6224 else
6226 first_stmt = stmt;
6227 first_dr = dr;
6228 group_size = vec_num = 1;
6231 if (!vec_stmt) /* transformation not required. */
6233 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
6235 if (loop_vinfo
6236 && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo))
6237 check_load_store_masking (loop_vinfo, vectype, vls_type, group_size,
6238 memory_access_type, &gs_info);
6240 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
6241 /* The SLP costs are calculated during SLP analysis. */
6242 if (!PURE_SLP_STMT (stmt_info))
6243 vect_model_store_cost (stmt_info, ncopies, memory_access_type,
6244 vls_type, NULL, NULL, NULL);
6245 return true;
6247 gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
6249 /* Transform. */
6251 ensure_base_align (dr);
6253 if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
6255 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, src;
6256 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
6257 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
6258 tree ptr, mask, var, scale, perm_mask = NULL_TREE;
6259 edge pe = loop_preheader_edge (loop);
6260 gimple_seq seq;
6261 basic_block new_bb;
6262 enum { NARROW, NONE, WIDEN } modifier;
6263 poly_uint64 scatter_off_nunits
6264 = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype);
6266 if (known_eq (nunits, scatter_off_nunits))
6267 modifier = NONE;
6268 else if (known_eq (nunits * 2, scatter_off_nunits))
6270 modifier = WIDEN;
6272 /* Currently gathers and scatters are only supported for
6273 fixed-length vectors. */
6274 unsigned int count = scatter_off_nunits.to_constant ();
6275 vec_perm_builder sel (count, count, 1);
6276 for (i = 0; i < (unsigned int) count; ++i)
6277 sel.quick_push (i | (count / 2));
6279 vec_perm_indices indices (sel, 1, count);
6280 perm_mask = vect_gen_perm_mask_checked (gs_info.offset_vectype,
6281 indices);
6282 gcc_assert (perm_mask != NULL_TREE);
6284 else if (known_eq (nunits, scatter_off_nunits * 2))
6286 modifier = NARROW;
6288 /* Currently gathers and scatters are only supported for
6289 fixed-length vectors. */
6290 unsigned int count = nunits.to_constant ();
6291 vec_perm_builder sel (count, count, 1);
6292 for (i = 0; i < (unsigned int) count; ++i)
6293 sel.quick_push (i | (count / 2));
6295 vec_perm_indices indices (sel, 2, count);
6296 perm_mask = vect_gen_perm_mask_checked (vectype, indices);
6297 gcc_assert (perm_mask != NULL_TREE);
6298 ncopies *= 2;
6300 else
6301 gcc_unreachable ();
6303 rettype = TREE_TYPE (TREE_TYPE (gs_info.decl));
6304 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6305 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6306 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6307 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6308 scaletype = TREE_VALUE (arglist);
6310 gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
6311 && TREE_CODE (rettype) == VOID_TYPE);
6313 ptr = fold_convert (ptrtype, gs_info.base);
6314 if (!is_gimple_min_invariant (ptr))
6316 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
6317 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
6318 gcc_assert (!new_bb);
6321 /* Currently we support only unconditional scatter stores,
6322 so mask should be all ones. */
6323 mask = build_int_cst (masktype, -1);
6324 mask = vect_init_vector (stmt, mask, masktype, NULL);
6326 scale = build_int_cst (scaletype, gs_info.scale);
6328 prev_stmt_info = NULL;
6329 for (j = 0; j < ncopies; ++j)
6331 if (j == 0)
6333 src = vec_oprnd1
6334 = vect_get_vec_def_for_operand (op, stmt);
6335 op = vec_oprnd0
6336 = vect_get_vec_def_for_operand (gs_info.offset, stmt);
6338 else if (modifier != NONE && (j & 1))
6340 if (modifier == WIDEN)
6342 src = vec_oprnd1
6343 = vect_get_vec_def_for_stmt_copy (rhs_dt, vec_oprnd1);
6344 op = permute_vec_elements (vec_oprnd0, vec_oprnd0, perm_mask,
6345 stmt, gsi);
6347 else if (modifier == NARROW)
6349 src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask,
6350 stmt, gsi);
6351 op = vec_oprnd0
6352 = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
6353 vec_oprnd0);
6355 else
6356 gcc_unreachable ();
6358 else
6360 src = vec_oprnd1
6361 = vect_get_vec_def_for_stmt_copy (rhs_dt, vec_oprnd1);
6362 op = vec_oprnd0
6363 = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
6364 vec_oprnd0);
6367 if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
6369 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src)),
6370 TYPE_VECTOR_SUBPARTS (srctype)));
6371 var = vect_get_new_ssa_name (srctype, vect_simple_var);
6372 src = build1 (VIEW_CONVERT_EXPR, srctype, src);
6373 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
6374 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6375 src = var;
6378 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
6380 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op)),
6381 TYPE_VECTOR_SUBPARTS (idxtype)));
6382 var = vect_get_new_ssa_name (idxtype, vect_simple_var);
6383 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
6384 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
6385 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6386 op = var;
6389 new_stmt
6390 = gimple_build_call (gs_info.decl, 5, ptr, mask, op, src, scale);
6392 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6394 if (prev_stmt_info == NULL)
6395 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6396 else
6397 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6398 prev_stmt_info = vinfo_for_stmt (new_stmt);
6400 return true;
6403 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
6405 gimple *group_stmt = GROUP_FIRST_ELEMENT (stmt_info);
6406 GROUP_STORE_COUNT (vinfo_for_stmt (group_stmt))++;
6409 if (grouped_store)
6411 /* FORNOW */
6412 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
6414 /* We vectorize all the stmts of the interleaving group when we
6415 reach the last stmt in the group. */
6416 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
6417 < GROUP_SIZE (vinfo_for_stmt (first_stmt))
6418 && !slp)
6420 *vec_stmt = NULL;
6421 return true;
6424 if (slp)
6426 grouped_store = false;
6427 /* VEC_NUM is the number of vect stmts to be created for this
6428 group. */
6429 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
6430 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
6431 gcc_assert (GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt)) == first_stmt);
6432 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
6433 op = vect_get_store_rhs (first_stmt);
6435 else
6436 /* VEC_NUM is the number of vect stmts to be created for this
6437 group. */
6438 vec_num = group_size;
6440 ref_type = get_group_alias_ptr_type (first_stmt);
6442 else
6443 ref_type = reference_alias_ptr_type (DR_REF (first_dr));
6445 if (dump_enabled_p ())
6446 dump_printf_loc (MSG_NOTE, vect_location,
6447 "transform store. ncopies = %d\n", ncopies);
6449 if (memory_access_type == VMAT_ELEMENTWISE
6450 || memory_access_type == VMAT_STRIDED_SLP)
6452 gimple_stmt_iterator incr_gsi;
6453 bool insert_after;
6454 gimple *incr;
6455 tree offvar;
6456 tree ivstep;
6457 tree running_off;
6458 gimple_seq stmts = NULL;
6459 tree stride_base, stride_step, alias_off;
6460 tree vec_oprnd;
6461 unsigned int g;
6462 /* Checked by get_load_store_type. */
6463 unsigned int const_nunits = nunits.to_constant ();
6465 gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
6466 gcc_assert (!nested_in_vect_loop_p (loop, stmt));
6468 stride_base
6469 = fold_build_pointer_plus
6470 (unshare_expr (DR_BASE_ADDRESS (first_dr)),
6471 size_binop (PLUS_EXPR,
6472 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr))),
6473 convert_to_ptrofftype (DR_INIT (first_dr))));
6474 stride_step = fold_convert (sizetype, unshare_expr (DR_STEP (first_dr)));
6476 /* For a store with loop-invariant (but other than power-of-2)
6477 stride (i.e. not a grouped access) like so:
6479 for (i = 0; i < n; i += stride)
6480 array[i] = ...;
6482 we generate a new induction variable and new stores from
6483 the components of the (vectorized) rhs:
6485 for (j = 0; ; j += VF*stride)
6486 vectemp = ...;
6487 tmp1 = vectemp[0];
6488 array[j] = tmp1;
6489 tmp2 = vectemp[1];
6490 array[j + stride] = tmp2;
6494 unsigned nstores = const_nunits;
6495 unsigned lnel = 1;
6496 tree ltype = elem_type;
6497 tree lvectype = vectype;
6498 if (slp)
6500 if (group_size < const_nunits
6501 && const_nunits % group_size == 0)
6503 nstores = const_nunits / group_size;
6504 lnel = group_size;
6505 ltype = build_vector_type (elem_type, group_size);
6506 lvectype = vectype;
6508 /* First check if vec_extract optab doesn't support extraction
6509 of vector elts directly. */
6510 scalar_mode elmode = SCALAR_TYPE_MODE (elem_type);
6511 machine_mode vmode;
6512 if (!mode_for_vector (elmode, group_size).exists (&vmode)
6513 || !VECTOR_MODE_P (vmode)
6514 || (convert_optab_handler (vec_extract_optab,
6515 TYPE_MODE (vectype), vmode)
6516 == CODE_FOR_nothing))
6518 /* Try to avoid emitting an extract of vector elements
6519 by performing the extracts using an integer type of the
6520 same size, extracting from a vector of those and then
6521 re-interpreting it as the original vector type if
6522 supported. */
6523 unsigned lsize
6524 = group_size * GET_MODE_BITSIZE (elmode);
6525 elmode = int_mode_for_size (lsize, 0).require ();
6526 unsigned int lnunits = const_nunits / group_size;
6527 /* If we can't construct such a vector fall back to
6528 element extracts from the original vector type and
6529 element size stores. */
6530 if (mode_for_vector (elmode, lnunits).exists (&vmode)
6531 && VECTOR_MODE_P (vmode)
6532 && (convert_optab_handler (vec_extract_optab,
6533 vmode, elmode)
6534 != CODE_FOR_nothing))
6536 nstores = lnunits;
6537 lnel = group_size;
6538 ltype = build_nonstandard_integer_type (lsize, 1);
6539 lvectype = build_vector_type (ltype, nstores);
6541 /* Else fall back to vector extraction anyway.
6542 Fewer stores are more important than avoiding spilling
6543 of the vector we extract from. Compared to the
6544 construction case in vectorizable_load no store-forwarding
6545 issue exists here for reasonable archs. */
6548 else if (group_size >= const_nunits
6549 && group_size % const_nunits == 0)
6551 nstores = 1;
6552 lnel = const_nunits;
6553 ltype = vectype;
6554 lvectype = vectype;
6556 ltype = build_aligned_type (ltype, TYPE_ALIGN (elem_type));
6557 ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
6560 ivstep = stride_step;
6561 ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
6562 build_int_cst (TREE_TYPE (ivstep), vf));
6564 standard_iv_increment_position (loop, &incr_gsi, &insert_after);
6566 create_iv (stride_base, ivstep, NULL,
6567 loop, &incr_gsi, insert_after,
6568 &offvar, NULL);
6569 incr = gsi_stmt (incr_gsi);
6570 set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
6572 stride_step = force_gimple_operand (stride_step, &stmts, true, NULL_TREE);
6573 if (stmts)
6574 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
6576 prev_stmt_info = NULL;
6577 alias_off = build_int_cst (ref_type, 0);
6578 next_stmt = first_stmt;
6579 for (g = 0; g < group_size; g++)
6581 running_off = offvar;
6582 if (g)
6584 tree size = TYPE_SIZE_UNIT (ltype);
6585 tree pos = fold_build2 (MULT_EXPR, sizetype, size_int (g),
6586 size);
6587 tree newoff = copy_ssa_name (running_off, NULL);
6588 incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
6589 running_off, pos);
6590 vect_finish_stmt_generation (stmt, incr, gsi);
6591 running_off = newoff;
6593 unsigned int group_el = 0;
6594 unsigned HOST_WIDE_INT
6595 elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
6596 for (j = 0; j < ncopies; j++)
6598 /* We've set op and dt above, from vect_get_store_rhs,
6599 and first_stmt == stmt. */
6600 if (j == 0)
6602 if (slp)
6604 vect_get_vec_defs (op, NULL_TREE, stmt, &vec_oprnds, NULL,
6605 slp_node);
6606 vec_oprnd = vec_oprnds[0];
6608 else
6610 op = vect_get_store_rhs (next_stmt);
6611 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
6614 else
6616 if (slp)
6617 vec_oprnd = vec_oprnds[j];
6618 else
6620 vect_is_simple_use (op, vinfo, &def_stmt, &rhs_dt);
6621 vec_oprnd = vect_get_vec_def_for_stmt_copy (rhs_dt,
6622 vec_oprnd);
6625 /* Pun the vector to extract from if necessary. */
6626 if (lvectype != vectype)
6628 tree tem = make_ssa_name (lvectype);
6629 gimple *pun
6630 = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR,
6631 lvectype, vec_oprnd));
6632 vect_finish_stmt_generation (stmt, pun, gsi);
6633 vec_oprnd = tem;
6635 for (i = 0; i < nstores; i++)
6637 tree newref, newoff;
6638 gimple *incr, *assign;
6639 tree size = TYPE_SIZE (ltype);
6640 /* Extract the i'th component. */
6641 tree pos = fold_build2 (MULT_EXPR, bitsizetype,
6642 bitsize_int (i), size);
6643 tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
6644 size, pos);
6646 elem = force_gimple_operand_gsi (gsi, elem, true,
6647 NULL_TREE, true,
6648 GSI_SAME_STMT);
6650 tree this_off = build_int_cst (TREE_TYPE (alias_off),
6651 group_el * elsz);
6652 newref = build2 (MEM_REF, ltype,
6653 running_off, this_off);
6655 /* And store it to *running_off. */
6656 assign = gimple_build_assign (newref, elem);
6657 vect_finish_stmt_generation (stmt, assign, gsi);
6659 group_el += lnel;
6660 if (! slp
6661 || group_el == group_size)
6663 newoff = copy_ssa_name (running_off, NULL);
6664 incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
6665 running_off, stride_step);
6666 vect_finish_stmt_generation (stmt, incr, gsi);
6668 running_off = newoff;
6669 group_el = 0;
6671 if (g == group_size - 1
6672 && !slp)
6674 if (j == 0 && i == 0)
6675 STMT_VINFO_VEC_STMT (stmt_info)
6676 = *vec_stmt = assign;
6677 else
6678 STMT_VINFO_RELATED_STMT (prev_stmt_info) = assign;
6679 prev_stmt_info = vinfo_for_stmt (assign);
6683 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
6684 if (slp)
6685 break;
6688 vec_oprnds.release ();
6689 return true;
6692 auto_vec<tree> dr_chain (group_size);
6693 oprnds.create (group_size);
6695 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
6696 gcc_assert (alignment_support_scheme);
6697 bool masked_loop_p = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
6698 /* Targets with store-lane instructions must not require explicit
6699 realignment. vect_supportable_dr_alignment always returns either
6700 dr_aligned or dr_unaligned_supported for masked operations. */
6701 gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
6702 && !mask
6703 && !masked_loop_p)
6704 || alignment_support_scheme == dr_aligned
6705 || alignment_support_scheme == dr_unaligned_supported);
6707 if (memory_access_type == VMAT_CONTIGUOUS_DOWN
6708 || memory_access_type == VMAT_CONTIGUOUS_REVERSE)
6709 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
6711 tree bump;
6712 tree vec_offset = NULL_TREE;
6713 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6715 aggr_type = NULL_TREE;
6716 bump = NULL_TREE;
6718 else if (memory_access_type == VMAT_GATHER_SCATTER)
6720 aggr_type = elem_type;
6721 vect_get_strided_load_store_ops (stmt, loop_vinfo, &gs_info,
6722 &bump, &vec_offset);
6724 else
6726 if (memory_access_type == VMAT_LOAD_STORE_LANES)
6727 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
6728 else
6729 aggr_type = vectype;
6730 bump = vect_get_data_ptr_increment (dr, aggr_type, memory_access_type);
6733 if (mask)
6734 LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
6736 /* In case the vectorization factor (VF) is bigger than the number
6737 of elements that we can fit in a vectype (nunits), we have to generate
6738 more than one vector stmt - i.e - we need to "unroll" the
6739 vector stmt by a factor VF/nunits. For more details see documentation in
6740 vect_get_vec_def_for_copy_stmt. */
6742 /* In case of interleaving (non-unit grouped access):
6744 S1: &base + 2 = x2
6745 S2: &base = x0
6746 S3: &base + 1 = x1
6747 S4: &base + 3 = x3
6749 We create vectorized stores starting from base address (the access of the
6750 first stmt in the chain (S2 in the above example), when the last store stmt
6751 of the chain (S4) is reached:
6753 VS1: &base = vx2
6754 VS2: &base + vec_size*1 = vx0
6755 VS3: &base + vec_size*2 = vx1
6756 VS4: &base + vec_size*3 = vx3
6758 Then permutation statements are generated:
6760 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
6761 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
6764 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6765 (the order of the data-refs in the output of vect_permute_store_chain
6766 corresponds to the order of scalar stmts in the interleaving chain - see
6767 the documentation of vect_permute_store_chain()).
6769 In case of both multiple types and interleaving, above vector stores and
6770 permutation stmts are created for every copy. The result vector stmts are
6771 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
6772 STMT_VINFO_RELATED_STMT for the next copies.
6775 prev_stmt_info = NULL;
6776 tree vec_mask = NULL_TREE;
6777 vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
6778 for (j = 0; j < ncopies; j++)
6781 if (j == 0)
6783 if (slp)
6785 /* Get vectorized arguments for SLP_NODE. */
6786 vect_get_vec_defs (op, NULL_TREE, stmt, &vec_oprnds,
6787 NULL, slp_node);
6789 vec_oprnd = vec_oprnds[0];
6791 else
6793 /* For interleaved stores we collect vectorized defs for all the
6794 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
6795 used as an input to vect_permute_store_chain(), and OPRNDS as
6796 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
6798 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
6799 OPRNDS are of size 1. */
6800 next_stmt = first_stmt;
6801 for (i = 0; i < group_size; i++)
6803 /* Since gaps are not supported for interleaved stores,
6804 GROUP_SIZE is the exact number of stmts in the chain.
6805 Therefore, NEXT_STMT can't be NULL_TREE. In case that
6806 there is no interleaving, GROUP_SIZE is 1, and only one
6807 iteration of the loop will be executed. */
6808 op = vect_get_store_rhs (next_stmt);
6809 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
6810 dr_chain.quick_push (vec_oprnd);
6811 oprnds.quick_push (vec_oprnd);
6812 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
6814 if (mask)
6815 vec_mask = vect_get_vec_def_for_operand (mask, stmt,
6816 mask_vectype);
6819 /* We should have catched mismatched types earlier. */
6820 gcc_assert (useless_type_conversion_p (vectype,
6821 TREE_TYPE (vec_oprnd)));
6822 bool simd_lane_access_p
6823 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
6824 if (simd_lane_access_p
6825 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
6826 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
6827 && integer_zerop (DR_OFFSET (first_dr))
6828 && integer_zerop (DR_INIT (first_dr))
6829 && alias_sets_conflict_p (get_alias_set (aggr_type),
6830 get_alias_set (TREE_TYPE (ref_type))))
6832 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
6833 dataref_offset = build_int_cst (ref_type, 0);
6834 inv_p = false;
6836 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6838 vect_get_gather_scatter_ops (loop, stmt, &gs_info,
6839 &dataref_ptr, &vec_offset);
6840 inv_p = false;
6842 else
6843 dataref_ptr
6844 = vect_create_data_ref_ptr (first_stmt, aggr_type,
6845 simd_lane_access_p ? loop : NULL,
6846 offset, &dummy, gsi, &ptr_incr,
6847 simd_lane_access_p, &inv_p,
6848 NULL_TREE, bump);
6849 gcc_assert (bb_vinfo || !inv_p);
6851 else
6853 /* For interleaved stores we created vectorized defs for all the
6854 defs stored in OPRNDS in the previous iteration (previous copy).
6855 DR_CHAIN is then used as an input to vect_permute_store_chain(),
6856 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
6857 next copy.
6858 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
6859 OPRNDS are of size 1. */
6860 for (i = 0; i < group_size; i++)
6862 op = oprnds[i];
6863 vect_is_simple_use (op, vinfo, &def_stmt, &rhs_dt);
6864 vec_oprnd = vect_get_vec_def_for_stmt_copy (rhs_dt, op);
6865 dr_chain[i] = vec_oprnd;
6866 oprnds[i] = vec_oprnd;
6868 if (mask)
6869 vec_mask = vect_get_vec_def_for_stmt_copy (mask_dt, vec_mask);
6870 if (dataref_offset)
6871 dataref_offset
6872 = int_const_binop (PLUS_EXPR, dataref_offset, bump);
6873 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6874 vec_offset = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
6875 vec_offset);
6876 else
6877 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
6878 bump);
6881 if (memory_access_type == VMAT_LOAD_STORE_LANES)
6883 tree vec_array;
6885 /* Combine all the vectors into an array. */
6886 vec_array = create_vector_array (vectype, vec_num);
6887 for (i = 0; i < vec_num; i++)
6889 vec_oprnd = dr_chain[i];
6890 write_vector_array (stmt, gsi, vec_oprnd, vec_array, i);
6893 tree final_mask = NULL;
6894 if (masked_loop_p)
6895 final_mask = vect_get_loop_mask (gsi, masks, ncopies, vectype, j);
6896 if (vec_mask)
6897 final_mask = prepare_load_store_mask (mask_vectype, final_mask,
6898 vec_mask, gsi);
6900 gcall *call;
6901 if (final_mask)
6903 /* Emit:
6904 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
6905 VEC_ARRAY). */
6906 unsigned int align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
6907 tree alias_ptr = build_int_cst (ref_type, align);
6908 call = gimple_build_call_internal (IFN_MASK_STORE_LANES, 4,
6909 dataref_ptr, alias_ptr,
6910 final_mask, vec_array);
6912 else
6914 /* Emit:
6915 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
6916 data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
6917 call = gimple_build_call_internal (IFN_STORE_LANES, 1,
6918 vec_array);
6919 gimple_call_set_lhs (call, data_ref);
6921 gimple_call_set_nothrow (call, true);
6922 new_stmt = call;
6923 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6925 else
6927 new_stmt = NULL;
6928 if (grouped_store)
6930 if (j == 0)
6931 result_chain.create (group_size);
6932 /* Permute. */
6933 vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
6934 &result_chain);
6937 next_stmt = first_stmt;
6938 for (i = 0; i < vec_num; i++)
6940 unsigned align, misalign;
6942 tree final_mask = NULL_TREE;
6943 if (masked_loop_p)
6944 final_mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
6945 vectype, vec_num * j + i);
6946 if (vec_mask)
6947 final_mask = prepare_load_store_mask (mask_vectype, final_mask,
6948 vec_mask, gsi);
6950 if (memory_access_type == VMAT_GATHER_SCATTER)
6952 tree scale = size_int (gs_info.scale);
6953 gcall *call;
6954 if (masked_loop_p)
6955 call = gimple_build_call_internal
6956 (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, vec_offset,
6957 scale, vec_oprnd, final_mask);
6958 else
6959 call = gimple_build_call_internal
6960 (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
6961 scale, vec_oprnd);
6962 gimple_call_set_nothrow (call, true);
6963 new_stmt = call;
6964 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6965 break;
6968 if (i > 0)
6969 /* Bump the vector pointer. */
6970 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
6971 stmt, bump);
6973 if (slp)
6974 vec_oprnd = vec_oprnds[i];
6975 else if (grouped_store)
6976 /* For grouped stores vectorized defs are interleaved in
6977 vect_permute_store_chain(). */
6978 vec_oprnd = result_chain[i];
6980 align = DR_TARGET_ALIGNMENT (first_dr);
6981 if (aligned_access_p (first_dr))
6982 misalign = 0;
6983 else if (DR_MISALIGNMENT (first_dr) == -1)
6985 align = dr_alignment (vect_dr_behavior (first_dr));
6986 misalign = 0;
6988 else
6989 misalign = DR_MISALIGNMENT (first_dr);
6990 if (dataref_offset == NULL_TREE
6991 && TREE_CODE (dataref_ptr) == SSA_NAME)
6992 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
6993 misalign);
6995 if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
6997 tree perm_mask = perm_mask_for_reverse (vectype);
6998 tree perm_dest
6999 = vect_create_destination_var (vect_get_store_rhs (stmt),
7000 vectype);
7001 tree new_temp = make_ssa_name (perm_dest);
7003 /* Generate the permute statement. */
7004 gimple *perm_stmt
7005 = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
7006 vec_oprnd, perm_mask);
7007 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
7009 perm_stmt = SSA_NAME_DEF_STMT (new_temp);
7010 vec_oprnd = new_temp;
7013 /* Arguments are ready. Create the new vector stmt. */
7014 if (final_mask)
7016 align = least_bit_hwi (misalign | align);
7017 tree ptr = build_int_cst (ref_type, align);
7018 gcall *call
7019 = gimple_build_call_internal (IFN_MASK_STORE, 4,
7020 dataref_ptr, ptr,
7021 final_mask, vec_oprnd);
7022 gimple_call_set_nothrow (call, true);
7023 new_stmt = call;
7025 else
7027 data_ref = fold_build2 (MEM_REF, vectype,
7028 dataref_ptr,
7029 dataref_offset
7030 ? dataref_offset
7031 : build_int_cst (ref_type, 0));
7032 if (aligned_access_p (first_dr))
7034 else if (DR_MISALIGNMENT (first_dr) == -1)
7035 TREE_TYPE (data_ref)
7036 = build_aligned_type (TREE_TYPE (data_ref),
7037 align * BITS_PER_UNIT);
7038 else
7039 TREE_TYPE (data_ref)
7040 = build_aligned_type (TREE_TYPE (data_ref),
7041 TYPE_ALIGN (elem_type));
7042 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
7044 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7046 if (slp)
7047 continue;
7049 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
7050 if (!next_stmt)
7051 break;
7054 if (!slp)
7056 if (j == 0)
7057 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
7058 else
7059 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
7060 prev_stmt_info = vinfo_for_stmt (new_stmt);
7064 oprnds.release ();
7065 result_chain.release ();
7066 vec_oprnds.release ();
7068 return true;
7071 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
7072 VECTOR_CST mask. No checks are made that the target platform supports the
7073 mask, so callers may wish to test can_vec_perm_const_p separately, or use
7074 vect_gen_perm_mask_checked. */
7076 tree
7077 vect_gen_perm_mask_any (tree vectype, const vec_perm_indices &sel)
7079 tree mask_type;
7081 poly_uint64 nunits = sel.length ();
7082 gcc_assert (known_eq (nunits, TYPE_VECTOR_SUBPARTS (vectype)));
7084 mask_type = build_vector_type (ssizetype, nunits);
7085 return vec_perm_indices_to_tree (mask_type, sel);
7088 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
7089 i.e. that the target supports the pattern _for arbitrary input vectors_. */
7091 tree
7092 vect_gen_perm_mask_checked (tree vectype, const vec_perm_indices &sel)
7094 gcc_assert (can_vec_perm_const_p (TYPE_MODE (vectype), sel));
7095 return vect_gen_perm_mask_any (vectype, sel);
7098 /* Given a vector variable X and Y, that was generated for the scalar
7099 STMT, generate instructions to permute the vector elements of X and Y
7100 using permutation mask MASK_VEC, insert them at *GSI and return the
7101 permuted vector variable. */
7103 static tree
7104 permute_vec_elements (tree x, tree y, tree mask_vec, gimple *stmt,
7105 gimple_stmt_iterator *gsi)
7107 tree vectype = TREE_TYPE (x);
7108 tree perm_dest, data_ref;
7109 gimple *perm_stmt;
7111 tree scalar_dest = gimple_get_lhs (stmt);
7112 if (TREE_CODE (scalar_dest) == SSA_NAME)
7113 perm_dest = vect_create_destination_var (scalar_dest, vectype);
7114 else
7115 perm_dest = vect_get_new_vect_var (vectype, vect_simple_var, NULL);
7116 data_ref = make_ssa_name (perm_dest);
7118 /* Generate the permute statement. */
7119 perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
7120 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
7122 return data_ref;
7125 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
7126 inserting them on the loops preheader edge. Returns true if we
7127 were successful in doing so (and thus STMT can be moved then),
7128 otherwise returns false. */
7130 static bool
7131 hoist_defs_of_uses (gimple *stmt, struct loop *loop)
7133 ssa_op_iter i;
7134 tree op;
7135 bool any = false;
7137 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
7139 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
7140 if (!gimple_nop_p (def_stmt)
7141 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
7143 /* Make sure we don't need to recurse. While we could do
7144 so in simple cases when there are more complex use webs
7145 we don't have an easy way to preserve stmt order to fulfil
7146 dependencies within them. */
7147 tree op2;
7148 ssa_op_iter i2;
7149 if (gimple_code (def_stmt) == GIMPLE_PHI)
7150 return false;
7151 FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
7153 gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
7154 if (!gimple_nop_p (def_stmt2)
7155 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
7156 return false;
7158 any = true;
7162 if (!any)
7163 return true;
7165 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
7167 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
7168 if (!gimple_nop_p (def_stmt)
7169 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
7171 gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
7172 gsi_remove (&gsi, false);
7173 gsi_insert_on_edge_immediate (loop_preheader_edge (loop), def_stmt);
7177 return true;
7180 /* vectorizable_load.
7182 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
7183 can be vectorized.
7184 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7185 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
7186 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7188 static bool
7189 vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
7190 slp_tree slp_node, slp_instance slp_node_instance)
7192 tree scalar_dest;
7193 tree vec_dest = NULL;
7194 tree data_ref = NULL;
7195 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7196 stmt_vec_info prev_stmt_info;
7197 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
7198 struct loop *loop = NULL;
7199 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
7200 bool nested_in_vect_loop = false;
7201 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
7202 tree elem_type;
7203 tree new_temp;
7204 machine_mode mode;
7205 gimple *new_stmt = NULL;
7206 tree dummy;
7207 enum dr_alignment_support alignment_support_scheme;
7208 tree dataref_ptr = NULL_TREE;
7209 tree dataref_offset = NULL_TREE;
7210 gimple *ptr_incr = NULL;
7211 int ncopies;
7212 int i, j;
7213 unsigned int group_size;
7214 poly_uint64 group_gap_adj;
7215 tree msq = NULL_TREE, lsq;
7216 tree offset = NULL_TREE;
7217 tree byte_offset = NULL_TREE;
7218 tree realignment_token = NULL_TREE;
7219 gphi *phi = NULL;
7220 vec<tree> dr_chain = vNULL;
7221 bool grouped_load = false;
7222 gimple *first_stmt;
7223 gimple *first_stmt_for_drptr = NULL;
7224 bool inv_p;
7225 bool compute_in_loop = false;
7226 struct loop *at_loop;
7227 int vec_num;
7228 bool slp = (slp_node != NULL);
7229 bool slp_perm = false;
7230 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
7231 poly_uint64 vf;
7232 tree aggr_type;
7233 gather_scatter_info gs_info;
7234 vec_info *vinfo = stmt_info->vinfo;
7235 tree ref_type;
7236 enum vect_def_type mask_dt = vect_unknown_def_type;
7238 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
7239 return false;
7241 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
7242 && ! vec_stmt)
7243 return false;
7245 tree mask = NULL_TREE, mask_vectype = NULL_TREE;
7246 if (is_gimple_assign (stmt))
7248 scalar_dest = gimple_assign_lhs (stmt);
7249 if (TREE_CODE (scalar_dest) != SSA_NAME)
7250 return false;
7252 tree_code code = gimple_assign_rhs_code (stmt);
7253 if (code != ARRAY_REF
7254 && code != BIT_FIELD_REF
7255 && code != INDIRECT_REF
7256 && code != COMPONENT_REF
7257 && code != IMAGPART_EXPR
7258 && code != REALPART_EXPR
7259 && code != MEM_REF
7260 && TREE_CODE_CLASS (code) != tcc_declaration)
7261 return false;
7263 else
7265 gcall *call = dyn_cast <gcall *> (stmt);
7266 if (!call || !gimple_call_internal_p (call))
7267 return false;
7269 internal_fn ifn = gimple_call_internal_fn (call);
7270 if (!internal_load_fn_p (ifn))
7271 return false;
7273 scalar_dest = gimple_call_lhs (call);
7274 if (!scalar_dest)
7275 return false;
7277 if (slp_node != NULL)
7279 if (dump_enabled_p ())
7280 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7281 "SLP of masked loads not supported.\n");
7282 return false;
7285 int mask_index = internal_fn_mask_index (ifn);
7286 if (mask_index >= 0)
7288 mask = gimple_call_arg (call, mask_index);
7289 if (!vect_check_load_store_mask (stmt, mask, &mask_dt,
7290 &mask_vectype))
7291 return false;
7295 if (!STMT_VINFO_DATA_REF (stmt_info))
7296 return false;
7298 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
7299 poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
7301 if (loop_vinfo)
7303 loop = LOOP_VINFO_LOOP (loop_vinfo);
7304 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
7305 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7307 else
7308 vf = 1;
7310 /* Multiple types in SLP are handled by creating the appropriate number of
7311 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
7312 case of SLP. */
7313 if (slp)
7314 ncopies = 1;
7315 else
7316 ncopies = vect_get_num_copies (loop_vinfo, vectype);
7318 gcc_assert (ncopies >= 1);
7320 /* FORNOW. This restriction should be relaxed. */
7321 if (nested_in_vect_loop && ncopies > 1)
7323 if (dump_enabled_p ())
7324 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7325 "multiple types in nested loop.\n");
7326 return false;
7329 /* Invalidate assumptions made by dependence analysis when vectorization
7330 on the unrolled body effectively re-orders stmts. */
7331 if (ncopies > 1
7332 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
7333 && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
7334 STMT_VINFO_MIN_NEG_DIST (stmt_info)))
7336 if (dump_enabled_p ())
7337 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7338 "cannot perform implicit CSE when unrolling "
7339 "with negative dependence distance\n");
7340 return false;
7343 elem_type = TREE_TYPE (vectype);
7344 mode = TYPE_MODE (vectype);
7346 /* FORNOW. In some cases can vectorize even if data-type not supported
7347 (e.g. - data copies). */
7348 if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
7350 if (dump_enabled_p ())
7351 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7352 "Aligned load, but unsupported type.\n");
7353 return false;
7356 /* Check if the load is a part of an interleaving chain. */
7357 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
7359 grouped_load = true;
7360 /* FORNOW */
7361 gcc_assert (!nested_in_vect_loop);
7362 gcc_assert (!STMT_VINFO_GATHER_SCATTER_P (stmt_info));
7364 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
7365 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
7367 if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
7368 slp_perm = true;
7370 /* Invalidate assumptions made by dependence analysis when vectorization
7371 on the unrolled body effectively re-orders stmts. */
7372 if (!PURE_SLP_STMT (stmt_info)
7373 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
7374 && maybe_gt (LOOP_VINFO_VECT_FACTOR (loop_vinfo),
7375 STMT_VINFO_MIN_NEG_DIST (stmt_info)))
7377 if (dump_enabled_p ())
7378 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7379 "cannot perform implicit CSE when performing "
7380 "group loads with negative dependence distance\n");
7381 return false;
7384 /* Similarly when the stmt is a load that is both part of a SLP
7385 instance and a loop vectorized stmt via the same-dr mechanism
7386 we have to give up. */
7387 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info)
7388 && (STMT_SLP_TYPE (stmt_info)
7389 != STMT_SLP_TYPE (vinfo_for_stmt
7390 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info)))))
7392 if (dump_enabled_p ())
7393 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7394 "conflicting SLP types for CSEd load\n");
7395 return false;
7398 else
7399 group_size = 1;
7401 vect_memory_access_type memory_access_type;
7402 if (!get_load_store_type (stmt, vectype, slp, mask, VLS_LOAD, ncopies,
7403 &memory_access_type, &gs_info))
7404 return false;
7406 if (mask)
7408 if (memory_access_type == VMAT_CONTIGUOUS)
7410 machine_mode vec_mode = TYPE_MODE (vectype);
7411 if (!VECTOR_MODE_P (vec_mode)
7412 || !can_vec_mask_load_store_p (vec_mode,
7413 TYPE_MODE (mask_vectype), true))
7414 return false;
7416 else if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
7418 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gs_info.decl));
7419 tree masktype
7420 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
7421 if (TREE_CODE (masktype) == INTEGER_TYPE)
7423 if (dump_enabled_p ())
7424 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7425 "masked gather with integer mask not"
7426 " supported.");
7427 return false;
7430 else if (memory_access_type != VMAT_LOAD_STORE_LANES
7431 && memory_access_type != VMAT_GATHER_SCATTER)
7433 if (dump_enabled_p ())
7434 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7435 "unsupported access type for masked load.\n");
7436 return false;
7440 if (!vec_stmt) /* transformation not required. */
7442 if (!slp)
7443 STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
7445 if (loop_vinfo
7446 && LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo))
7447 check_load_store_masking (loop_vinfo, vectype, VLS_LOAD, group_size,
7448 memory_access_type, &gs_info);
7450 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
7451 /* The SLP costs are calculated during SLP analysis. */
7452 if (!PURE_SLP_STMT (stmt_info))
7453 vect_model_load_cost (stmt_info, ncopies, memory_access_type,
7454 NULL, NULL, NULL);
7455 return true;
7458 if (!slp)
7459 gcc_assert (memory_access_type
7460 == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
7462 if (dump_enabled_p ())
7463 dump_printf_loc (MSG_NOTE, vect_location,
7464 "transform load. ncopies = %d\n", ncopies);
7466 /* Transform. */
7468 ensure_base_align (dr);
7470 if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
7472 vect_build_gather_load_calls (stmt, gsi, vec_stmt, &gs_info, mask,
7473 mask_dt);
7474 return true;
7477 if (memory_access_type == VMAT_ELEMENTWISE
7478 || memory_access_type == VMAT_STRIDED_SLP)
7480 gimple_stmt_iterator incr_gsi;
7481 bool insert_after;
7482 gimple *incr;
7483 tree offvar;
7484 tree ivstep;
7485 tree running_off;
7486 vec<constructor_elt, va_gc> *v = NULL;
7487 gimple_seq stmts = NULL;
7488 tree stride_base, stride_step, alias_off;
7489 /* Checked by get_load_store_type. */
7490 unsigned int const_nunits = nunits.to_constant ();
7492 gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
7493 gcc_assert (!nested_in_vect_loop);
7495 if (slp && grouped_load)
7497 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
7498 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
7499 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
7500 ref_type = get_group_alias_ptr_type (first_stmt);
7502 else
7504 first_stmt = stmt;
7505 first_dr = dr;
7506 group_size = 1;
7507 ref_type = reference_alias_ptr_type (DR_REF (first_dr));
7510 stride_base
7511 = fold_build_pointer_plus
7512 (DR_BASE_ADDRESS (first_dr),
7513 size_binop (PLUS_EXPR,
7514 convert_to_ptrofftype (DR_OFFSET (first_dr)),
7515 convert_to_ptrofftype (DR_INIT (first_dr))));
7516 stride_step = fold_convert (sizetype, DR_STEP (first_dr));
7518 /* For a load with loop-invariant (but other than power-of-2)
7519 stride (i.e. not a grouped access) like so:
7521 for (i = 0; i < n; i += stride)
7522 ... = array[i];
7524 we generate a new induction variable and new accesses to
7525 form a new vector (or vectors, depending on ncopies):
7527 for (j = 0; ; j += VF*stride)
7528 tmp1 = array[j];
7529 tmp2 = array[j + stride];
7531 vectemp = {tmp1, tmp2, ...}
7534 ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step), stride_step,
7535 build_int_cst (TREE_TYPE (stride_step), vf));
7537 standard_iv_increment_position (loop, &incr_gsi, &insert_after);
7539 create_iv (unshare_expr (stride_base), unshare_expr (ivstep), NULL,
7540 loop, &incr_gsi, insert_after,
7541 &offvar, NULL);
7542 incr = gsi_stmt (incr_gsi);
7543 set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
7545 stride_step = force_gimple_operand (unshare_expr (stride_step),
7546 &stmts, true, NULL_TREE);
7547 if (stmts)
7548 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
7550 prev_stmt_info = NULL;
7551 running_off = offvar;
7552 alias_off = build_int_cst (ref_type, 0);
7553 int nloads = const_nunits;
7554 int lnel = 1;
7555 tree ltype = TREE_TYPE (vectype);
7556 tree lvectype = vectype;
7557 auto_vec<tree> dr_chain;
7558 if (memory_access_type == VMAT_STRIDED_SLP)
7560 if (group_size < const_nunits)
7562 /* First check if vec_init optab supports construction from
7563 vector elts directly. */
7564 scalar_mode elmode = SCALAR_TYPE_MODE (TREE_TYPE (vectype));
7565 machine_mode vmode;
7566 if (mode_for_vector (elmode, group_size).exists (&vmode)
7567 && VECTOR_MODE_P (vmode)
7568 && (convert_optab_handler (vec_init_optab,
7569 TYPE_MODE (vectype), vmode)
7570 != CODE_FOR_nothing))
7572 nloads = const_nunits / group_size;
7573 lnel = group_size;
7574 ltype = build_vector_type (TREE_TYPE (vectype), group_size);
7576 else
7578 /* Otherwise avoid emitting a constructor of vector elements
7579 by performing the loads using an integer type of the same
7580 size, constructing a vector of those and then
7581 re-interpreting it as the original vector type.
7582 This avoids a huge runtime penalty due to the general
7583 inability to perform store forwarding from smaller stores
7584 to a larger load. */
7585 unsigned lsize
7586 = group_size * TYPE_PRECISION (TREE_TYPE (vectype));
7587 elmode = int_mode_for_size (lsize, 0).require ();
7588 unsigned int lnunits = const_nunits / group_size;
7589 /* If we can't construct such a vector fall back to
7590 element loads of the original vector type. */
7591 if (mode_for_vector (elmode, lnunits).exists (&vmode)
7592 && VECTOR_MODE_P (vmode)
7593 && (convert_optab_handler (vec_init_optab, vmode, elmode)
7594 != CODE_FOR_nothing))
7596 nloads = lnunits;
7597 lnel = group_size;
7598 ltype = build_nonstandard_integer_type (lsize, 1);
7599 lvectype = build_vector_type (ltype, nloads);
7603 else
7605 nloads = 1;
7606 lnel = const_nunits;
7607 ltype = vectype;
7609 ltype = build_aligned_type (ltype, TYPE_ALIGN (TREE_TYPE (vectype)));
7611 if (slp)
7613 /* For SLP permutation support we need to load the whole group,
7614 not only the number of vector stmts the permutation result
7615 fits in. */
7616 if (slp_perm)
7618 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
7619 variable VF. */
7620 unsigned int const_vf = vf.to_constant ();
7621 ncopies = CEIL (group_size * const_vf, const_nunits);
7622 dr_chain.create (ncopies);
7624 else
7625 ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
7627 unsigned int group_el = 0;
7628 unsigned HOST_WIDE_INT
7629 elsz = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
7630 for (j = 0; j < ncopies; j++)
7632 if (nloads > 1)
7633 vec_alloc (v, nloads);
7634 for (i = 0; i < nloads; i++)
7636 tree this_off = build_int_cst (TREE_TYPE (alias_off),
7637 group_el * elsz);
7638 new_stmt = gimple_build_assign (make_ssa_name (ltype),
7639 build2 (MEM_REF, ltype,
7640 running_off, this_off));
7641 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7642 if (nloads > 1)
7643 CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
7644 gimple_assign_lhs (new_stmt));
7646 group_el += lnel;
7647 if (! slp
7648 || group_el == group_size)
7650 tree newoff = copy_ssa_name (running_off);
7651 gimple *incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
7652 running_off, stride_step);
7653 vect_finish_stmt_generation (stmt, incr, gsi);
7655 running_off = newoff;
7656 group_el = 0;
7659 if (nloads > 1)
7661 tree vec_inv = build_constructor (lvectype, v);
7662 new_temp = vect_init_vector (stmt, vec_inv, lvectype, gsi);
7663 new_stmt = SSA_NAME_DEF_STMT (new_temp);
7664 if (lvectype != vectype)
7666 new_stmt = gimple_build_assign (make_ssa_name (vectype),
7667 VIEW_CONVERT_EXPR,
7668 build1 (VIEW_CONVERT_EXPR,
7669 vectype, new_temp));
7670 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7674 if (slp)
7676 if (slp_perm)
7677 dr_chain.quick_push (gimple_assign_lhs (new_stmt));
7678 else
7679 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
7681 else
7683 if (j == 0)
7684 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
7685 else
7686 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
7687 prev_stmt_info = vinfo_for_stmt (new_stmt);
7690 if (slp_perm)
7692 unsigned n_perms;
7693 vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
7694 slp_node_instance, false, &n_perms);
7696 return true;
7699 if (memory_access_type == VMAT_GATHER_SCATTER)
7700 grouped_load = false;
7702 if (grouped_load)
7704 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
7705 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
7706 /* For SLP vectorization we directly vectorize a subchain
7707 without permutation. */
7708 if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
7709 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
7710 /* For BB vectorization always use the first stmt to base
7711 the data ref pointer on. */
7712 if (bb_vinfo)
7713 first_stmt_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0];
7715 /* Check if the chain of loads is already vectorized. */
7716 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt))
7717 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
7718 ??? But we can only do so if there is exactly one
7719 as we have no way to get at the rest. Leave the CSE
7720 opportunity alone.
7721 ??? With the group load eventually participating
7722 in multiple different permutations (having multiple
7723 slp nodes which refer to the same group) the CSE
7724 is even wrong code. See PR56270. */
7725 && !slp)
7727 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
7728 return true;
7730 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
7731 group_gap_adj = 0;
7733 /* VEC_NUM is the number of vect stmts to be created for this group. */
7734 if (slp)
7736 grouped_load = false;
7737 /* For SLP permutation support we need to load the whole group,
7738 not only the number of vector stmts the permutation result
7739 fits in. */
7740 if (slp_perm)
7742 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
7743 variable VF. */
7744 unsigned int const_vf = vf.to_constant ();
7745 unsigned int const_nunits = nunits.to_constant ();
7746 vec_num = CEIL (group_size * const_vf, const_nunits);
7747 group_gap_adj = vf * group_size - nunits * vec_num;
7749 else
7751 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
7752 group_gap_adj
7753 = group_size - SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
7756 else
7757 vec_num = group_size;
7759 ref_type = get_group_alias_ptr_type (first_stmt);
7761 else
7763 first_stmt = stmt;
7764 first_dr = dr;
7765 group_size = vec_num = 1;
7766 group_gap_adj = 0;
7767 ref_type = reference_alias_ptr_type (DR_REF (first_dr));
7770 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
7771 gcc_assert (alignment_support_scheme);
7772 bool masked_loop_p = (loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
7773 /* Targets with store-lane instructions must not require explicit
7774 realignment. vect_supportable_dr_alignment always returns either
7775 dr_aligned or dr_unaligned_supported for masked operations. */
7776 gcc_assert ((memory_access_type != VMAT_LOAD_STORE_LANES
7777 && !mask
7778 && !masked_loop_p)
7779 || alignment_support_scheme == dr_aligned
7780 || alignment_support_scheme == dr_unaligned_supported);
7782 /* In case the vectorization factor (VF) is bigger than the number
7783 of elements that we can fit in a vectype (nunits), we have to generate
7784 more than one vector stmt - i.e - we need to "unroll" the
7785 vector stmt by a factor VF/nunits. In doing so, we record a pointer
7786 from one copy of the vector stmt to the next, in the field
7787 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
7788 stages to find the correct vector defs to be used when vectorizing
7789 stmts that use the defs of the current stmt. The example below
7790 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
7791 need to create 4 vectorized stmts):
7793 before vectorization:
7794 RELATED_STMT VEC_STMT
7795 S1: x = memref - -
7796 S2: z = x + 1 - -
7798 step 1: vectorize stmt S1:
7799 We first create the vector stmt VS1_0, and, as usual, record a
7800 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
7801 Next, we create the vector stmt VS1_1, and record a pointer to
7802 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
7803 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
7804 stmts and pointers:
7805 RELATED_STMT VEC_STMT
7806 VS1_0: vx0 = memref0 VS1_1 -
7807 VS1_1: vx1 = memref1 VS1_2 -
7808 VS1_2: vx2 = memref2 VS1_3 -
7809 VS1_3: vx3 = memref3 - -
7810 S1: x = load - VS1_0
7811 S2: z = x + 1 - -
7813 See in documentation in vect_get_vec_def_for_stmt_copy for how the
7814 information we recorded in RELATED_STMT field is used to vectorize
7815 stmt S2. */
7817 /* In case of interleaving (non-unit grouped access):
7819 S1: x2 = &base + 2
7820 S2: x0 = &base
7821 S3: x1 = &base + 1
7822 S4: x3 = &base + 3
7824 Vectorized loads are created in the order of memory accesses
7825 starting from the access of the first stmt of the chain:
7827 VS1: vx0 = &base
7828 VS2: vx1 = &base + vec_size*1
7829 VS3: vx3 = &base + vec_size*2
7830 VS4: vx4 = &base + vec_size*3
7832 Then permutation statements are generated:
7834 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
7835 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
7838 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
7839 (the order of the data-refs in the output of vect_permute_load_chain
7840 corresponds to the order of scalar stmts in the interleaving chain - see
7841 the documentation of vect_permute_load_chain()).
7842 The generation of permutation stmts and recording them in
7843 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
7845 In case of both multiple types and interleaving, the vector loads and
7846 permutation stmts above are created for every copy. The result vector
7847 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
7848 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
7850 /* If the data reference is aligned (dr_aligned) or potentially unaligned
7851 on a target that supports unaligned accesses (dr_unaligned_supported)
7852 we generate the following code:
7853 p = initial_addr;
7854 indx = 0;
7855 loop {
7856 p = p + indx * vectype_size;
7857 vec_dest = *(p);
7858 indx = indx + 1;
7861 Otherwise, the data reference is potentially unaligned on a target that
7862 does not support unaligned accesses (dr_explicit_realign_optimized) -
7863 then generate the following code, in which the data in each iteration is
7864 obtained by two vector loads, one from the previous iteration, and one
7865 from the current iteration:
7866 p1 = initial_addr;
7867 msq_init = *(floor(p1))
7868 p2 = initial_addr + VS - 1;
7869 realignment_token = call target_builtin;
7870 indx = 0;
7871 loop {
7872 p2 = p2 + indx * vectype_size
7873 lsq = *(floor(p2))
7874 vec_dest = realign_load (msq, lsq, realignment_token)
7875 indx = indx + 1;
7876 msq = lsq;
7877 } */
7879 /* If the misalignment remains the same throughout the execution of the
7880 loop, we can create the init_addr and permutation mask at the loop
7881 preheader. Otherwise, it needs to be created inside the loop.
7882 This can only occur when vectorizing memory accesses in the inner-loop
7883 nested within an outer-loop that is being vectorized. */
7885 if (nested_in_vect_loop
7886 && !multiple_p (DR_STEP_ALIGNMENT (dr),
7887 GET_MODE_SIZE (TYPE_MODE (vectype))))
7889 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
7890 compute_in_loop = true;
7893 if ((alignment_support_scheme == dr_explicit_realign_optimized
7894 || alignment_support_scheme == dr_explicit_realign)
7895 && !compute_in_loop)
7897 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
7898 alignment_support_scheme, NULL_TREE,
7899 &at_loop);
7900 if (alignment_support_scheme == dr_explicit_realign_optimized)
7902 phi = as_a <gphi *> (SSA_NAME_DEF_STMT (msq));
7903 byte_offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
7904 size_one_node);
7907 else
7908 at_loop = loop;
7910 if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
7911 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
7913 tree bump;
7914 tree vec_offset = NULL_TREE;
7915 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
7917 aggr_type = NULL_TREE;
7918 bump = NULL_TREE;
7920 else if (memory_access_type == VMAT_GATHER_SCATTER)
7922 aggr_type = elem_type;
7923 vect_get_strided_load_store_ops (stmt, loop_vinfo, &gs_info,
7924 &bump, &vec_offset);
7926 else
7928 if (memory_access_type == VMAT_LOAD_STORE_LANES)
7929 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
7930 else
7931 aggr_type = vectype;
7932 bump = vect_get_data_ptr_increment (dr, aggr_type, memory_access_type);
7935 tree vec_mask = NULL_TREE;
7936 prev_stmt_info = NULL;
7937 poly_uint64 group_elt = 0;
7938 vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo);
7939 for (j = 0; j < ncopies; j++)
7941 /* 1. Create the vector or array pointer update chain. */
7942 if (j == 0)
7944 bool simd_lane_access_p
7945 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
7946 if (simd_lane_access_p
7947 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
7948 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
7949 && integer_zerop (DR_OFFSET (first_dr))
7950 && integer_zerop (DR_INIT (first_dr))
7951 && alias_sets_conflict_p (get_alias_set (aggr_type),
7952 get_alias_set (TREE_TYPE (ref_type)))
7953 && (alignment_support_scheme == dr_aligned
7954 || alignment_support_scheme == dr_unaligned_supported))
7956 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
7957 dataref_offset = build_int_cst (ref_type, 0);
7958 inv_p = false;
7960 else if (first_stmt_for_drptr
7961 && first_stmt != first_stmt_for_drptr)
7963 dataref_ptr
7964 = vect_create_data_ref_ptr (first_stmt_for_drptr, aggr_type,
7965 at_loop, offset, &dummy, gsi,
7966 &ptr_incr, simd_lane_access_p,
7967 &inv_p, byte_offset, bump);
7968 /* Adjust the pointer by the difference to first_stmt. */
7969 data_reference_p ptrdr
7970 = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr));
7971 tree diff = fold_convert (sizetype,
7972 size_binop (MINUS_EXPR,
7973 DR_INIT (first_dr),
7974 DR_INIT (ptrdr)));
7975 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
7976 stmt, diff);
7978 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
7980 vect_get_gather_scatter_ops (loop, stmt, &gs_info,
7981 &dataref_ptr, &vec_offset);
7982 inv_p = false;
7984 else
7985 dataref_ptr
7986 = vect_create_data_ref_ptr (first_stmt, aggr_type, at_loop,
7987 offset, &dummy, gsi, &ptr_incr,
7988 simd_lane_access_p, &inv_p,
7989 byte_offset, bump);
7990 if (mask)
7991 vec_mask = vect_get_vec_def_for_operand (mask, stmt,
7992 mask_vectype);
7994 else
7996 if (dataref_offset)
7997 dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
7998 bump);
7999 else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
8000 vec_offset = vect_get_vec_def_for_stmt_copy (gs_info.offset_dt,
8001 vec_offset);
8002 else
8003 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
8004 stmt, bump);
8005 if (mask)
8006 vec_mask = vect_get_vec_def_for_stmt_copy (mask_dt, vec_mask);
8009 if (grouped_load || slp_perm)
8010 dr_chain.create (vec_num);
8012 if (memory_access_type == VMAT_LOAD_STORE_LANES)
8014 tree vec_array;
8016 vec_array = create_vector_array (vectype, vec_num);
8018 tree final_mask = NULL_TREE;
8019 if (masked_loop_p)
8020 final_mask = vect_get_loop_mask (gsi, masks, ncopies, vectype, j);
8021 if (vec_mask)
8022 final_mask = prepare_load_store_mask (mask_vectype, final_mask,
8023 vec_mask, gsi);
8025 gcall *call;
8026 if (final_mask)
8028 /* Emit:
8029 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
8030 VEC_MASK). */
8031 unsigned int align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype));
8032 tree alias_ptr = build_int_cst (ref_type, align);
8033 call = gimple_build_call_internal (IFN_MASK_LOAD_LANES, 3,
8034 dataref_ptr, alias_ptr,
8035 final_mask);
8037 else
8039 /* Emit:
8040 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
8041 data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
8042 call = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
8044 gimple_call_set_lhs (call, vec_array);
8045 gimple_call_set_nothrow (call, true);
8046 new_stmt = call;
8047 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8049 /* Extract each vector into an SSA_NAME. */
8050 for (i = 0; i < vec_num; i++)
8052 new_temp = read_vector_array (stmt, gsi, scalar_dest,
8053 vec_array, i);
8054 dr_chain.quick_push (new_temp);
8057 /* Record the mapping between SSA_NAMEs and statements. */
8058 vect_record_grouped_load_vectors (stmt, dr_chain);
8060 else
8062 for (i = 0; i < vec_num; i++)
8064 tree final_mask = NULL_TREE;
8065 if (masked_loop_p
8066 && memory_access_type != VMAT_INVARIANT)
8067 final_mask = vect_get_loop_mask (gsi, masks, vec_num * ncopies,
8068 vectype, vec_num * j + i);
8069 if (vec_mask)
8070 final_mask = prepare_load_store_mask (mask_vectype, final_mask,
8071 vec_mask, gsi);
8073 if (i > 0)
8074 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
8075 stmt, bump);
8077 /* 2. Create the vector-load in the loop. */
8078 switch (alignment_support_scheme)
8080 case dr_aligned:
8081 case dr_unaligned_supported:
8083 unsigned int align, misalign;
8085 if (memory_access_type == VMAT_GATHER_SCATTER)
8087 tree scale = size_int (gs_info.scale);
8088 gcall *call;
8089 if (masked_loop_p)
8090 call = gimple_build_call_internal
8091 (IFN_MASK_GATHER_LOAD, 4, dataref_ptr,
8092 vec_offset, scale, final_mask);
8093 else
8094 call = gimple_build_call_internal
8095 (IFN_GATHER_LOAD, 3, dataref_ptr,
8096 vec_offset, scale);
8097 gimple_call_set_nothrow (call, true);
8098 new_stmt = call;
8099 data_ref = NULL_TREE;
8100 break;
8103 align = DR_TARGET_ALIGNMENT (dr);
8104 if (alignment_support_scheme == dr_aligned)
8106 gcc_assert (aligned_access_p (first_dr));
8107 misalign = 0;
8109 else if (DR_MISALIGNMENT (first_dr) == -1)
8111 align = dr_alignment (vect_dr_behavior (first_dr));
8112 misalign = 0;
8114 else
8115 misalign = DR_MISALIGNMENT (first_dr);
8116 if (dataref_offset == NULL_TREE
8117 && TREE_CODE (dataref_ptr) == SSA_NAME)
8118 set_ptr_info_alignment (get_ptr_info (dataref_ptr),
8119 align, misalign);
8121 if (final_mask)
8123 align = least_bit_hwi (misalign | align);
8124 tree ptr = build_int_cst (ref_type, align);
8125 gcall *call
8126 = gimple_build_call_internal (IFN_MASK_LOAD, 3,
8127 dataref_ptr, ptr,
8128 final_mask);
8129 gimple_call_set_nothrow (call, true);
8130 new_stmt = call;
8131 data_ref = NULL_TREE;
8133 else
8135 data_ref
8136 = fold_build2 (MEM_REF, vectype, dataref_ptr,
8137 dataref_offset
8138 ? dataref_offset
8139 : build_int_cst (ref_type, 0));
8140 if (alignment_support_scheme == dr_aligned)
8142 else if (DR_MISALIGNMENT (first_dr) == -1)
8143 TREE_TYPE (data_ref)
8144 = build_aligned_type (TREE_TYPE (data_ref),
8145 align * BITS_PER_UNIT);
8146 else
8147 TREE_TYPE (data_ref)
8148 = build_aligned_type (TREE_TYPE (data_ref),
8149 TYPE_ALIGN (elem_type));
8151 break;
8153 case dr_explicit_realign:
8155 tree ptr, bump;
8157 tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
8159 if (compute_in_loop)
8160 msq = vect_setup_realignment (first_stmt, gsi,
8161 &realignment_token,
8162 dr_explicit_realign,
8163 dataref_ptr, NULL);
8165 if (TREE_CODE (dataref_ptr) == SSA_NAME)
8166 ptr = copy_ssa_name (dataref_ptr);
8167 else
8168 ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
8169 unsigned int align = DR_TARGET_ALIGNMENT (first_dr);
8170 new_stmt = gimple_build_assign
8171 (ptr, BIT_AND_EXPR, dataref_ptr,
8172 build_int_cst
8173 (TREE_TYPE (dataref_ptr),
8174 -(HOST_WIDE_INT) align));
8175 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8176 data_ref
8177 = build2 (MEM_REF, vectype, ptr,
8178 build_int_cst (ref_type, 0));
8179 vec_dest = vect_create_destination_var (scalar_dest,
8180 vectype);
8181 new_stmt = gimple_build_assign (vec_dest, data_ref);
8182 new_temp = make_ssa_name (vec_dest, new_stmt);
8183 gimple_assign_set_lhs (new_stmt, new_temp);
8184 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
8185 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
8186 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8187 msq = new_temp;
8189 bump = size_binop (MULT_EXPR, vs,
8190 TYPE_SIZE_UNIT (elem_type));
8191 bump = size_binop (MINUS_EXPR, bump, size_one_node);
8192 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
8193 new_stmt = gimple_build_assign
8194 (NULL_TREE, BIT_AND_EXPR, ptr,
8195 build_int_cst
8196 (TREE_TYPE (ptr), -(HOST_WIDE_INT) align));
8197 ptr = copy_ssa_name (ptr, new_stmt);
8198 gimple_assign_set_lhs (new_stmt, ptr);
8199 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8200 data_ref
8201 = build2 (MEM_REF, vectype, ptr,
8202 build_int_cst (ref_type, 0));
8203 break;
8205 case dr_explicit_realign_optimized:
8207 if (TREE_CODE (dataref_ptr) == SSA_NAME)
8208 new_temp = copy_ssa_name (dataref_ptr);
8209 else
8210 new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
8211 unsigned int align = DR_TARGET_ALIGNMENT (first_dr);
8212 new_stmt = gimple_build_assign
8213 (new_temp, BIT_AND_EXPR, dataref_ptr,
8214 build_int_cst (TREE_TYPE (dataref_ptr),
8215 -(HOST_WIDE_INT) align));
8216 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8217 data_ref
8218 = build2 (MEM_REF, vectype, new_temp,
8219 build_int_cst (ref_type, 0));
8220 break;
8222 default:
8223 gcc_unreachable ();
8225 vec_dest = vect_create_destination_var (scalar_dest, vectype);
8226 /* DATA_REF is null if we've already built the statement. */
8227 if (data_ref)
8228 new_stmt = gimple_build_assign (vec_dest, data_ref);
8229 new_temp = make_ssa_name (vec_dest, new_stmt);
8230 gimple_set_lhs (new_stmt, new_temp);
8231 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8233 /* 3. Handle explicit realignment if necessary/supported.
8234 Create in loop:
8235 vec_dest = realign_load (msq, lsq, realignment_token) */
8236 if (alignment_support_scheme == dr_explicit_realign_optimized
8237 || alignment_support_scheme == dr_explicit_realign)
8239 lsq = gimple_assign_lhs (new_stmt);
8240 if (!realignment_token)
8241 realignment_token = dataref_ptr;
8242 vec_dest = vect_create_destination_var (scalar_dest, vectype);
8243 new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR,
8244 msq, lsq, realignment_token);
8245 new_temp = make_ssa_name (vec_dest, new_stmt);
8246 gimple_assign_set_lhs (new_stmt, new_temp);
8247 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8249 if (alignment_support_scheme == dr_explicit_realign_optimized)
8251 gcc_assert (phi);
8252 if (i == vec_num - 1 && j == ncopies - 1)
8253 add_phi_arg (phi, lsq,
8254 loop_latch_edge (containing_loop),
8255 UNKNOWN_LOCATION);
8256 msq = lsq;
8260 /* 4. Handle invariant-load. */
8261 if (inv_p && !bb_vinfo)
8263 gcc_assert (!grouped_load);
8264 /* If we have versioned for aliasing or the loop doesn't
8265 have any data dependencies that would preclude this,
8266 then we are sure this is a loop invariant load and
8267 thus we can insert it on the preheader edge. */
8268 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
8269 && !nested_in_vect_loop
8270 && hoist_defs_of_uses (stmt, loop))
8272 if (dump_enabled_p ())
8274 dump_printf_loc (MSG_NOTE, vect_location,
8275 "hoisting out of the vectorized "
8276 "loop: ");
8277 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
8279 tree tem = copy_ssa_name (scalar_dest);
8280 gsi_insert_on_edge_immediate
8281 (loop_preheader_edge (loop),
8282 gimple_build_assign (tem,
8283 unshare_expr
8284 (gimple_assign_rhs1 (stmt))));
8285 new_temp = vect_init_vector (stmt, tem, vectype, NULL);
8286 new_stmt = SSA_NAME_DEF_STMT (new_temp);
8287 set_vinfo_for_stmt (new_stmt,
8288 new_stmt_vec_info (new_stmt, vinfo));
8290 else
8292 gimple_stmt_iterator gsi2 = *gsi;
8293 gsi_next (&gsi2);
8294 new_temp = vect_init_vector (stmt, scalar_dest,
8295 vectype, &gsi2);
8296 new_stmt = SSA_NAME_DEF_STMT (new_temp);
8300 if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
8302 tree perm_mask = perm_mask_for_reverse (vectype);
8303 new_temp = permute_vec_elements (new_temp, new_temp,
8304 perm_mask, stmt, gsi);
8305 new_stmt = SSA_NAME_DEF_STMT (new_temp);
8308 /* Collect vector loads and later create their permutation in
8309 vect_transform_grouped_load (). */
8310 if (grouped_load || slp_perm)
8311 dr_chain.quick_push (new_temp);
8313 /* Store vector loads in the corresponding SLP_NODE. */
8314 if (slp && !slp_perm)
8315 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
8317 /* With SLP permutation we load the gaps as well, without
8318 we need to skip the gaps after we manage to fully load
8319 all elements. group_gap_adj is GROUP_SIZE here. */
8320 group_elt += nunits;
8321 if (maybe_ne (group_gap_adj, 0U)
8322 && !slp_perm
8323 && known_eq (group_elt, group_size - group_gap_adj))
8325 poly_wide_int bump_val
8326 = (wi::to_wide (TYPE_SIZE_UNIT (elem_type))
8327 * group_gap_adj);
8328 tree bump = wide_int_to_tree (sizetype, bump_val);
8329 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
8330 stmt, bump);
8331 group_elt = 0;
8334 /* Bump the vector pointer to account for a gap or for excess
8335 elements loaded for a permuted SLP load. */
8336 if (maybe_ne (group_gap_adj, 0U) && slp_perm)
8338 poly_wide_int bump_val
8339 = (wi::to_wide (TYPE_SIZE_UNIT (elem_type))
8340 * group_gap_adj);
8341 tree bump = wide_int_to_tree (sizetype, bump_val);
8342 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
8343 stmt, bump);
8347 if (slp && !slp_perm)
8348 continue;
8350 if (slp_perm)
8352 unsigned n_perms;
8353 if (!vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
8354 slp_node_instance, false,
8355 &n_perms))
8357 dr_chain.release ();
8358 return false;
8361 else
8363 if (grouped_load)
8365 if (memory_access_type != VMAT_LOAD_STORE_LANES)
8366 vect_transform_grouped_load (stmt, dr_chain, group_size, gsi);
8367 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
8369 else
8371 if (j == 0)
8372 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
8373 else
8374 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
8375 prev_stmt_info = vinfo_for_stmt (new_stmt);
8378 dr_chain.release ();
8381 return true;
8384 /* Function vect_is_simple_cond.
8386 Input:
8387 LOOP - the loop that is being vectorized.
8388 COND - Condition that is checked for simple use.
8390 Output:
8391 *COMP_VECTYPE - the vector type for the comparison.
8392 *DTS - The def types for the arguments of the comparison
8394 Returns whether a COND can be vectorized. Checks whether
8395 condition operands are supportable using vec_is_simple_use. */
8397 static bool
8398 vect_is_simple_cond (tree cond, vec_info *vinfo,
8399 tree *comp_vectype, enum vect_def_type *dts,
8400 tree vectype)
8402 tree lhs, rhs;
8403 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
8405 /* Mask case. */
8406 if (TREE_CODE (cond) == SSA_NAME
8407 && VECT_SCALAR_BOOLEAN_TYPE_P (TREE_TYPE (cond)))
8409 gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
8410 if (!vect_is_simple_use (cond, vinfo, &lhs_def_stmt,
8411 &dts[0], comp_vectype)
8412 || !*comp_vectype
8413 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
8414 return false;
8415 return true;
8418 if (!COMPARISON_CLASS_P (cond))
8419 return false;
8421 lhs = TREE_OPERAND (cond, 0);
8422 rhs = TREE_OPERAND (cond, 1);
8424 if (TREE_CODE (lhs) == SSA_NAME)
8426 gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
8427 if (!vect_is_simple_use (lhs, vinfo, &lhs_def_stmt, &dts[0], &vectype1))
8428 return false;
8430 else if (TREE_CODE (lhs) == INTEGER_CST || TREE_CODE (lhs) == REAL_CST
8431 || TREE_CODE (lhs) == FIXED_CST)
8432 dts[0] = vect_constant_def;
8433 else
8434 return false;
8436 if (TREE_CODE (rhs) == SSA_NAME)
8438 gimple *rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
8439 if (!vect_is_simple_use (rhs, vinfo, &rhs_def_stmt, &dts[1], &vectype2))
8440 return false;
8442 else if (TREE_CODE (rhs) == INTEGER_CST || TREE_CODE (rhs) == REAL_CST
8443 || TREE_CODE (rhs) == FIXED_CST)
8444 dts[1] = vect_constant_def;
8445 else
8446 return false;
8448 if (vectype1 && vectype2
8449 && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
8450 TYPE_VECTOR_SUBPARTS (vectype2)))
8451 return false;
8453 *comp_vectype = vectype1 ? vectype1 : vectype2;
8454 /* Invariant comparison. */
8455 if (! *comp_vectype)
8457 tree scalar_type = TREE_TYPE (lhs);
8458 /* If we can widen the comparison to match vectype do so. */
8459 if (INTEGRAL_TYPE_P (scalar_type)
8460 && tree_int_cst_lt (TYPE_SIZE (scalar_type),
8461 TYPE_SIZE (TREE_TYPE (vectype))))
8462 scalar_type = build_nonstandard_integer_type
8463 (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype))),
8464 TYPE_UNSIGNED (scalar_type));
8465 *comp_vectype = get_vectype_for_scalar_type (scalar_type);
8468 return true;
8471 /* vectorizable_condition.
8473 Check if STMT is conditional modify expression that can be vectorized.
8474 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
8475 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
8476 at GSI.
8478 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
8479 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
8480 else clause if it is 2).
8482 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
8484 bool
8485 vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
8486 gimple **vec_stmt, tree reduc_def, int reduc_index,
8487 slp_tree slp_node)
8489 tree scalar_dest = NULL_TREE;
8490 tree vec_dest = NULL_TREE;
8491 tree cond_expr, cond_expr0 = NULL_TREE, cond_expr1 = NULL_TREE;
8492 tree then_clause, else_clause;
8493 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
8494 tree comp_vectype = NULL_TREE;
8495 tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
8496 tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
8497 tree vec_compare;
8498 tree new_temp;
8499 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
8500 enum vect_def_type dts[4]
8501 = {vect_unknown_def_type, vect_unknown_def_type,
8502 vect_unknown_def_type, vect_unknown_def_type};
8503 int ndts = 4;
8504 int ncopies;
8505 enum tree_code code, cond_code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
8506 stmt_vec_info prev_stmt_info = NULL;
8507 int i, j;
8508 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
8509 vec<tree> vec_oprnds0 = vNULL;
8510 vec<tree> vec_oprnds1 = vNULL;
8511 vec<tree> vec_oprnds2 = vNULL;
8512 vec<tree> vec_oprnds3 = vNULL;
8513 tree vec_cmp_type;
8514 bool masked = false;
8516 if (reduc_index && STMT_SLP_TYPE (stmt_info))
8517 return false;
8519 vect_reduction_type reduction_type
8520 = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info);
8521 if (reduction_type == TREE_CODE_REDUCTION)
8523 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8524 return false;
8526 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8527 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
8528 && reduc_def))
8529 return false;
8531 /* FORNOW: not yet supported. */
8532 if (STMT_VINFO_LIVE_P (stmt_info))
8534 if (dump_enabled_p ())
8535 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8536 "value used after loop.\n");
8537 return false;
8541 /* Is vectorizable conditional operation? */
8542 if (!is_gimple_assign (stmt))
8543 return false;
8545 code = gimple_assign_rhs_code (stmt);
8547 if (code != COND_EXPR)
8548 return false;
8550 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
8551 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
8553 if (slp_node)
8554 ncopies = 1;
8555 else
8556 ncopies = vect_get_num_copies (loop_vinfo, vectype);
8558 gcc_assert (ncopies >= 1);
8559 if (reduc_index && ncopies > 1)
8560 return false; /* FORNOW */
8562 cond_expr = gimple_assign_rhs1 (stmt);
8563 then_clause = gimple_assign_rhs2 (stmt);
8564 else_clause = gimple_assign_rhs3 (stmt);
8566 if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo,
8567 &comp_vectype, &dts[0], vectype)
8568 || !comp_vectype)
8569 return false;
8571 gimple *def_stmt;
8572 if (!vect_is_simple_use (then_clause, stmt_info->vinfo, &def_stmt, &dts[2],
8573 &vectype1))
8574 return false;
8575 if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &def_stmt, &dts[3],
8576 &vectype2))
8577 return false;
8579 if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
8580 return false;
8582 if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
8583 return false;
8585 masked = !COMPARISON_CLASS_P (cond_expr);
8586 vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
8588 if (vec_cmp_type == NULL_TREE)
8589 return false;
8591 cond_code = TREE_CODE (cond_expr);
8592 if (!masked)
8594 cond_expr0 = TREE_OPERAND (cond_expr, 0);
8595 cond_expr1 = TREE_OPERAND (cond_expr, 1);
8598 if (!masked && VECTOR_BOOLEAN_TYPE_P (comp_vectype))
8600 /* Boolean values may have another representation in vectors
8601 and therefore we prefer bit operations over comparison for
8602 them (which also works for scalar masks). We store opcodes
8603 to use in bitop1 and bitop2. Statement is vectorized as
8604 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
8605 depending on bitop1 and bitop2 arity. */
8606 switch (cond_code)
8608 case GT_EXPR:
8609 bitop1 = BIT_NOT_EXPR;
8610 bitop2 = BIT_AND_EXPR;
8611 break;
8612 case GE_EXPR:
8613 bitop1 = BIT_NOT_EXPR;
8614 bitop2 = BIT_IOR_EXPR;
8615 break;
8616 case LT_EXPR:
8617 bitop1 = BIT_NOT_EXPR;
8618 bitop2 = BIT_AND_EXPR;
8619 std::swap (cond_expr0, cond_expr1);
8620 break;
8621 case LE_EXPR:
8622 bitop1 = BIT_NOT_EXPR;
8623 bitop2 = BIT_IOR_EXPR;
8624 std::swap (cond_expr0, cond_expr1);
8625 break;
8626 case NE_EXPR:
8627 bitop1 = BIT_XOR_EXPR;
8628 break;
8629 case EQ_EXPR:
8630 bitop1 = BIT_XOR_EXPR;
8631 bitop2 = BIT_NOT_EXPR;
8632 break;
8633 default:
8634 return false;
8636 cond_code = SSA_NAME;
8639 if (!vec_stmt)
8641 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
8642 if (bitop1 != NOP_EXPR)
8644 machine_mode mode = TYPE_MODE (comp_vectype);
8645 optab optab;
8647 optab = optab_for_tree_code (bitop1, comp_vectype, optab_default);
8648 if (!optab || optab_handler (optab, mode) == CODE_FOR_nothing)
8649 return false;
8651 if (bitop2 != NOP_EXPR)
8653 optab = optab_for_tree_code (bitop2, comp_vectype,
8654 optab_default);
8655 if (!optab || optab_handler (optab, mode) == CODE_FOR_nothing)
8656 return false;
8659 if (expand_vec_cond_expr_p (vectype, comp_vectype,
8660 cond_code))
8662 vect_model_simple_cost (stmt_info, ncopies, dts, ndts, NULL, NULL);
8663 return true;
8665 return false;
8668 /* Transform. */
8670 if (!slp_node)
8672 vec_oprnds0.create (1);
8673 vec_oprnds1.create (1);
8674 vec_oprnds2.create (1);
8675 vec_oprnds3.create (1);
8678 /* Handle def. */
8679 scalar_dest = gimple_assign_lhs (stmt);
8680 if (reduction_type != EXTRACT_LAST_REDUCTION)
8681 vec_dest = vect_create_destination_var (scalar_dest, vectype);
8683 /* Handle cond expr. */
8684 for (j = 0; j < ncopies; j++)
8686 gimple *new_stmt = NULL;
8687 if (j == 0)
8689 if (slp_node)
8691 auto_vec<tree, 4> ops;
8692 auto_vec<vec<tree>, 4> vec_defs;
8694 if (masked)
8695 ops.safe_push (cond_expr);
8696 else
8698 ops.safe_push (cond_expr0);
8699 ops.safe_push (cond_expr1);
8701 ops.safe_push (then_clause);
8702 ops.safe_push (else_clause);
8703 vect_get_slp_defs (ops, slp_node, &vec_defs);
8704 vec_oprnds3 = vec_defs.pop ();
8705 vec_oprnds2 = vec_defs.pop ();
8706 if (!masked)
8707 vec_oprnds1 = vec_defs.pop ();
8708 vec_oprnds0 = vec_defs.pop ();
8710 else
8712 gimple *gtemp;
8713 if (masked)
8715 vec_cond_lhs
8716 = vect_get_vec_def_for_operand (cond_expr, stmt,
8717 comp_vectype);
8718 vect_is_simple_use (cond_expr, stmt_info->vinfo,
8719 &gtemp, &dts[0]);
8721 else
8723 vec_cond_lhs
8724 = vect_get_vec_def_for_operand (cond_expr0,
8725 stmt, comp_vectype);
8726 vect_is_simple_use (cond_expr0, loop_vinfo, &gtemp, &dts[0]);
8728 vec_cond_rhs
8729 = vect_get_vec_def_for_operand (cond_expr1,
8730 stmt, comp_vectype);
8731 vect_is_simple_use (cond_expr1, loop_vinfo, &gtemp, &dts[1]);
8733 if (reduc_index == 1)
8734 vec_then_clause = reduc_def;
8735 else
8737 vec_then_clause = vect_get_vec_def_for_operand (then_clause,
8738 stmt);
8739 vect_is_simple_use (then_clause, loop_vinfo,
8740 &gtemp, &dts[2]);
8742 if (reduc_index == 2)
8743 vec_else_clause = reduc_def;
8744 else
8746 vec_else_clause = vect_get_vec_def_for_operand (else_clause,
8747 stmt);
8748 vect_is_simple_use (else_clause, loop_vinfo, &gtemp, &dts[3]);
8752 else
8754 vec_cond_lhs
8755 = vect_get_vec_def_for_stmt_copy (dts[0],
8756 vec_oprnds0.pop ());
8757 if (!masked)
8758 vec_cond_rhs
8759 = vect_get_vec_def_for_stmt_copy (dts[1],
8760 vec_oprnds1.pop ());
8762 vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
8763 vec_oprnds2.pop ());
8764 vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
8765 vec_oprnds3.pop ());
8768 if (!slp_node)
8770 vec_oprnds0.quick_push (vec_cond_lhs);
8771 if (!masked)
8772 vec_oprnds1.quick_push (vec_cond_rhs);
8773 vec_oprnds2.quick_push (vec_then_clause);
8774 vec_oprnds3.quick_push (vec_else_clause);
8777 /* Arguments are ready. Create the new vector stmt. */
8778 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
8780 vec_then_clause = vec_oprnds2[i];
8781 vec_else_clause = vec_oprnds3[i];
8783 if (masked)
8784 vec_compare = vec_cond_lhs;
8785 else
8787 vec_cond_rhs = vec_oprnds1[i];
8788 if (bitop1 == NOP_EXPR)
8789 vec_compare = build2 (cond_code, vec_cmp_type,
8790 vec_cond_lhs, vec_cond_rhs);
8791 else
8793 new_temp = make_ssa_name (vec_cmp_type);
8794 if (bitop1 == BIT_NOT_EXPR)
8795 new_stmt = gimple_build_assign (new_temp, bitop1,
8796 vec_cond_rhs);
8797 else
8798 new_stmt
8799 = gimple_build_assign (new_temp, bitop1, vec_cond_lhs,
8800 vec_cond_rhs);
8801 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8802 if (bitop2 == NOP_EXPR)
8803 vec_compare = new_temp;
8804 else if (bitop2 == BIT_NOT_EXPR)
8806 /* Instead of doing ~x ? y : z do x ? z : y. */
8807 vec_compare = new_temp;
8808 std::swap (vec_then_clause, vec_else_clause);
8810 else
8812 vec_compare = make_ssa_name (vec_cmp_type);
8813 new_stmt
8814 = gimple_build_assign (vec_compare, bitop2,
8815 vec_cond_lhs, new_temp);
8816 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8820 if (reduction_type == EXTRACT_LAST_REDUCTION)
8822 if (!is_gimple_val (vec_compare))
8824 tree vec_compare_name = make_ssa_name (vec_cmp_type);
8825 new_stmt = gimple_build_assign (vec_compare_name,
8826 vec_compare);
8827 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8828 vec_compare = vec_compare_name;
8830 gcc_assert (reduc_index == 2);
8831 new_stmt = gimple_build_call_internal
8832 (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare,
8833 vec_then_clause);
8834 gimple_call_set_lhs (new_stmt, scalar_dest);
8835 SSA_NAME_DEF_STMT (scalar_dest) = new_stmt;
8836 if (stmt == gsi_stmt (*gsi))
8837 vect_finish_replace_stmt (stmt, new_stmt);
8838 else
8840 /* In this case we're moving the definition to later in the
8841 block. That doesn't matter because the only uses of the
8842 lhs are in phi statements. */
8843 gimple_stmt_iterator old_gsi = gsi_for_stmt (stmt);
8844 gsi_remove (&old_gsi, true);
8845 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8848 else
8850 new_temp = make_ssa_name (vec_dest);
8851 new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR,
8852 vec_compare, vec_then_clause,
8853 vec_else_clause);
8854 vect_finish_stmt_generation (stmt, new_stmt, gsi);
8856 if (slp_node)
8857 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
8860 if (slp_node)
8861 continue;
8863 if (j == 0)
8864 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
8865 else
8866 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
8868 prev_stmt_info = vinfo_for_stmt (new_stmt);
8871 vec_oprnds0.release ();
8872 vec_oprnds1.release ();
8873 vec_oprnds2.release ();
8874 vec_oprnds3.release ();
8876 return true;
8879 /* vectorizable_comparison.
8881 Check if STMT is comparison expression that can be vectorized.
8882 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
8883 comparison, put it in VEC_STMT, and insert it at GSI.
8885 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
8887 static bool
8888 vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi,
8889 gimple **vec_stmt, tree reduc_def,
8890 slp_tree slp_node)
8892 tree lhs, rhs1, rhs2;
8893 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
8894 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
8895 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
8896 tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
8897 tree new_temp;
8898 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
8899 enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
8900 int ndts = 2;
8901 poly_uint64 nunits;
8902 int ncopies;
8903 enum tree_code code, bitop1 = NOP_EXPR, bitop2 = NOP_EXPR;
8904 stmt_vec_info prev_stmt_info = NULL;
8905 int i, j;
8906 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
8907 vec<tree> vec_oprnds0 = vNULL;
8908 vec<tree> vec_oprnds1 = vNULL;
8909 gimple *def_stmt;
8910 tree mask_type;
8911 tree mask;
8913 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
8914 return false;
8916 if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
8917 return false;
8919 mask_type = vectype;
8920 nunits = TYPE_VECTOR_SUBPARTS (vectype);
8922 if (slp_node)
8923 ncopies = 1;
8924 else
8925 ncopies = vect_get_num_copies (loop_vinfo, vectype);
8927 gcc_assert (ncopies >= 1);
8928 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
8929 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
8930 && reduc_def))
8931 return false;
8933 if (STMT_VINFO_LIVE_P (stmt_info))
8935 if (dump_enabled_p ())
8936 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8937 "value used after loop.\n");
8938 return false;
8941 if (!is_gimple_assign (stmt))
8942 return false;
8944 code = gimple_assign_rhs_code (stmt);
8946 if (TREE_CODE_CLASS (code) != tcc_comparison)
8947 return false;
8949 rhs1 = gimple_assign_rhs1 (stmt);
8950 rhs2 = gimple_assign_rhs2 (stmt);
8952 if (!vect_is_simple_use (rhs1, stmt_info->vinfo, &def_stmt,
8953 &dts[0], &vectype1))
8954 return false;
8956 if (!vect_is_simple_use (rhs2, stmt_info->vinfo, &def_stmt,
8957 &dts[1], &vectype2))
8958 return false;
8960 if (vectype1 && vectype2
8961 && maybe_ne (TYPE_VECTOR_SUBPARTS (vectype1),
8962 TYPE_VECTOR_SUBPARTS (vectype2)))
8963 return false;
8965 vectype = vectype1 ? vectype1 : vectype2;
8967 /* Invariant comparison. */
8968 if (!vectype)
8970 vectype = get_vectype_for_scalar_type (TREE_TYPE (rhs1));
8971 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), nunits))
8972 return false;
8974 else if (maybe_ne (nunits, TYPE_VECTOR_SUBPARTS (vectype)))
8975 return false;
8977 /* Can't compare mask and non-mask types. */
8978 if (vectype1 && vectype2
8979 && (VECTOR_BOOLEAN_TYPE_P (vectype1) ^ VECTOR_BOOLEAN_TYPE_P (vectype2)))
8980 return false;
8982 /* Boolean values may have another representation in vectors
8983 and therefore we prefer bit operations over comparison for
8984 them (which also works for scalar masks). We store opcodes
8985 to use in bitop1 and bitop2. Statement is vectorized as
8986 BITOP2 (rhs1 BITOP1 rhs2) or
8987 rhs1 BITOP2 (BITOP1 rhs2)
8988 depending on bitop1 and bitop2 arity. */
8989 if (VECTOR_BOOLEAN_TYPE_P (vectype))
8991 if (code == GT_EXPR)
8993 bitop1 = BIT_NOT_EXPR;
8994 bitop2 = BIT_AND_EXPR;
8996 else if (code == GE_EXPR)
8998 bitop1 = BIT_NOT_EXPR;
8999 bitop2 = BIT_IOR_EXPR;
9001 else if (code == LT_EXPR)
9003 bitop1 = BIT_NOT_EXPR;
9004 bitop2 = BIT_AND_EXPR;
9005 std::swap (rhs1, rhs2);
9006 std::swap (dts[0], dts[1]);
9008 else if (code == LE_EXPR)
9010 bitop1 = BIT_NOT_EXPR;
9011 bitop2 = BIT_IOR_EXPR;
9012 std::swap (rhs1, rhs2);
9013 std::swap (dts[0], dts[1]);
9015 else
9017 bitop1 = BIT_XOR_EXPR;
9018 if (code == EQ_EXPR)
9019 bitop2 = BIT_NOT_EXPR;
9023 if (!vec_stmt)
9025 STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
9026 vect_model_simple_cost (stmt_info, ncopies * (1 + (bitop2 != NOP_EXPR)),
9027 dts, ndts, NULL, NULL);
9028 if (bitop1 == NOP_EXPR)
9029 return expand_vec_cmp_expr_p (vectype, mask_type, code);
9030 else
9032 machine_mode mode = TYPE_MODE (vectype);
9033 optab optab;
9035 optab = optab_for_tree_code (bitop1, vectype, optab_default);
9036 if (!optab || optab_handler (optab, mode) == CODE_FOR_nothing)
9037 return false;
9039 if (bitop2 != NOP_EXPR)
9041 optab = optab_for_tree_code (bitop2, vectype, optab_default);
9042 if (!optab || optab_handler (optab, mode) == CODE_FOR_nothing)
9043 return false;
9045 return true;
9049 /* Transform. */
9050 if (!slp_node)
9052 vec_oprnds0.create (1);
9053 vec_oprnds1.create (1);
9056 /* Handle def. */
9057 lhs = gimple_assign_lhs (stmt);
9058 mask = vect_create_destination_var (lhs, mask_type);
9060 /* Handle cmp expr. */
9061 for (j = 0; j < ncopies; j++)
9063 gassign *new_stmt = NULL;
9064 if (j == 0)
9066 if (slp_node)
9068 auto_vec<tree, 2> ops;
9069 auto_vec<vec<tree>, 2> vec_defs;
9071 ops.safe_push (rhs1);
9072 ops.safe_push (rhs2);
9073 vect_get_slp_defs (ops, slp_node, &vec_defs);
9074 vec_oprnds1 = vec_defs.pop ();
9075 vec_oprnds0 = vec_defs.pop ();
9077 else
9079 vec_rhs1 = vect_get_vec_def_for_operand (rhs1, stmt, vectype);
9080 vec_rhs2 = vect_get_vec_def_for_operand (rhs2, stmt, vectype);
9083 else
9085 vec_rhs1 = vect_get_vec_def_for_stmt_copy (dts[0],
9086 vec_oprnds0.pop ());
9087 vec_rhs2 = vect_get_vec_def_for_stmt_copy (dts[1],
9088 vec_oprnds1.pop ());
9091 if (!slp_node)
9093 vec_oprnds0.quick_push (vec_rhs1);
9094 vec_oprnds1.quick_push (vec_rhs2);
9097 /* Arguments are ready. Create the new vector stmt. */
9098 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
9100 vec_rhs2 = vec_oprnds1[i];
9102 new_temp = make_ssa_name (mask);
9103 if (bitop1 == NOP_EXPR)
9105 new_stmt = gimple_build_assign (new_temp, code,
9106 vec_rhs1, vec_rhs2);
9107 vect_finish_stmt_generation (stmt, new_stmt, gsi);
9109 else
9111 if (bitop1 == BIT_NOT_EXPR)
9112 new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2);
9113 else
9114 new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1,
9115 vec_rhs2);
9116 vect_finish_stmt_generation (stmt, new_stmt, gsi);
9117 if (bitop2 != NOP_EXPR)
9119 tree res = make_ssa_name (mask);
9120 if (bitop2 == BIT_NOT_EXPR)
9121 new_stmt = gimple_build_assign (res, bitop2, new_temp);
9122 else
9123 new_stmt = gimple_build_assign (res, bitop2, vec_rhs1,
9124 new_temp);
9125 vect_finish_stmt_generation (stmt, new_stmt, gsi);
9128 if (slp_node)
9129 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
9132 if (slp_node)
9133 continue;
9135 if (j == 0)
9136 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
9137 else
9138 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
9140 prev_stmt_info = vinfo_for_stmt (new_stmt);
9143 vec_oprnds0.release ();
9144 vec_oprnds1.release ();
9146 return true;
9149 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
9150 can handle all live statements in the node. Otherwise return true
9151 if STMT is not live or if vectorizable_live_operation can handle it.
9152 GSI and VEC_STMT are as for vectorizable_live_operation. */
9154 static bool
9155 can_vectorize_live_stmts (gimple *stmt, gimple_stmt_iterator *gsi,
9156 slp_tree slp_node, gimple **vec_stmt)
9158 if (slp_node)
9160 gimple *slp_stmt;
9161 unsigned int i;
9162 FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt)
9164 stmt_vec_info slp_stmt_info = vinfo_for_stmt (slp_stmt);
9165 if (STMT_VINFO_LIVE_P (slp_stmt_info)
9166 && !vectorizable_live_operation (slp_stmt, gsi, slp_node, i,
9167 vec_stmt))
9168 return false;
9171 else if (STMT_VINFO_LIVE_P (vinfo_for_stmt (stmt))
9172 && !vectorizable_live_operation (stmt, gsi, slp_node, -1, vec_stmt))
9173 return false;
9175 return true;
9178 /* Make sure the statement is vectorizable. */
9180 bool
9181 vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node,
9182 slp_instance node_instance)
9184 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
9185 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
9186 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
9187 bool ok;
9188 gimple *pattern_stmt;
9189 gimple_seq pattern_def_seq;
9191 if (dump_enabled_p ())
9193 dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
9194 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
9197 if (gimple_has_volatile_ops (stmt))
9199 if (dump_enabled_p ())
9200 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9201 "not vectorized: stmt has volatile operands\n");
9203 return false;
9206 /* Skip stmts that do not need to be vectorized. In loops this is expected
9207 to include:
9208 - the COND_EXPR which is the loop exit condition
9209 - any LABEL_EXPRs in the loop
9210 - computations that are used only for array indexing or loop control.
9211 In basic blocks we only analyze statements that are a part of some SLP
9212 instance, therefore, all the statements are relevant.
9214 Pattern statement needs to be analyzed instead of the original statement
9215 if the original statement is not relevant. Otherwise, we analyze both
9216 statements. In basic blocks we are called from some SLP instance
9217 traversal, don't analyze pattern stmts instead, the pattern stmts
9218 already will be part of SLP instance. */
9220 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
9221 if (!STMT_VINFO_RELEVANT_P (stmt_info)
9222 && !STMT_VINFO_LIVE_P (stmt_info))
9224 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
9225 && pattern_stmt
9226 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
9227 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
9229 /* Analyze PATTERN_STMT instead of the original stmt. */
9230 stmt = pattern_stmt;
9231 stmt_info = vinfo_for_stmt (pattern_stmt);
9232 if (dump_enabled_p ())
9234 dump_printf_loc (MSG_NOTE, vect_location,
9235 "==> examining pattern statement: ");
9236 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
9239 else
9241 if (dump_enabled_p ())
9242 dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
9244 return true;
9247 else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
9248 && node == NULL
9249 && pattern_stmt
9250 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
9251 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
9253 /* Analyze PATTERN_STMT too. */
9254 if (dump_enabled_p ())
9256 dump_printf_loc (MSG_NOTE, vect_location,
9257 "==> examining pattern statement: ");
9258 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
9261 if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node,
9262 node_instance))
9263 return false;
9266 if (is_pattern_stmt_p (stmt_info)
9267 && node == NULL
9268 && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
9270 gimple_stmt_iterator si;
9272 for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
9274 gimple *pattern_def_stmt = gsi_stmt (si);
9275 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
9276 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
9278 /* Analyze def stmt of STMT if it's a pattern stmt. */
9279 if (dump_enabled_p ())
9281 dump_printf_loc (MSG_NOTE, vect_location,
9282 "==> examining pattern def statement: ");
9283 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
9286 if (!vect_analyze_stmt (pattern_def_stmt,
9287 need_to_vectorize, node, node_instance))
9288 return false;
9293 switch (STMT_VINFO_DEF_TYPE (stmt_info))
9295 case vect_internal_def:
9296 break;
9298 case vect_reduction_def:
9299 case vect_nested_cycle:
9300 gcc_assert (!bb_vinfo
9301 && (relevance == vect_used_in_outer
9302 || relevance == vect_used_in_outer_by_reduction
9303 || relevance == vect_used_by_reduction
9304 || relevance == vect_unused_in_scope
9305 || relevance == vect_used_only_live));
9306 break;
9308 case vect_induction_def:
9309 gcc_assert (!bb_vinfo);
9310 break;
9312 case vect_constant_def:
9313 case vect_external_def:
9314 case vect_unknown_def_type:
9315 default:
9316 gcc_unreachable ();
9319 if (STMT_VINFO_RELEVANT_P (stmt_info))
9321 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
9322 gcc_assert (STMT_VINFO_VECTYPE (stmt_info)
9323 || (is_gimple_call (stmt)
9324 && gimple_call_lhs (stmt) == NULL_TREE));
9325 *need_to_vectorize = true;
9328 if (PURE_SLP_STMT (stmt_info) && !node)
9330 dump_printf_loc (MSG_NOTE, vect_location,
9331 "handled only by SLP analysis\n");
9332 return true;
9335 ok = true;
9336 if (!bb_vinfo
9337 && (STMT_VINFO_RELEVANT_P (stmt_info)
9338 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
9339 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, node)
9340 || vectorizable_conversion (stmt, NULL, NULL, node)
9341 || vectorizable_shift (stmt, NULL, NULL, node)
9342 || vectorizable_operation (stmt, NULL, NULL, node)
9343 || vectorizable_assignment (stmt, NULL, NULL, node)
9344 || vectorizable_load (stmt, NULL, NULL, node, NULL)
9345 || vectorizable_call (stmt, NULL, NULL, node)
9346 || vectorizable_store (stmt, NULL, NULL, node)
9347 || vectorizable_reduction (stmt, NULL, NULL, node, node_instance)
9348 || vectorizable_induction (stmt, NULL, NULL, node)
9349 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
9350 || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
9351 else
9353 if (bb_vinfo)
9354 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, node)
9355 || vectorizable_conversion (stmt, NULL, NULL, node)
9356 || vectorizable_shift (stmt, NULL, NULL, node)
9357 || vectorizable_operation (stmt, NULL, NULL, node)
9358 || vectorizable_assignment (stmt, NULL, NULL, node)
9359 || vectorizable_load (stmt, NULL, NULL, node, NULL)
9360 || vectorizable_call (stmt, NULL, NULL, node)
9361 || vectorizable_store (stmt, NULL, NULL, node)
9362 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
9363 || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
9366 if (!ok)
9368 if (dump_enabled_p ())
9370 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9371 "not vectorized: relevant stmt not ");
9372 dump_printf (MSG_MISSED_OPTIMIZATION, "supported: ");
9373 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
9376 return false;
9379 if (bb_vinfo)
9380 return true;
9382 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
9383 need extra handling, except for vectorizable reductions. */
9384 if (STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
9385 && !can_vectorize_live_stmts (stmt, NULL, node, NULL))
9387 if (dump_enabled_p ())
9389 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9390 "not vectorized: live stmt not supported: ");
9391 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
9394 return false;
9397 return true;
9401 /* Function vect_transform_stmt.
9403 Create a vectorized stmt to replace STMT, and insert it at BSI. */
9405 bool
9406 vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
9407 bool *grouped_store, slp_tree slp_node,
9408 slp_instance slp_node_instance)
9410 bool is_store = false;
9411 gimple *vec_stmt = NULL;
9412 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
9413 bool done;
9415 gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
9416 gimple *old_vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
9418 bool nested_p = (STMT_VINFO_LOOP_VINFO (stmt_info)
9419 && nested_in_vect_loop_p
9420 (LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info)),
9421 stmt));
9423 switch (STMT_VINFO_TYPE (stmt_info))
9425 case type_demotion_vec_info_type:
9426 case type_promotion_vec_info_type:
9427 case type_conversion_vec_info_type:
9428 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
9429 gcc_assert (done);
9430 break;
9432 case induc_vec_info_type:
9433 done = vectorizable_induction (stmt, gsi, &vec_stmt, slp_node);
9434 gcc_assert (done);
9435 break;
9437 case shift_vec_info_type:
9438 done = vectorizable_shift (stmt, gsi, &vec_stmt, slp_node);
9439 gcc_assert (done);
9440 break;
9442 case op_vec_info_type:
9443 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
9444 gcc_assert (done);
9445 break;
9447 case assignment_vec_info_type:
9448 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
9449 gcc_assert (done);
9450 break;
9452 case load_vec_info_type:
9453 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
9454 slp_node_instance);
9455 gcc_assert (done);
9456 break;
9458 case store_vec_info_type:
9459 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
9460 gcc_assert (done);
9461 if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && !slp_node)
9463 /* In case of interleaving, the whole chain is vectorized when the
9464 last store in the chain is reached. Store stmts before the last
9465 one are skipped, and there vec_stmt_info shouldn't be freed
9466 meanwhile. */
9467 *grouped_store = true;
9468 stmt_vec_info group_info
9469 = vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info));
9470 if (GROUP_STORE_COUNT (group_info) == GROUP_SIZE (group_info))
9471 is_store = true;
9473 else
9474 is_store = true;
9475 break;
9477 case condition_vec_info_type:
9478 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0, slp_node);
9479 gcc_assert (done);
9480 break;
9482 case comparison_vec_info_type:
9483 done = vectorizable_comparison (stmt, gsi, &vec_stmt, NULL, slp_node);
9484 gcc_assert (done);
9485 break;
9487 case call_vec_info_type:
9488 done = vectorizable_call (stmt, gsi, &vec_stmt, slp_node);
9489 stmt = gsi_stmt (*gsi);
9490 break;
9492 case call_simd_clone_vec_info_type:
9493 done = vectorizable_simd_clone_call (stmt, gsi, &vec_stmt, slp_node);
9494 stmt = gsi_stmt (*gsi);
9495 break;
9497 case reduc_vec_info_type:
9498 done = vectorizable_reduction (stmt, gsi, &vec_stmt, slp_node,
9499 slp_node_instance);
9500 gcc_assert (done);
9501 break;
9503 default:
9504 if (!STMT_VINFO_LIVE_P (stmt_info))
9506 if (dump_enabled_p ())
9507 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9508 "stmt not supported.\n");
9509 gcc_unreachable ();
9513 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
9514 This would break hybrid SLP vectorization. */
9515 if (slp_node)
9516 gcc_assert (!vec_stmt
9517 && STMT_VINFO_VEC_STMT (stmt_info) == old_vec_stmt);
9519 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
9520 is being vectorized, but outside the immediately enclosing loop. */
9521 if (vec_stmt
9522 && nested_p
9523 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
9524 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
9525 || STMT_VINFO_RELEVANT (stmt_info) ==
9526 vect_used_in_outer_by_reduction))
9528 struct loop *innerloop = LOOP_VINFO_LOOP (
9529 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
9530 imm_use_iterator imm_iter;
9531 use_operand_p use_p;
9532 tree scalar_dest;
9533 gimple *exit_phi;
9535 if (dump_enabled_p ())
9536 dump_printf_loc (MSG_NOTE, vect_location,
9537 "Record the vdef for outer-loop vectorization.\n");
9539 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
9540 (to be used when vectorizing outer-loop stmts that use the DEF of
9541 STMT). */
9542 if (gimple_code (stmt) == GIMPLE_PHI)
9543 scalar_dest = PHI_RESULT (stmt);
9544 else
9545 scalar_dest = gimple_assign_lhs (stmt);
9547 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
9549 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
9551 exit_phi = USE_STMT (use_p);
9552 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
9557 /* Handle stmts whose DEF is used outside the loop-nest that is
9558 being vectorized. */
9559 if (STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
9561 done = can_vectorize_live_stmts (stmt, gsi, slp_node, &vec_stmt);
9562 gcc_assert (done);
9565 if (vec_stmt)
9566 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
9568 return is_store;
9572 /* Remove a group of stores (for SLP or interleaving), free their
9573 stmt_vec_info. */
9575 void
9576 vect_remove_stores (gimple *first_stmt)
9578 gimple *next = first_stmt;
9579 gimple *tmp;
9580 gimple_stmt_iterator next_si;
9582 while (next)
9584 stmt_vec_info stmt_info = vinfo_for_stmt (next);
9586 tmp = GROUP_NEXT_ELEMENT (stmt_info);
9587 if (is_pattern_stmt_p (stmt_info))
9588 next = STMT_VINFO_RELATED_STMT (stmt_info);
9589 /* Free the attached stmt_vec_info and remove the stmt. */
9590 next_si = gsi_for_stmt (next);
9591 unlink_stmt_vdef (next);
9592 gsi_remove (&next_si, true);
9593 release_defs (next);
9594 free_stmt_vec_info (next);
9595 next = tmp;
9600 /* Function new_stmt_vec_info.
9602 Create and initialize a new stmt_vec_info struct for STMT. */
9604 stmt_vec_info
9605 new_stmt_vec_info (gimple *stmt, vec_info *vinfo)
9607 stmt_vec_info res;
9608 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
9610 STMT_VINFO_TYPE (res) = undef_vec_info_type;
9611 STMT_VINFO_STMT (res) = stmt;
9612 res->vinfo = vinfo;
9613 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
9614 STMT_VINFO_LIVE_P (res) = false;
9615 STMT_VINFO_VECTYPE (res) = NULL;
9616 STMT_VINFO_VEC_STMT (res) = NULL;
9617 STMT_VINFO_VECTORIZABLE (res) = true;
9618 STMT_VINFO_IN_PATTERN_P (res) = false;
9619 STMT_VINFO_RELATED_STMT (res) = NULL;
9620 STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
9621 STMT_VINFO_DATA_REF (res) = NULL;
9622 STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
9623 STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
9625 if (gimple_code (stmt) == GIMPLE_PHI
9626 && is_loop_header_bb_p (gimple_bb (stmt)))
9627 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
9628 else
9629 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
9631 STMT_VINFO_SAME_ALIGN_REFS (res).create (0);
9632 STMT_SLP_TYPE (res) = loop_vect;
9633 STMT_VINFO_NUM_SLP_USES (res) = 0;
9635 GROUP_FIRST_ELEMENT (res) = NULL;
9636 GROUP_NEXT_ELEMENT (res) = NULL;
9637 GROUP_SIZE (res) = 0;
9638 GROUP_STORE_COUNT (res) = 0;
9639 GROUP_GAP (res) = 0;
9640 GROUP_SAME_DR_STMT (res) = NULL;
9642 return res;
9646 /* Create a hash table for stmt_vec_info. */
9648 void
9649 init_stmt_vec_info_vec (void)
9651 gcc_assert (!stmt_vec_info_vec.exists ());
9652 stmt_vec_info_vec.create (50);
9656 /* Free hash table for stmt_vec_info. */
9658 void
9659 free_stmt_vec_info_vec (void)
9661 unsigned int i;
9662 stmt_vec_info info;
9663 FOR_EACH_VEC_ELT (stmt_vec_info_vec, i, info)
9664 if (info != NULL)
9665 free_stmt_vec_info (STMT_VINFO_STMT (info));
9666 gcc_assert (stmt_vec_info_vec.exists ());
9667 stmt_vec_info_vec.release ();
9671 /* Free stmt vectorization related info. */
9673 void
9674 free_stmt_vec_info (gimple *stmt)
9676 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
9678 if (!stmt_info)
9679 return;
9681 /* Check if this statement has a related "pattern stmt"
9682 (introduced by the vectorizer during the pattern recognition
9683 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
9684 too. */
9685 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
9687 stmt_vec_info patt_info
9688 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
9689 if (patt_info)
9691 gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (patt_info);
9692 gimple *patt_stmt = STMT_VINFO_STMT (patt_info);
9693 gimple_set_bb (patt_stmt, NULL);
9694 tree lhs = gimple_get_lhs (patt_stmt);
9695 if (lhs && TREE_CODE (lhs) == SSA_NAME)
9696 release_ssa_name (lhs);
9697 if (seq)
9699 gimple_stmt_iterator si;
9700 for (si = gsi_start (seq); !gsi_end_p (si); gsi_next (&si))
9702 gimple *seq_stmt = gsi_stmt (si);
9703 gimple_set_bb (seq_stmt, NULL);
9704 lhs = gimple_get_lhs (seq_stmt);
9705 if (lhs && TREE_CODE (lhs) == SSA_NAME)
9706 release_ssa_name (lhs);
9707 free_stmt_vec_info (seq_stmt);
9710 free_stmt_vec_info (patt_stmt);
9714 STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release ();
9715 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
9716 set_vinfo_for_stmt (stmt, NULL);
9717 free (stmt_info);
9721 /* Function get_vectype_for_scalar_type_and_size.
9723 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
9724 by the target. */
9726 tree
9727 get_vectype_for_scalar_type_and_size (tree scalar_type, poly_uint64 size)
9729 tree orig_scalar_type = scalar_type;
9730 scalar_mode inner_mode;
9731 machine_mode simd_mode;
9732 poly_uint64 nunits;
9733 tree vectype;
9735 if (!is_int_mode (TYPE_MODE (scalar_type), &inner_mode)
9736 && !is_float_mode (TYPE_MODE (scalar_type), &inner_mode))
9737 return NULL_TREE;
9739 unsigned int nbytes = GET_MODE_SIZE (inner_mode);
9741 /* For vector types of elements whose mode precision doesn't
9742 match their types precision we use a element type of mode
9743 precision. The vectorization routines will have to make sure
9744 they support the proper result truncation/extension.
9745 We also make sure to build vector types with INTEGER_TYPE
9746 component type only. */
9747 if (INTEGRAL_TYPE_P (scalar_type)
9748 && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
9749 || TREE_CODE (scalar_type) != INTEGER_TYPE))
9750 scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
9751 TYPE_UNSIGNED (scalar_type));
9753 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
9754 When the component mode passes the above test simply use a type
9755 corresponding to that mode. The theory is that any use that
9756 would cause problems with this will disable vectorization anyway. */
9757 else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
9758 && !INTEGRAL_TYPE_P (scalar_type))
9759 scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
9761 /* We can't build a vector type of elements with alignment bigger than
9762 their size. */
9763 else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
9764 scalar_type = lang_hooks.types.type_for_mode (inner_mode,
9765 TYPE_UNSIGNED (scalar_type));
9767 /* If we felt back to using the mode fail if there was
9768 no scalar type for it. */
9769 if (scalar_type == NULL_TREE)
9770 return NULL_TREE;
9772 /* If no size was supplied use the mode the target prefers. Otherwise
9773 lookup a vector mode of the specified size. */
9774 if (known_eq (size, 0U))
9775 simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
9776 else if (!multiple_p (size, nbytes, &nunits)
9777 || !mode_for_vector (inner_mode, nunits).exists (&simd_mode))
9778 return NULL_TREE;
9779 /* NOTE: nunits == 1 is allowed to support single element vector types. */
9780 if (!multiple_p (GET_MODE_SIZE (simd_mode), nbytes, &nunits))
9781 return NULL_TREE;
9783 vectype = build_vector_type (scalar_type, nunits);
9785 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
9786 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
9787 return NULL_TREE;
9789 /* Re-attach the address-space qualifier if we canonicalized the scalar
9790 type. */
9791 if (TYPE_ADDR_SPACE (orig_scalar_type) != TYPE_ADDR_SPACE (vectype))
9792 return build_qualified_type
9793 (vectype, KEEP_QUAL_ADDR_SPACE (TYPE_QUALS (orig_scalar_type)));
9795 return vectype;
9798 poly_uint64 current_vector_size;
9800 /* Function get_vectype_for_scalar_type.
9802 Returns the vector type corresponding to SCALAR_TYPE as supported
9803 by the target. */
9805 tree
9806 get_vectype_for_scalar_type (tree scalar_type)
9808 tree vectype;
9809 vectype = get_vectype_for_scalar_type_and_size (scalar_type,
9810 current_vector_size);
9811 if (vectype
9812 && known_eq (current_vector_size, 0U))
9813 current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
9814 return vectype;
9817 /* Function get_mask_type_for_scalar_type.
9819 Returns the mask type corresponding to a result of comparison
9820 of vectors of specified SCALAR_TYPE as supported by target. */
9822 tree
9823 get_mask_type_for_scalar_type (tree scalar_type)
9825 tree vectype = get_vectype_for_scalar_type (scalar_type);
9827 if (!vectype)
9828 return NULL;
9830 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype),
9831 current_vector_size);
9834 /* Function get_same_sized_vectype
9836 Returns a vector type corresponding to SCALAR_TYPE of size
9837 VECTOR_TYPE if supported by the target. */
9839 tree
9840 get_same_sized_vectype (tree scalar_type, tree vector_type)
9842 if (VECT_SCALAR_BOOLEAN_TYPE_P (scalar_type))
9843 return build_same_sized_truth_vector_type (vector_type);
9845 return get_vectype_for_scalar_type_and_size
9846 (scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
9849 /* Function vect_is_simple_use.
9851 Input:
9852 VINFO - the vect info of the loop or basic block that is being vectorized.
9853 OPERAND - operand in the loop or bb.
9854 Output:
9855 DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
9856 DT - the type of definition
9858 Returns whether a stmt with OPERAND can be vectorized.
9859 For loops, supportable operands are constants, loop invariants, and operands
9860 that are defined by the current iteration of the loop. Unsupportable
9861 operands are those that are defined by a previous iteration of the loop (as
9862 is the case in reduction/induction computations).
9863 For basic blocks, supportable operands are constants and bb invariants.
9864 For now, operands defined outside the basic block are not supported. */
9866 bool
9867 vect_is_simple_use (tree operand, vec_info *vinfo,
9868 gimple **def_stmt, enum vect_def_type *dt)
9870 *def_stmt = NULL;
9871 *dt = vect_unknown_def_type;
9873 if (dump_enabled_p ())
9875 dump_printf_loc (MSG_NOTE, vect_location,
9876 "vect_is_simple_use: operand ");
9877 dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
9878 dump_printf (MSG_NOTE, "\n");
9881 if (CONSTANT_CLASS_P (operand))
9883 *dt = vect_constant_def;
9884 return true;
9887 if (is_gimple_min_invariant (operand))
9889 *dt = vect_external_def;
9890 return true;
9893 if (TREE_CODE (operand) != SSA_NAME)
9895 if (dump_enabled_p ())
9896 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9897 "not ssa-name.\n");
9898 return false;
9901 if (SSA_NAME_IS_DEFAULT_DEF (operand))
9903 *dt = vect_external_def;
9904 return true;
9907 *def_stmt = SSA_NAME_DEF_STMT (operand);
9908 if (dump_enabled_p ())
9910 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt: ");
9911 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0);
9914 if (! vect_stmt_in_region_p (vinfo, *def_stmt))
9915 *dt = vect_external_def;
9916 else
9918 stmt_vec_info stmt_vinfo = vinfo_for_stmt (*def_stmt);
9919 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
9922 if (dump_enabled_p ())
9924 dump_printf_loc (MSG_NOTE, vect_location, "type of def: ");
9925 switch (*dt)
9927 case vect_uninitialized_def:
9928 dump_printf (MSG_NOTE, "uninitialized\n");
9929 break;
9930 case vect_constant_def:
9931 dump_printf (MSG_NOTE, "constant\n");
9932 break;
9933 case vect_external_def:
9934 dump_printf (MSG_NOTE, "external\n");
9935 break;
9936 case vect_internal_def:
9937 dump_printf (MSG_NOTE, "internal\n");
9938 break;
9939 case vect_induction_def:
9940 dump_printf (MSG_NOTE, "induction\n");
9941 break;
9942 case vect_reduction_def:
9943 dump_printf (MSG_NOTE, "reduction\n");
9944 break;
9945 case vect_double_reduction_def:
9946 dump_printf (MSG_NOTE, "double reduction\n");
9947 break;
9948 case vect_nested_cycle:
9949 dump_printf (MSG_NOTE, "nested cycle\n");
9950 break;
9951 case vect_unknown_def_type:
9952 dump_printf (MSG_NOTE, "unknown\n");
9953 break;
9957 if (*dt == vect_unknown_def_type)
9959 if (dump_enabled_p ())
9960 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9961 "Unsupported pattern.\n");
9962 return false;
9965 switch (gimple_code (*def_stmt))
9967 case GIMPLE_PHI:
9968 case GIMPLE_ASSIGN:
9969 case GIMPLE_CALL:
9970 break;
9971 default:
9972 if (dump_enabled_p ())
9973 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9974 "unsupported defining stmt:\n");
9975 return false;
9978 return true;
9981 /* Function vect_is_simple_use.
9983 Same as vect_is_simple_use but also determines the vector operand
9984 type of OPERAND and stores it to *VECTYPE. If the definition of
9985 OPERAND is vect_uninitialized_def, vect_constant_def or
9986 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
9987 is responsible to compute the best suited vector type for the
9988 scalar operand. */
9990 bool
9991 vect_is_simple_use (tree operand, vec_info *vinfo,
9992 gimple **def_stmt, enum vect_def_type *dt, tree *vectype)
9994 if (!vect_is_simple_use (operand, vinfo, def_stmt, dt))
9995 return false;
9997 /* Now get a vector type if the def is internal, otherwise supply
9998 NULL_TREE and leave it up to the caller to figure out a proper
9999 type for the use stmt. */
10000 if (*dt == vect_internal_def
10001 || *dt == vect_induction_def
10002 || *dt == vect_reduction_def
10003 || *dt == vect_double_reduction_def
10004 || *dt == vect_nested_cycle)
10006 stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
10008 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
10009 && !STMT_VINFO_RELEVANT (stmt_info)
10010 && !STMT_VINFO_LIVE_P (stmt_info))
10011 stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
10013 *vectype = STMT_VINFO_VECTYPE (stmt_info);
10014 gcc_assert (*vectype != NULL_TREE);
10016 else if (*dt == vect_uninitialized_def
10017 || *dt == vect_constant_def
10018 || *dt == vect_external_def)
10019 *vectype = NULL_TREE;
10020 else
10021 gcc_unreachable ();
10023 return true;
10027 /* Function supportable_widening_operation
10029 Check whether an operation represented by the code CODE is a
10030 widening operation that is supported by the target platform in
10031 vector form (i.e., when operating on arguments of type VECTYPE_IN
10032 producing a result of type VECTYPE_OUT).
10034 Widening operations we currently support are NOP (CONVERT), FLOAT
10035 and WIDEN_MULT. This function checks if these operations are supported
10036 by the target platform either directly (via vector tree-codes), or via
10037 target builtins.
10039 Output:
10040 - CODE1 and CODE2 are codes of vector operations to be used when
10041 vectorizing the operation, if available.
10042 - MULTI_STEP_CVT determines the number of required intermediate steps in
10043 case of multi-step conversion (like char->short->int - in that case
10044 MULTI_STEP_CVT will be 1).
10045 - INTERM_TYPES contains the intermediate type required to perform the
10046 widening operation (short in the above example). */
10048 bool
10049 supportable_widening_operation (enum tree_code code, gimple *stmt,
10050 tree vectype_out, tree vectype_in,
10051 enum tree_code *code1, enum tree_code *code2,
10052 int *multi_step_cvt,
10053 vec<tree> *interm_types)
10055 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
10056 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
10057 struct loop *vect_loop = NULL;
10058 machine_mode vec_mode;
10059 enum insn_code icode1, icode2;
10060 optab optab1, optab2;
10061 tree vectype = vectype_in;
10062 tree wide_vectype = vectype_out;
10063 enum tree_code c1, c2;
10064 int i;
10065 tree prev_type, intermediate_type;
10066 machine_mode intermediate_mode, prev_mode;
10067 optab optab3, optab4;
10069 *multi_step_cvt = 0;
10070 if (loop_info)
10071 vect_loop = LOOP_VINFO_LOOP (loop_info);
10073 switch (code)
10075 case WIDEN_MULT_EXPR:
10076 /* The result of a vectorized widening operation usually requires
10077 two vectors (because the widened results do not fit into one vector).
10078 The generated vector results would normally be expected to be
10079 generated in the same order as in the original scalar computation,
10080 i.e. if 8 results are generated in each vector iteration, they are
10081 to be organized as follows:
10082 vect1: [res1,res2,res3,res4],
10083 vect2: [res5,res6,res7,res8].
10085 However, in the special case that the result of the widening
10086 operation is used in a reduction computation only, the order doesn't
10087 matter (because when vectorizing a reduction we change the order of
10088 the computation). Some targets can take advantage of this and
10089 generate more efficient code. For example, targets like Altivec,
10090 that support widen_mult using a sequence of {mult_even,mult_odd}
10091 generate the following vectors:
10092 vect1: [res1,res3,res5,res7],
10093 vect2: [res2,res4,res6,res8].
10095 When vectorizing outer-loops, we execute the inner-loop sequentially
10096 (each vectorized inner-loop iteration contributes to VF outer-loop
10097 iterations in parallel). We therefore don't allow to change the
10098 order of the computation in the inner-loop during outer-loop
10099 vectorization. */
10100 /* TODO: Another case in which order doesn't *really* matter is when we
10101 widen and then contract again, e.g. (short)((int)x * y >> 8).
10102 Normally, pack_trunc performs an even/odd permute, whereas the
10103 repack from an even/odd expansion would be an interleave, which
10104 would be significantly simpler for e.g. AVX2. */
10105 /* In any case, in order to avoid duplicating the code below, recurse
10106 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
10107 are properly set up for the caller. If we fail, we'll continue with
10108 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
10109 if (vect_loop
10110 && STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
10111 && !nested_in_vect_loop_p (vect_loop, stmt)
10112 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
10113 stmt, vectype_out, vectype_in,
10114 code1, code2, multi_step_cvt,
10115 interm_types))
10117 /* Elements in a vector with vect_used_by_reduction property cannot
10118 be reordered if the use chain with this property does not have the
10119 same operation. One such an example is s += a * b, where elements
10120 in a and b cannot be reordered. Here we check if the vector defined
10121 by STMT is only directly used in the reduction statement. */
10122 tree lhs = gimple_assign_lhs (stmt);
10123 use_operand_p dummy;
10124 gimple *use_stmt;
10125 stmt_vec_info use_stmt_info = NULL;
10126 if (single_imm_use (lhs, &dummy, &use_stmt)
10127 && (use_stmt_info = vinfo_for_stmt (use_stmt))
10128 && STMT_VINFO_DEF_TYPE (use_stmt_info) == vect_reduction_def)
10129 return true;
10131 c1 = VEC_WIDEN_MULT_LO_EXPR;
10132 c2 = VEC_WIDEN_MULT_HI_EXPR;
10133 break;
10135 case DOT_PROD_EXPR:
10136 c1 = DOT_PROD_EXPR;
10137 c2 = DOT_PROD_EXPR;
10138 break;
10140 case SAD_EXPR:
10141 c1 = SAD_EXPR;
10142 c2 = SAD_EXPR;
10143 break;
10145 case VEC_WIDEN_MULT_EVEN_EXPR:
10146 /* Support the recursion induced just above. */
10147 c1 = VEC_WIDEN_MULT_EVEN_EXPR;
10148 c2 = VEC_WIDEN_MULT_ODD_EXPR;
10149 break;
10151 case WIDEN_LSHIFT_EXPR:
10152 c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
10153 c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
10154 break;
10156 CASE_CONVERT:
10157 c1 = VEC_UNPACK_LO_EXPR;
10158 c2 = VEC_UNPACK_HI_EXPR;
10159 break;
10161 case FLOAT_EXPR:
10162 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
10163 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
10164 break;
10166 case FIX_TRUNC_EXPR:
10167 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
10168 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
10169 computing the operation. */
10170 return false;
10172 default:
10173 gcc_unreachable ();
10176 if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
10177 std::swap (c1, c2);
10179 if (code == FIX_TRUNC_EXPR)
10181 /* The signedness is determined from output operand. */
10182 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
10183 optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
10185 else
10187 optab1 = optab_for_tree_code (c1, vectype, optab_default);
10188 optab2 = optab_for_tree_code (c2, vectype, optab_default);
10191 if (!optab1 || !optab2)
10192 return false;
10194 vec_mode = TYPE_MODE (vectype);
10195 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
10196 || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
10197 return false;
10199 *code1 = c1;
10200 *code2 = c2;
10202 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
10203 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
10204 /* For scalar masks we may have different boolean
10205 vector types having the same QImode. Thus we
10206 add additional check for elements number. */
10207 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
10208 || known_eq (TYPE_VECTOR_SUBPARTS (vectype),
10209 TYPE_VECTOR_SUBPARTS (wide_vectype) * 2));
10211 /* Check if it's a multi-step conversion that can be done using intermediate
10212 types. */
10214 prev_type = vectype;
10215 prev_mode = vec_mode;
10217 if (!CONVERT_EXPR_CODE_P (code))
10218 return false;
10220 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
10221 intermediate steps in promotion sequence. We try
10222 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
10223 not. */
10224 interm_types->create (MAX_INTERM_CVT_STEPS);
10225 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
10227 intermediate_mode = insn_data[icode1].operand[0].mode;
10228 if (VECTOR_BOOLEAN_TYPE_P (prev_type))
10230 intermediate_type = vect_halve_mask_nunits (prev_type);
10231 if (intermediate_mode != TYPE_MODE (intermediate_type))
10232 return false;
10234 else
10235 intermediate_type
10236 = lang_hooks.types.type_for_mode (intermediate_mode,
10237 TYPE_UNSIGNED (prev_type));
10239 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
10240 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
10242 if (!optab3 || !optab4
10243 || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
10244 || insn_data[icode1].operand[0].mode != intermediate_mode
10245 || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
10246 || insn_data[icode2].operand[0].mode != intermediate_mode
10247 || ((icode1 = optab_handler (optab3, intermediate_mode))
10248 == CODE_FOR_nothing)
10249 || ((icode2 = optab_handler (optab4, intermediate_mode))
10250 == CODE_FOR_nothing))
10251 break;
10253 interm_types->quick_push (intermediate_type);
10254 (*multi_step_cvt)++;
10256 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
10257 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
10258 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
10259 || known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type),
10260 TYPE_VECTOR_SUBPARTS (wide_vectype) * 2));
10262 prev_type = intermediate_type;
10263 prev_mode = intermediate_mode;
10266 interm_types->release ();
10267 return false;
10271 /* Function supportable_narrowing_operation
10273 Check whether an operation represented by the code CODE is a
10274 narrowing operation that is supported by the target platform in
10275 vector form (i.e., when operating on arguments of type VECTYPE_IN
10276 and producing a result of type VECTYPE_OUT).
10278 Narrowing operations we currently support are NOP (CONVERT) and
10279 FIX_TRUNC. This function checks if these operations are supported by
10280 the target platform directly via vector tree-codes.
10282 Output:
10283 - CODE1 is the code of a vector operation to be used when
10284 vectorizing the operation, if available.
10285 - MULTI_STEP_CVT determines the number of required intermediate steps in
10286 case of multi-step conversion (like int->short->char - in that case
10287 MULTI_STEP_CVT will be 1).
10288 - INTERM_TYPES contains the intermediate type required to perform the
10289 narrowing operation (short in the above example). */
10291 bool
10292 supportable_narrowing_operation (enum tree_code code,
10293 tree vectype_out, tree vectype_in,
10294 enum tree_code *code1, int *multi_step_cvt,
10295 vec<tree> *interm_types)
10297 machine_mode vec_mode;
10298 enum insn_code icode1;
10299 optab optab1, interm_optab;
10300 tree vectype = vectype_in;
10301 tree narrow_vectype = vectype_out;
10302 enum tree_code c1;
10303 tree intermediate_type, prev_type;
10304 machine_mode intermediate_mode, prev_mode;
10305 int i;
10306 bool uns;
10308 *multi_step_cvt = 0;
10309 switch (code)
10311 CASE_CONVERT:
10312 c1 = VEC_PACK_TRUNC_EXPR;
10313 break;
10315 case FIX_TRUNC_EXPR:
10316 c1 = VEC_PACK_FIX_TRUNC_EXPR;
10317 break;
10319 case FLOAT_EXPR:
10320 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
10321 tree code and optabs used for computing the operation. */
10322 return false;
10324 default:
10325 gcc_unreachable ();
10328 if (code == FIX_TRUNC_EXPR)
10329 /* The signedness is determined from output operand. */
10330 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
10331 else
10332 optab1 = optab_for_tree_code (c1, vectype, optab_default);
10334 if (!optab1)
10335 return false;
10337 vec_mode = TYPE_MODE (vectype);
10338 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
10339 return false;
10341 *code1 = c1;
10343 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
10344 /* For scalar masks we may have different boolean
10345 vector types having the same QImode. Thus we
10346 add additional check for elements number. */
10347 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
10348 || known_eq (TYPE_VECTOR_SUBPARTS (vectype) * 2,
10349 TYPE_VECTOR_SUBPARTS (narrow_vectype)));
10351 /* Check if it's a multi-step conversion that can be done using intermediate
10352 types. */
10353 prev_mode = vec_mode;
10354 prev_type = vectype;
10355 if (code == FIX_TRUNC_EXPR)
10356 uns = TYPE_UNSIGNED (vectype_out);
10357 else
10358 uns = TYPE_UNSIGNED (vectype);
10360 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
10361 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
10362 costly than signed. */
10363 if (code == FIX_TRUNC_EXPR && uns)
10365 enum insn_code icode2;
10367 intermediate_type
10368 = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
10369 interm_optab
10370 = optab_for_tree_code (c1, intermediate_type, optab_default);
10371 if (interm_optab != unknown_optab
10372 && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
10373 && insn_data[icode1].operand[0].mode
10374 == insn_data[icode2].operand[0].mode)
10376 uns = false;
10377 optab1 = interm_optab;
10378 icode1 = icode2;
10382 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
10383 intermediate steps in promotion sequence. We try
10384 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
10385 interm_types->create (MAX_INTERM_CVT_STEPS);
10386 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
10388 intermediate_mode = insn_data[icode1].operand[0].mode;
10389 if (VECTOR_BOOLEAN_TYPE_P (prev_type))
10391 intermediate_type = vect_double_mask_nunits (prev_type);
10392 if (intermediate_mode != TYPE_MODE (intermediate_type))
10393 return false;
10395 else
10396 intermediate_type
10397 = lang_hooks.types.type_for_mode (intermediate_mode, uns);
10398 interm_optab
10399 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
10400 optab_default);
10401 if (!interm_optab
10402 || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
10403 || insn_data[icode1].operand[0].mode != intermediate_mode
10404 || ((icode1 = optab_handler (interm_optab, intermediate_mode))
10405 == CODE_FOR_nothing))
10406 break;
10408 interm_types->quick_push (intermediate_type);
10409 (*multi_step_cvt)++;
10411 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
10412 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
10413 || known_eq (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2,
10414 TYPE_VECTOR_SUBPARTS (narrow_vectype)));
10416 prev_mode = intermediate_mode;
10417 prev_type = intermediate_type;
10418 optab1 = interm_optab;
10421 interm_types->release ();
10422 return false;
10425 /* Generate and return a statement that sets vector mask MASK such that
10426 MASK[I] is true iff J + START_INDEX < END_INDEX for all J <= I. */
10428 gcall *
10429 vect_gen_while (tree mask, tree start_index, tree end_index)
10431 tree cmp_type = TREE_TYPE (start_index);
10432 tree mask_type = TREE_TYPE (mask);
10433 gcc_checking_assert (direct_internal_fn_supported_p (IFN_WHILE_ULT,
10434 cmp_type, mask_type,
10435 OPTIMIZE_FOR_SPEED));
10436 gcall *call = gimple_build_call_internal (IFN_WHILE_ULT, 3,
10437 start_index, end_index,
10438 build_zero_cst (mask_type));
10439 gimple_call_set_lhs (call, mask);
10440 return call;
10443 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
10444 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
10446 tree
10447 vect_gen_while_not (gimple_seq *seq, tree mask_type, tree start_index,
10448 tree end_index)
10450 tree tmp = make_ssa_name (mask_type);
10451 gcall *call = vect_gen_while (tmp, start_index, end_index);
10452 gimple_seq_add_stmt (seq, call);
10453 return gimple_build (seq, BIT_NOT_EXPR, mask_type, tmp);