PR target/47230
[official-gcc.git] / gcc / tree-vect-stmts.c
blob26eb2d40979aa4d15d195dbe94400f3368060093
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2014 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #include "config.h"
23 #include "system.h"
24 #include "coretypes.h"
25 #include "dumpfile.h"
26 #include "tm.h"
27 #include "tree.h"
28 #include "stor-layout.h"
29 #include "target.h"
30 #include "basic-block.h"
31 #include "gimple-pretty-print.h"
32 #include "tree-ssa-alias.h"
33 #include "internal-fn.h"
34 #include "tree-eh.h"
35 #include "gimple-expr.h"
36 #include "is-a.h"
37 #include "gimple.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-ssa.h"
42 #include "tree-cfg.h"
43 #include "tree-phinodes.h"
44 #include "ssa-iterators.h"
45 #include "stringpool.h"
46 #include "tree-ssanames.h"
47 #include "tree-ssa-loop-manip.h"
48 #include "cfgloop.h"
49 #include "tree-ssa-loop.h"
50 #include "tree-scalar-evolution.h"
51 #include "expr.h"
52 #include "recog.h" /* FIXME: for insn_data */
53 #include "optabs.h"
54 #include "diagnostic-core.h"
55 #include "tree-vectorizer.h"
56 #include "dumpfile.h"
57 #include "cgraph.h"
58 #include "builtins.h"
60 /* For lang_hooks.types.type_for_mode. */
61 #include "langhooks.h"
63 /* Return the vectorized type for the given statement. */
65 tree
66 stmt_vectype (struct _stmt_vec_info *stmt_info)
68 return STMT_VINFO_VECTYPE (stmt_info);
71 /* Return TRUE iff the given statement is in an inner loop relative to
72 the loop being vectorized. */
73 bool
74 stmt_in_inner_loop_p (struct _stmt_vec_info *stmt_info)
76 gimple stmt = STMT_VINFO_STMT (stmt_info);
77 basic_block bb = gimple_bb (stmt);
78 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
79 struct loop* loop;
81 if (!loop_vinfo)
82 return false;
84 loop = LOOP_VINFO_LOOP (loop_vinfo);
86 return (bb->loop_father == loop->inner);
89 /* Record the cost of a statement, either by directly informing the
90 target model or by saving it in a vector for later processing.
91 Return a preliminary estimate of the statement's cost. */
93 unsigned
94 record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
95 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
96 int misalign, enum vect_cost_model_location where)
98 if (body_cost_vec)
100 tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
101 add_stmt_info_to_vec (body_cost_vec, count, kind,
102 stmt_info ? STMT_VINFO_STMT (stmt_info) : NULL,
103 misalign);
104 return (unsigned)
105 (builtin_vectorization_cost (kind, vectype, misalign) * count);
108 else
110 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
111 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
112 void *target_cost_data;
114 if (loop_vinfo)
115 target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
116 else
117 target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
119 return add_stmt_cost (target_cost_data, count, kind, stmt_info,
120 misalign, where);
124 /* Return a variable of type ELEM_TYPE[NELEMS]. */
126 static tree
127 create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
129 return create_tmp_var (build_array_type_nelts (elem_type, nelems),
130 "vect_array");
133 /* ARRAY is an array of vectors created by create_vector_array.
134 Return an SSA_NAME for the vector in index N. The reference
135 is part of the vectorization of STMT and the vector is associated
136 with scalar destination SCALAR_DEST. */
138 static tree
139 read_vector_array (gimple stmt, gimple_stmt_iterator *gsi, tree scalar_dest,
140 tree array, unsigned HOST_WIDE_INT n)
142 tree vect_type, vect, vect_name, array_ref;
143 gimple new_stmt;
145 gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
146 vect_type = TREE_TYPE (TREE_TYPE (array));
147 vect = vect_create_destination_var (scalar_dest, vect_type);
148 array_ref = build4 (ARRAY_REF, vect_type, array,
149 build_int_cst (size_type_node, n),
150 NULL_TREE, NULL_TREE);
152 new_stmt = gimple_build_assign (vect, array_ref);
153 vect_name = make_ssa_name (vect, new_stmt);
154 gimple_assign_set_lhs (new_stmt, vect_name);
155 vect_finish_stmt_generation (stmt, new_stmt, gsi);
157 return vect_name;
160 /* ARRAY is an array of vectors created by create_vector_array.
161 Emit code to store SSA_NAME VECT in index N of the array.
162 The store is part of the vectorization of STMT. */
164 static void
165 write_vector_array (gimple stmt, gimple_stmt_iterator *gsi, tree vect,
166 tree array, unsigned HOST_WIDE_INT n)
168 tree array_ref;
169 gimple new_stmt;
171 array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
172 build_int_cst (size_type_node, n),
173 NULL_TREE, NULL_TREE);
175 new_stmt = gimple_build_assign (array_ref, vect);
176 vect_finish_stmt_generation (stmt, new_stmt, gsi);
179 /* PTR is a pointer to an array of type TYPE. Return a representation
180 of *PTR. The memory reference replaces those in FIRST_DR
181 (and its group). */
183 static tree
184 create_array_ref (tree type, tree ptr, struct data_reference *first_dr)
186 tree mem_ref, alias_ptr_type;
188 alias_ptr_type = reference_alias_ptr_type (DR_REF (first_dr));
189 mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
190 /* Arrays have the same alignment as their type. */
191 set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
192 return mem_ref;
195 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
197 /* Function vect_mark_relevant.
199 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
201 static void
202 vect_mark_relevant (vec<gimple> *worklist, gimple stmt,
203 enum vect_relevant relevant, bool live_p,
204 bool used_in_pattern)
206 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
207 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
208 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
209 gimple pattern_stmt;
211 if (dump_enabled_p ())
212 dump_printf_loc (MSG_NOTE, vect_location,
213 "mark relevant %d, live %d.\n", relevant, live_p);
215 /* If this stmt is an original stmt in a pattern, we might need to mark its
216 related pattern stmt instead of the original stmt. However, such stmts
217 may have their own uses that are not in any pattern, in such cases the
218 stmt itself should be marked. */
219 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
221 bool found = false;
222 if (!used_in_pattern)
224 imm_use_iterator imm_iter;
225 use_operand_p use_p;
226 gimple use_stmt;
227 tree lhs;
228 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
229 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
231 if (is_gimple_assign (stmt))
232 lhs = gimple_assign_lhs (stmt);
233 else
234 lhs = gimple_call_lhs (stmt);
236 /* This use is out of pattern use, if LHS has other uses that are
237 pattern uses, we should mark the stmt itself, and not the pattern
238 stmt. */
239 if (lhs && TREE_CODE (lhs) == SSA_NAME)
240 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
242 if (is_gimple_debug (USE_STMT (use_p)))
243 continue;
244 use_stmt = USE_STMT (use_p);
246 if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
247 continue;
249 if (vinfo_for_stmt (use_stmt)
250 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt)))
252 found = true;
253 break;
258 if (!found)
260 /* This is the last stmt in a sequence that was detected as a
261 pattern that can potentially be vectorized. Don't mark the stmt
262 as relevant/live because it's not going to be vectorized.
263 Instead mark the pattern-stmt that replaces it. */
265 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
267 if (dump_enabled_p ())
268 dump_printf_loc (MSG_NOTE, vect_location,
269 "last stmt in pattern. don't mark"
270 " relevant/live.\n");
271 stmt_info = vinfo_for_stmt (pattern_stmt);
272 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
273 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
274 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
275 stmt = pattern_stmt;
279 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
280 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
281 STMT_VINFO_RELEVANT (stmt_info) = relevant;
283 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
284 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
286 if (dump_enabled_p ())
287 dump_printf_loc (MSG_NOTE, vect_location,
288 "already marked relevant/live.\n");
289 return;
292 worklist->safe_push (stmt);
296 /* Function vect_stmt_relevant_p.
298 Return true if STMT in loop that is represented by LOOP_VINFO is
299 "relevant for vectorization".
301 A stmt is considered "relevant for vectorization" if:
302 - it has uses outside the loop.
303 - it has vdefs (it alters memory).
304 - control stmts in the loop (except for the exit condition).
306 CHECKME: what other side effects would the vectorizer allow? */
308 static bool
309 vect_stmt_relevant_p (gimple stmt, loop_vec_info loop_vinfo,
310 enum vect_relevant *relevant, bool *live_p)
312 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
313 ssa_op_iter op_iter;
314 imm_use_iterator imm_iter;
315 use_operand_p use_p;
316 def_operand_p def_p;
318 *relevant = vect_unused_in_scope;
319 *live_p = false;
321 /* cond stmt other than loop exit cond. */
322 if (is_ctrl_stmt (stmt)
323 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
324 != loop_exit_ctrl_vec_info_type)
325 *relevant = vect_used_in_scope;
327 /* changing memory. */
328 if (gimple_code (stmt) != GIMPLE_PHI)
329 if (gimple_vdef (stmt))
331 if (dump_enabled_p ())
332 dump_printf_loc (MSG_NOTE, vect_location,
333 "vec_stmt_relevant_p: stmt has vdefs.\n");
334 *relevant = vect_used_in_scope;
337 /* uses outside the loop. */
338 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
340 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
342 basic_block bb = gimple_bb (USE_STMT (use_p));
343 if (!flow_bb_inside_loop_p (loop, bb))
345 if (dump_enabled_p ())
346 dump_printf_loc (MSG_NOTE, vect_location,
347 "vec_stmt_relevant_p: used out of loop.\n");
349 if (is_gimple_debug (USE_STMT (use_p)))
350 continue;
352 /* We expect all such uses to be in the loop exit phis
353 (because of loop closed form) */
354 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
355 gcc_assert (bb == single_exit (loop)->dest);
357 *live_p = true;
362 return (*live_p || *relevant);
366 /* Function exist_non_indexing_operands_for_use_p
368 USE is one of the uses attached to STMT. Check if USE is
369 used in STMT for anything other than indexing an array. */
371 static bool
372 exist_non_indexing_operands_for_use_p (tree use, gimple stmt)
374 tree operand;
375 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
377 /* USE corresponds to some operand in STMT. If there is no data
378 reference in STMT, then any operand that corresponds to USE
379 is not indexing an array. */
380 if (!STMT_VINFO_DATA_REF (stmt_info))
381 return true;
383 /* STMT has a data_ref. FORNOW this means that its of one of
384 the following forms:
385 -1- ARRAY_REF = var
386 -2- var = ARRAY_REF
387 (This should have been verified in analyze_data_refs).
389 'var' in the second case corresponds to a def, not a use,
390 so USE cannot correspond to any operands that are not used
391 for array indexing.
393 Therefore, all we need to check is if STMT falls into the
394 first case, and whether var corresponds to USE. */
396 if (!gimple_assign_copy_p (stmt))
398 if (is_gimple_call (stmt)
399 && gimple_call_internal_p (stmt))
400 switch (gimple_call_internal_fn (stmt))
402 case IFN_MASK_STORE:
403 operand = gimple_call_arg (stmt, 3);
404 if (operand == use)
405 return true;
406 /* FALLTHRU */
407 case IFN_MASK_LOAD:
408 operand = gimple_call_arg (stmt, 2);
409 if (operand == use)
410 return true;
411 break;
412 default:
413 break;
415 return false;
418 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
419 return false;
420 operand = gimple_assign_rhs1 (stmt);
421 if (TREE_CODE (operand) != SSA_NAME)
422 return false;
424 if (operand == use)
425 return true;
427 return false;
432 Function process_use.
434 Inputs:
435 - a USE in STMT in a loop represented by LOOP_VINFO
436 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
437 that defined USE. This is done by calling mark_relevant and passing it
438 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
439 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
440 be performed.
442 Outputs:
443 Generally, LIVE_P and RELEVANT are used to define the liveness and
444 relevance info of the DEF_STMT of this USE:
445 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
446 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
447 Exceptions:
448 - case 1: If USE is used only for address computations (e.g. array indexing),
449 which does not need to be directly vectorized, then the liveness/relevance
450 of the respective DEF_STMT is left unchanged.
451 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
452 skip DEF_STMT cause it had already been processed.
453 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
454 be modified accordingly.
456 Return true if everything is as expected. Return false otherwise. */
458 static bool
459 process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
460 enum vect_relevant relevant, vec<gimple> *worklist,
461 bool force)
463 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
464 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
465 stmt_vec_info dstmt_vinfo;
466 basic_block bb, def_bb;
467 tree def;
468 gimple def_stmt;
469 enum vect_def_type dt;
471 /* case 1: we are only interested in uses that need to be vectorized. Uses
472 that are used for address computation are not considered relevant. */
473 if (!force && !exist_non_indexing_operands_for_use_p (use, stmt))
474 return true;
476 if (!vect_is_simple_use (use, stmt, loop_vinfo, NULL, &def_stmt, &def, &dt))
478 if (dump_enabled_p ())
479 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
480 "not vectorized: unsupported use in stmt.\n");
481 return false;
484 if (!def_stmt || gimple_nop_p (def_stmt))
485 return true;
487 def_bb = gimple_bb (def_stmt);
488 if (!flow_bb_inside_loop_p (loop, def_bb))
490 if (dump_enabled_p ())
491 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt is out of loop.\n");
492 return true;
495 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
496 DEF_STMT must have already been processed, because this should be the
497 only way that STMT, which is a reduction-phi, was put in the worklist,
498 as there should be no other uses for DEF_STMT in the loop. So we just
499 check that everything is as expected, and we are done. */
500 dstmt_vinfo = vinfo_for_stmt (def_stmt);
501 bb = gimple_bb (stmt);
502 if (gimple_code (stmt) == GIMPLE_PHI
503 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
504 && gimple_code (def_stmt) != GIMPLE_PHI
505 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
506 && bb->loop_father == def_bb->loop_father)
508 if (dump_enabled_p ())
509 dump_printf_loc (MSG_NOTE, vect_location,
510 "reduc-stmt defining reduc-phi in the same nest.\n");
511 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
512 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
513 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
514 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
515 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
516 return true;
519 /* case 3a: outer-loop stmt defining an inner-loop stmt:
520 outer-loop-header-bb:
521 d = def_stmt
522 inner-loop:
523 stmt # use (d)
524 outer-loop-tail-bb:
525 ... */
526 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
528 if (dump_enabled_p ())
529 dump_printf_loc (MSG_NOTE, vect_location,
530 "outer-loop def-stmt defining inner-loop stmt.\n");
532 switch (relevant)
534 case vect_unused_in_scope:
535 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
536 vect_used_in_scope : vect_unused_in_scope;
537 break;
539 case vect_used_in_outer_by_reduction:
540 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
541 relevant = vect_used_by_reduction;
542 break;
544 case vect_used_in_outer:
545 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
546 relevant = vect_used_in_scope;
547 break;
549 case vect_used_in_scope:
550 break;
552 default:
553 gcc_unreachable ();
557 /* case 3b: inner-loop stmt defining an outer-loop stmt:
558 outer-loop-header-bb:
560 inner-loop:
561 d = def_stmt
562 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
563 stmt # use (d) */
564 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
566 if (dump_enabled_p ())
567 dump_printf_loc (MSG_NOTE, vect_location,
568 "inner-loop def-stmt defining outer-loop stmt.\n");
570 switch (relevant)
572 case vect_unused_in_scope:
573 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
574 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
575 vect_used_in_outer_by_reduction : vect_unused_in_scope;
576 break;
578 case vect_used_by_reduction:
579 relevant = vect_used_in_outer_by_reduction;
580 break;
582 case vect_used_in_scope:
583 relevant = vect_used_in_outer;
584 break;
586 default:
587 gcc_unreachable ();
591 vect_mark_relevant (worklist, def_stmt, relevant, live_p,
592 is_pattern_stmt_p (stmt_vinfo));
593 return true;
597 /* Function vect_mark_stmts_to_be_vectorized.
599 Not all stmts in the loop need to be vectorized. For example:
601 for i...
602 for j...
603 1. T0 = i + j
604 2. T1 = a[T0]
606 3. j = j + 1
608 Stmt 1 and 3 do not need to be vectorized, because loop control and
609 addressing of vectorized data-refs are handled differently.
611 This pass detects such stmts. */
613 bool
614 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
616 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
617 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
618 unsigned int nbbs = loop->num_nodes;
619 gimple_stmt_iterator si;
620 gimple stmt;
621 unsigned int i;
622 stmt_vec_info stmt_vinfo;
623 basic_block bb;
624 gimple phi;
625 bool live_p;
626 enum vect_relevant relevant, tmp_relevant;
627 enum vect_def_type def_type;
629 if (dump_enabled_p ())
630 dump_printf_loc (MSG_NOTE, vect_location,
631 "=== vect_mark_stmts_to_be_vectorized ===\n");
633 auto_vec<gimple, 64> worklist;
635 /* 1. Init worklist. */
636 for (i = 0; i < nbbs; i++)
638 bb = bbs[i];
639 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
641 phi = gsi_stmt (si);
642 if (dump_enabled_p ())
644 dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? ");
645 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
646 dump_printf (MSG_NOTE, "\n");
649 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
650 vect_mark_relevant (&worklist, phi, relevant, live_p, false);
652 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
654 stmt = gsi_stmt (si);
655 if (dump_enabled_p ())
657 dump_printf_loc (MSG_NOTE, vect_location, "init: stmt relevant? ");
658 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
659 dump_printf (MSG_NOTE, "\n");
662 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
663 vect_mark_relevant (&worklist, stmt, relevant, live_p, false);
667 /* 2. Process_worklist */
668 while (worklist.length () > 0)
670 use_operand_p use_p;
671 ssa_op_iter iter;
673 stmt = worklist.pop ();
674 if (dump_enabled_p ())
676 dump_printf_loc (MSG_NOTE, vect_location, "worklist: examine stmt: ");
677 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
678 dump_printf (MSG_NOTE, "\n");
681 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
682 (DEF_STMT) as relevant/irrelevant and live/dead according to the
683 liveness and relevance properties of STMT. */
684 stmt_vinfo = vinfo_for_stmt (stmt);
685 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
686 live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
688 /* Generally, the liveness and relevance properties of STMT are
689 propagated as is to the DEF_STMTs of its USEs:
690 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
691 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
693 One exception is when STMT has been identified as defining a reduction
694 variable; in this case we set the liveness/relevance as follows:
695 live_p = false
696 relevant = vect_used_by_reduction
697 This is because we distinguish between two kinds of relevant stmts -
698 those that are used by a reduction computation, and those that are
699 (also) used by a regular computation. This allows us later on to
700 identify stmts that are used solely by a reduction, and therefore the
701 order of the results that they produce does not have to be kept. */
703 def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
704 tmp_relevant = relevant;
705 switch (def_type)
707 case vect_reduction_def:
708 switch (tmp_relevant)
710 case vect_unused_in_scope:
711 relevant = vect_used_by_reduction;
712 break;
714 case vect_used_by_reduction:
715 if (gimple_code (stmt) == GIMPLE_PHI)
716 break;
717 /* fall through */
719 default:
720 if (dump_enabled_p ())
721 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
722 "unsupported use of reduction.\n");
723 return false;
726 live_p = false;
727 break;
729 case vect_nested_cycle:
730 if (tmp_relevant != vect_unused_in_scope
731 && tmp_relevant != vect_used_in_outer_by_reduction
732 && tmp_relevant != vect_used_in_outer)
734 if (dump_enabled_p ())
735 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
736 "unsupported use of nested cycle.\n");
738 return false;
741 live_p = false;
742 break;
744 case vect_double_reduction_def:
745 if (tmp_relevant != vect_unused_in_scope
746 && tmp_relevant != vect_used_by_reduction)
748 if (dump_enabled_p ())
749 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
750 "unsupported use of double reduction.\n");
752 return false;
755 live_p = false;
756 break;
758 default:
759 break;
762 if (is_pattern_stmt_p (stmt_vinfo))
764 /* Pattern statements are not inserted into the code, so
765 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
766 have to scan the RHS or function arguments instead. */
767 if (is_gimple_assign (stmt))
769 enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
770 tree op = gimple_assign_rhs1 (stmt);
772 i = 1;
773 if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
775 if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo,
776 live_p, relevant, &worklist, false)
777 || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo,
778 live_p, relevant, &worklist, false))
779 return false;
780 i = 2;
782 for (; i < gimple_num_ops (stmt); i++)
784 op = gimple_op (stmt, i);
785 if (!process_use (stmt, op, loop_vinfo, live_p, relevant,
786 &worklist, false))
787 return false;
790 else if (is_gimple_call (stmt))
792 for (i = 0; i < gimple_call_num_args (stmt); i++)
794 tree arg = gimple_call_arg (stmt, i);
795 if (!process_use (stmt, arg, loop_vinfo, live_p, relevant,
796 &worklist, false))
797 return false;
801 else
802 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
804 tree op = USE_FROM_PTR (use_p);
805 if (!process_use (stmt, op, loop_vinfo, live_p, relevant,
806 &worklist, false))
807 return false;
810 if (STMT_VINFO_GATHER_P (stmt_vinfo))
812 tree off;
813 tree decl = vect_check_gather (stmt, loop_vinfo, NULL, &off, NULL);
814 gcc_assert (decl);
815 if (!process_use (stmt, off, loop_vinfo, live_p, relevant,
816 &worklist, true))
817 return false;
819 } /* while worklist */
821 return true;
825 /* Function vect_model_simple_cost.
827 Models cost for simple operations, i.e. those that only emit ncopies of a
828 single op. Right now, this does not account for multiple insns that could
829 be generated for the single vector op. We will handle that shortly. */
831 void
832 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
833 enum vect_def_type *dt,
834 stmt_vector_for_cost *prologue_cost_vec,
835 stmt_vector_for_cost *body_cost_vec)
837 int i;
838 int inside_cost = 0, prologue_cost = 0;
840 /* The SLP costs were already calculated during SLP tree build. */
841 if (PURE_SLP_STMT (stmt_info))
842 return;
844 /* FORNOW: Assuming maximum 2 args per stmts. */
845 for (i = 0; i < 2; i++)
846 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
847 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, vector_stmt,
848 stmt_info, 0, vect_prologue);
850 /* Pass the inside-of-loop statements to the target-specific cost model. */
851 inside_cost = record_stmt_cost (body_cost_vec, ncopies, vector_stmt,
852 stmt_info, 0, vect_body);
854 if (dump_enabled_p ())
855 dump_printf_loc (MSG_NOTE, vect_location,
856 "vect_model_simple_cost: inside_cost = %d, "
857 "prologue_cost = %d .\n", inside_cost, prologue_cost);
861 /* Model cost for type demotion and promotion operations. PWR is normally
862 zero for single-step promotions and demotions. It will be one if
863 two-step promotion/demotion is required, and so on. Each additional
864 step doubles the number of instructions required. */
866 static void
867 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
868 enum vect_def_type *dt, int pwr)
870 int i, tmp;
871 int inside_cost = 0, prologue_cost = 0;
872 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
873 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
874 void *target_cost_data;
876 /* The SLP costs were already calculated during SLP tree build. */
877 if (PURE_SLP_STMT (stmt_info))
878 return;
880 if (loop_vinfo)
881 target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
882 else
883 target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
885 for (i = 0; i < pwr + 1; i++)
887 tmp = (STMT_VINFO_TYPE (stmt_info) == type_promotion_vec_info_type) ?
888 (i + 1) : i;
889 inside_cost += add_stmt_cost (target_cost_data, vect_pow2 (tmp),
890 vec_promote_demote, stmt_info, 0,
891 vect_body);
894 /* FORNOW: Assuming maximum 2 args per stmts. */
895 for (i = 0; i < 2; i++)
896 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
897 prologue_cost += add_stmt_cost (target_cost_data, 1, vector_stmt,
898 stmt_info, 0, vect_prologue);
900 if (dump_enabled_p ())
901 dump_printf_loc (MSG_NOTE, vect_location,
902 "vect_model_promotion_demotion_cost: inside_cost = %d, "
903 "prologue_cost = %d .\n", inside_cost, prologue_cost);
906 /* Function vect_cost_group_size
908 For grouped load or store, return the group_size only if it is the first
909 load or store of a group, else return 1. This ensures that group size is
910 only returned once per group. */
912 static int
913 vect_cost_group_size (stmt_vec_info stmt_info)
915 gimple first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
917 if (first_stmt == STMT_VINFO_STMT (stmt_info))
918 return GROUP_SIZE (stmt_info);
920 return 1;
924 /* Function vect_model_store_cost
926 Models cost for stores. In the case of grouped accesses, one access
927 has the overhead of the grouped access attributed to it. */
929 void
930 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
931 bool store_lanes_p, enum vect_def_type dt,
932 slp_tree slp_node,
933 stmt_vector_for_cost *prologue_cost_vec,
934 stmt_vector_for_cost *body_cost_vec)
936 int group_size;
937 unsigned int inside_cost = 0, prologue_cost = 0;
938 struct data_reference *first_dr;
939 gimple first_stmt;
941 /* The SLP costs were already calculated during SLP tree build. */
942 if (PURE_SLP_STMT (stmt_info))
943 return;
945 if (dt == vect_constant_def || dt == vect_external_def)
946 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, scalar_to_vec,
947 stmt_info, 0, vect_prologue);
949 /* Grouped access? */
950 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
952 if (slp_node)
954 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
955 group_size = 1;
957 else
959 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
960 group_size = vect_cost_group_size (stmt_info);
963 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
965 /* Not a grouped access. */
966 else
968 group_size = 1;
969 first_dr = STMT_VINFO_DATA_REF (stmt_info);
972 /* We assume that the cost of a single store-lanes instruction is
973 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
974 access is instead being provided by a permute-and-store operation,
975 include the cost of the permutes. */
976 if (!store_lanes_p && group_size > 1)
978 /* Uses a high and low interleave or shuffle operations for each
979 needed permute. */
980 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
981 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
982 stmt_info, 0, vect_body);
984 if (dump_enabled_p ())
985 dump_printf_loc (MSG_NOTE, vect_location,
986 "vect_model_store_cost: strided group_size = %d .\n",
987 group_size);
990 /* Costs of the stores. */
991 vect_get_store_cost (first_dr, ncopies, &inside_cost, body_cost_vec);
993 if (dump_enabled_p ())
994 dump_printf_loc (MSG_NOTE, vect_location,
995 "vect_model_store_cost: inside_cost = %d, "
996 "prologue_cost = %d .\n", inside_cost, prologue_cost);
1000 /* Calculate cost of DR's memory access. */
1001 void
1002 vect_get_store_cost (struct data_reference *dr, int ncopies,
1003 unsigned int *inside_cost,
1004 stmt_vector_for_cost *body_cost_vec)
1006 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
1007 gimple stmt = DR_STMT (dr);
1008 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1010 switch (alignment_support_scheme)
1012 case dr_aligned:
1014 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1015 vector_store, stmt_info, 0,
1016 vect_body);
1018 if (dump_enabled_p ())
1019 dump_printf_loc (MSG_NOTE, vect_location,
1020 "vect_model_store_cost: aligned.\n");
1021 break;
1024 case dr_unaligned_supported:
1026 /* Here, we assign an additional cost for the unaligned store. */
1027 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1028 unaligned_store, stmt_info,
1029 DR_MISALIGNMENT (dr), vect_body);
1030 if (dump_enabled_p ())
1031 dump_printf_loc (MSG_NOTE, vect_location,
1032 "vect_model_store_cost: unaligned supported by "
1033 "hardware.\n");
1034 break;
1037 case dr_unaligned_unsupported:
1039 *inside_cost = VECT_MAX_COST;
1041 if (dump_enabled_p ())
1042 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1043 "vect_model_store_cost: unsupported access.\n");
1044 break;
1047 default:
1048 gcc_unreachable ();
1053 /* Function vect_model_load_cost
1055 Models cost for loads. In the case of grouped accesses, the last access
1056 has the overhead of the grouped access attributed to it. Since unaligned
1057 accesses are supported for loads, we also account for the costs of the
1058 access scheme chosen. */
1060 void
1061 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
1062 bool load_lanes_p, slp_tree slp_node,
1063 stmt_vector_for_cost *prologue_cost_vec,
1064 stmt_vector_for_cost *body_cost_vec)
1066 int group_size;
1067 gimple first_stmt;
1068 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
1069 unsigned int inside_cost = 0, prologue_cost = 0;
1071 /* The SLP costs were already calculated during SLP tree build. */
1072 if (PURE_SLP_STMT (stmt_info))
1073 return;
1075 /* Grouped accesses? */
1076 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
1077 if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && first_stmt && !slp_node)
1079 group_size = vect_cost_group_size (stmt_info);
1080 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
1082 /* Not a grouped access. */
1083 else
1085 group_size = 1;
1086 first_dr = dr;
1089 /* We assume that the cost of a single load-lanes instruction is
1090 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1091 access is instead being provided by a load-and-permute operation,
1092 include the cost of the permutes. */
1093 if (!load_lanes_p && group_size > 1)
1095 /* Uses an even and odd extract operations or shuffle operations
1096 for each needed permute. */
1097 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
1098 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
1099 stmt_info, 0, vect_body);
1101 if (dump_enabled_p ())
1102 dump_printf_loc (MSG_NOTE, vect_location,
1103 "vect_model_load_cost: strided group_size = %d .\n",
1104 group_size);
1107 /* The loads themselves. */
1108 if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
1110 /* N scalar loads plus gathering them into a vector. */
1111 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1112 inside_cost += record_stmt_cost (body_cost_vec,
1113 ncopies * TYPE_VECTOR_SUBPARTS (vectype),
1114 scalar_load, stmt_info, 0, vect_body);
1115 inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_construct,
1116 stmt_info, 0, vect_body);
1118 else
1119 vect_get_load_cost (first_dr, ncopies,
1120 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info))
1121 || group_size > 1 || slp_node),
1122 &inside_cost, &prologue_cost,
1123 prologue_cost_vec, body_cost_vec, true);
1125 if (dump_enabled_p ())
1126 dump_printf_loc (MSG_NOTE, vect_location,
1127 "vect_model_load_cost: inside_cost = %d, "
1128 "prologue_cost = %d .\n", inside_cost, prologue_cost);
1132 /* Calculate cost of DR's memory access. */
1133 void
1134 vect_get_load_cost (struct data_reference *dr, int ncopies,
1135 bool add_realign_cost, unsigned int *inside_cost,
1136 unsigned int *prologue_cost,
1137 stmt_vector_for_cost *prologue_cost_vec,
1138 stmt_vector_for_cost *body_cost_vec,
1139 bool record_prologue_costs)
1141 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
1142 gimple stmt = DR_STMT (dr);
1143 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1145 switch (alignment_support_scheme)
1147 case dr_aligned:
1149 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1150 stmt_info, 0, vect_body);
1152 if (dump_enabled_p ())
1153 dump_printf_loc (MSG_NOTE, vect_location,
1154 "vect_model_load_cost: aligned.\n");
1156 break;
1158 case dr_unaligned_supported:
1160 /* Here, we assign an additional cost for the unaligned load. */
1161 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1162 unaligned_load, stmt_info,
1163 DR_MISALIGNMENT (dr), vect_body);
1165 if (dump_enabled_p ())
1166 dump_printf_loc (MSG_NOTE, vect_location,
1167 "vect_model_load_cost: unaligned supported by "
1168 "hardware.\n");
1170 break;
1172 case dr_explicit_realign:
1174 *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1175 vector_load, stmt_info, 0, vect_body);
1176 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1177 vec_perm, stmt_info, 0, vect_body);
1179 /* FIXME: If the misalignment remains fixed across the iterations of
1180 the containing loop, the following cost should be added to the
1181 prologue costs. */
1182 if (targetm.vectorize.builtin_mask_for_load)
1183 *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1184 stmt_info, 0, vect_body);
1186 if (dump_enabled_p ())
1187 dump_printf_loc (MSG_NOTE, vect_location,
1188 "vect_model_load_cost: explicit realign\n");
1190 break;
1192 case dr_explicit_realign_optimized:
1194 if (dump_enabled_p ())
1195 dump_printf_loc (MSG_NOTE, vect_location,
1196 "vect_model_load_cost: unaligned software "
1197 "pipelined.\n");
1199 /* Unaligned software pipeline has a load of an address, an initial
1200 load, and possibly a mask operation to "prime" the loop. However,
1201 if this is an access in a group of loads, which provide grouped
1202 access, then the above cost should only be considered for one
1203 access in the group. Inside the loop, there is a load op
1204 and a realignment op. */
1206 if (add_realign_cost && record_prologue_costs)
1208 *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1209 vector_stmt, stmt_info,
1210 0, vect_prologue);
1211 if (targetm.vectorize.builtin_mask_for_load)
1212 *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1213 vector_stmt, stmt_info,
1214 0, vect_prologue);
1217 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1218 stmt_info, 0, vect_body);
1219 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1220 stmt_info, 0, vect_body);
1222 if (dump_enabled_p ())
1223 dump_printf_loc (MSG_NOTE, vect_location,
1224 "vect_model_load_cost: explicit realign optimized"
1225 "\n");
1227 break;
1230 case dr_unaligned_unsupported:
1232 *inside_cost = VECT_MAX_COST;
1234 if (dump_enabled_p ())
1235 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1236 "vect_model_load_cost: unsupported access.\n");
1237 break;
1240 default:
1241 gcc_unreachable ();
1245 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1246 the loop preheader for the vectorized stmt STMT. */
1248 static void
1249 vect_init_vector_1 (gimple stmt, gimple new_stmt, gimple_stmt_iterator *gsi)
1251 if (gsi)
1252 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1253 else
1255 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1256 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1258 if (loop_vinfo)
1260 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1261 basic_block new_bb;
1262 edge pe;
1264 if (nested_in_vect_loop_p (loop, stmt))
1265 loop = loop->inner;
1267 pe = loop_preheader_edge (loop);
1268 new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
1269 gcc_assert (!new_bb);
1271 else
1273 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
1274 basic_block bb;
1275 gimple_stmt_iterator gsi_bb_start;
1277 gcc_assert (bb_vinfo);
1278 bb = BB_VINFO_BB (bb_vinfo);
1279 gsi_bb_start = gsi_after_labels (bb);
1280 gsi_insert_before (&gsi_bb_start, new_stmt, GSI_SAME_STMT);
1284 if (dump_enabled_p ())
1286 dump_printf_loc (MSG_NOTE, vect_location,
1287 "created new init_stmt: ");
1288 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, new_stmt, 0);
1289 dump_printf (MSG_NOTE, "\n");
1293 /* Function vect_init_vector.
1295 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1296 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1297 vector type a vector with all elements equal to VAL is created first.
1298 Place the initialization at BSI if it is not NULL. Otherwise, place the
1299 initialization at the loop preheader.
1300 Return the DEF of INIT_STMT.
1301 It will be used in the vectorization of STMT. */
1303 tree
1304 vect_init_vector (gimple stmt, tree val, tree type, gimple_stmt_iterator *gsi)
1306 tree new_var;
1307 gimple init_stmt;
1308 tree vec_oprnd;
1309 tree new_temp;
1311 if (TREE_CODE (type) == VECTOR_TYPE
1312 && TREE_CODE (TREE_TYPE (val)) != VECTOR_TYPE)
1314 if (!types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1316 if (CONSTANT_CLASS_P (val))
1317 val = fold_unary (VIEW_CONVERT_EXPR, TREE_TYPE (type), val);
1318 else
1320 new_temp = make_ssa_name (TREE_TYPE (type), NULL);
1321 init_stmt = gimple_build_assign_with_ops (NOP_EXPR,
1322 new_temp, val,
1323 NULL_TREE);
1324 vect_init_vector_1 (stmt, init_stmt, gsi);
1325 val = new_temp;
1328 val = build_vector_from_val (type, val);
1331 new_var = vect_get_new_vect_var (type, vect_simple_var, "cst_");
1332 init_stmt = gimple_build_assign (new_var, val);
1333 new_temp = make_ssa_name (new_var, init_stmt);
1334 gimple_assign_set_lhs (init_stmt, new_temp);
1335 vect_init_vector_1 (stmt, init_stmt, gsi);
1336 vec_oprnd = gimple_assign_lhs (init_stmt);
1337 return vec_oprnd;
1341 /* Function vect_get_vec_def_for_operand.
1343 OP is an operand in STMT. This function returns a (vector) def that will be
1344 used in the vectorized stmt for STMT.
1346 In the case that OP is an SSA_NAME which is defined in the loop, then
1347 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1349 In case OP is an invariant or constant, a new stmt that creates a vector def
1350 needs to be introduced. */
1352 tree
1353 vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def)
1355 tree vec_oprnd;
1356 gimple vec_stmt;
1357 gimple def_stmt;
1358 stmt_vec_info def_stmt_info = NULL;
1359 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1360 unsigned int nunits;
1361 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1362 tree def;
1363 enum vect_def_type dt;
1364 bool is_simple_use;
1365 tree vector_type;
1367 if (dump_enabled_p ())
1369 dump_printf_loc (MSG_NOTE, vect_location,
1370 "vect_get_vec_def_for_operand: ");
1371 dump_generic_expr (MSG_NOTE, TDF_SLIM, op);
1372 dump_printf (MSG_NOTE, "\n");
1375 is_simple_use = vect_is_simple_use (op, stmt, loop_vinfo, NULL,
1376 &def_stmt, &def, &dt);
1377 gcc_assert (is_simple_use);
1378 if (dump_enabled_p ())
1380 int loc_printed = 0;
1381 if (def)
1383 dump_printf_loc (MSG_NOTE, vect_location, "def = ");
1384 loc_printed = 1;
1385 dump_generic_expr (MSG_NOTE, TDF_SLIM, def);
1386 dump_printf (MSG_NOTE, "\n");
1388 if (def_stmt)
1390 if (loc_printed)
1391 dump_printf (MSG_NOTE, " def_stmt = ");
1392 else
1393 dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = ");
1394 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0);
1395 dump_printf (MSG_NOTE, "\n");
1399 switch (dt)
1401 /* Case 1: operand is a constant. */
1402 case vect_constant_def:
1404 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
1405 gcc_assert (vector_type);
1406 nunits = TYPE_VECTOR_SUBPARTS (vector_type);
1408 if (scalar_def)
1409 *scalar_def = op;
1411 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1412 if (dump_enabled_p ())
1413 dump_printf_loc (MSG_NOTE, vect_location,
1414 "Create vector_cst. nunits = %d\n", nunits);
1416 return vect_init_vector (stmt, op, vector_type, NULL);
1419 /* Case 2: operand is defined outside the loop - loop invariant. */
1420 case vect_external_def:
1422 vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
1423 gcc_assert (vector_type);
1425 if (scalar_def)
1426 *scalar_def = def;
1428 /* Create 'vec_inv = {inv,inv,..,inv}' */
1429 if (dump_enabled_p ())
1430 dump_printf_loc (MSG_NOTE, vect_location, "Create vector_inv.\n");
1432 return vect_init_vector (stmt, def, vector_type, NULL);
1435 /* Case 3: operand is defined inside the loop. */
1436 case vect_internal_def:
1438 if (scalar_def)
1439 *scalar_def = NULL/* FIXME tuples: def_stmt*/;
1441 /* Get the def from the vectorized stmt. */
1442 def_stmt_info = vinfo_for_stmt (def_stmt);
1444 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1445 /* Get vectorized pattern statement. */
1446 if (!vec_stmt
1447 && STMT_VINFO_IN_PATTERN_P (def_stmt_info)
1448 && !STMT_VINFO_RELEVANT (def_stmt_info))
1449 vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1450 STMT_VINFO_RELATED_STMT (def_stmt_info)));
1451 gcc_assert (vec_stmt);
1452 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1453 vec_oprnd = PHI_RESULT (vec_stmt);
1454 else if (is_gimple_call (vec_stmt))
1455 vec_oprnd = gimple_call_lhs (vec_stmt);
1456 else
1457 vec_oprnd = gimple_assign_lhs (vec_stmt);
1458 return vec_oprnd;
1461 /* Case 4: operand is defined by a loop header phi - reduction */
1462 case vect_reduction_def:
1463 case vect_double_reduction_def:
1464 case vect_nested_cycle:
1466 struct loop *loop;
1468 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1469 loop = (gimple_bb (def_stmt))->loop_father;
1471 /* Get the def before the loop */
1472 op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));
1473 return get_initial_def_for_reduction (stmt, op, scalar_def);
1476 /* Case 5: operand is defined by loop-header phi - induction. */
1477 case vect_induction_def:
1479 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1481 /* Get the def from the vectorized stmt. */
1482 def_stmt_info = vinfo_for_stmt (def_stmt);
1483 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1484 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1485 vec_oprnd = PHI_RESULT (vec_stmt);
1486 else
1487 vec_oprnd = gimple_get_lhs (vec_stmt);
1488 return vec_oprnd;
1491 default:
1492 gcc_unreachable ();
1497 /* Function vect_get_vec_def_for_stmt_copy
1499 Return a vector-def for an operand. This function is used when the
1500 vectorized stmt to be created (by the caller to this function) is a "copy"
1501 created in case the vectorized result cannot fit in one vector, and several
1502 copies of the vector-stmt are required. In this case the vector-def is
1503 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1504 of the stmt that defines VEC_OPRND.
1505 DT is the type of the vector def VEC_OPRND.
1507 Context:
1508 In case the vectorization factor (VF) is bigger than the number
1509 of elements that can fit in a vectype (nunits), we have to generate
1510 more than one vector stmt to vectorize the scalar stmt. This situation
1511 arises when there are multiple data-types operated upon in the loop; the
1512 smallest data-type determines the VF, and as a result, when vectorizing
1513 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1514 vector stmt (each computing a vector of 'nunits' results, and together
1515 computing 'VF' results in each iteration). This function is called when
1516 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1517 which VF=16 and nunits=4, so the number of copies required is 4):
1519 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1521 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1522 VS1.1: vx.1 = memref1 VS1.2
1523 VS1.2: vx.2 = memref2 VS1.3
1524 VS1.3: vx.3 = memref3
1526 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1527 VSnew.1: vz1 = vx.1 + ... VSnew.2
1528 VSnew.2: vz2 = vx.2 + ... VSnew.3
1529 VSnew.3: vz3 = vx.3 + ...
1531 The vectorization of S1 is explained in vectorizable_load.
1532 The vectorization of S2:
1533 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1534 the function 'vect_get_vec_def_for_operand' is called to
1535 get the relevant vector-def for each operand of S2. For operand x it
1536 returns the vector-def 'vx.0'.
1538 To create the remaining copies of the vector-stmt (VSnew.j), this
1539 function is called to get the relevant vector-def for each operand. It is
1540 obtained from the respective VS1.j stmt, which is recorded in the
1541 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1543 For example, to obtain the vector-def 'vx.1' in order to create the
1544 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1545 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1546 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1547 and return its def ('vx.1').
1548 Overall, to create the above sequence this function will be called 3 times:
1549 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1550 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1551 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1553 tree
1554 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1556 gimple vec_stmt_for_operand;
1557 stmt_vec_info def_stmt_info;
1559 /* Do nothing; can reuse same def. */
1560 if (dt == vect_external_def || dt == vect_constant_def )
1561 return vec_oprnd;
1563 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1564 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1565 gcc_assert (def_stmt_info);
1566 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1567 gcc_assert (vec_stmt_for_operand);
1568 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1569 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1570 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1571 else
1572 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1573 return vec_oprnd;
1577 /* Get vectorized definitions for the operands to create a copy of an original
1578 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1580 static void
1581 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1582 vec<tree> *vec_oprnds0,
1583 vec<tree> *vec_oprnds1)
1585 tree vec_oprnd = vec_oprnds0->pop ();
1587 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1588 vec_oprnds0->quick_push (vec_oprnd);
1590 if (vec_oprnds1 && vec_oprnds1->length ())
1592 vec_oprnd = vec_oprnds1->pop ();
1593 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1594 vec_oprnds1->quick_push (vec_oprnd);
1599 /* Get vectorized definitions for OP0 and OP1.
1600 REDUC_INDEX is the index of reduction operand in case of reduction,
1601 and -1 otherwise. */
1603 void
1604 vect_get_vec_defs (tree op0, tree op1, gimple stmt,
1605 vec<tree> *vec_oprnds0,
1606 vec<tree> *vec_oprnds1,
1607 slp_tree slp_node, int reduc_index)
1609 if (slp_node)
1611 int nops = (op1 == NULL_TREE) ? 1 : 2;
1612 auto_vec<tree> ops (nops);
1613 auto_vec<vec<tree> > vec_defs (nops);
1615 ops.quick_push (op0);
1616 if (op1)
1617 ops.quick_push (op1);
1619 vect_get_slp_defs (ops, slp_node, &vec_defs, reduc_index);
1621 *vec_oprnds0 = vec_defs[0];
1622 if (op1)
1623 *vec_oprnds1 = vec_defs[1];
1625 else
1627 tree vec_oprnd;
1629 vec_oprnds0->create (1);
1630 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt, NULL);
1631 vec_oprnds0->quick_push (vec_oprnd);
1633 if (op1)
1635 vec_oprnds1->create (1);
1636 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt, NULL);
1637 vec_oprnds1->quick_push (vec_oprnd);
1643 /* Function vect_finish_stmt_generation.
1645 Insert a new stmt. */
1647 void
1648 vect_finish_stmt_generation (gimple stmt, gimple vec_stmt,
1649 gimple_stmt_iterator *gsi)
1651 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1652 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1653 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
1655 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1657 if (!gsi_end_p (*gsi)
1658 && gimple_has_mem_ops (vec_stmt))
1660 gimple at_stmt = gsi_stmt (*gsi);
1661 tree vuse = gimple_vuse (at_stmt);
1662 if (vuse && TREE_CODE (vuse) == SSA_NAME)
1664 tree vdef = gimple_vdef (at_stmt);
1665 gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1666 /* If we have an SSA vuse and insert a store, update virtual
1667 SSA form to avoid triggering the renamer. Do so only
1668 if we can easily see all uses - which is what almost always
1669 happens with the way vectorized stmts are inserted. */
1670 if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1671 && ((is_gimple_assign (vec_stmt)
1672 && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1673 || (is_gimple_call (vec_stmt)
1674 && !(gimple_call_flags (vec_stmt)
1675 & (ECF_CONST|ECF_PURE|ECF_NOVOPS)))))
1677 tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1678 gimple_set_vdef (vec_stmt, new_vdef);
1679 SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1683 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1685 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, loop_vinfo,
1686 bb_vinfo));
1688 if (dump_enabled_p ())
1690 dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: ");
1691 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, vec_stmt, 0);
1692 dump_printf (MSG_NOTE, "\n");
1695 gimple_set_location (vec_stmt, gimple_location (stmt));
1697 /* While EH edges will generally prevent vectorization, stmt might
1698 e.g. be in a must-not-throw region. Ensure newly created stmts
1699 that could throw are part of the same region. */
1700 int lp_nr = lookup_stmt_eh_lp (stmt);
1701 if (lp_nr != 0 && stmt_could_throw_p (vec_stmt))
1702 add_stmt_to_eh_lp (vec_stmt, lp_nr);
1705 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1706 a function declaration if the target has a vectorized version
1707 of the function, or NULL_TREE if the function cannot be vectorized. */
1709 tree
1710 vectorizable_function (gimple call, tree vectype_out, tree vectype_in)
1712 tree fndecl = gimple_call_fndecl (call);
1714 /* We only handle functions that do not read or clobber memory -- i.e.
1715 const or novops ones. */
1716 if (!(gimple_call_flags (call) & (ECF_CONST | ECF_NOVOPS)))
1717 return NULL_TREE;
1719 if (!fndecl
1720 || TREE_CODE (fndecl) != FUNCTION_DECL
1721 || !DECL_BUILT_IN (fndecl))
1722 return NULL_TREE;
1724 return targetm.vectorize.builtin_vectorized_function (fndecl, vectype_out,
1725 vectype_in);
1729 static tree permute_vec_elements (tree, tree, tree, gimple,
1730 gimple_stmt_iterator *);
1733 /* Function vectorizable_mask_load_store.
1735 Check if STMT performs a conditional load or store that can be vectorized.
1736 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1737 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1738 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1740 static bool
1741 vectorizable_mask_load_store (gimple stmt, gimple_stmt_iterator *gsi,
1742 gimple *vec_stmt, slp_tree slp_node)
1744 tree vec_dest = NULL;
1745 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1746 stmt_vec_info prev_stmt_info;
1747 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1748 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1749 bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
1750 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
1751 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1752 tree elem_type;
1753 gimple new_stmt;
1754 tree dummy;
1755 tree dataref_ptr = NULL_TREE;
1756 gimple ptr_incr;
1757 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
1758 int ncopies;
1759 int i, j;
1760 bool inv_p;
1761 tree gather_base = NULL_TREE, gather_off = NULL_TREE;
1762 tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
1763 int gather_scale = 1;
1764 enum vect_def_type gather_dt = vect_unknown_def_type;
1765 bool is_store;
1766 tree mask;
1767 gimple def_stmt;
1768 tree def;
1769 enum vect_def_type dt;
1771 if (slp_node != NULL)
1772 return false;
1774 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
1775 gcc_assert (ncopies >= 1);
1777 is_store = gimple_call_internal_fn (stmt) == IFN_MASK_STORE;
1778 mask = gimple_call_arg (stmt, 2);
1779 if (TYPE_PRECISION (TREE_TYPE (mask))
1780 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype))))
1781 return false;
1783 /* FORNOW. This restriction should be relaxed. */
1784 if (nested_in_vect_loop && ncopies > 1)
1786 if (dump_enabled_p ())
1787 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1788 "multiple types in nested loop.");
1789 return false;
1792 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1793 return false;
1795 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
1796 return false;
1798 if (!STMT_VINFO_DATA_REF (stmt_info))
1799 return false;
1801 elem_type = TREE_TYPE (vectype);
1803 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1804 return false;
1806 if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
1807 return false;
1809 if (STMT_VINFO_GATHER_P (stmt_info))
1811 gimple def_stmt;
1812 tree def;
1813 gather_decl = vect_check_gather (stmt, loop_vinfo, &gather_base,
1814 &gather_off, &gather_scale);
1815 gcc_assert (gather_decl);
1816 if (!vect_is_simple_use_1 (gather_off, NULL, loop_vinfo, NULL,
1817 &def_stmt, &def, &gather_dt,
1818 &gather_off_vectype))
1820 if (dump_enabled_p ())
1821 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1822 "gather index use not simple.");
1823 return false;
1826 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
1827 tree masktype
1828 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
1829 if (TREE_CODE (masktype) == INTEGER_TYPE)
1831 if (dump_enabled_p ())
1832 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1833 "masked gather with integer mask not supported.");
1834 return false;
1837 else if (tree_int_cst_compare (nested_in_vect_loop
1838 ? STMT_VINFO_DR_STEP (stmt_info)
1839 : DR_STEP (dr), size_zero_node) <= 0)
1840 return false;
1841 else if (!VECTOR_MODE_P (TYPE_MODE (vectype))
1842 || !can_vec_mask_load_store_p (TYPE_MODE (vectype), !is_store))
1843 return false;
1845 if (TREE_CODE (mask) != SSA_NAME)
1846 return false;
1848 if (!vect_is_simple_use (mask, stmt, loop_vinfo, NULL,
1849 &def_stmt, &def, &dt))
1850 return false;
1852 if (is_store)
1854 tree rhs = gimple_call_arg (stmt, 3);
1855 if (!vect_is_simple_use (rhs, stmt, loop_vinfo, NULL,
1856 &def_stmt, &def, &dt))
1857 return false;
1860 if (!vec_stmt) /* transformation not required. */
1862 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
1863 if (is_store)
1864 vect_model_store_cost (stmt_info, ncopies, false, dt,
1865 NULL, NULL, NULL);
1866 else
1867 vect_model_load_cost (stmt_info, ncopies, false, NULL, NULL, NULL);
1868 return true;
1871 /** Transform. **/
1873 if (STMT_VINFO_GATHER_P (stmt_info))
1875 tree vec_oprnd0 = NULL_TREE, op;
1876 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
1877 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
1878 tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale;
1879 tree perm_mask = NULL_TREE, prev_res = NULL_TREE;
1880 tree mask_perm_mask = NULL_TREE;
1881 edge pe = loop_preheader_edge (loop);
1882 gimple_seq seq;
1883 basic_block new_bb;
1884 enum { NARROW, NONE, WIDEN } modifier;
1885 int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
1887 rettype = TREE_TYPE (TREE_TYPE (gather_decl));
1888 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1889 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1890 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1891 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1892 scaletype = TREE_VALUE (arglist);
1893 gcc_checking_assert (types_compatible_p (srctype, rettype)
1894 && types_compatible_p (srctype, masktype));
1896 if (nunits == gather_off_nunits)
1897 modifier = NONE;
1898 else if (nunits == gather_off_nunits / 2)
1900 unsigned char *sel = XALLOCAVEC (unsigned char, gather_off_nunits);
1901 modifier = WIDEN;
1903 for (i = 0; i < gather_off_nunits; ++i)
1904 sel[i] = i | nunits;
1906 perm_mask = vect_gen_perm_mask (gather_off_vectype, sel);
1907 gcc_assert (perm_mask != NULL_TREE);
1909 else if (nunits == gather_off_nunits * 2)
1911 unsigned char *sel = XALLOCAVEC (unsigned char, nunits);
1912 modifier = NARROW;
1914 for (i = 0; i < nunits; ++i)
1915 sel[i] = i < gather_off_nunits
1916 ? i : i + nunits - gather_off_nunits;
1918 perm_mask = vect_gen_perm_mask (vectype, sel);
1919 gcc_assert (perm_mask != NULL_TREE);
1920 ncopies *= 2;
1921 for (i = 0; i < nunits; ++i)
1922 sel[i] = i | gather_off_nunits;
1923 mask_perm_mask = vect_gen_perm_mask (masktype, sel);
1924 gcc_assert (mask_perm_mask != NULL_TREE);
1926 else
1927 gcc_unreachable ();
1929 vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype);
1931 ptr = fold_convert (ptrtype, gather_base);
1932 if (!is_gimple_min_invariant (ptr))
1934 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
1935 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
1936 gcc_assert (!new_bb);
1939 scale = build_int_cst (scaletype, gather_scale);
1941 prev_stmt_info = NULL;
1942 for (j = 0; j < ncopies; ++j)
1944 if (modifier == WIDEN && (j & 1))
1945 op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
1946 perm_mask, stmt, gsi);
1947 else if (j == 0)
1948 op = vec_oprnd0
1949 = vect_get_vec_def_for_operand (gather_off, stmt, NULL);
1950 else
1951 op = vec_oprnd0
1952 = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
1954 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
1956 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
1957 == TYPE_VECTOR_SUBPARTS (idxtype));
1958 var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL);
1959 var = make_ssa_name (var, NULL);
1960 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
1961 new_stmt
1962 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR, var,
1963 op, NULL_TREE);
1964 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1965 op = var;
1968 if (mask_perm_mask && (j & 1))
1969 mask_op = permute_vec_elements (mask_op, mask_op,
1970 mask_perm_mask, stmt, gsi);
1971 else
1973 if (j == 0)
1974 vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
1975 else
1977 vect_is_simple_use (vec_mask, NULL, loop_vinfo, NULL,
1978 &def_stmt, &def, &dt);
1979 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
1982 mask_op = vec_mask;
1983 if (!useless_type_conversion_p (masktype, TREE_TYPE (vec_mask)))
1985 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op))
1986 == TYPE_VECTOR_SUBPARTS (masktype));
1987 var = vect_get_new_vect_var (masktype, vect_simple_var,
1988 NULL);
1989 var = make_ssa_name (var, NULL);
1990 mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op);
1991 new_stmt
1992 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR, var,
1993 mask_op, NULL_TREE);
1994 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1995 mask_op = var;
1999 new_stmt
2000 = gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op,
2001 scale);
2003 if (!useless_type_conversion_p (vectype, rettype))
2005 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
2006 == TYPE_VECTOR_SUBPARTS (rettype));
2007 var = vect_get_new_vect_var (rettype, vect_simple_var, NULL);
2008 op = make_ssa_name (var, new_stmt);
2009 gimple_call_set_lhs (new_stmt, op);
2010 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2011 var = make_ssa_name (vec_dest, NULL);
2012 op = build1 (VIEW_CONVERT_EXPR, vectype, op);
2013 new_stmt
2014 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR, var, op,
2015 NULL_TREE);
2017 else
2019 var = make_ssa_name (vec_dest, new_stmt);
2020 gimple_call_set_lhs (new_stmt, var);
2023 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2025 if (modifier == NARROW)
2027 if ((j & 1) == 0)
2029 prev_res = var;
2030 continue;
2032 var = permute_vec_elements (prev_res, var,
2033 perm_mask, stmt, gsi);
2034 new_stmt = SSA_NAME_DEF_STMT (var);
2037 if (prev_stmt_info == NULL)
2038 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2039 else
2040 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2041 prev_stmt_info = vinfo_for_stmt (new_stmt);
2044 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2045 from the IL. */
2046 tree lhs = gimple_call_lhs (stmt);
2047 new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
2048 set_vinfo_for_stmt (new_stmt, stmt_info);
2049 set_vinfo_for_stmt (stmt, NULL);
2050 STMT_VINFO_STMT (stmt_info) = new_stmt;
2051 gsi_replace (gsi, new_stmt, true);
2052 return true;
2054 else if (is_store)
2056 tree vec_rhs = NULL_TREE, vec_mask = NULL_TREE;
2057 prev_stmt_info = NULL;
2058 for (i = 0; i < ncopies; i++)
2060 unsigned align, misalign;
2062 if (i == 0)
2064 tree rhs = gimple_call_arg (stmt, 3);
2065 vec_rhs = vect_get_vec_def_for_operand (rhs, stmt, NULL);
2066 vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
2067 /* We should have catched mismatched types earlier. */
2068 gcc_assert (useless_type_conversion_p (vectype,
2069 TREE_TYPE (vec_rhs)));
2070 dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL,
2071 NULL_TREE, &dummy, gsi,
2072 &ptr_incr, false, &inv_p);
2073 gcc_assert (!inv_p);
2075 else
2077 vect_is_simple_use (vec_rhs, NULL, loop_vinfo, NULL, &def_stmt,
2078 &def, &dt);
2079 vec_rhs = vect_get_vec_def_for_stmt_copy (dt, vec_rhs);
2080 vect_is_simple_use (vec_mask, NULL, loop_vinfo, NULL, &def_stmt,
2081 &def, &dt);
2082 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
2083 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
2084 TYPE_SIZE_UNIT (vectype));
2087 align = TYPE_ALIGN_UNIT (vectype);
2088 if (aligned_access_p (dr))
2089 misalign = 0;
2090 else if (DR_MISALIGNMENT (dr) == -1)
2092 align = TYPE_ALIGN_UNIT (elem_type);
2093 misalign = 0;
2095 else
2096 misalign = DR_MISALIGNMENT (dr);
2097 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
2098 misalign);
2099 new_stmt
2100 = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
2101 gimple_call_arg (stmt, 1),
2102 vec_mask, vec_rhs);
2103 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2104 if (i == 0)
2105 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2106 else
2107 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2108 prev_stmt_info = vinfo_for_stmt (new_stmt);
2111 else
2113 tree vec_mask = NULL_TREE;
2114 prev_stmt_info = NULL;
2115 vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype);
2116 for (i = 0; i < ncopies; i++)
2118 unsigned align, misalign;
2120 if (i == 0)
2122 vec_mask = vect_get_vec_def_for_operand (mask, stmt, NULL);
2123 dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL,
2124 NULL_TREE, &dummy, gsi,
2125 &ptr_incr, false, &inv_p);
2126 gcc_assert (!inv_p);
2128 else
2130 vect_is_simple_use (vec_mask, NULL, loop_vinfo, NULL, &def_stmt,
2131 &def, &dt);
2132 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
2133 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
2134 TYPE_SIZE_UNIT (vectype));
2137 align = TYPE_ALIGN_UNIT (vectype);
2138 if (aligned_access_p (dr))
2139 misalign = 0;
2140 else if (DR_MISALIGNMENT (dr) == -1)
2142 align = TYPE_ALIGN_UNIT (elem_type);
2143 misalign = 0;
2145 else
2146 misalign = DR_MISALIGNMENT (dr);
2147 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
2148 misalign);
2149 new_stmt
2150 = gimple_build_call_internal (IFN_MASK_LOAD, 3, dataref_ptr,
2151 gimple_call_arg (stmt, 1),
2152 vec_mask);
2153 gimple_call_set_lhs (new_stmt, make_ssa_name (vec_dest, NULL));
2154 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2155 if (i == 0)
2156 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2157 else
2158 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2159 prev_stmt_info = vinfo_for_stmt (new_stmt);
2163 if (!is_store)
2165 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2166 from the IL. */
2167 tree lhs = gimple_call_lhs (stmt);
2168 new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
2169 set_vinfo_for_stmt (new_stmt, stmt_info);
2170 set_vinfo_for_stmt (stmt, NULL);
2171 STMT_VINFO_STMT (stmt_info) = new_stmt;
2172 gsi_replace (gsi, new_stmt, true);
2175 return true;
2179 /* Function vectorizable_call.
2181 Check if STMT performs a function call that can be vectorized.
2182 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2183 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2184 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2186 static bool
2187 vectorizable_call (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
2188 slp_tree slp_node)
2190 tree vec_dest;
2191 tree scalar_dest;
2192 tree op, type;
2193 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
2194 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
2195 tree vectype_out, vectype_in;
2196 int nunits_in;
2197 int nunits_out;
2198 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2199 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2200 tree fndecl, new_temp, def, rhs_type;
2201 gimple def_stmt;
2202 enum vect_def_type dt[3]
2203 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
2204 gimple new_stmt = NULL;
2205 int ncopies, j;
2206 vec<tree> vargs = vNULL;
2207 enum { NARROW, NONE, WIDEN } modifier;
2208 size_t i, nargs;
2209 tree lhs;
2211 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2212 return false;
2214 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2215 return false;
2217 /* Is STMT a vectorizable call? */
2218 if (!is_gimple_call (stmt))
2219 return false;
2221 if (gimple_call_internal_p (stmt)
2222 && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
2223 || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
2224 return vectorizable_mask_load_store (stmt, gsi, vec_stmt,
2225 slp_node);
2227 if (gimple_call_lhs (stmt) == NULL_TREE
2228 || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
2229 return false;
2231 gcc_checking_assert (!stmt_can_throw_internal (stmt));
2233 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2235 /* Process function arguments. */
2236 rhs_type = NULL_TREE;
2237 vectype_in = NULL_TREE;
2238 nargs = gimple_call_num_args (stmt);
2240 /* Bail out if the function has more than three arguments, we do not have
2241 interesting builtin functions to vectorize with more than two arguments
2242 except for fma. No arguments is also not good. */
2243 if (nargs == 0 || nargs > 3)
2244 return false;
2246 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2247 if (gimple_call_internal_p (stmt)
2248 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
2250 nargs = 0;
2251 rhs_type = unsigned_type_node;
2254 for (i = 0; i < nargs; i++)
2256 tree opvectype;
2258 op = gimple_call_arg (stmt, i);
2260 /* We can only handle calls with arguments of the same type. */
2261 if (rhs_type
2262 && !types_compatible_p (rhs_type, TREE_TYPE (op)))
2264 if (dump_enabled_p ())
2265 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2266 "argument types differ.\n");
2267 return false;
2269 if (!rhs_type)
2270 rhs_type = TREE_TYPE (op);
2272 if (!vect_is_simple_use_1 (op, stmt, loop_vinfo, bb_vinfo,
2273 &def_stmt, &def, &dt[i], &opvectype))
2275 if (dump_enabled_p ())
2276 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2277 "use not simple.\n");
2278 return false;
2281 if (!vectype_in)
2282 vectype_in = opvectype;
2283 else if (opvectype
2284 && opvectype != vectype_in)
2286 if (dump_enabled_p ())
2287 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2288 "argument vector types differ.\n");
2289 return false;
2292 /* If all arguments are external or constant defs use a vector type with
2293 the same size as the output vector type. */
2294 if (!vectype_in)
2295 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
2296 if (vec_stmt)
2297 gcc_assert (vectype_in);
2298 if (!vectype_in)
2300 if (dump_enabled_p ())
2302 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2303 "no vectype for scalar type ");
2304 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
2305 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
2308 return false;
2311 /* FORNOW */
2312 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2313 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2314 if (nunits_in == nunits_out / 2)
2315 modifier = NARROW;
2316 else if (nunits_out == nunits_in)
2317 modifier = NONE;
2318 else if (nunits_out == nunits_in / 2)
2319 modifier = WIDEN;
2320 else
2321 return false;
2323 /* For now, we only vectorize functions if a target specific builtin
2324 is available. TODO -- in some cases, it might be profitable to
2325 insert the calls for pieces of the vector, in order to be able
2326 to vectorize other operations in the loop. */
2327 fndecl = vectorizable_function (stmt, vectype_out, vectype_in);
2328 if (fndecl == NULL_TREE)
2330 if (gimple_call_internal_p (stmt)
2331 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE
2332 && !slp_node
2333 && loop_vinfo
2334 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
2335 && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
2336 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
2337 == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
2339 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2340 { 0, 1, 2, ... vf - 1 } vector. */
2341 gcc_assert (nargs == 0);
2343 else
2345 if (dump_enabled_p ())
2346 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2347 "function is not vectorizable.\n");
2348 return false;
2352 gcc_assert (!gimple_vuse (stmt));
2354 if (slp_node || PURE_SLP_STMT (stmt_info))
2355 ncopies = 1;
2356 else if (modifier == NARROW)
2357 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
2358 else
2359 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2361 /* Sanity check: make sure that at least one copy of the vectorized stmt
2362 needs to be generated. */
2363 gcc_assert (ncopies >= 1);
2365 if (!vec_stmt) /* transformation not required. */
2367 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
2368 if (dump_enabled_p ())
2369 dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_call ==="
2370 "\n");
2371 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
2372 return true;
2375 /** Transform. **/
2377 if (dump_enabled_p ())
2378 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
2380 /* Handle def. */
2381 scalar_dest = gimple_call_lhs (stmt);
2382 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2384 prev_stmt_info = NULL;
2385 switch (modifier)
2387 case NONE:
2388 for (j = 0; j < ncopies; ++j)
2390 /* Build argument list for the vectorized call. */
2391 if (j == 0)
2392 vargs.create (nargs);
2393 else
2394 vargs.truncate (0);
2396 if (slp_node)
2398 auto_vec<vec<tree> > vec_defs (nargs);
2399 vec<tree> vec_oprnds0;
2401 for (i = 0; i < nargs; i++)
2402 vargs.quick_push (gimple_call_arg (stmt, i));
2403 vect_get_slp_defs (vargs, slp_node, &vec_defs, -1);
2404 vec_oprnds0 = vec_defs[0];
2406 /* Arguments are ready. Create the new vector stmt. */
2407 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
2409 size_t k;
2410 for (k = 0; k < nargs; k++)
2412 vec<tree> vec_oprndsk = vec_defs[k];
2413 vargs[k] = vec_oprndsk[i];
2415 new_stmt = gimple_build_call_vec (fndecl, vargs);
2416 new_temp = make_ssa_name (vec_dest, new_stmt);
2417 gimple_call_set_lhs (new_stmt, new_temp);
2418 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2419 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
2422 for (i = 0; i < nargs; i++)
2424 vec<tree> vec_oprndsi = vec_defs[i];
2425 vec_oprndsi.release ();
2427 continue;
2430 for (i = 0; i < nargs; i++)
2432 op = gimple_call_arg (stmt, i);
2433 if (j == 0)
2434 vec_oprnd0
2435 = vect_get_vec_def_for_operand (op, stmt, NULL);
2436 else
2438 vec_oprnd0 = gimple_call_arg (new_stmt, i);
2439 vec_oprnd0
2440 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2443 vargs.quick_push (vec_oprnd0);
2446 if (gimple_call_internal_p (stmt)
2447 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
2449 tree *v = XALLOCAVEC (tree, nunits_out);
2450 int k;
2451 for (k = 0; k < nunits_out; ++k)
2452 v[k] = build_int_cst (unsigned_type_node, j * nunits_out + k);
2453 tree cst = build_vector (vectype_out, v);
2454 tree new_var
2455 = vect_get_new_vect_var (vectype_out, vect_simple_var, "cst_");
2456 gimple init_stmt = gimple_build_assign (new_var, cst);
2457 new_temp = make_ssa_name (new_var, init_stmt);
2458 gimple_assign_set_lhs (init_stmt, new_temp);
2459 vect_init_vector_1 (stmt, init_stmt, NULL);
2460 new_temp = make_ssa_name (vec_dest, NULL);
2461 new_stmt = gimple_build_assign (new_temp,
2462 gimple_assign_lhs (init_stmt));
2464 else
2466 new_stmt = gimple_build_call_vec (fndecl, vargs);
2467 new_temp = make_ssa_name (vec_dest, new_stmt);
2468 gimple_call_set_lhs (new_stmt, new_temp);
2470 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2472 if (j == 0)
2473 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2474 else
2475 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2477 prev_stmt_info = vinfo_for_stmt (new_stmt);
2480 break;
2482 case NARROW:
2483 for (j = 0; j < ncopies; ++j)
2485 /* Build argument list for the vectorized call. */
2486 if (j == 0)
2487 vargs.create (nargs * 2);
2488 else
2489 vargs.truncate (0);
2491 if (slp_node)
2493 auto_vec<vec<tree> > vec_defs (nargs);
2494 vec<tree> vec_oprnds0;
2496 for (i = 0; i < nargs; i++)
2497 vargs.quick_push (gimple_call_arg (stmt, i));
2498 vect_get_slp_defs (vargs, slp_node, &vec_defs, -1);
2499 vec_oprnds0 = vec_defs[0];
2501 /* Arguments are ready. Create the new vector stmt. */
2502 for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
2504 size_t k;
2505 vargs.truncate (0);
2506 for (k = 0; k < nargs; k++)
2508 vec<tree> vec_oprndsk = vec_defs[k];
2509 vargs.quick_push (vec_oprndsk[i]);
2510 vargs.quick_push (vec_oprndsk[i + 1]);
2512 new_stmt = gimple_build_call_vec (fndecl, vargs);
2513 new_temp = make_ssa_name (vec_dest, new_stmt);
2514 gimple_call_set_lhs (new_stmt, new_temp);
2515 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2516 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
2519 for (i = 0; i < nargs; i++)
2521 vec<tree> vec_oprndsi = vec_defs[i];
2522 vec_oprndsi.release ();
2524 continue;
2527 for (i = 0; i < nargs; i++)
2529 op = gimple_call_arg (stmt, i);
2530 if (j == 0)
2532 vec_oprnd0
2533 = vect_get_vec_def_for_operand (op, stmt, NULL);
2534 vec_oprnd1
2535 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2537 else
2539 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1);
2540 vec_oprnd0
2541 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
2542 vec_oprnd1
2543 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2546 vargs.quick_push (vec_oprnd0);
2547 vargs.quick_push (vec_oprnd1);
2550 new_stmt = gimple_build_call_vec (fndecl, vargs);
2551 new_temp = make_ssa_name (vec_dest, new_stmt);
2552 gimple_call_set_lhs (new_stmt, new_temp);
2553 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2555 if (j == 0)
2556 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
2557 else
2558 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2560 prev_stmt_info = vinfo_for_stmt (new_stmt);
2563 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2565 break;
2567 case WIDEN:
2568 /* No current target implements this case. */
2569 return false;
2572 vargs.release ();
2574 /* The call in STMT might prevent it from being removed in dce.
2575 We however cannot remove it here, due to the way the ssa name
2576 it defines is mapped to the new definition. So just replace
2577 rhs of the statement with something harmless. */
2579 if (slp_node)
2580 return true;
2582 type = TREE_TYPE (scalar_dest);
2583 if (is_pattern_stmt_p (stmt_info))
2584 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
2585 else
2586 lhs = gimple_call_lhs (stmt);
2587 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
2588 set_vinfo_for_stmt (new_stmt, stmt_info);
2589 set_vinfo_for_stmt (stmt, NULL);
2590 STMT_VINFO_STMT (stmt_info) = new_stmt;
2591 gsi_replace (gsi, new_stmt, false);
2593 return true;
2597 struct simd_call_arg_info
2599 tree vectype;
2600 tree op;
2601 enum vect_def_type dt;
2602 HOST_WIDE_INT linear_step;
2603 unsigned int align;
2606 /* Function vectorizable_simd_clone_call.
2608 Check if STMT performs a function call that can be vectorized
2609 by calling a simd clone of the function.
2610 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2611 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2612 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2614 static bool
2615 vectorizable_simd_clone_call (gimple stmt, gimple_stmt_iterator *gsi,
2616 gimple *vec_stmt, slp_tree slp_node)
2618 tree vec_dest;
2619 tree scalar_dest;
2620 tree op, type;
2621 tree vec_oprnd0 = NULL_TREE;
2622 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
2623 tree vectype;
2624 unsigned int nunits;
2625 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2626 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2627 struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2628 tree fndecl, new_temp, def;
2629 gimple def_stmt;
2630 gimple new_stmt = NULL;
2631 int ncopies, j;
2632 vec<simd_call_arg_info> arginfo = vNULL;
2633 vec<tree> vargs = vNULL;
2634 size_t i, nargs;
2635 tree lhs, rtype, ratype;
2636 vec<constructor_elt, va_gc> *ret_ctor_elts;
2638 /* Is STMT a vectorizable call? */
2639 if (!is_gimple_call (stmt))
2640 return false;
2642 fndecl = gimple_call_fndecl (stmt);
2643 if (fndecl == NULL_TREE)
2644 return false;
2646 struct cgraph_node *node = cgraph_node::get (fndecl);
2647 if (node == NULL || node->simd_clones == NULL)
2648 return false;
2650 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2651 return false;
2653 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
2654 return false;
2656 if (gimple_call_lhs (stmt)
2657 && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
2658 return false;
2660 gcc_checking_assert (!stmt_can_throw_internal (stmt));
2662 vectype = STMT_VINFO_VECTYPE (stmt_info);
2664 if (loop_vinfo && nested_in_vect_loop_p (loop, stmt))
2665 return false;
2667 /* FORNOW */
2668 if (slp_node || PURE_SLP_STMT (stmt_info))
2669 return false;
2671 /* Process function arguments. */
2672 nargs = gimple_call_num_args (stmt);
2674 /* Bail out if the function has zero arguments. */
2675 if (nargs == 0)
2676 return false;
2678 arginfo.create (nargs);
2680 for (i = 0; i < nargs; i++)
2682 simd_call_arg_info thisarginfo;
2683 affine_iv iv;
2685 thisarginfo.linear_step = 0;
2686 thisarginfo.align = 0;
2687 thisarginfo.op = NULL_TREE;
2689 op = gimple_call_arg (stmt, i);
2690 if (!vect_is_simple_use_1 (op, stmt, loop_vinfo, bb_vinfo,
2691 &def_stmt, &def, &thisarginfo.dt,
2692 &thisarginfo.vectype)
2693 || thisarginfo.dt == vect_uninitialized_def)
2695 if (dump_enabled_p ())
2696 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2697 "use not simple.\n");
2698 arginfo.release ();
2699 return false;
2702 if (thisarginfo.dt == vect_constant_def
2703 || thisarginfo.dt == vect_external_def)
2704 gcc_assert (thisarginfo.vectype == NULL_TREE);
2705 else
2706 gcc_assert (thisarginfo.vectype != NULL_TREE);
2708 if (thisarginfo.dt != vect_constant_def
2709 && thisarginfo.dt != vect_external_def
2710 && loop_vinfo
2711 && TREE_CODE (op) == SSA_NAME
2712 && simple_iv (loop, loop_containing_stmt (stmt), op, &iv, false)
2713 && tree_fits_shwi_p (iv.step))
2715 thisarginfo.linear_step = tree_to_shwi (iv.step);
2716 thisarginfo.op = iv.base;
2718 else if ((thisarginfo.dt == vect_constant_def
2719 || thisarginfo.dt == vect_external_def)
2720 && POINTER_TYPE_P (TREE_TYPE (op)))
2721 thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
2723 arginfo.quick_push (thisarginfo);
2726 unsigned int badness = 0;
2727 struct cgraph_node *bestn = NULL;
2728 if (STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info))
2729 bestn = cgraph_node::get (STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info));
2730 else
2731 for (struct cgraph_node *n = node->simd_clones; n != NULL;
2732 n = n->simdclone->next_clone)
2734 unsigned int this_badness = 0;
2735 if (n->simdclone->simdlen
2736 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo)
2737 || n->simdclone->nargs != nargs)
2738 continue;
2739 if (n->simdclone->simdlen
2740 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo))
2741 this_badness += (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo))
2742 - exact_log2 (n->simdclone->simdlen)) * 1024;
2743 if (n->simdclone->inbranch)
2744 this_badness += 2048;
2745 int target_badness = targetm.simd_clone.usable (n);
2746 if (target_badness < 0)
2747 continue;
2748 this_badness += target_badness * 512;
2749 /* FORNOW: Have to add code to add the mask argument. */
2750 if (n->simdclone->inbranch)
2751 continue;
2752 for (i = 0; i < nargs; i++)
2754 switch (n->simdclone->args[i].arg_type)
2756 case SIMD_CLONE_ARG_TYPE_VECTOR:
2757 if (!useless_type_conversion_p
2758 (n->simdclone->args[i].orig_type,
2759 TREE_TYPE (gimple_call_arg (stmt, i))))
2760 i = -1;
2761 else if (arginfo[i].dt == vect_constant_def
2762 || arginfo[i].dt == vect_external_def
2763 || arginfo[i].linear_step)
2764 this_badness += 64;
2765 break;
2766 case SIMD_CLONE_ARG_TYPE_UNIFORM:
2767 if (arginfo[i].dt != vect_constant_def
2768 && arginfo[i].dt != vect_external_def)
2769 i = -1;
2770 break;
2771 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
2772 if (arginfo[i].dt == vect_constant_def
2773 || arginfo[i].dt == vect_external_def
2774 || (arginfo[i].linear_step
2775 != n->simdclone->args[i].linear_step))
2776 i = -1;
2777 break;
2778 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
2779 /* FORNOW */
2780 i = -1;
2781 break;
2782 case SIMD_CLONE_ARG_TYPE_MASK:
2783 gcc_unreachable ();
2785 if (i == (size_t) -1)
2786 break;
2787 if (n->simdclone->args[i].alignment > arginfo[i].align)
2789 i = -1;
2790 break;
2792 if (arginfo[i].align)
2793 this_badness += (exact_log2 (arginfo[i].align)
2794 - exact_log2 (n->simdclone->args[i].alignment));
2796 if (i == (size_t) -1)
2797 continue;
2798 if (bestn == NULL || this_badness < badness)
2800 bestn = n;
2801 badness = this_badness;
2805 if (bestn == NULL)
2807 arginfo.release ();
2808 return false;
2811 for (i = 0; i < nargs; i++)
2812 if ((arginfo[i].dt == vect_constant_def
2813 || arginfo[i].dt == vect_external_def)
2814 && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
2816 arginfo[i].vectype
2817 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt,
2818 i)));
2819 if (arginfo[i].vectype == NULL
2820 || (TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)
2821 > bestn->simdclone->simdlen))
2823 arginfo.release ();
2824 return false;
2828 fndecl = bestn->decl;
2829 nunits = bestn->simdclone->simdlen;
2830 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
2832 /* If the function isn't const, only allow it in simd loops where user
2833 has asserted that at least nunits consecutive iterations can be
2834 performed using SIMD instructions. */
2835 if ((loop == NULL || (unsigned) loop->safelen < nunits)
2836 && gimple_vuse (stmt))
2838 arginfo.release ();
2839 return false;
2842 /* Sanity check: make sure that at least one copy of the vectorized stmt
2843 needs to be generated. */
2844 gcc_assert (ncopies >= 1);
2846 if (!vec_stmt) /* transformation not required. */
2848 STMT_VINFO_SIMD_CLONE_FNDECL (stmt_info) = bestn->decl;
2849 STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
2850 if (dump_enabled_p ())
2851 dump_printf_loc (MSG_NOTE, vect_location,
2852 "=== vectorizable_simd_clone_call ===\n");
2853 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2854 arginfo.release ();
2855 return true;
2858 /** Transform. **/
2860 if (dump_enabled_p ())
2861 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
2863 /* Handle def. */
2864 scalar_dest = gimple_call_lhs (stmt);
2865 vec_dest = NULL_TREE;
2866 rtype = NULL_TREE;
2867 ratype = NULL_TREE;
2868 if (scalar_dest)
2870 vec_dest = vect_create_destination_var (scalar_dest, vectype);
2871 rtype = TREE_TYPE (TREE_TYPE (fndecl));
2872 if (TREE_CODE (rtype) == ARRAY_TYPE)
2874 ratype = rtype;
2875 rtype = TREE_TYPE (ratype);
2879 prev_stmt_info = NULL;
2880 for (j = 0; j < ncopies; ++j)
2882 /* Build argument list for the vectorized call. */
2883 if (j == 0)
2884 vargs.create (nargs);
2885 else
2886 vargs.truncate (0);
2888 for (i = 0; i < nargs; i++)
2890 unsigned int k, l, m, o;
2891 tree atype;
2892 op = gimple_call_arg (stmt, i);
2893 switch (bestn->simdclone->args[i].arg_type)
2895 case SIMD_CLONE_ARG_TYPE_VECTOR:
2896 atype = bestn->simdclone->args[i].vector_type;
2897 o = nunits / TYPE_VECTOR_SUBPARTS (atype);
2898 for (m = j * o; m < (j + 1) * o; m++)
2900 if (TYPE_VECTOR_SUBPARTS (atype)
2901 < TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
2903 unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
2904 k = (TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)
2905 / TYPE_VECTOR_SUBPARTS (atype));
2906 gcc_assert ((k & (k - 1)) == 0);
2907 if (m == 0)
2908 vec_oprnd0
2909 = vect_get_vec_def_for_operand (op, stmt, NULL);
2910 else
2912 vec_oprnd0 = arginfo[i].op;
2913 if ((m & (k - 1)) == 0)
2914 vec_oprnd0
2915 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
2916 vec_oprnd0);
2918 arginfo[i].op = vec_oprnd0;
2919 vec_oprnd0
2920 = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
2921 size_int (prec),
2922 bitsize_int ((m & (k - 1)) * prec));
2923 new_stmt
2924 = gimple_build_assign (make_ssa_name (atype, NULL),
2925 vec_oprnd0);
2926 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2927 vargs.safe_push (gimple_assign_lhs (new_stmt));
2929 else
2931 k = (TYPE_VECTOR_SUBPARTS (atype)
2932 / TYPE_VECTOR_SUBPARTS (arginfo[i].vectype));
2933 gcc_assert ((k & (k - 1)) == 0);
2934 vec<constructor_elt, va_gc> *ctor_elts;
2935 if (k != 1)
2936 vec_alloc (ctor_elts, k);
2937 else
2938 ctor_elts = NULL;
2939 for (l = 0; l < k; l++)
2941 if (m == 0 && l == 0)
2942 vec_oprnd0
2943 = vect_get_vec_def_for_operand (op, stmt, NULL);
2944 else
2945 vec_oprnd0
2946 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
2947 arginfo[i].op);
2948 arginfo[i].op = vec_oprnd0;
2949 if (k == 1)
2950 break;
2951 CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
2952 vec_oprnd0);
2954 if (k == 1)
2955 vargs.safe_push (vec_oprnd0);
2956 else
2958 vec_oprnd0 = build_constructor (atype, ctor_elts);
2959 new_stmt
2960 = gimple_build_assign (make_ssa_name (atype, NULL),
2961 vec_oprnd0);
2962 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2963 vargs.safe_push (gimple_assign_lhs (new_stmt));
2967 break;
2968 case SIMD_CLONE_ARG_TYPE_UNIFORM:
2969 vargs.safe_push (op);
2970 break;
2971 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
2972 if (j == 0)
2974 gimple_seq stmts;
2975 arginfo[i].op
2976 = force_gimple_operand (arginfo[i].op, &stmts, true,
2977 NULL_TREE);
2978 if (stmts != NULL)
2980 basic_block new_bb;
2981 edge pe = loop_preheader_edge (loop);
2982 new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
2983 gcc_assert (!new_bb);
2985 tree phi_res = copy_ssa_name (op, NULL);
2986 gimple new_phi = create_phi_node (phi_res, loop->header);
2987 set_vinfo_for_stmt (new_phi,
2988 new_stmt_vec_info (new_phi, loop_vinfo,
2989 NULL));
2990 add_phi_arg (new_phi, arginfo[i].op,
2991 loop_preheader_edge (loop), UNKNOWN_LOCATION);
2992 enum tree_code code
2993 = POINTER_TYPE_P (TREE_TYPE (op))
2994 ? POINTER_PLUS_EXPR : PLUS_EXPR;
2995 tree type = POINTER_TYPE_P (TREE_TYPE (op))
2996 ? sizetype : TREE_TYPE (op);
2997 widest_int cst
2998 = wi::mul (bestn->simdclone->args[i].linear_step,
2999 ncopies * nunits);
3000 tree tcst = wide_int_to_tree (type, cst);
3001 tree phi_arg = copy_ssa_name (op, NULL);
3002 new_stmt = gimple_build_assign_with_ops (code, phi_arg,
3003 phi_res, tcst);
3004 gimple_stmt_iterator si = gsi_after_labels (loop->header);
3005 gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
3006 set_vinfo_for_stmt (new_stmt,
3007 new_stmt_vec_info (new_stmt, loop_vinfo,
3008 NULL));
3009 add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
3010 UNKNOWN_LOCATION);
3011 arginfo[i].op = phi_res;
3012 vargs.safe_push (phi_res);
3014 else
3016 enum tree_code code
3017 = POINTER_TYPE_P (TREE_TYPE (op))
3018 ? POINTER_PLUS_EXPR : PLUS_EXPR;
3019 tree type = POINTER_TYPE_P (TREE_TYPE (op))
3020 ? sizetype : TREE_TYPE (op);
3021 widest_int cst
3022 = wi::mul (bestn->simdclone->args[i].linear_step,
3023 j * nunits);
3024 tree tcst = wide_int_to_tree (type, cst);
3025 new_temp = make_ssa_name (TREE_TYPE (op), NULL);
3026 new_stmt
3027 = gimple_build_assign_with_ops (code, new_temp,
3028 arginfo[i].op, tcst);
3029 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3030 vargs.safe_push (new_temp);
3032 break;
3033 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
3034 default:
3035 gcc_unreachable ();
3039 new_stmt = gimple_build_call_vec (fndecl, vargs);
3040 if (vec_dest)
3042 gcc_assert (ratype || TYPE_VECTOR_SUBPARTS (rtype) == nunits);
3043 if (ratype)
3044 new_temp = create_tmp_var (ratype, NULL);
3045 else if (TYPE_VECTOR_SUBPARTS (vectype)
3046 == TYPE_VECTOR_SUBPARTS (rtype))
3047 new_temp = make_ssa_name (vec_dest, new_stmt);
3048 else
3049 new_temp = make_ssa_name (rtype, new_stmt);
3050 gimple_call_set_lhs (new_stmt, new_temp);
3052 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3054 if (vec_dest)
3056 if (TYPE_VECTOR_SUBPARTS (vectype) < nunits)
3058 unsigned int k, l;
3059 unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
3060 k = nunits / TYPE_VECTOR_SUBPARTS (vectype);
3061 gcc_assert ((k & (k - 1)) == 0);
3062 for (l = 0; l < k; l++)
3064 tree t;
3065 if (ratype)
3067 t = build_fold_addr_expr (new_temp);
3068 t = build2 (MEM_REF, vectype, t,
3069 build_int_cst (TREE_TYPE (t),
3070 l * prec / BITS_PER_UNIT));
3072 else
3073 t = build3 (BIT_FIELD_REF, vectype, new_temp,
3074 size_int (prec), bitsize_int (l * prec));
3075 new_stmt
3076 = gimple_build_assign (make_ssa_name (vectype, NULL), t);
3077 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3078 if (j == 0 && l == 0)
3079 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3080 else
3081 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3083 prev_stmt_info = vinfo_for_stmt (new_stmt);
3086 if (ratype)
3088 tree clobber = build_constructor (ratype, NULL);
3089 TREE_THIS_VOLATILE (clobber) = 1;
3090 new_stmt = gimple_build_assign (new_temp, clobber);
3091 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3093 continue;
3095 else if (TYPE_VECTOR_SUBPARTS (vectype) > nunits)
3097 unsigned int k = (TYPE_VECTOR_SUBPARTS (vectype)
3098 / TYPE_VECTOR_SUBPARTS (rtype));
3099 gcc_assert ((k & (k - 1)) == 0);
3100 if ((j & (k - 1)) == 0)
3101 vec_alloc (ret_ctor_elts, k);
3102 if (ratype)
3104 unsigned int m, o = nunits / TYPE_VECTOR_SUBPARTS (rtype);
3105 for (m = 0; m < o; m++)
3107 tree tem = build4 (ARRAY_REF, rtype, new_temp,
3108 size_int (m), NULL_TREE, NULL_TREE);
3109 new_stmt
3110 = gimple_build_assign (make_ssa_name (rtype, NULL),
3111 tem);
3112 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3113 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
3114 gimple_assign_lhs (new_stmt));
3116 tree clobber = build_constructor (ratype, NULL);
3117 TREE_THIS_VOLATILE (clobber) = 1;
3118 new_stmt = gimple_build_assign (new_temp, clobber);
3119 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3121 else
3122 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
3123 if ((j & (k - 1)) != k - 1)
3124 continue;
3125 vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
3126 new_stmt
3127 = gimple_build_assign (make_ssa_name (vec_dest, NULL),
3128 vec_oprnd0);
3129 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3131 if ((unsigned) j == k - 1)
3132 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3133 else
3134 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3136 prev_stmt_info = vinfo_for_stmt (new_stmt);
3137 continue;
3139 else if (ratype)
3141 tree t = build_fold_addr_expr (new_temp);
3142 t = build2 (MEM_REF, vectype, t,
3143 build_int_cst (TREE_TYPE (t), 0));
3144 new_stmt
3145 = gimple_build_assign (make_ssa_name (vec_dest, NULL), t);
3146 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3147 tree clobber = build_constructor (ratype, NULL);
3148 TREE_THIS_VOLATILE (clobber) = 1;
3149 vect_finish_stmt_generation (stmt,
3150 gimple_build_assign (new_temp,
3151 clobber), gsi);
3155 if (j == 0)
3156 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3157 else
3158 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3160 prev_stmt_info = vinfo_for_stmt (new_stmt);
3163 vargs.release ();
3165 /* The call in STMT might prevent it from being removed in dce.
3166 We however cannot remove it here, due to the way the ssa name
3167 it defines is mapped to the new definition. So just replace
3168 rhs of the statement with something harmless. */
3170 if (slp_node)
3171 return true;
3173 if (scalar_dest)
3175 type = TREE_TYPE (scalar_dest);
3176 if (is_pattern_stmt_p (stmt_info))
3177 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
3178 else
3179 lhs = gimple_call_lhs (stmt);
3180 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
3182 else
3183 new_stmt = gimple_build_nop ();
3184 set_vinfo_for_stmt (new_stmt, stmt_info);
3185 set_vinfo_for_stmt (stmt, NULL);
3186 STMT_VINFO_STMT (stmt_info) = new_stmt;
3187 gsi_replace (gsi, new_stmt, false);
3188 unlink_stmt_vdef (stmt);
3190 return true;
3194 /* Function vect_gen_widened_results_half
3196 Create a vector stmt whose code, type, number of arguments, and result
3197 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3198 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3199 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3200 needs to be created (DECL is a function-decl of a target-builtin).
3201 STMT is the original scalar stmt that we are vectorizing. */
3203 static gimple
3204 vect_gen_widened_results_half (enum tree_code code,
3205 tree decl,
3206 tree vec_oprnd0, tree vec_oprnd1, int op_type,
3207 tree vec_dest, gimple_stmt_iterator *gsi,
3208 gimple stmt)
3210 gimple new_stmt;
3211 tree new_temp;
3213 /* Generate half of the widened result: */
3214 if (code == CALL_EXPR)
3216 /* Target specific support */
3217 if (op_type == binary_op)
3218 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
3219 else
3220 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
3221 new_temp = make_ssa_name (vec_dest, new_stmt);
3222 gimple_call_set_lhs (new_stmt, new_temp);
3224 else
3226 /* Generic support */
3227 gcc_assert (op_type == TREE_CODE_LENGTH (code));
3228 if (op_type != binary_op)
3229 vec_oprnd1 = NULL;
3230 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vec_oprnd0,
3231 vec_oprnd1);
3232 new_temp = make_ssa_name (vec_dest, new_stmt);
3233 gimple_assign_set_lhs (new_stmt, new_temp);
3235 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3237 return new_stmt;
3241 /* Get vectorized definitions for loop-based vectorization. For the first
3242 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3243 scalar operand), and for the rest we get a copy with
3244 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3245 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3246 The vectors are collected into VEC_OPRNDS. */
3248 static void
3249 vect_get_loop_based_defs (tree *oprnd, gimple stmt, enum vect_def_type dt,
3250 vec<tree> *vec_oprnds, int multi_step_cvt)
3252 tree vec_oprnd;
3254 /* Get first vector operand. */
3255 /* All the vector operands except the very first one (that is scalar oprnd)
3256 are stmt copies. */
3257 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
3258 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt, NULL);
3259 else
3260 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
3262 vec_oprnds->quick_push (vec_oprnd);
3264 /* Get second vector operand. */
3265 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
3266 vec_oprnds->quick_push (vec_oprnd);
3268 *oprnd = vec_oprnd;
3270 /* For conversion in multiple steps, continue to get operands
3271 recursively. */
3272 if (multi_step_cvt)
3273 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
3277 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3278 For multi-step conversions store the resulting vectors and call the function
3279 recursively. */
3281 static void
3282 vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
3283 int multi_step_cvt, gimple stmt,
3284 vec<tree> vec_dsts,
3285 gimple_stmt_iterator *gsi,
3286 slp_tree slp_node, enum tree_code code,
3287 stmt_vec_info *prev_stmt_info)
3289 unsigned int i;
3290 tree vop0, vop1, new_tmp, vec_dest;
3291 gimple new_stmt;
3292 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3294 vec_dest = vec_dsts.pop ();
3296 for (i = 0; i < vec_oprnds->length (); i += 2)
3298 /* Create demotion operation. */
3299 vop0 = (*vec_oprnds)[i];
3300 vop1 = (*vec_oprnds)[i + 1];
3301 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
3302 new_tmp = make_ssa_name (vec_dest, new_stmt);
3303 gimple_assign_set_lhs (new_stmt, new_tmp);
3304 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3306 if (multi_step_cvt)
3307 /* Store the resulting vector for next recursive call. */
3308 (*vec_oprnds)[i/2] = new_tmp;
3309 else
3311 /* This is the last step of the conversion sequence. Store the
3312 vectors in SLP_NODE or in vector info of the scalar statement
3313 (or in STMT_VINFO_RELATED_STMT chain). */
3314 if (slp_node)
3315 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3316 else
3318 if (!*prev_stmt_info)
3319 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
3320 else
3321 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
3323 *prev_stmt_info = vinfo_for_stmt (new_stmt);
3328 /* For multi-step demotion operations we first generate demotion operations
3329 from the source type to the intermediate types, and then combine the
3330 results (stored in VEC_OPRNDS) in demotion operation to the destination
3331 type. */
3332 if (multi_step_cvt)
3334 /* At each level of recursion we have half of the operands we had at the
3335 previous level. */
3336 vec_oprnds->truncate ((i+1)/2);
3337 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
3338 stmt, vec_dsts, gsi, slp_node,
3339 VEC_PACK_TRUNC_EXPR,
3340 prev_stmt_info);
3343 vec_dsts.quick_push (vec_dest);
3347 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3348 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3349 the resulting vectors and call the function recursively. */
3351 static void
3352 vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
3353 vec<tree> *vec_oprnds1,
3354 gimple stmt, tree vec_dest,
3355 gimple_stmt_iterator *gsi,
3356 enum tree_code code1,
3357 enum tree_code code2, tree decl1,
3358 tree decl2, int op_type)
3360 int i;
3361 tree vop0, vop1, new_tmp1, new_tmp2;
3362 gimple new_stmt1, new_stmt2;
3363 vec<tree> vec_tmp = vNULL;
3365 vec_tmp.create (vec_oprnds0->length () * 2);
3366 FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
3368 if (op_type == binary_op)
3369 vop1 = (*vec_oprnds1)[i];
3370 else
3371 vop1 = NULL_TREE;
3373 /* Generate the two halves of promotion operation. */
3374 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
3375 op_type, vec_dest, gsi, stmt);
3376 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
3377 op_type, vec_dest, gsi, stmt);
3378 if (is_gimple_call (new_stmt1))
3380 new_tmp1 = gimple_call_lhs (new_stmt1);
3381 new_tmp2 = gimple_call_lhs (new_stmt2);
3383 else
3385 new_tmp1 = gimple_assign_lhs (new_stmt1);
3386 new_tmp2 = gimple_assign_lhs (new_stmt2);
3389 /* Store the results for the next step. */
3390 vec_tmp.quick_push (new_tmp1);
3391 vec_tmp.quick_push (new_tmp2);
3394 vec_oprnds0->release ();
3395 *vec_oprnds0 = vec_tmp;
3399 /* Check if STMT performs a conversion operation, that can be vectorized.
3400 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3401 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3402 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3404 static bool
3405 vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi,
3406 gimple *vec_stmt, slp_tree slp_node)
3408 tree vec_dest;
3409 tree scalar_dest;
3410 tree op0, op1 = NULL_TREE;
3411 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
3412 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3413 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3414 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
3415 enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
3416 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
3417 tree new_temp;
3418 tree def;
3419 gimple def_stmt;
3420 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
3421 gimple new_stmt = NULL;
3422 stmt_vec_info prev_stmt_info;
3423 int nunits_in;
3424 int nunits_out;
3425 tree vectype_out, vectype_in;
3426 int ncopies, i, j;
3427 tree lhs_type, rhs_type;
3428 enum { NARROW, NONE, WIDEN } modifier;
3429 vec<tree> vec_oprnds0 = vNULL;
3430 vec<tree> vec_oprnds1 = vNULL;
3431 tree vop0;
3432 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3433 int multi_step_cvt = 0;
3434 vec<tree> vec_dsts = vNULL;
3435 vec<tree> interm_types = vNULL;
3436 tree last_oprnd, intermediate_type, cvt_type = NULL_TREE;
3437 int op_type;
3438 enum machine_mode rhs_mode;
3439 unsigned short fltsz;
3441 /* Is STMT a vectorizable conversion? */
3443 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3444 return false;
3446 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
3447 return false;
3449 if (!is_gimple_assign (stmt))
3450 return false;
3452 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
3453 return false;
3455 code = gimple_assign_rhs_code (stmt);
3456 if (!CONVERT_EXPR_CODE_P (code)
3457 && code != FIX_TRUNC_EXPR
3458 && code != FLOAT_EXPR
3459 && code != WIDEN_MULT_EXPR
3460 && code != WIDEN_LSHIFT_EXPR)
3461 return false;
3463 op_type = TREE_CODE_LENGTH (code);
3465 /* Check types of lhs and rhs. */
3466 scalar_dest = gimple_assign_lhs (stmt);
3467 lhs_type = TREE_TYPE (scalar_dest);
3468 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
3470 op0 = gimple_assign_rhs1 (stmt);
3471 rhs_type = TREE_TYPE (op0);
3473 if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
3474 && !((INTEGRAL_TYPE_P (lhs_type)
3475 && INTEGRAL_TYPE_P (rhs_type))
3476 || (SCALAR_FLOAT_TYPE_P (lhs_type)
3477 && SCALAR_FLOAT_TYPE_P (rhs_type))))
3478 return false;
3480 if ((INTEGRAL_TYPE_P (lhs_type)
3481 && (TYPE_PRECISION (lhs_type)
3482 != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
3483 || (INTEGRAL_TYPE_P (rhs_type)
3484 && (TYPE_PRECISION (rhs_type)
3485 != GET_MODE_PRECISION (TYPE_MODE (rhs_type)))))
3487 if (dump_enabled_p ())
3488 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3489 "type conversion to/from bit-precision unsupported."
3490 "\n");
3491 return false;
3494 /* Check the operands of the operation. */
3495 if (!vect_is_simple_use_1 (op0, stmt, loop_vinfo, bb_vinfo,
3496 &def_stmt, &def, &dt[0], &vectype_in))
3498 if (dump_enabled_p ())
3499 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3500 "use not simple.\n");
3501 return false;
3503 if (op_type == binary_op)
3505 bool ok;
3507 op1 = gimple_assign_rhs2 (stmt);
3508 gcc_assert (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR);
3509 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3510 OP1. */
3511 if (CONSTANT_CLASS_P (op0))
3512 ok = vect_is_simple_use_1 (op1, stmt, loop_vinfo, bb_vinfo,
3513 &def_stmt, &def, &dt[1], &vectype_in);
3514 else
3515 ok = vect_is_simple_use (op1, stmt, loop_vinfo, bb_vinfo, &def_stmt,
3516 &def, &dt[1]);
3518 if (!ok)
3520 if (dump_enabled_p ())
3521 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3522 "use not simple.\n");
3523 return false;
3527 /* If op0 is an external or constant defs use a vector type of
3528 the same size as the output vector type. */
3529 if (!vectype_in)
3530 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
3531 if (vec_stmt)
3532 gcc_assert (vectype_in);
3533 if (!vectype_in)
3535 if (dump_enabled_p ())
3537 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3538 "no vectype for scalar type ");
3539 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
3540 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
3543 return false;
3546 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3547 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3548 if (nunits_in < nunits_out)
3549 modifier = NARROW;
3550 else if (nunits_out == nunits_in)
3551 modifier = NONE;
3552 else
3553 modifier = WIDEN;
3555 /* Multiple types in SLP are handled by creating the appropriate number of
3556 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3557 case of SLP. */
3558 if (slp_node || PURE_SLP_STMT (stmt_info))
3559 ncopies = 1;
3560 else if (modifier == NARROW)
3561 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
3562 else
3563 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
3565 /* Sanity check: make sure that at least one copy of the vectorized stmt
3566 needs to be generated. */
3567 gcc_assert (ncopies >= 1);
3569 /* Supportable by target? */
3570 switch (modifier)
3572 case NONE:
3573 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
3574 return false;
3575 if (supportable_convert_operation (code, vectype_out, vectype_in,
3576 &decl1, &code1))
3577 break;
3578 /* FALLTHRU */
3579 unsupported:
3580 if (dump_enabled_p ())
3581 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3582 "conversion not supported by target.\n");
3583 return false;
3585 case WIDEN:
3586 if (supportable_widening_operation (code, stmt, vectype_out, vectype_in,
3587 &code1, &code2, &multi_step_cvt,
3588 &interm_types))
3590 /* Binary widening operation can only be supported directly by the
3591 architecture. */
3592 gcc_assert (!(multi_step_cvt && op_type == binary_op));
3593 break;
3596 if (code != FLOAT_EXPR
3597 || (GET_MODE_SIZE (TYPE_MODE (lhs_type))
3598 <= GET_MODE_SIZE (TYPE_MODE (rhs_type))))
3599 goto unsupported;
3601 rhs_mode = TYPE_MODE (rhs_type);
3602 fltsz = GET_MODE_SIZE (TYPE_MODE (lhs_type));
3603 for (rhs_mode = GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type));
3604 rhs_mode != VOIDmode && GET_MODE_SIZE (rhs_mode) <= fltsz;
3605 rhs_mode = GET_MODE_2XWIDER_MODE (rhs_mode))
3607 cvt_type
3608 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
3609 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
3610 if (cvt_type == NULL_TREE)
3611 goto unsupported;
3613 if (GET_MODE_SIZE (rhs_mode) == fltsz)
3615 if (!supportable_convert_operation (code, vectype_out,
3616 cvt_type, &decl1, &codecvt1))
3617 goto unsupported;
3619 else if (!supportable_widening_operation (code, stmt, vectype_out,
3620 cvt_type, &codecvt1,
3621 &codecvt2, &multi_step_cvt,
3622 &interm_types))
3623 continue;
3624 else
3625 gcc_assert (multi_step_cvt == 0);
3627 if (supportable_widening_operation (NOP_EXPR, stmt, cvt_type,
3628 vectype_in, &code1, &code2,
3629 &multi_step_cvt, &interm_types))
3630 break;
3633 if (rhs_mode == VOIDmode || GET_MODE_SIZE (rhs_mode) > fltsz)
3634 goto unsupported;
3636 if (GET_MODE_SIZE (rhs_mode) == fltsz)
3637 codecvt2 = ERROR_MARK;
3638 else
3640 multi_step_cvt++;
3641 interm_types.safe_push (cvt_type);
3642 cvt_type = NULL_TREE;
3644 break;
3646 case NARROW:
3647 gcc_assert (op_type == unary_op);
3648 if (supportable_narrowing_operation (code, vectype_out, vectype_in,
3649 &code1, &multi_step_cvt,
3650 &interm_types))
3651 break;
3653 if (code != FIX_TRUNC_EXPR
3654 || (GET_MODE_SIZE (TYPE_MODE (lhs_type))
3655 >= GET_MODE_SIZE (TYPE_MODE (rhs_type))))
3656 goto unsupported;
3658 rhs_mode = TYPE_MODE (rhs_type);
3659 cvt_type
3660 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
3661 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
3662 if (cvt_type == NULL_TREE)
3663 goto unsupported;
3664 if (!supportable_convert_operation (code, cvt_type, vectype_in,
3665 &decl1, &codecvt1))
3666 goto unsupported;
3667 if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
3668 &code1, &multi_step_cvt,
3669 &interm_types))
3670 break;
3671 goto unsupported;
3673 default:
3674 gcc_unreachable ();
3677 if (!vec_stmt) /* transformation not required. */
3679 if (dump_enabled_p ())
3680 dump_printf_loc (MSG_NOTE, vect_location,
3681 "=== vectorizable_conversion ===\n");
3682 if (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR)
3684 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
3685 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
3687 else if (modifier == NARROW)
3689 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
3690 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
3692 else
3694 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
3695 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
3697 interm_types.release ();
3698 return true;
3701 /** Transform. **/
3702 if (dump_enabled_p ())
3703 dump_printf_loc (MSG_NOTE, vect_location,
3704 "transform conversion. ncopies = %d.\n", ncopies);
3706 if (op_type == binary_op)
3708 if (CONSTANT_CLASS_P (op0))
3709 op0 = fold_convert (TREE_TYPE (op1), op0);
3710 else if (CONSTANT_CLASS_P (op1))
3711 op1 = fold_convert (TREE_TYPE (op0), op1);
3714 /* In case of multi-step conversion, we first generate conversion operations
3715 to the intermediate types, and then from that types to the final one.
3716 We create vector destinations for the intermediate type (TYPES) received
3717 from supportable_*_operation, and store them in the correct order
3718 for future use in vect_create_vectorized_*_stmts (). */
3719 vec_dsts.create (multi_step_cvt + 1);
3720 vec_dest = vect_create_destination_var (scalar_dest,
3721 (cvt_type && modifier == WIDEN)
3722 ? cvt_type : vectype_out);
3723 vec_dsts.quick_push (vec_dest);
3725 if (multi_step_cvt)
3727 for (i = interm_types.length () - 1;
3728 interm_types.iterate (i, &intermediate_type); i--)
3730 vec_dest = vect_create_destination_var (scalar_dest,
3731 intermediate_type);
3732 vec_dsts.quick_push (vec_dest);
3736 if (cvt_type)
3737 vec_dest = vect_create_destination_var (scalar_dest,
3738 modifier == WIDEN
3739 ? vectype_out : cvt_type);
3741 if (!slp_node)
3743 if (modifier == WIDEN)
3745 vec_oprnds0.create (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1);
3746 if (op_type == binary_op)
3747 vec_oprnds1.create (1);
3749 else if (modifier == NARROW)
3750 vec_oprnds0.create (
3751 2 * (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
3753 else if (code == WIDEN_LSHIFT_EXPR)
3754 vec_oprnds1.create (slp_node->vec_stmts_size);
3756 last_oprnd = op0;
3757 prev_stmt_info = NULL;
3758 switch (modifier)
3760 case NONE:
3761 for (j = 0; j < ncopies; j++)
3763 if (j == 0)
3764 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node,
3765 -1);
3766 else
3767 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
3769 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
3771 /* Arguments are ready, create the new vector stmt. */
3772 if (code1 == CALL_EXPR)
3774 new_stmt = gimple_build_call (decl1, 1, vop0);
3775 new_temp = make_ssa_name (vec_dest, new_stmt);
3776 gimple_call_set_lhs (new_stmt, new_temp);
3778 else
3780 gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
3781 new_stmt = gimple_build_assign_with_ops (code1, vec_dest,
3782 vop0, NULL);
3783 new_temp = make_ssa_name (vec_dest, new_stmt);
3784 gimple_assign_set_lhs (new_stmt, new_temp);
3787 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3788 if (slp_node)
3789 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3792 if (j == 0)
3793 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3794 else
3795 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3796 prev_stmt_info = vinfo_for_stmt (new_stmt);
3798 break;
3800 case WIDEN:
3801 /* In case the vectorization factor (VF) is bigger than the number
3802 of elements that we can fit in a vectype (nunits), we have to
3803 generate more than one vector stmt - i.e - we need to "unroll"
3804 the vector stmt by a factor VF/nunits. */
3805 for (j = 0; j < ncopies; j++)
3807 /* Handle uses. */
3808 if (j == 0)
3810 if (slp_node)
3812 if (code == WIDEN_LSHIFT_EXPR)
3814 unsigned int k;
3816 vec_oprnd1 = op1;
3817 /* Store vec_oprnd1 for every vector stmt to be created
3818 for SLP_NODE. We check during the analysis that all
3819 the shift arguments are the same. */
3820 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
3821 vec_oprnds1.quick_push (vec_oprnd1);
3823 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
3824 slp_node, -1);
3826 else
3827 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0,
3828 &vec_oprnds1, slp_node, -1);
3830 else
3832 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
3833 vec_oprnds0.quick_push (vec_oprnd0);
3834 if (op_type == binary_op)
3836 if (code == WIDEN_LSHIFT_EXPR)
3837 vec_oprnd1 = op1;
3838 else
3839 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt,
3840 NULL);
3841 vec_oprnds1.quick_push (vec_oprnd1);
3845 else
3847 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
3848 vec_oprnds0.truncate (0);
3849 vec_oprnds0.quick_push (vec_oprnd0);
3850 if (op_type == binary_op)
3852 if (code == WIDEN_LSHIFT_EXPR)
3853 vec_oprnd1 = op1;
3854 else
3855 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1],
3856 vec_oprnd1);
3857 vec_oprnds1.truncate (0);
3858 vec_oprnds1.quick_push (vec_oprnd1);
3862 /* Arguments are ready. Create the new vector stmts. */
3863 for (i = multi_step_cvt; i >= 0; i--)
3865 tree this_dest = vec_dsts[i];
3866 enum tree_code c1 = code1, c2 = code2;
3867 if (i == 0 && codecvt2 != ERROR_MARK)
3869 c1 = codecvt1;
3870 c2 = codecvt2;
3872 vect_create_vectorized_promotion_stmts (&vec_oprnds0,
3873 &vec_oprnds1,
3874 stmt, this_dest, gsi,
3875 c1, c2, decl1, decl2,
3876 op_type);
3879 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
3881 if (cvt_type)
3883 if (codecvt1 == CALL_EXPR)
3885 new_stmt = gimple_build_call (decl1, 1, vop0);
3886 new_temp = make_ssa_name (vec_dest, new_stmt);
3887 gimple_call_set_lhs (new_stmt, new_temp);
3889 else
3891 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
3892 new_temp = make_ssa_name (vec_dest, NULL);
3893 new_stmt = gimple_build_assign_with_ops (codecvt1,
3894 new_temp,
3895 vop0, NULL);
3898 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3900 else
3901 new_stmt = SSA_NAME_DEF_STMT (vop0);
3903 if (slp_node)
3904 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3905 else
3907 if (!prev_stmt_info)
3908 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
3909 else
3910 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3911 prev_stmt_info = vinfo_for_stmt (new_stmt);
3916 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3917 break;
3919 case NARROW:
3920 /* In case the vectorization factor (VF) is bigger than the number
3921 of elements that we can fit in a vectype (nunits), we have to
3922 generate more than one vector stmt - i.e - we need to "unroll"
3923 the vector stmt by a factor VF/nunits. */
3924 for (j = 0; j < ncopies; j++)
3926 /* Handle uses. */
3927 if (slp_node)
3928 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
3929 slp_node, -1);
3930 else
3932 vec_oprnds0.truncate (0);
3933 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
3934 vect_pow2 (multi_step_cvt) - 1);
3937 /* Arguments are ready. Create the new vector stmts. */
3938 if (cvt_type)
3939 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
3941 if (codecvt1 == CALL_EXPR)
3943 new_stmt = gimple_build_call (decl1, 1, vop0);
3944 new_temp = make_ssa_name (vec_dest, new_stmt);
3945 gimple_call_set_lhs (new_stmt, new_temp);
3947 else
3949 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
3950 new_temp = make_ssa_name (vec_dest, NULL);
3951 new_stmt = gimple_build_assign_with_ops (codecvt1, new_temp,
3952 vop0, NULL);
3955 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3956 vec_oprnds0[i] = new_temp;
3959 vect_create_vectorized_demotion_stmts (&vec_oprnds0, multi_step_cvt,
3960 stmt, vec_dsts, gsi,
3961 slp_node, code1,
3962 &prev_stmt_info);
3965 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
3966 break;
3969 vec_oprnds0.release ();
3970 vec_oprnds1.release ();
3971 vec_dsts.release ();
3972 interm_types.release ();
3974 return true;
3978 /* Function vectorizable_assignment.
3980 Check if STMT performs an assignment (copy) that can be vectorized.
3981 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3982 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3983 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3985 static bool
3986 vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi,
3987 gimple *vec_stmt, slp_tree slp_node)
3989 tree vec_dest;
3990 tree scalar_dest;
3991 tree op;
3992 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3993 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
3994 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3995 tree new_temp;
3996 tree def;
3997 gimple def_stmt;
3998 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
3999 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
4000 int ncopies;
4001 int i, j;
4002 vec<tree> vec_oprnds = vNULL;
4003 tree vop;
4004 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4005 gimple new_stmt = NULL;
4006 stmt_vec_info prev_stmt_info = NULL;
4007 enum tree_code code;
4008 tree vectype_in;
4010 /* Multiple types in SLP are handled by creating the appropriate number of
4011 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4012 case of SLP. */
4013 if (slp_node || PURE_SLP_STMT (stmt_info))
4014 ncopies = 1;
4015 else
4016 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
4018 gcc_assert (ncopies >= 1);
4020 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4021 return false;
4023 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
4024 return false;
4026 /* Is vectorizable assignment? */
4027 if (!is_gimple_assign (stmt))
4028 return false;
4030 scalar_dest = gimple_assign_lhs (stmt);
4031 if (TREE_CODE (scalar_dest) != SSA_NAME)
4032 return false;
4034 code = gimple_assign_rhs_code (stmt);
4035 if (gimple_assign_single_p (stmt)
4036 || code == PAREN_EXPR
4037 || CONVERT_EXPR_CODE_P (code))
4038 op = gimple_assign_rhs1 (stmt);
4039 else
4040 return false;
4042 if (code == VIEW_CONVERT_EXPR)
4043 op = TREE_OPERAND (op, 0);
4045 if (!vect_is_simple_use_1 (op, stmt, loop_vinfo, bb_vinfo,
4046 &def_stmt, &def, &dt[0], &vectype_in))
4048 if (dump_enabled_p ())
4049 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4050 "use not simple.\n");
4051 return false;
4054 /* We can handle NOP_EXPR conversions that do not change the number
4055 of elements or the vector size. */
4056 if ((CONVERT_EXPR_CODE_P (code)
4057 || code == VIEW_CONVERT_EXPR)
4058 && (!vectype_in
4059 || TYPE_VECTOR_SUBPARTS (vectype_in) != nunits
4060 || (GET_MODE_SIZE (TYPE_MODE (vectype))
4061 != GET_MODE_SIZE (TYPE_MODE (vectype_in)))))
4062 return false;
4064 /* We do not handle bit-precision changes. */
4065 if ((CONVERT_EXPR_CODE_P (code)
4066 || code == VIEW_CONVERT_EXPR)
4067 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
4068 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest))
4069 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4070 || ((TYPE_PRECISION (TREE_TYPE (op))
4071 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op))))))
4072 /* But a conversion that does not change the bit-pattern is ok. */
4073 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest))
4074 > TYPE_PRECISION (TREE_TYPE (op)))
4075 && TYPE_UNSIGNED (TREE_TYPE (op))))
4077 if (dump_enabled_p ())
4078 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4079 "type conversion to/from bit-precision "
4080 "unsupported.\n");
4081 return false;
4084 if (!vec_stmt) /* transformation not required. */
4086 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
4087 if (dump_enabled_p ())
4088 dump_printf_loc (MSG_NOTE, vect_location,
4089 "=== vectorizable_assignment ===\n");
4090 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4091 return true;
4094 /** Transform. **/
4095 if (dump_enabled_p ())
4096 dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
4098 /* Handle def. */
4099 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4101 /* Handle use. */
4102 for (j = 0; j < ncopies; j++)
4104 /* Handle uses. */
4105 if (j == 0)
4106 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node, -1);
4107 else
4108 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
4110 /* Arguments are ready. create the new vector stmt. */
4111 FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
4113 if (CONVERT_EXPR_CODE_P (code)
4114 || code == VIEW_CONVERT_EXPR)
4115 vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
4116 new_stmt = gimple_build_assign (vec_dest, vop);
4117 new_temp = make_ssa_name (vec_dest, new_stmt);
4118 gimple_assign_set_lhs (new_stmt, new_temp);
4119 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4120 if (slp_node)
4121 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4124 if (slp_node)
4125 continue;
4127 if (j == 0)
4128 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4129 else
4130 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4132 prev_stmt_info = vinfo_for_stmt (new_stmt);
4135 vec_oprnds.release ();
4136 return true;
4140 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4141 either as shift by a scalar or by a vector. */
4143 bool
4144 vect_supportable_shift (enum tree_code code, tree scalar_type)
4147 enum machine_mode vec_mode;
4148 optab optab;
4149 int icode;
4150 tree vectype;
4152 vectype = get_vectype_for_scalar_type (scalar_type);
4153 if (!vectype)
4154 return false;
4156 optab = optab_for_tree_code (code, vectype, optab_scalar);
4157 if (!optab
4158 || optab_handler (optab, TYPE_MODE (vectype)) == CODE_FOR_nothing)
4160 optab = optab_for_tree_code (code, vectype, optab_vector);
4161 if (!optab
4162 || (optab_handler (optab, TYPE_MODE (vectype))
4163 == CODE_FOR_nothing))
4164 return false;
4167 vec_mode = TYPE_MODE (vectype);
4168 icode = (int) optab_handler (optab, vec_mode);
4169 if (icode == CODE_FOR_nothing)
4170 return false;
4172 return true;
4176 /* Function vectorizable_shift.
4178 Check if STMT performs a shift operation that can be vectorized.
4179 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4180 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4181 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4183 static bool
4184 vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi,
4185 gimple *vec_stmt, slp_tree slp_node)
4187 tree vec_dest;
4188 tree scalar_dest;
4189 tree op0, op1 = NULL;
4190 tree vec_oprnd1 = NULL_TREE;
4191 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4192 tree vectype;
4193 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4194 enum tree_code code;
4195 enum machine_mode vec_mode;
4196 tree new_temp;
4197 optab optab;
4198 int icode;
4199 enum machine_mode optab_op2_mode;
4200 tree def;
4201 gimple def_stmt;
4202 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
4203 gimple new_stmt = NULL;
4204 stmt_vec_info prev_stmt_info;
4205 int nunits_in;
4206 int nunits_out;
4207 tree vectype_out;
4208 tree op1_vectype;
4209 int ncopies;
4210 int j, i;
4211 vec<tree> vec_oprnds0 = vNULL;
4212 vec<tree> vec_oprnds1 = vNULL;
4213 tree vop0, vop1;
4214 unsigned int k;
4215 bool scalar_shift_arg = true;
4216 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4217 int vf;
4219 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4220 return false;
4222 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
4223 return false;
4225 /* Is STMT a vectorizable binary/unary operation? */
4226 if (!is_gimple_assign (stmt))
4227 return false;
4229 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
4230 return false;
4232 code = gimple_assign_rhs_code (stmt);
4234 if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
4235 || code == RROTATE_EXPR))
4236 return false;
4238 scalar_dest = gimple_assign_lhs (stmt);
4239 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
4240 if (TYPE_PRECISION (TREE_TYPE (scalar_dest))
4241 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4243 if (dump_enabled_p ())
4244 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4245 "bit-precision shifts not supported.\n");
4246 return false;
4249 op0 = gimple_assign_rhs1 (stmt);
4250 if (!vect_is_simple_use_1 (op0, stmt, loop_vinfo, bb_vinfo,
4251 &def_stmt, &def, &dt[0], &vectype))
4253 if (dump_enabled_p ())
4254 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4255 "use not simple.\n");
4256 return false;
4258 /* If op0 is an external or constant def use a vector type with
4259 the same size as the output vector type. */
4260 if (!vectype)
4261 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
4262 if (vec_stmt)
4263 gcc_assert (vectype);
4264 if (!vectype)
4266 if (dump_enabled_p ())
4267 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4268 "no vectype for scalar type\n");
4269 return false;
4272 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
4273 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
4274 if (nunits_out != nunits_in)
4275 return false;
4277 op1 = gimple_assign_rhs2 (stmt);
4278 if (!vect_is_simple_use_1 (op1, stmt, loop_vinfo, bb_vinfo, &def_stmt,
4279 &def, &dt[1], &op1_vectype))
4281 if (dump_enabled_p ())
4282 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4283 "use not simple.\n");
4284 return false;
4287 if (loop_vinfo)
4288 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4289 else
4290 vf = 1;
4292 /* Multiple types in SLP are handled by creating the appropriate number of
4293 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4294 case of SLP. */
4295 if (slp_node || PURE_SLP_STMT (stmt_info))
4296 ncopies = 1;
4297 else
4298 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
4300 gcc_assert (ncopies >= 1);
4302 /* Determine whether the shift amount is a vector, or scalar. If the
4303 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4305 if (dt[1] == vect_internal_def && !slp_node)
4306 scalar_shift_arg = false;
4307 else if (dt[1] == vect_constant_def
4308 || dt[1] == vect_external_def
4309 || dt[1] == vect_internal_def)
4311 /* In SLP, need to check whether the shift count is the same,
4312 in loops if it is a constant or invariant, it is always
4313 a scalar shift. */
4314 if (slp_node)
4316 vec<gimple> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
4317 gimple slpstmt;
4319 FOR_EACH_VEC_ELT (stmts, k, slpstmt)
4320 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
4321 scalar_shift_arg = false;
4324 else
4326 if (dump_enabled_p ())
4327 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4328 "operand mode requires invariant argument.\n");
4329 return false;
4332 /* Vector shifted by vector. */
4333 if (!scalar_shift_arg)
4335 optab = optab_for_tree_code (code, vectype, optab_vector);
4336 if (dump_enabled_p ())
4337 dump_printf_loc (MSG_NOTE, vect_location,
4338 "vector/vector shift/rotate found.\n");
4340 if (!op1_vectype)
4341 op1_vectype = get_same_sized_vectype (TREE_TYPE (op1), vectype_out);
4342 if (op1_vectype == NULL_TREE
4343 || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype))
4345 if (dump_enabled_p ())
4346 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4347 "unusable type for last operand in"
4348 " vector/vector shift/rotate.\n");
4349 return false;
4352 /* See if the machine has a vector shifted by scalar insn and if not
4353 then see if it has a vector shifted by vector insn. */
4354 else
4356 optab = optab_for_tree_code (code, vectype, optab_scalar);
4357 if (optab
4358 && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
4360 if (dump_enabled_p ())
4361 dump_printf_loc (MSG_NOTE, vect_location,
4362 "vector/scalar shift/rotate found.\n");
4364 else
4366 optab = optab_for_tree_code (code, vectype, optab_vector);
4367 if (optab
4368 && (optab_handler (optab, TYPE_MODE (vectype))
4369 != CODE_FOR_nothing))
4371 scalar_shift_arg = false;
4373 if (dump_enabled_p ())
4374 dump_printf_loc (MSG_NOTE, vect_location,
4375 "vector/vector shift/rotate found.\n");
4377 /* Unlike the other binary operators, shifts/rotates have
4378 the rhs being int, instead of the same type as the lhs,
4379 so make sure the scalar is the right type if we are
4380 dealing with vectors of long long/long/short/char. */
4381 if (dt[1] == vect_constant_def)
4382 op1 = fold_convert (TREE_TYPE (vectype), op1);
4383 else if (!useless_type_conversion_p (TREE_TYPE (vectype),
4384 TREE_TYPE (op1)))
4386 if (slp_node
4387 && TYPE_MODE (TREE_TYPE (vectype))
4388 != TYPE_MODE (TREE_TYPE (op1)))
4390 if (dump_enabled_p ())
4391 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4392 "unusable type for last operand in"
4393 " vector/vector shift/rotate.\n");
4394 return false;
4396 if (vec_stmt && !slp_node)
4398 op1 = fold_convert (TREE_TYPE (vectype), op1);
4399 op1 = vect_init_vector (stmt, op1,
4400 TREE_TYPE (vectype), NULL);
4407 /* Supportable by target? */
4408 if (!optab)
4410 if (dump_enabled_p ())
4411 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4412 "no optab.\n");
4413 return false;
4415 vec_mode = TYPE_MODE (vectype);
4416 icode = (int) optab_handler (optab, vec_mode);
4417 if (icode == CODE_FOR_nothing)
4419 if (dump_enabled_p ())
4420 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4421 "op not supported by target.\n");
4422 /* Check only during analysis. */
4423 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
4424 || (vf < vect_min_worthwhile_factor (code)
4425 && !vec_stmt))
4426 return false;
4427 if (dump_enabled_p ())
4428 dump_printf_loc (MSG_NOTE, vect_location,
4429 "proceeding using word mode.\n");
4432 /* Worthwhile without SIMD support? Check only during analysis. */
4433 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
4434 && vf < vect_min_worthwhile_factor (code)
4435 && !vec_stmt)
4437 if (dump_enabled_p ())
4438 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4439 "not worthwhile without SIMD support.\n");
4440 return false;
4443 if (!vec_stmt) /* transformation not required. */
4445 STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
4446 if (dump_enabled_p ())
4447 dump_printf_loc (MSG_NOTE, vect_location,
4448 "=== vectorizable_shift ===\n");
4449 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4450 return true;
4453 /** Transform. **/
4455 if (dump_enabled_p ())
4456 dump_printf_loc (MSG_NOTE, vect_location,
4457 "transform binary/unary operation.\n");
4459 /* Handle def. */
4460 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4462 prev_stmt_info = NULL;
4463 for (j = 0; j < ncopies; j++)
4465 /* Handle uses. */
4466 if (j == 0)
4468 if (scalar_shift_arg)
4470 /* Vector shl and shr insn patterns can be defined with scalar
4471 operand 2 (shift operand). In this case, use constant or loop
4472 invariant op1 directly, without extending it to vector mode
4473 first. */
4474 optab_op2_mode = insn_data[icode].operand[2].mode;
4475 if (!VECTOR_MODE_P (optab_op2_mode))
4477 if (dump_enabled_p ())
4478 dump_printf_loc (MSG_NOTE, vect_location,
4479 "operand 1 using scalar mode.\n");
4480 vec_oprnd1 = op1;
4481 vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : 1);
4482 vec_oprnds1.quick_push (vec_oprnd1);
4483 if (slp_node)
4485 /* Store vec_oprnd1 for every vector stmt to be created
4486 for SLP_NODE. We check during the analysis that all
4487 the shift arguments are the same.
4488 TODO: Allow different constants for different vector
4489 stmts generated for an SLP instance. */
4490 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
4491 vec_oprnds1.quick_push (vec_oprnd1);
4496 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4497 (a special case for certain kind of vector shifts); otherwise,
4498 operand 1 should be of a vector type (the usual case). */
4499 if (vec_oprnd1)
4500 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4501 slp_node, -1);
4502 else
4503 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
4504 slp_node, -1);
4506 else
4507 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
4509 /* Arguments are ready. Create the new vector stmt. */
4510 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4512 vop1 = vec_oprnds1[i];
4513 new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1);
4514 new_temp = make_ssa_name (vec_dest, new_stmt);
4515 gimple_assign_set_lhs (new_stmt, new_temp);
4516 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4517 if (slp_node)
4518 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4521 if (slp_node)
4522 continue;
4524 if (j == 0)
4525 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4526 else
4527 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4528 prev_stmt_info = vinfo_for_stmt (new_stmt);
4531 vec_oprnds0.release ();
4532 vec_oprnds1.release ();
4534 return true;
4538 /* Function vectorizable_operation.
4540 Check if STMT performs a binary, unary or ternary operation that can
4541 be vectorized.
4542 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4543 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4544 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4546 static bool
4547 vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi,
4548 gimple *vec_stmt, slp_tree slp_node)
4550 tree vec_dest;
4551 tree scalar_dest;
4552 tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
4553 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4554 tree vectype;
4555 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4556 enum tree_code code;
4557 enum machine_mode vec_mode;
4558 tree new_temp;
4559 int op_type;
4560 optab optab;
4561 int icode;
4562 tree def;
4563 gimple def_stmt;
4564 enum vect_def_type dt[3]
4565 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
4566 gimple new_stmt = NULL;
4567 stmt_vec_info prev_stmt_info;
4568 int nunits_in;
4569 int nunits_out;
4570 tree vectype_out;
4571 int ncopies;
4572 int j, i;
4573 vec<tree> vec_oprnds0 = vNULL;
4574 vec<tree> vec_oprnds1 = vNULL;
4575 vec<tree> vec_oprnds2 = vNULL;
4576 tree vop0, vop1, vop2;
4577 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4578 int vf;
4580 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4581 return false;
4583 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
4584 return false;
4586 /* Is STMT a vectorizable binary/unary operation? */
4587 if (!is_gimple_assign (stmt))
4588 return false;
4590 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
4591 return false;
4593 code = gimple_assign_rhs_code (stmt);
4595 /* For pointer addition, we should use the normal plus for
4596 the vector addition. */
4597 if (code == POINTER_PLUS_EXPR)
4598 code = PLUS_EXPR;
4600 /* Support only unary or binary operations. */
4601 op_type = TREE_CODE_LENGTH (code);
4602 if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
4604 if (dump_enabled_p ())
4605 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4606 "num. args = %d (not unary/binary/ternary op).\n",
4607 op_type);
4608 return false;
4611 scalar_dest = gimple_assign_lhs (stmt);
4612 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
4614 /* Most operations cannot handle bit-precision types without extra
4615 truncations. */
4616 if ((TYPE_PRECISION (TREE_TYPE (scalar_dest))
4617 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4618 /* Exception are bitwise binary operations. */
4619 && code != BIT_IOR_EXPR
4620 && code != BIT_XOR_EXPR
4621 && code != BIT_AND_EXPR)
4623 if (dump_enabled_p ())
4624 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4625 "bit-precision arithmetic not supported.\n");
4626 return false;
4629 op0 = gimple_assign_rhs1 (stmt);
4630 if (!vect_is_simple_use_1 (op0, stmt, loop_vinfo, bb_vinfo,
4631 &def_stmt, &def, &dt[0], &vectype))
4633 if (dump_enabled_p ())
4634 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4635 "use not simple.\n");
4636 return false;
4638 /* If op0 is an external or constant def use a vector type with
4639 the same size as the output vector type. */
4640 if (!vectype)
4641 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
4642 if (vec_stmt)
4643 gcc_assert (vectype);
4644 if (!vectype)
4646 if (dump_enabled_p ())
4648 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4649 "no vectype for scalar type ");
4650 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
4651 TREE_TYPE (op0));
4652 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
4655 return false;
4658 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
4659 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
4660 if (nunits_out != nunits_in)
4661 return false;
4663 if (op_type == binary_op || op_type == ternary_op)
4665 op1 = gimple_assign_rhs2 (stmt);
4666 if (!vect_is_simple_use (op1, stmt, loop_vinfo, bb_vinfo, &def_stmt,
4667 &def, &dt[1]))
4669 if (dump_enabled_p ())
4670 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4671 "use not simple.\n");
4672 return false;
4675 if (op_type == ternary_op)
4677 op2 = gimple_assign_rhs3 (stmt);
4678 if (!vect_is_simple_use (op2, stmt, loop_vinfo, bb_vinfo, &def_stmt,
4679 &def, &dt[2]))
4681 if (dump_enabled_p ())
4682 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4683 "use not simple.\n");
4684 return false;
4688 if (loop_vinfo)
4689 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4690 else
4691 vf = 1;
4693 /* Multiple types in SLP are handled by creating the appropriate number of
4694 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4695 case of SLP. */
4696 if (slp_node || PURE_SLP_STMT (stmt_info))
4697 ncopies = 1;
4698 else
4699 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
4701 gcc_assert (ncopies >= 1);
4703 /* Shifts are handled in vectorizable_shift (). */
4704 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
4705 || code == RROTATE_EXPR)
4706 return false;
4708 /* Supportable by target? */
4710 vec_mode = TYPE_MODE (vectype);
4711 if (code == MULT_HIGHPART_EXPR)
4713 if (can_mult_highpart_p (vec_mode, TYPE_UNSIGNED (vectype)))
4714 icode = LAST_INSN_CODE;
4715 else
4716 icode = CODE_FOR_nothing;
4718 else
4720 optab = optab_for_tree_code (code, vectype, optab_default);
4721 if (!optab)
4723 if (dump_enabled_p ())
4724 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4725 "no optab.\n");
4726 return false;
4728 icode = (int) optab_handler (optab, vec_mode);
4731 if (icode == CODE_FOR_nothing)
4733 if (dump_enabled_p ())
4734 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4735 "op not supported by target.\n");
4736 /* Check only during analysis. */
4737 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
4738 || (!vec_stmt && vf < vect_min_worthwhile_factor (code)))
4739 return false;
4740 if (dump_enabled_p ())
4741 dump_printf_loc (MSG_NOTE, vect_location,
4742 "proceeding using word mode.\n");
4745 /* Worthwhile without SIMD support? Check only during analysis. */
4746 if (!VECTOR_MODE_P (vec_mode)
4747 && !vec_stmt
4748 && vf < vect_min_worthwhile_factor (code))
4750 if (dump_enabled_p ())
4751 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4752 "not worthwhile without SIMD support.\n");
4753 return false;
4756 if (!vec_stmt) /* transformation not required. */
4758 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
4759 if (dump_enabled_p ())
4760 dump_printf_loc (MSG_NOTE, vect_location,
4761 "=== vectorizable_operation ===\n");
4762 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4763 return true;
4766 /** Transform. **/
4768 if (dump_enabled_p ())
4769 dump_printf_loc (MSG_NOTE, vect_location,
4770 "transform binary/unary operation.\n");
4772 /* Handle def. */
4773 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4775 /* In case the vectorization factor (VF) is bigger than the number
4776 of elements that we can fit in a vectype (nunits), we have to generate
4777 more than one vector stmt - i.e - we need to "unroll" the
4778 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4779 from one copy of the vector stmt to the next, in the field
4780 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4781 stages to find the correct vector defs to be used when vectorizing
4782 stmts that use the defs of the current stmt. The example below
4783 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4784 we need to create 4 vectorized stmts):
4786 before vectorization:
4787 RELATED_STMT VEC_STMT
4788 S1: x = memref - -
4789 S2: z = x + 1 - -
4791 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4792 there):
4793 RELATED_STMT VEC_STMT
4794 VS1_0: vx0 = memref0 VS1_1 -
4795 VS1_1: vx1 = memref1 VS1_2 -
4796 VS1_2: vx2 = memref2 VS1_3 -
4797 VS1_3: vx3 = memref3 - -
4798 S1: x = load - VS1_0
4799 S2: z = x + 1 - -
4801 step2: vectorize stmt S2 (done here):
4802 To vectorize stmt S2 we first need to find the relevant vector
4803 def for the first operand 'x'. This is, as usual, obtained from
4804 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4805 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4806 relevant vector def 'vx0'. Having found 'vx0' we can generate
4807 the vector stmt VS2_0, and as usual, record it in the
4808 STMT_VINFO_VEC_STMT of stmt S2.
4809 When creating the second copy (VS2_1), we obtain the relevant vector
4810 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4811 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4812 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4813 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4814 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4815 chain of stmts and pointers:
4816 RELATED_STMT VEC_STMT
4817 VS1_0: vx0 = memref0 VS1_1 -
4818 VS1_1: vx1 = memref1 VS1_2 -
4819 VS1_2: vx2 = memref2 VS1_3 -
4820 VS1_3: vx3 = memref3 - -
4821 S1: x = load - VS1_0
4822 VS2_0: vz0 = vx0 + v1 VS2_1 -
4823 VS2_1: vz1 = vx1 + v1 VS2_2 -
4824 VS2_2: vz2 = vx2 + v1 VS2_3 -
4825 VS2_3: vz3 = vx3 + v1 - -
4826 S2: z = x + 1 - VS2_0 */
4828 prev_stmt_info = NULL;
4829 for (j = 0; j < ncopies; j++)
4831 /* Handle uses. */
4832 if (j == 0)
4834 if (op_type == binary_op || op_type == ternary_op)
4835 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
4836 slp_node, -1);
4837 else
4838 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4839 slp_node, -1);
4840 if (op_type == ternary_op)
4842 vec_oprnds2.create (1);
4843 vec_oprnds2.quick_push (vect_get_vec_def_for_operand (op2,
4844 stmt,
4845 NULL));
4848 else
4850 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
4851 if (op_type == ternary_op)
4853 tree vec_oprnd = vec_oprnds2.pop ();
4854 vec_oprnds2.quick_push (vect_get_vec_def_for_stmt_copy (dt[2],
4855 vec_oprnd));
4859 /* Arguments are ready. Create the new vector stmt. */
4860 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4862 vop1 = ((op_type == binary_op || op_type == ternary_op)
4863 ? vec_oprnds1[i] : NULL_TREE);
4864 vop2 = ((op_type == ternary_op)
4865 ? vec_oprnds2[i] : NULL_TREE);
4866 new_stmt = gimple_build_assign_with_ops (code, vec_dest,
4867 vop0, vop1, vop2);
4868 new_temp = make_ssa_name (vec_dest, new_stmt);
4869 gimple_assign_set_lhs (new_stmt, new_temp);
4870 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4871 if (slp_node)
4872 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4875 if (slp_node)
4876 continue;
4878 if (j == 0)
4879 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4880 else
4881 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4882 prev_stmt_info = vinfo_for_stmt (new_stmt);
4885 vec_oprnds0.release ();
4886 vec_oprnds1.release ();
4887 vec_oprnds2.release ();
4889 return true;
4892 /* A helper function to ensure data reference DR's base alignment
4893 for STMT_INFO. */
4895 static void
4896 ensure_base_align (stmt_vec_info stmt_info, struct data_reference *dr)
4898 if (!dr->aux)
4899 return;
4901 if (((dataref_aux *)dr->aux)->base_misaligned)
4903 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4904 tree base_decl = ((dataref_aux *)dr->aux)->base_decl;
4906 DECL_ALIGN (base_decl) = TYPE_ALIGN (vectype);
4907 DECL_USER_ALIGN (base_decl) = 1;
4908 ((dataref_aux *)dr->aux)->base_misaligned = false;
4913 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
4914 reversal of the vector elements. If that is impossible to do,
4915 returns NULL. */
4917 static tree
4918 perm_mask_for_reverse (tree vectype)
4920 int i, nunits;
4921 unsigned char *sel;
4923 nunits = TYPE_VECTOR_SUBPARTS (vectype);
4924 sel = XALLOCAVEC (unsigned char, nunits);
4926 for (i = 0; i < nunits; ++i)
4927 sel[i] = nunits - 1 - i;
4929 return vect_gen_perm_mask (vectype, sel);
4932 /* Function vectorizable_store.
4934 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
4935 can be vectorized.
4936 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4937 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4938 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4940 static bool
4941 vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
4942 slp_tree slp_node)
4944 tree scalar_dest;
4945 tree data_ref;
4946 tree op;
4947 tree vec_oprnd = NULL_TREE;
4948 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4949 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
4950 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4951 tree elem_type;
4952 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4953 struct loop *loop = NULL;
4954 enum machine_mode vec_mode;
4955 tree dummy;
4956 enum dr_alignment_support alignment_support_scheme;
4957 tree def;
4958 gimple def_stmt;
4959 enum vect_def_type dt;
4960 stmt_vec_info prev_stmt_info = NULL;
4961 tree dataref_ptr = NULL_TREE;
4962 tree dataref_offset = NULL_TREE;
4963 gimple ptr_incr = NULL;
4964 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
4965 int ncopies;
4966 int j;
4967 gimple next_stmt, first_stmt = NULL;
4968 bool grouped_store = false;
4969 bool store_lanes_p = false;
4970 unsigned int group_size, i;
4971 vec<tree> dr_chain = vNULL;
4972 vec<tree> oprnds = vNULL;
4973 vec<tree> result_chain = vNULL;
4974 bool inv_p;
4975 bool negative = false;
4976 tree offset = NULL_TREE;
4977 vec<tree> vec_oprnds = vNULL;
4978 bool slp = (slp_node != NULL);
4979 unsigned int vec_num;
4980 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4981 tree aggr_type;
4983 if (loop_vinfo)
4984 loop = LOOP_VINFO_LOOP (loop_vinfo);
4986 /* Multiple types in SLP are handled by creating the appropriate number of
4987 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4988 case of SLP. */
4989 if (slp || PURE_SLP_STMT (stmt_info))
4990 ncopies = 1;
4991 else
4992 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
4994 gcc_assert (ncopies >= 1);
4996 /* FORNOW. This restriction should be relaxed. */
4997 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
4999 if (dump_enabled_p ())
5000 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5001 "multiple types in nested loop.\n");
5002 return false;
5005 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5006 return false;
5008 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
5009 return false;
5011 /* Is vectorizable store? */
5013 if (!is_gimple_assign (stmt))
5014 return false;
5016 scalar_dest = gimple_assign_lhs (stmt);
5017 if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
5018 && is_pattern_stmt_p (stmt_info))
5019 scalar_dest = TREE_OPERAND (scalar_dest, 0);
5020 if (TREE_CODE (scalar_dest) != ARRAY_REF
5021 && TREE_CODE (scalar_dest) != BIT_FIELD_REF
5022 && TREE_CODE (scalar_dest) != INDIRECT_REF
5023 && TREE_CODE (scalar_dest) != COMPONENT_REF
5024 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
5025 && TREE_CODE (scalar_dest) != REALPART_EXPR
5026 && TREE_CODE (scalar_dest) != MEM_REF)
5027 return false;
5029 gcc_assert (gimple_assign_single_p (stmt));
5030 op = gimple_assign_rhs1 (stmt);
5031 if (!vect_is_simple_use (op, stmt, loop_vinfo, bb_vinfo, &def_stmt,
5032 &def, &dt))
5034 if (dump_enabled_p ())
5035 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5036 "use not simple.\n");
5037 return false;
5040 elem_type = TREE_TYPE (vectype);
5041 vec_mode = TYPE_MODE (vectype);
5043 /* FORNOW. In some cases can vectorize even if data-type not supported
5044 (e.g. - array initialization with 0). */
5045 if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing)
5046 return false;
5048 if (!STMT_VINFO_DATA_REF (stmt_info))
5049 return false;
5051 negative =
5052 tree_int_cst_compare (loop && nested_in_vect_loop_p (loop, stmt)
5053 ? STMT_VINFO_DR_STEP (stmt_info) : DR_STEP (dr),
5054 size_zero_node) < 0;
5055 if (negative && ncopies > 1)
5057 if (dump_enabled_p ())
5058 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5059 "multiple types with negative step.\n");
5060 return false;
5063 if (negative)
5065 gcc_assert (!grouped_store);
5066 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
5067 if (alignment_support_scheme != dr_aligned
5068 && alignment_support_scheme != dr_unaligned_supported)
5070 if (dump_enabled_p ())
5071 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5072 "negative step but alignment required.\n");
5073 return false;
5075 if (dt != vect_constant_def
5076 && dt != vect_external_def
5077 && !perm_mask_for_reverse (vectype))
5079 if (dump_enabled_p ())
5080 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5081 "negative step and reversing not supported.\n");
5082 return false;
5086 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5088 grouped_store = true;
5089 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
5090 if (!slp && !PURE_SLP_STMT (stmt_info))
5092 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
5093 if (vect_store_lanes_supported (vectype, group_size))
5094 store_lanes_p = true;
5095 else if (!vect_grouped_store_supported (vectype, group_size))
5096 return false;
5099 if (first_stmt == stmt)
5101 /* STMT is the leader of the group. Check the operands of all the
5102 stmts of the group. */
5103 next_stmt = GROUP_NEXT_ELEMENT (stmt_info);
5104 while (next_stmt)
5106 gcc_assert (gimple_assign_single_p (next_stmt));
5107 op = gimple_assign_rhs1 (next_stmt);
5108 if (!vect_is_simple_use (op, next_stmt, loop_vinfo, bb_vinfo,
5109 &def_stmt, &def, &dt))
5111 if (dump_enabled_p ())
5112 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5113 "use not simple.\n");
5114 return false;
5116 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5121 if (!vec_stmt) /* transformation not required. */
5123 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
5124 vect_model_store_cost (stmt_info, ncopies, store_lanes_p, dt,
5125 NULL, NULL, NULL);
5126 return true;
5129 /** Transform. **/
5131 ensure_base_align (stmt_info, dr);
5133 if (grouped_store)
5135 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
5136 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
5138 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
5140 /* FORNOW */
5141 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
5143 /* We vectorize all the stmts of the interleaving group when we
5144 reach the last stmt in the group. */
5145 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
5146 < GROUP_SIZE (vinfo_for_stmt (first_stmt))
5147 && !slp)
5149 *vec_stmt = NULL;
5150 return true;
5153 if (slp)
5155 grouped_store = false;
5156 /* VEC_NUM is the number of vect stmts to be created for this
5157 group. */
5158 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
5159 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
5160 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
5161 op = gimple_assign_rhs1 (first_stmt);
5163 else
5164 /* VEC_NUM is the number of vect stmts to be created for this
5165 group. */
5166 vec_num = group_size;
5168 else
5170 first_stmt = stmt;
5171 first_dr = dr;
5172 group_size = vec_num = 1;
5175 if (dump_enabled_p ())
5176 dump_printf_loc (MSG_NOTE, vect_location,
5177 "transform store. ncopies = %d\n", ncopies);
5179 dr_chain.create (group_size);
5180 oprnds.create (group_size);
5182 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
5183 gcc_assert (alignment_support_scheme);
5184 /* Targets with store-lane instructions must not require explicit
5185 realignment. */
5186 gcc_assert (!store_lanes_p
5187 || alignment_support_scheme == dr_aligned
5188 || alignment_support_scheme == dr_unaligned_supported);
5190 if (negative)
5191 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
5193 if (store_lanes_p)
5194 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
5195 else
5196 aggr_type = vectype;
5198 /* In case the vectorization factor (VF) is bigger than the number
5199 of elements that we can fit in a vectype (nunits), we have to generate
5200 more than one vector stmt - i.e - we need to "unroll" the
5201 vector stmt by a factor VF/nunits. For more details see documentation in
5202 vect_get_vec_def_for_copy_stmt. */
5204 /* In case of interleaving (non-unit grouped access):
5206 S1: &base + 2 = x2
5207 S2: &base = x0
5208 S3: &base + 1 = x1
5209 S4: &base + 3 = x3
5211 We create vectorized stores starting from base address (the access of the
5212 first stmt in the chain (S2 in the above example), when the last store stmt
5213 of the chain (S4) is reached:
5215 VS1: &base = vx2
5216 VS2: &base + vec_size*1 = vx0
5217 VS3: &base + vec_size*2 = vx1
5218 VS4: &base + vec_size*3 = vx3
5220 Then permutation statements are generated:
5222 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5223 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5226 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5227 (the order of the data-refs in the output of vect_permute_store_chain
5228 corresponds to the order of scalar stmts in the interleaving chain - see
5229 the documentation of vect_permute_store_chain()).
5231 In case of both multiple types and interleaving, above vector stores and
5232 permutation stmts are created for every copy. The result vector stmts are
5233 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5234 STMT_VINFO_RELATED_STMT for the next copies.
5237 prev_stmt_info = NULL;
5238 for (j = 0; j < ncopies; j++)
5240 gimple new_stmt;
5242 if (j == 0)
5244 if (slp)
5246 /* Get vectorized arguments for SLP_NODE. */
5247 vect_get_vec_defs (op, NULL_TREE, stmt, &vec_oprnds,
5248 NULL, slp_node, -1);
5250 vec_oprnd = vec_oprnds[0];
5252 else
5254 /* For interleaved stores we collect vectorized defs for all the
5255 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5256 used as an input to vect_permute_store_chain(), and OPRNDS as
5257 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5259 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5260 OPRNDS are of size 1. */
5261 next_stmt = first_stmt;
5262 for (i = 0; i < group_size; i++)
5264 /* Since gaps are not supported for interleaved stores,
5265 GROUP_SIZE is the exact number of stmts in the chain.
5266 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5267 there is no interleaving, GROUP_SIZE is 1, and only one
5268 iteration of the loop will be executed. */
5269 gcc_assert (next_stmt
5270 && gimple_assign_single_p (next_stmt));
5271 op = gimple_assign_rhs1 (next_stmt);
5273 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt,
5274 NULL);
5275 dr_chain.quick_push (vec_oprnd);
5276 oprnds.quick_push (vec_oprnd);
5277 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5281 /* We should have catched mismatched types earlier. */
5282 gcc_assert (useless_type_conversion_p (vectype,
5283 TREE_TYPE (vec_oprnd)));
5284 bool simd_lane_access_p
5285 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
5286 if (simd_lane_access_p
5287 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
5288 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
5289 && integer_zerop (DR_OFFSET (first_dr))
5290 && integer_zerop (DR_INIT (first_dr))
5291 && alias_sets_conflict_p (get_alias_set (aggr_type),
5292 get_alias_set (DR_REF (first_dr))))
5294 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
5295 dataref_offset = build_int_cst (reference_alias_ptr_type
5296 (DR_REF (first_dr)), 0);
5297 inv_p = false;
5299 else
5300 dataref_ptr
5301 = vect_create_data_ref_ptr (first_stmt, aggr_type,
5302 simd_lane_access_p ? loop : NULL,
5303 offset, &dummy, gsi, &ptr_incr,
5304 simd_lane_access_p, &inv_p);
5305 gcc_assert (bb_vinfo || !inv_p);
5307 else
5309 /* For interleaved stores we created vectorized defs for all the
5310 defs stored in OPRNDS in the previous iteration (previous copy).
5311 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5312 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5313 next copy.
5314 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5315 OPRNDS are of size 1. */
5316 for (i = 0; i < group_size; i++)
5318 op = oprnds[i];
5319 vect_is_simple_use (op, NULL, loop_vinfo, bb_vinfo, &def_stmt,
5320 &def, &dt);
5321 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
5322 dr_chain[i] = vec_oprnd;
5323 oprnds[i] = vec_oprnd;
5325 if (dataref_offset)
5326 dataref_offset
5327 = int_const_binop (PLUS_EXPR, dataref_offset,
5328 TYPE_SIZE_UNIT (aggr_type));
5329 else
5330 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
5331 TYPE_SIZE_UNIT (aggr_type));
5334 if (store_lanes_p)
5336 tree vec_array;
5338 /* Combine all the vectors into an array. */
5339 vec_array = create_vector_array (vectype, vec_num);
5340 for (i = 0; i < vec_num; i++)
5342 vec_oprnd = dr_chain[i];
5343 write_vector_array (stmt, gsi, vec_oprnd, vec_array, i);
5346 /* Emit:
5347 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5348 data_ref = create_array_ref (aggr_type, dataref_ptr, first_dr);
5349 new_stmt = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
5350 gimple_call_set_lhs (new_stmt, data_ref);
5351 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5353 else
5355 new_stmt = NULL;
5356 if (grouped_store)
5358 if (j == 0)
5359 result_chain.create (group_size);
5360 /* Permute. */
5361 vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
5362 &result_chain);
5365 next_stmt = first_stmt;
5366 for (i = 0; i < vec_num; i++)
5368 unsigned align, misalign;
5370 if (i > 0)
5371 /* Bump the vector pointer. */
5372 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
5373 stmt, NULL_TREE);
5375 if (slp)
5376 vec_oprnd = vec_oprnds[i];
5377 else if (grouped_store)
5378 /* For grouped stores vectorized defs are interleaved in
5379 vect_permute_store_chain(). */
5380 vec_oprnd = result_chain[i];
5382 data_ref = build2 (MEM_REF, TREE_TYPE (vec_oprnd), dataref_ptr,
5383 dataref_offset
5384 ? dataref_offset
5385 : build_int_cst (reference_alias_ptr_type
5386 (DR_REF (first_dr)), 0));
5387 align = TYPE_ALIGN_UNIT (vectype);
5388 if (aligned_access_p (first_dr))
5389 misalign = 0;
5390 else if (DR_MISALIGNMENT (first_dr) == -1)
5392 TREE_TYPE (data_ref)
5393 = build_aligned_type (TREE_TYPE (data_ref),
5394 TYPE_ALIGN (elem_type));
5395 align = TYPE_ALIGN_UNIT (elem_type);
5396 misalign = 0;
5398 else
5400 TREE_TYPE (data_ref)
5401 = build_aligned_type (TREE_TYPE (data_ref),
5402 TYPE_ALIGN (elem_type));
5403 misalign = DR_MISALIGNMENT (first_dr);
5405 if (dataref_offset == NULL_TREE)
5406 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
5407 misalign);
5409 if (negative
5410 && dt != vect_constant_def
5411 && dt != vect_external_def)
5413 tree perm_mask = perm_mask_for_reverse (vectype);
5414 tree perm_dest
5415 = vect_create_destination_var (gimple_assign_rhs1 (stmt),
5416 vectype);
5417 tree new_temp = make_ssa_name (perm_dest, NULL);
5419 /* Generate the permute statement. */
5420 gimple perm_stmt
5421 = gimple_build_assign_with_ops (VEC_PERM_EXPR, new_temp,
5422 vec_oprnd, vec_oprnd,
5423 perm_mask);
5424 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
5426 perm_stmt = SSA_NAME_DEF_STMT (new_temp);
5427 vec_oprnd = new_temp;
5430 /* Arguments are ready. Create the new vector stmt. */
5431 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
5432 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5434 if (slp)
5435 continue;
5437 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5438 if (!next_stmt)
5439 break;
5442 if (!slp)
5444 if (j == 0)
5445 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5446 else
5447 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5448 prev_stmt_info = vinfo_for_stmt (new_stmt);
5452 dr_chain.release ();
5453 oprnds.release ();
5454 result_chain.release ();
5455 vec_oprnds.release ();
5457 return true;
5460 /* Given a vector type VECTYPE and permutation SEL returns
5461 the VECTOR_CST mask that implements the permutation of the
5462 vector elements. If that is impossible to do, returns NULL. */
5464 tree
5465 vect_gen_perm_mask (tree vectype, unsigned char *sel)
5467 tree mask_elt_type, mask_type, mask_vec, *mask_elts;
5468 int i, nunits;
5470 nunits = TYPE_VECTOR_SUBPARTS (vectype);
5472 if (!can_vec_perm_p (TYPE_MODE (vectype), false, sel))
5473 return NULL;
5475 mask_elt_type = lang_hooks.types.type_for_mode
5476 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype))), 1);
5477 mask_type = get_vectype_for_scalar_type (mask_elt_type);
5479 mask_elts = XALLOCAVEC (tree, nunits);
5480 for (i = nunits - 1; i >= 0; i--)
5481 mask_elts[i] = build_int_cst (mask_elt_type, sel[i]);
5482 mask_vec = build_vector (mask_type, mask_elts);
5484 return mask_vec;
5487 /* Given a vector variable X and Y, that was generated for the scalar
5488 STMT, generate instructions to permute the vector elements of X and Y
5489 using permutation mask MASK_VEC, insert them at *GSI and return the
5490 permuted vector variable. */
5492 static tree
5493 permute_vec_elements (tree x, tree y, tree mask_vec, gimple stmt,
5494 gimple_stmt_iterator *gsi)
5496 tree vectype = TREE_TYPE (x);
5497 tree perm_dest, data_ref;
5498 gimple perm_stmt;
5500 perm_dest = vect_create_destination_var (gimple_get_lhs (stmt), vectype);
5501 data_ref = make_ssa_name (perm_dest, NULL);
5503 /* Generate the permute statement. */
5504 perm_stmt = gimple_build_assign_with_ops (VEC_PERM_EXPR, data_ref,
5505 x, y, mask_vec);
5506 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
5508 return data_ref;
5511 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5512 inserting them on the loops preheader edge. Returns true if we
5513 were successful in doing so (and thus STMT can be moved then),
5514 otherwise returns false. */
5516 static bool
5517 hoist_defs_of_uses (gimple stmt, struct loop *loop)
5519 ssa_op_iter i;
5520 tree op;
5521 bool any = false;
5523 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
5525 gimple def_stmt = SSA_NAME_DEF_STMT (op);
5526 if (!gimple_nop_p (def_stmt)
5527 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
5529 /* Make sure we don't need to recurse. While we could do
5530 so in simple cases when there are more complex use webs
5531 we don't have an easy way to preserve stmt order to fulfil
5532 dependencies within them. */
5533 tree op2;
5534 ssa_op_iter i2;
5535 if (gimple_code (def_stmt) == GIMPLE_PHI)
5536 return false;
5537 FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
5539 gimple def_stmt2 = SSA_NAME_DEF_STMT (op2);
5540 if (!gimple_nop_p (def_stmt2)
5541 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
5542 return false;
5544 any = true;
5548 if (!any)
5549 return true;
5551 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
5553 gimple def_stmt = SSA_NAME_DEF_STMT (op);
5554 if (!gimple_nop_p (def_stmt)
5555 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
5557 gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
5558 gsi_remove (&gsi, false);
5559 gsi_insert_on_edge_immediate (loop_preheader_edge (loop), def_stmt);
5563 return true;
5566 /* vectorizable_load.
5568 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
5569 can be vectorized.
5570 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5571 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5572 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5574 static bool
5575 vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt,
5576 slp_tree slp_node, slp_instance slp_node_instance)
5578 tree scalar_dest;
5579 tree vec_dest = NULL;
5580 tree data_ref = NULL;
5581 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5582 stmt_vec_info prev_stmt_info;
5583 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
5584 struct loop *loop = NULL;
5585 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
5586 bool nested_in_vect_loop = false;
5587 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
5588 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
5589 tree elem_type;
5590 tree new_temp;
5591 enum machine_mode mode;
5592 gimple new_stmt = NULL;
5593 tree dummy;
5594 enum dr_alignment_support alignment_support_scheme;
5595 tree dataref_ptr = NULL_TREE;
5596 tree dataref_offset = NULL_TREE;
5597 gimple ptr_incr = NULL;
5598 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
5599 int ncopies;
5600 int i, j, group_size, group_gap;
5601 tree msq = NULL_TREE, lsq;
5602 tree offset = NULL_TREE;
5603 tree realignment_token = NULL_TREE;
5604 gimple phi = NULL;
5605 vec<tree> dr_chain = vNULL;
5606 bool grouped_load = false;
5607 bool load_lanes_p = false;
5608 gimple first_stmt;
5609 bool inv_p;
5610 bool negative = false;
5611 bool compute_in_loop = false;
5612 struct loop *at_loop;
5613 int vec_num;
5614 bool slp = (slp_node != NULL);
5615 bool slp_perm = false;
5616 enum tree_code code;
5617 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
5618 int vf;
5619 tree aggr_type;
5620 tree gather_base = NULL_TREE, gather_off = NULL_TREE;
5621 tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
5622 int gather_scale = 1;
5623 enum vect_def_type gather_dt = vect_unknown_def_type;
5625 if (loop_vinfo)
5627 loop = LOOP_VINFO_LOOP (loop_vinfo);
5628 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
5629 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
5631 else
5632 vf = 1;
5634 /* Multiple types in SLP are handled by creating the appropriate number of
5635 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5636 case of SLP. */
5637 if (slp || PURE_SLP_STMT (stmt_info))
5638 ncopies = 1;
5639 else
5640 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
5642 gcc_assert (ncopies >= 1);
5644 /* FORNOW. This restriction should be relaxed. */
5645 if (nested_in_vect_loop && ncopies > 1)
5647 if (dump_enabled_p ())
5648 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5649 "multiple types in nested loop.\n");
5650 return false;
5653 /* Invalidate assumptions made by dependence analysis when vectorization
5654 on the unrolled body effectively re-orders stmts. */
5655 if (ncopies > 1
5656 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
5657 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo)
5658 > STMT_VINFO_MIN_NEG_DIST (stmt_info)))
5660 if (dump_enabled_p ())
5661 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5662 "cannot perform implicit CSE when unrolling "
5663 "with negative dependence distance\n");
5664 return false;
5667 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5668 return false;
5670 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def)
5671 return false;
5673 /* Is vectorizable load? */
5674 if (!is_gimple_assign (stmt))
5675 return false;
5677 scalar_dest = gimple_assign_lhs (stmt);
5678 if (TREE_CODE (scalar_dest) != SSA_NAME)
5679 return false;
5681 code = gimple_assign_rhs_code (stmt);
5682 if (code != ARRAY_REF
5683 && code != BIT_FIELD_REF
5684 && code != INDIRECT_REF
5685 && code != COMPONENT_REF
5686 && code != IMAGPART_EXPR
5687 && code != REALPART_EXPR
5688 && code != MEM_REF
5689 && TREE_CODE_CLASS (code) != tcc_declaration)
5690 return false;
5692 if (!STMT_VINFO_DATA_REF (stmt_info))
5693 return false;
5695 elem_type = TREE_TYPE (vectype);
5696 mode = TYPE_MODE (vectype);
5698 /* FORNOW. In some cases can vectorize even if data-type not supported
5699 (e.g. - data copies). */
5700 if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
5702 if (dump_enabled_p ())
5703 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5704 "Aligned load, but unsupported type.\n");
5705 return false;
5708 /* Check if the load is a part of an interleaving chain. */
5709 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5711 grouped_load = true;
5712 /* FORNOW */
5713 gcc_assert (! nested_in_vect_loop && !STMT_VINFO_GATHER_P (stmt_info));
5715 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
5716 if (!slp && !PURE_SLP_STMT (stmt_info))
5718 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
5719 if (vect_load_lanes_supported (vectype, group_size))
5720 load_lanes_p = true;
5721 else if (!vect_grouped_load_supported (vectype, group_size))
5722 return false;
5725 /* Invalidate assumptions made by dependence analysis when vectorization
5726 on the unrolled body effectively re-orders stmts. */
5727 if (!PURE_SLP_STMT (stmt_info)
5728 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
5729 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo)
5730 > STMT_VINFO_MIN_NEG_DIST (stmt_info)))
5732 if (dump_enabled_p ())
5733 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5734 "cannot perform implicit CSE when performing "
5735 "group loads with negative dependence distance\n");
5736 return false;
5741 if (STMT_VINFO_GATHER_P (stmt_info))
5743 gimple def_stmt;
5744 tree def;
5745 gather_decl = vect_check_gather (stmt, loop_vinfo, &gather_base,
5746 &gather_off, &gather_scale);
5747 gcc_assert (gather_decl);
5748 if (!vect_is_simple_use_1 (gather_off, NULL, loop_vinfo, bb_vinfo,
5749 &def_stmt, &def, &gather_dt,
5750 &gather_off_vectype))
5752 if (dump_enabled_p ())
5753 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5754 "gather index use not simple.\n");
5755 return false;
5758 else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
5760 else
5762 negative = tree_int_cst_compare (nested_in_vect_loop
5763 ? STMT_VINFO_DR_STEP (stmt_info)
5764 : DR_STEP (dr),
5765 size_zero_node) < 0;
5766 if (negative && ncopies > 1)
5768 if (dump_enabled_p ())
5769 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5770 "multiple types with negative step.\n");
5771 return false;
5774 if (negative)
5776 if (grouped_load)
5778 if (dump_enabled_p ())
5779 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5780 "negative step for group load not supported"
5781 "\n");
5782 return false;
5784 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
5785 if (alignment_support_scheme != dr_aligned
5786 && alignment_support_scheme != dr_unaligned_supported)
5788 if (dump_enabled_p ())
5789 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5790 "negative step but alignment required.\n");
5791 return false;
5793 if (!perm_mask_for_reverse (vectype))
5795 if (dump_enabled_p ())
5796 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5797 "negative step and reversing not supported."
5798 "\n");
5799 return false;
5804 if (!vec_stmt) /* transformation not required. */
5806 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
5807 vect_model_load_cost (stmt_info, ncopies, load_lanes_p, NULL, NULL, NULL);
5808 return true;
5811 if (dump_enabled_p ())
5812 dump_printf_loc (MSG_NOTE, vect_location,
5813 "transform load. ncopies = %d\n", ncopies);
5815 /** Transform. **/
5817 ensure_base_align (stmt_info, dr);
5819 if (STMT_VINFO_GATHER_P (stmt_info))
5821 tree vec_oprnd0 = NULL_TREE, op;
5822 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
5823 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
5824 tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE;
5825 edge pe = loop_preheader_edge (loop);
5826 gimple_seq seq;
5827 basic_block new_bb;
5828 enum { NARROW, NONE, WIDEN } modifier;
5829 int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
5831 if (nunits == gather_off_nunits)
5832 modifier = NONE;
5833 else if (nunits == gather_off_nunits / 2)
5835 unsigned char *sel = XALLOCAVEC (unsigned char, gather_off_nunits);
5836 modifier = WIDEN;
5838 for (i = 0; i < gather_off_nunits; ++i)
5839 sel[i] = i | nunits;
5841 perm_mask = vect_gen_perm_mask (gather_off_vectype, sel);
5842 gcc_assert (perm_mask != NULL_TREE);
5844 else if (nunits == gather_off_nunits * 2)
5846 unsigned char *sel = XALLOCAVEC (unsigned char, nunits);
5847 modifier = NARROW;
5849 for (i = 0; i < nunits; ++i)
5850 sel[i] = i < gather_off_nunits
5851 ? i : i + nunits - gather_off_nunits;
5853 perm_mask = vect_gen_perm_mask (vectype, sel);
5854 gcc_assert (perm_mask != NULL_TREE);
5855 ncopies *= 2;
5857 else
5858 gcc_unreachable ();
5860 rettype = TREE_TYPE (TREE_TYPE (gather_decl));
5861 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5862 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5863 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5864 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5865 scaletype = TREE_VALUE (arglist);
5866 gcc_checking_assert (types_compatible_p (srctype, rettype));
5868 vec_dest = vect_create_destination_var (scalar_dest, vectype);
5870 ptr = fold_convert (ptrtype, gather_base);
5871 if (!is_gimple_min_invariant (ptr))
5873 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
5874 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
5875 gcc_assert (!new_bb);
5878 /* Currently we support only unconditional gather loads,
5879 so mask should be all ones. */
5880 if (TREE_CODE (masktype) == INTEGER_TYPE)
5881 mask = build_int_cst (masktype, -1);
5882 else if (TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
5884 mask = build_int_cst (TREE_TYPE (masktype), -1);
5885 mask = build_vector_from_val (masktype, mask);
5886 mask = vect_init_vector (stmt, mask, masktype, NULL);
5888 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
5890 REAL_VALUE_TYPE r;
5891 long tmp[6];
5892 for (j = 0; j < 6; ++j)
5893 tmp[j] = -1;
5894 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
5895 mask = build_real (TREE_TYPE (masktype), r);
5896 mask = build_vector_from_val (masktype, mask);
5897 mask = vect_init_vector (stmt, mask, masktype, NULL);
5899 else
5900 gcc_unreachable ();
5902 scale = build_int_cst (scaletype, gather_scale);
5904 if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE)
5905 merge = build_int_cst (TREE_TYPE (rettype), 0);
5906 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype)))
5908 REAL_VALUE_TYPE r;
5909 long tmp[6];
5910 for (j = 0; j < 6; ++j)
5911 tmp[j] = 0;
5912 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (rettype)));
5913 merge = build_real (TREE_TYPE (rettype), r);
5915 else
5916 gcc_unreachable ();
5917 merge = build_vector_from_val (rettype, merge);
5918 merge = vect_init_vector (stmt, merge, rettype, NULL);
5920 prev_stmt_info = NULL;
5921 for (j = 0; j < ncopies; ++j)
5923 if (modifier == WIDEN && (j & 1))
5924 op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
5925 perm_mask, stmt, gsi);
5926 else if (j == 0)
5927 op = vec_oprnd0
5928 = vect_get_vec_def_for_operand (gather_off, stmt, NULL);
5929 else
5930 op = vec_oprnd0
5931 = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
5933 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
5935 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
5936 == TYPE_VECTOR_SUBPARTS (idxtype));
5937 var = vect_get_new_vect_var (idxtype, vect_simple_var, NULL);
5938 var = make_ssa_name (var, NULL);
5939 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
5940 new_stmt
5941 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR, var,
5942 op, NULL_TREE);
5943 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5944 op = var;
5947 new_stmt
5948 = gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale);
5950 if (!useless_type_conversion_p (vectype, rettype))
5952 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
5953 == TYPE_VECTOR_SUBPARTS (rettype));
5954 var = vect_get_new_vect_var (rettype, vect_simple_var, NULL);
5955 op = make_ssa_name (var, new_stmt);
5956 gimple_call_set_lhs (new_stmt, op);
5957 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5958 var = make_ssa_name (vec_dest, NULL);
5959 op = build1 (VIEW_CONVERT_EXPR, vectype, op);
5960 new_stmt
5961 = gimple_build_assign_with_ops (VIEW_CONVERT_EXPR, var, op,
5962 NULL_TREE);
5964 else
5966 var = make_ssa_name (vec_dest, new_stmt);
5967 gimple_call_set_lhs (new_stmt, var);
5970 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5972 if (modifier == NARROW)
5974 if ((j & 1) == 0)
5976 prev_res = var;
5977 continue;
5979 var = permute_vec_elements (prev_res, var,
5980 perm_mask, stmt, gsi);
5981 new_stmt = SSA_NAME_DEF_STMT (var);
5984 if (prev_stmt_info == NULL)
5985 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5986 else
5987 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5988 prev_stmt_info = vinfo_for_stmt (new_stmt);
5990 return true;
5992 else if (STMT_VINFO_STRIDE_LOAD_P (stmt_info))
5994 gimple_stmt_iterator incr_gsi;
5995 bool insert_after;
5996 gimple incr;
5997 tree offvar;
5998 tree ivstep;
5999 tree running_off;
6000 vec<constructor_elt, va_gc> *v = NULL;
6001 gimple_seq stmts = NULL;
6002 tree stride_base, stride_step, alias_off;
6004 gcc_assert (!nested_in_vect_loop);
6006 stride_base
6007 = fold_build_pointer_plus
6008 (unshare_expr (DR_BASE_ADDRESS (dr)),
6009 size_binop (PLUS_EXPR,
6010 convert_to_ptrofftype (unshare_expr (DR_OFFSET (dr))),
6011 convert_to_ptrofftype (DR_INIT (dr))));
6012 stride_step = fold_convert (sizetype, unshare_expr (DR_STEP (dr)));
6014 /* For a load with loop-invariant (but other than power-of-2)
6015 stride (i.e. not a grouped access) like so:
6017 for (i = 0; i < n; i += stride)
6018 ... = array[i];
6020 we generate a new induction variable and new accesses to
6021 form a new vector (or vectors, depending on ncopies):
6023 for (j = 0; ; j += VF*stride)
6024 tmp1 = array[j];
6025 tmp2 = array[j + stride];
6027 vectemp = {tmp1, tmp2, ...}
6030 ivstep = stride_step;
6031 ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
6032 build_int_cst (TREE_TYPE (ivstep), vf));
6034 standard_iv_increment_position (loop, &incr_gsi, &insert_after);
6036 create_iv (stride_base, ivstep, NULL,
6037 loop, &incr_gsi, insert_after,
6038 &offvar, NULL);
6039 incr = gsi_stmt (incr_gsi);
6040 set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo, NULL));
6042 stride_step = force_gimple_operand (stride_step, &stmts, true, NULL_TREE);
6043 if (stmts)
6044 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
6046 prev_stmt_info = NULL;
6047 running_off = offvar;
6048 alias_off = build_int_cst (reference_alias_ptr_type (DR_REF (dr)), 0);
6049 for (j = 0; j < ncopies; j++)
6051 tree vec_inv;
6053 vec_alloc (v, nunits);
6054 for (i = 0; i < nunits; i++)
6056 tree newref, newoff;
6057 gimple incr;
6058 newref = build2 (MEM_REF, TREE_TYPE (vectype),
6059 running_off, alias_off);
6061 newref = force_gimple_operand_gsi (gsi, newref, true,
6062 NULL_TREE, true,
6063 GSI_SAME_STMT);
6064 CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, newref);
6065 newoff = copy_ssa_name (running_off, NULL);
6066 incr = gimple_build_assign_with_ops (POINTER_PLUS_EXPR, newoff,
6067 running_off, stride_step);
6068 vect_finish_stmt_generation (stmt, incr, gsi);
6070 running_off = newoff;
6073 vec_inv = build_constructor (vectype, v);
6074 new_temp = vect_init_vector (stmt, vec_inv, vectype, gsi);
6075 new_stmt = SSA_NAME_DEF_STMT (new_temp);
6077 if (j == 0)
6078 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6079 else
6080 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6081 prev_stmt_info = vinfo_for_stmt (new_stmt);
6083 return true;
6086 if (grouped_load)
6088 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
6089 if (slp
6090 && !SLP_TREE_LOAD_PERMUTATION (slp_node).exists ()
6091 && first_stmt != SLP_TREE_SCALAR_STMTS (slp_node)[0])
6092 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
6094 /* Check if the chain of loads is already vectorized. */
6095 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt))
6096 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6097 ??? But we can only do so if there is exactly one
6098 as we have no way to get at the rest. Leave the CSE
6099 opportunity alone.
6100 ??? With the group load eventually participating
6101 in multiple different permutations (having multiple
6102 slp nodes which refer to the same group) the CSE
6103 is even wrong code. See PR56270. */
6104 && !slp)
6106 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
6107 return true;
6109 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
6110 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
6112 /* VEC_NUM is the number of vect stmts to be created for this group. */
6113 if (slp)
6115 grouped_load = false;
6116 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
6117 if (SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
6118 slp_perm = true;
6119 group_gap = GROUP_GAP (vinfo_for_stmt (first_stmt));
6121 else
6123 vec_num = group_size;
6124 group_gap = 0;
6127 else
6129 first_stmt = stmt;
6130 first_dr = dr;
6131 group_size = vec_num = 1;
6132 group_gap = 0;
6135 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
6136 gcc_assert (alignment_support_scheme);
6137 /* Targets with load-lane instructions must not require explicit
6138 realignment. */
6139 gcc_assert (!load_lanes_p
6140 || alignment_support_scheme == dr_aligned
6141 || alignment_support_scheme == dr_unaligned_supported);
6143 /* In case the vectorization factor (VF) is bigger than the number
6144 of elements that we can fit in a vectype (nunits), we have to generate
6145 more than one vector stmt - i.e - we need to "unroll" the
6146 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6147 from one copy of the vector stmt to the next, in the field
6148 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6149 stages to find the correct vector defs to be used when vectorizing
6150 stmts that use the defs of the current stmt. The example below
6151 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6152 need to create 4 vectorized stmts):
6154 before vectorization:
6155 RELATED_STMT VEC_STMT
6156 S1: x = memref - -
6157 S2: z = x + 1 - -
6159 step 1: vectorize stmt S1:
6160 We first create the vector stmt VS1_0, and, as usual, record a
6161 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6162 Next, we create the vector stmt VS1_1, and record a pointer to
6163 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6164 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6165 stmts and pointers:
6166 RELATED_STMT VEC_STMT
6167 VS1_0: vx0 = memref0 VS1_1 -
6168 VS1_1: vx1 = memref1 VS1_2 -
6169 VS1_2: vx2 = memref2 VS1_3 -
6170 VS1_3: vx3 = memref3 - -
6171 S1: x = load - VS1_0
6172 S2: z = x + 1 - -
6174 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6175 information we recorded in RELATED_STMT field is used to vectorize
6176 stmt S2. */
6178 /* In case of interleaving (non-unit grouped access):
6180 S1: x2 = &base + 2
6181 S2: x0 = &base
6182 S3: x1 = &base + 1
6183 S4: x3 = &base + 3
6185 Vectorized loads are created in the order of memory accesses
6186 starting from the access of the first stmt of the chain:
6188 VS1: vx0 = &base
6189 VS2: vx1 = &base + vec_size*1
6190 VS3: vx3 = &base + vec_size*2
6191 VS4: vx4 = &base + vec_size*3
6193 Then permutation statements are generated:
6195 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6196 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6199 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6200 (the order of the data-refs in the output of vect_permute_load_chain
6201 corresponds to the order of scalar stmts in the interleaving chain - see
6202 the documentation of vect_permute_load_chain()).
6203 The generation of permutation stmts and recording them in
6204 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6206 In case of both multiple types and interleaving, the vector loads and
6207 permutation stmts above are created for every copy. The result vector
6208 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6209 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6211 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6212 on a target that supports unaligned accesses (dr_unaligned_supported)
6213 we generate the following code:
6214 p = initial_addr;
6215 indx = 0;
6216 loop {
6217 p = p + indx * vectype_size;
6218 vec_dest = *(p);
6219 indx = indx + 1;
6222 Otherwise, the data reference is potentially unaligned on a target that
6223 does not support unaligned accesses (dr_explicit_realign_optimized) -
6224 then generate the following code, in which the data in each iteration is
6225 obtained by two vector loads, one from the previous iteration, and one
6226 from the current iteration:
6227 p1 = initial_addr;
6228 msq_init = *(floor(p1))
6229 p2 = initial_addr + VS - 1;
6230 realignment_token = call target_builtin;
6231 indx = 0;
6232 loop {
6233 p2 = p2 + indx * vectype_size
6234 lsq = *(floor(p2))
6235 vec_dest = realign_load (msq, lsq, realignment_token)
6236 indx = indx + 1;
6237 msq = lsq;
6238 } */
6240 /* If the misalignment remains the same throughout the execution of the
6241 loop, we can create the init_addr and permutation mask at the loop
6242 preheader. Otherwise, it needs to be created inside the loop.
6243 This can only occur when vectorizing memory accesses in the inner-loop
6244 nested within an outer-loop that is being vectorized. */
6246 if (nested_in_vect_loop
6247 && (TREE_INT_CST_LOW (DR_STEP (dr))
6248 % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
6250 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
6251 compute_in_loop = true;
6254 if ((alignment_support_scheme == dr_explicit_realign_optimized
6255 || alignment_support_scheme == dr_explicit_realign)
6256 && !compute_in_loop)
6258 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
6259 alignment_support_scheme, NULL_TREE,
6260 &at_loop);
6261 if (alignment_support_scheme == dr_explicit_realign_optimized)
6263 phi = SSA_NAME_DEF_STMT (msq);
6264 offset = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
6267 else
6268 at_loop = loop;
6270 if (negative)
6271 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
6273 if (load_lanes_p)
6274 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
6275 else
6276 aggr_type = vectype;
6278 prev_stmt_info = NULL;
6279 for (j = 0; j < ncopies; j++)
6281 /* 1. Create the vector or array pointer update chain. */
6282 if (j == 0)
6284 bool simd_lane_access_p
6285 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
6286 if (simd_lane_access_p
6287 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
6288 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
6289 && integer_zerop (DR_OFFSET (first_dr))
6290 && integer_zerop (DR_INIT (first_dr))
6291 && alias_sets_conflict_p (get_alias_set (aggr_type),
6292 get_alias_set (DR_REF (first_dr)))
6293 && (alignment_support_scheme == dr_aligned
6294 || alignment_support_scheme == dr_unaligned_supported))
6296 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
6297 dataref_offset = build_int_cst (reference_alias_ptr_type
6298 (DR_REF (first_dr)), 0);
6299 inv_p = false;
6301 else
6302 dataref_ptr
6303 = vect_create_data_ref_ptr (first_stmt, aggr_type, at_loop,
6304 offset, &dummy, gsi, &ptr_incr,
6305 simd_lane_access_p, &inv_p);
6307 else if (dataref_offset)
6308 dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
6309 TYPE_SIZE_UNIT (aggr_type));
6310 else
6311 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
6312 TYPE_SIZE_UNIT (aggr_type));
6314 if (grouped_load || slp_perm)
6315 dr_chain.create (vec_num);
6317 if (load_lanes_p)
6319 tree vec_array;
6321 vec_array = create_vector_array (vectype, vec_num);
6323 /* Emit:
6324 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6325 data_ref = create_array_ref (aggr_type, dataref_ptr, first_dr);
6326 new_stmt = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
6327 gimple_call_set_lhs (new_stmt, vec_array);
6328 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6330 /* Extract each vector into an SSA_NAME. */
6331 for (i = 0; i < vec_num; i++)
6333 new_temp = read_vector_array (stmt, gsi, scalar_dest,
6334 vec_array, i);
6335 dr_chain.quick_push (new_temp);
6338 /* Record the mapping between SSA_NAMEs and statements. */
6339 vect_record_grouped_load_vectors (stmt, dr_chain);
6341 else
6343 for (i = 0; i < vec_num; i++)
6345 if (i > 0)
6346 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
6347 stmt, NULL_TREE);
6349 /* 2. Create the vector-load in the loop. */
6350 switch (alignment_support_scheme)
6352 case dr_aligned:
6353 case dr_unaligned_supported:
6355 unsigned int align, misalign;
6357 data_ref
6358 = build2 (MEM_REF, vectype, dataref_ptr,
6359 dataref_offset
6360 ? dataref_offset
6361 : build_int_cst (reference_alias_ptr_type
6362 (DR_REF (first_dr)), 0));
6363 align = TYPE_ALIGN_UNIT (vectype);
6364 if (alignment_support_scheme == dr_aligned)
6366 gcc_assert (aligned_access_p (first_dr));
6367 misalign = 0;
6369 else if (DR_MISALIGNMENT (first_dr) == -1)
6371 TREE_TYPE (data_ref)
6372 = build_aligned_type (TREE_TYPE (data_ref),
6373 TYPE_ALIGN (elem_type));
6374 align = TYPE_ALIGN_UNIT (elem_type);
6375 misalign = 0;
6377 else
6379 TREE_TYPE (data_ref)
6380 = build_aligned_type (TREE_TYPE (data_ref),
6381 TYPE_ALIGN (elem_type));
6382 misalign = DR_MISALIGNMENT (first_dr);
6384 if (dataref_offset == NULL_TREE)
6385 set_ptr_info_alignment (get_ptr_info (dataref_ptr),
6386 align, misalign);
6387 break;
6389 case dr_explicit_realign:
6391 tree ptr, bump;
6392 tree vs_minus_1;
6394 vs_minus_1 = size_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
6396 if (compute_in_loop)
6397 msq = vect_setup_realignment (first_stmt, gsi,
6398 &realignment_token,
6399 dr_explicit_realign,
6400 dataref_ptr, NULL);
6402 ptr = copy_ssa_name (dataref_ptr, NULL);
6403 new_stmt = gimple_build_assign_with_ops
6404 (BIT_AND_EXPR, ptr, dataref_ptr,
6405 build_int_cst
6406 (TREE_TYPE (dataref_ptr),
6407 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
6408 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6409 data_ref
6410 = build2 (MEM_REF, vectype, ptr,
6411 build_int_cst (reference_alias_ptr_type
6412 (DR_REF (first_dr)), 0));
6413 vec_dest = vect_create_destination_var (scalar_dest,
6414 vectype);
6415 new_stmt = gimple_build_assign (vec_dest, data_ref);
6416 new_temp = make_ssa_name (vec_dest, new_stmt);
6417 gimple_assign_set_lhs (new_stmt, new_temp);
6418 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
6419 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
6420 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6421 msq = new_temp;
6423 bump = size_binop (MULT_EXPR, vs_minus_1,
6424 TYPE_SIZE_UNIT (elem_type));
6425 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
6426 new_stmt = gimple_build_assign_with_ops
6427 (BIT_AND_EXPR, NULL_TREE, ptr,
6428 build_int_cst
6429 (TREE_TYPE (ptr),
6430 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
6431 ptr = copy_ssa_name (dataref_ptr, new_stmt);
6432 gimple_assign_set_lhs (new_stmt, ptr);
6433 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6434 data_ref
6435 = build2 (MEM_REF, vectype, ptr,
6436 build_int_cst (reference_alias_ptr_type
6437 (DR_REF (first_dr)), 0));
6438 break;
6440 case dr_explicit_realign_optimized:
6441 new_temp = copy_ssa_name (dataref_ptr, NULL);
6442 new_stmt = gimple_build_assign_with_ops
6443 (BIT_AND_EXPR, new_temp, dataref_ptr,
6444 build_int_cst
6445 (TREE_TYPE (dataref_ptr),
6446 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
6447 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6448 data_ref
6449 = build2 (MEM_REF, vectype, new_temp,
6450 build_int_cst (reference_alias_ptr_type
6451 (DR_REF (first_dr)), 0));
6452 break;
6453 default:
6454 gcc_unreachable ();
6456 vec_dest = vect_create_destination_var (scalar_dest, vectype);
6457 new_stmt = gimple_build_assign (vec_dest, data_ref);
6458 new_temp = make_ssa_name (vec_dest, new_stmt);
6459 gimple_assign_set_lhs (new_stmt, new_temp);
6460 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6462 /* 3. Handle explicit realignment if necessary/supported.
6463 Create in loop:
6464 vec_dest = realign_load (msq, lsq, realignment_token) */
6465 if (alignment_support_scheme == dr_explicit_realign_optimized
6466 || alignment_support_scheme == dr_explicit_realign)
6468 lsq = gimple_assign_lhs (new_stmt);
6469 if (!realignment_token)
6470 realignment_token = dataref_ptr;
6471 vec_dest = vect_create_destination_var (scalar_dest, vectype);
6472 new_stmt
6473 = gimple_build_assign_with_ops (REALIGN_LOAD_EXPR,
6474 vec_dest, msq, lsq,
6475 realignment_token);
6476 new_temp = make_ssa_name (vec_dest, new_stmt);
6477 gimple_assign_set_lhs (new_stmt, new_temp);
6478 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6480 if (alignment_support_scheme == dr_explicit_realign_optimized)
6482 gcc_assert (phi);
6483 if (i == vec_num - 1 && j == ncopies - 1)
6484 add_phi_arg (phi, lsq,
6485 loop_latch_edge (containing_loop),
6486 UNKNOWN_LOCATION);
6487 msq = lsq;
6491 /* 4. Handle invariant-load. */
6492 if (inv_p && !bb_vinfo)
6494 gcc_assert (!grouped_load);
6495 /* If we have versioned for aliasing or the loop doesn't
6496 have any data dependencies that would preclude this,
6497 then we are sure this is a loop invariant load and
6498 thus we can insert it on the preheader edge. */
6499 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
6500 && !nested_in_vect_loop
6501 && hoist_defs_of_uses (stmt, loop))
6503 if (dump_enabled_p ())
6505 dump_printf_loc (MSG_NOTE, vect_location,
6506 "hoisting out of the vectorized "
6507 "loop: ");
6508 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
6509 dump_printf (MSG_NOTE, "\n");
6511 tree tem = copy_ssa_name (scalar_dest, NULL);
6512 gsi_insert_on_edge_immediate
6513 (loop_preheader_edge (loop),
6514 gimple_build_assign (tem,
6515 unshare_expr
6516 (gimple_assign_rhs1 (stmt))));
6517 new_temp = vect_init_vector (stmt, tem, vectype, NULL);
6519 else
6521 gimple_stmt_iterator gsi2 = *gsi;
6522 gsi_next (&gsi2);
6523 new_temp = vect_init_vector (stmt, scalar_dest,
6524 vectype, &gsi2);
6526 new_stmt = SSA_NAME_DEF_STMT (new_temp);
6527 set_vinfo_for_stmt (new_stmt,
6528 new_stmt_vec_info (new_stmt, loop_vinfo,
6529 bb_vinfo));
6532 if (negative)
6534 tree perm_mask = perm_mask_for_reverse (vectype);
6535 new_temp = permute_vec_elements (new_temp, new_temp,
6536 perm_mask, stmt, gsi);
6537 new_stmt = SSA_NAME_DEF_STMT (new_temp);
6540 /* Collect vector loads and later create their permutation in
6541 vect_transform_grouped_load (). */
6542 if (grouped_load || slp_perm)
6543 dr_chain.quick_push (new_temp);
6545 /* Store vector loads in the corresponding SLP_NODE. */
6546 if (slp && !slp_perm)
6547 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
6549 /* Bump the vector pointer to account for a gap. */
6550 if (slp && group_gap != 0)
6552 tree bump = size_binop (MULT_EXPR,
6553 TYPE_SIZE_UNIT (elem_type),
6554 size_int (group_gap));
6555 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
6556 stmt, bump);
6560 if (slp && !slp_perm)
6561 continue;
6563 if (slp_perm)
6565 if (!vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
6566 slp_node_instance, false))
6568 dr_chain.release ();
6569 return false;
6572 else
6574 if (grouped_load)
6576 if (!load_lanes_p)
6577 vect_transform_grouped_load (stmt, dr_chain, group_size, gsi);
6578 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
6580 else
6582 if (j == 0)
6583 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6584 else
6585 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6586 prev_stmt_info = vinfo_for_stmt (new_stmt);
6589 dr_chain.release ();
6592 return true;
6595 /* Function vect_is_simple_cond.
6597 Input:
6598 LOOP - the loop that is being vectorized.
6599 COND - Condition that is checked for simple use.
6601 Output:
6602 *COMP_VECTYPE - the vector type for the comparison.
6604 Returns whether a COND can be vectorized. Checks whether
6605 condition operands are supportable using vec_is_simple_use. */
6607 static bool
6608 vect_is_simple_cond (tree cond, gimple stmt, loop_vec_info loop_vinfo,
6609 bb_vec_info bb_vinfo, tree *comp_vectype)
6611 tree lhs, rhs;
6612 tree def;
6613 enum vect_def_type dt;
6614 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
6616 if (!COMPARISON_CLASS_P (cond))
6617 return false;
6619 lhs = TREE_OPERAND (cond, 0);
6620 rhs = TREE_OPERAND (cond, 1);
6622 if (TREE_CODE (lhs) == SSA_NAME)
6624 gimple lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
6625 if (!vect_is_simple_use_1 (lhs, stmt, loop_vinfo, bb_vinfo,
6626 &lhs_def_stmt, &def, &dt, &vectype1))
6627 return false;
6629 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
6630 && TREE_CODE (lhs) != FIXED_CST)
6631 return false;
6633 if (TREE_CODE (rhs) == SSA_NAME)
6635 gimple rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
6636 if (!vect_is_simple_use_1 (rhs, stmt, loop_vinfo, bb_vinfo,
6637 &rhs_def_stmt, &def, &dt, &vectype2))
6638 return false;
6640 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
6641 && TREE_CODE (rhs) != FIXED_CST)
6642 return false;
6644 *comp_vectype = vectype1 ? vectype1 : vectype2;
6645 return true;
6648 /* vectorizable_condition.
6650 Check if STMT is conditional modify expression that can be vectorized.
6651 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6652 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
6653 at GSI.
6655 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
6656 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
6657 else caluse if it is 2).
6659 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6661 bool
6662 vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi,
6663 gimple *vec_stmt, tree reduc_def, int reduc_index,
6664 slp_tree slp_node)
6666 tree scalar_dest = NULL_TREE;
6667 tree vec_dest = NULL_TREE;
6668 tree cond_expr, then_clause, else_clause;
6669 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
6670 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6671 tree comp_vectype = NULL_TREE;
6672 tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
6673 tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
6674 tree vec_compare, vec_cond_expr;
6675 tree new_temp;
6676 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
6677 tree def;
6678 enum vect_def_type dt, dts[4];
6679 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
6680 int ncopies;
6681 enum tree_code code;
6682 stmt_vec_info prev_stmt_info = NULL;
6683 int i, j;
6684 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
6685 vec<tree> vec_oprnds0 = vNULL;
6686 vec<tree> vec_oprnds1 = vNULL;
6687 vec<tree> vec_oprnds2 = vNULL;
6688 vec<tree> vec_oprnds3 = vNULL;
6689 tree vec_cmp_type;
6691 if (slp_node || PURE_SLP_STMT (stmt_info))
6692 ncopies = 1;
6693 else
6694 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
6696 gcc_assert (ncopies >= 1);
6697 if (reduc_index && ncopies > 1)
6698 return false; /* FORNOW */
6700 if (reduc_index && STMT_SLP_TYPE (stmt_info))
6701 return false;
6703 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6704 return false;
6706 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6707 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
6708 && reduc_def))
6709 return false;
6711 /* FORNOW: not yet supported. */
6712 if (STMT_VINFO_LIVE_P (stmt_info))
6714 if (dump_enabled_p ())
6715 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6716 "value used after loop.\n");
6717 return false;
6720 /* Is vectorizable conditional operation? */
6721 if (!is_gimple_assign (stmt))
6722 return false;
6724 code = gimple_assign_rhs_code (stmt);
6726 if (code != COND_EXPR)
6727 return false;
6729 cond_expr = gimple_assign_rhs1 (stmt);
6730 then_clause = gimple_assign_rhs2 (stmt);
6731 else_clause = gimple_assign_rhs3 (stmt);
6733 if (!vect_is_simple_cond (cond_expr, stmt, loop_vinfo, bb_vinfo,
6734 &comp_vectype)
6735 || !comp_vectype)
6736 return false;
6738 if (TREE_CODE (then_clause) == SSA_NAME)
6740 gimple then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
6741 if (!vect_is_simple_use (then_clause, stmt, loop_vinfo, bb_vinfo,
6742 &then_def_stmt, &def, &dt))
6743 return false;
6745 else if (TREE_CODE (then_clause) != INTEGER_CST
6746 && TREE_CODE (then_clause) != REAL_CST
6747 && TREE_CODE (then_clause) != FIXED_CST)
6748 return false;
6750 if (TREE_CODE (else_clause) == SSA_NAME)
6752 gimple else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
6753 if (!vect_is_simple_use (else_clause, stmt, loop_vinfo, bb_vinfo,
6754 &else_def_stmt, &def, &dt))
6755 return false;
6757 else if (TREE_CODE (else_clause) != INTEGER_CST
6758 && TREE_CODE (else_clause) != REAL_CST
6759 && TREE_CODE (else_clause) != FIXED_CST)
6760 return false;
6762 unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype)));
6763 /* The result of a vector comparison should be signed type. */
6764 tree cmp_type = build_nonstandard_integer_type (prec, 0);
6765 vec_cmp_type = get_same_sized_vectype (cmp_type, vectype);
6766 if (vec_cmp_type == NULL_TREE)
6767 return false;
6769 if (!vec_stmt)
6771 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
6772 return expand_vec_cond_expr_p (vectype, comp_vectype);
6775 /* Transform. */
6777 if (!slp_node)
6779 vec_oprnds0.create (1);
6780 vec_oprnds1.create (1);
6781 vec_oprnds2.create (1);
6782 vec_oprnds3.create (1);
6785 /* Handle def. */
6786 scalar_dest = gimple_assign_lhs (stmt);
6787 vec_dest = vect_create_destination_var (scalar_dest, vectype);
6789 /* Handle cond expr. */
6790 for (j = 0; j < ncopies; j++)
6792 gimple new_stmt = NULL;
6793 if (j == 0)
6795 if (slp_node)
6797 auto_vec<tree, 4> ops;
6798 auto_vec<vec<tree>, 4> vec_defs;
6800 ops.safe_push (TREE_OPERAND (cond_expr, 0));
6801 ops.safe_push (TREE_OPERAND (cond_expr, 1));
6802 ops.safe_push (then_clause);
6803 ops.safe_push (else_clause);
6804 vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
6805 vec_oprnds3 = vec_defs.pop ();
6806 vec_oprnds2 = vec_defs.pop ();
6807 vec_oprnds1 = vec_defs.pop ();
6808 vec_oprnds0 = vec_defs.pop ();
6810 ops.release ();
6811 vec_defs.release ();
6813 else
6815 gimple gtemp;
6816 vec_cond_lhs =
6817 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
6818 stmt, NULL);
6819 vect_is_simple_use (TREE_OPERAND (cond_expr, 0), stmt,
6820 loop_vinfo, NULL, &gtemp, &def, &dts[0]);
6822 vec_cond_rhs =
6823 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
6824 stmt, NULL);
6825 vect_is_simple_use (TREE_OPERAND (cond_expr, 1), stmt,
6826 loop_vinfo, NULL, &gtemp, &def, &dts[1]);
6827 if (reduc_index == 1)
6828 vec_then_clause = reduc_def;
6829 else
6831 vec_then_clause = vect_get_vec_def_for_operand (then_clause,
6832 stmt, NULL);
6833 vect_is_simple_use (then_clause, stmt, loop_vinfo,
6834 NULL, &gtemp, &def, &dts[2]);
6836 if (reduc_index == 2)
6837 vec_else_clause = reduc_def;
6838 else
6840 vec_else_clause = vect_get_vec_def_for_operand (else_clause,
6841 stmt, NULL);
6842 vect_is_simple_use (else_clause, stmt, loop_vinfo,
6843 NULL, &gtemp, &def, &dts[3]);
6847 else
6849 vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0],
6850 vec_oprnds0.pop ());
6851 vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1],
6852 vec_oprnds1.pop ());
6853 vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
6854 vec_oprnds2.pop ());
6855 vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
6856 vec_oprnds3.pop ());
6859 if (!slp_node)
6861 vec_oprnds0.quick_push (vec_cond_lhs);
6862 vec_oprnds1.quick_push (vec_cond_rhs);
6863 vec_oprnds2.quick_push (vec_then_clause);
6864 vec_oprnds3.quick_push (vec_else_clause);
6867 /* Arguments are ready. Create the new vector stmt. */
6868 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
6870 vec_cond_rhs = vec_oprnds1[i];
6871 vec_then_clause = vec_oprnds2[i];
6872 vec_else_clause = vec_oprnds3[i];
6874 vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
6875 vec_cond_lhs, vec_cond_rhs);
6876 vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
6877 vec_compare, vec_then_clause, vec_else_clause);
6879 new_stmt = gimple_build_assign (vec_dest, vec_cond_expr);
6880 new_temp = make_ssa_name (vec_dest, new_stmt);
6881 gimple_assign_set_lhs (new_stmt, new_temp);
6882 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6883 if (slp_node)
6884 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
6887 if (slp_node)
6888 continue;
6890 if (j == 0)
6891 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6892 else
6893 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6895 prev_stmt_info = vinfo_for_stmt (new_stmt);
6898 vec_oprnds0.release ();
6899 vec_oprnds1.release ();
6900 vec_oprnds2.release ();
6901 vec_oprnds3.release ();
6903 return true;
6907 /* Make sure the statement is vectorizable. */
6909 bool
6910 vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node)
6912 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
6913 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
6914 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
6915 bool ok;
6916 tree scalar_type, vectype;
6917 gimple pattern_stmt;
6918 gimple_seq pattern_def_seq;
6920 if (dump_enabled_p ())
6922 dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
6923 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
6924 dump_printf (MSG_NOTE, "\n");
6927 if (gimple_has_volatile_ops (stmt))
6929 if (dump_enabled_p ())
6930 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6931 "not vectorized: stmt has volatile operands\n");
6933 return false;
6936 /* Skip stmts that do not need to be vectorized. In loops this is expected
6937 to include:
6938 - the COND_EXPR which is the loop exit condition
6939 - any LABEL_EXPRs in the loop
6940 - computations that are used only for array indexing or loop control.
6941 In basic blocks we only analyze statements that are a part of some SLP
6942 instance, therefore, all the statements are relevant.
6944 Pattern statement needs to be analyzed instead of the original statement
6945 if the original statement is not relevant. Otherwise, we analyze both
6946 statements. In basic blocks we are called from some SLP instance
6947 traversal, don't analyze pattern stmts instead, the pattern stmts
6948 already will be part of SLP instance. */
6950 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
6951 if (!STMT_VINFO_RELEVANT_P (stmt_info)
6952 && !STMT_VINFO_LIVE_P (stmt_info))
6954 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
6955 && pattern_stmt
6956 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
6957 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
6959 /* Analyze PATTERN_STMT instead of the original stmt. */
6960 stmt = pattern_stmt;
6961 stmt_info = vinfo_for_stmt (pattern_stmt);
6962 if (dump_enabled_p ())
6964 dump_printf_loc (MSG_NOTE, vect_location,
6965 "==> examining pattern statement: ");
6966 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
6967 dump_printf (MSG_NOTE, "\n");
6970 else
6972 if (dump_enabled_p ())
6973 dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
6975 return true;
6978 else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
6979 && node == NULL
6980 && pattern_stmt
6981 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
6982 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
6984 /* Analyze PATTERN_STMT too. */
6985 if (dump_enabled_p ())
6987 dump_printf_loc (MSG_NOTE, vect_location,
6988 "==> examining pattern statement: ");
6989 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
6990 dump_printf (MSG_NOTE, "\n");
6993 if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node))
6994 return false;
6997 if (is_pattern_stmt_p (stmt_info)
6998 && node == NULL
6999 && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
7001 gimple_stmt_iterator si;
7003 for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
7005 gimple pattern_def_stmt = gsi_stmt (si);
7006 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
7007 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
7009 /* Analyze def stmt of STMT if it's a pattern stmt. */
7010 if (dump_enabled_p ())
7012 dump_printf_loc (MSG_NOTE, vect_location,
7013 "==> examining pattern def statement: ");
7014 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
7015 dump_printf (MSG_NOTE, "\n");
7018 if (!vect_analyze_stmt (pattern_def_stmt,
7019 need_to_vectorize, node))
7020 return false;
7025 switch (STMT_VINFO_DEF_TYPE (stmt_info))
7027 case vect_internal_def:
7028 break;
7030 case vect_reduction_def:
7031 case vect_nested_cycle:
7032 gcc_assert (!bb_vinfo && (relevance == vect_used_in_outer
7033 || relevance == vect_used_in_outer_by_reduction
7034 || relevance == vect_unused_in_scope));
7035 break;
7037 case vect_induction_def:
7038 case vect_constant_def:
7039 case vect_external_def:
7040 case vect_unknown_def_type:
7041 default:
7042 gcc_unreachable ();
7045 if (bb_vinfo)
7047 gcc_assert (PURE_SLP_STMT (stmt_info));
7049 scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
7050 if (dump_enabled_p ())
7052 dump_printf_loc (MSG_NOTE, vect_location,
7053 "get vectype for scalar type: ");
7054 dump_generic_expr (MSG_NOTE, TDF_SLIM, scalar_type);
7055 dump_printf (MSG_NOTE, "\n");
7058 vectype = get_vectype_for_scalar_type (scalar_type);
7059 if (!vectype)
7061 if (dump_enabled_p ())
7063 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7064 "not SLPed: unsupported data-type ");
7065 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
7066 scalar_type);
7067 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
7069 return false;
7072 if (dump_enabled_p ())
7074 dump_printf_loc (MSG_NOTE, vect_location, "vectype: ");
7075 dump_generic_expr (MSG_NOTE, TDF_SLIM, vectype);
7076 dump_printf (MSG_NOTE, "\n");
7079 STMT_VINFO_VECTYPE (stmt_info) = vectype;
7082 if (STMT_VINFO_RELEVANT_P (stmt_info))
7084 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
7085 gcc_assert (STMT_VINFO_VECTYPE (stmt_info)
7086 || (is_gimple_call (stmt)
7087 && gimple_call_lhs (stmt) == NULL_TREE));
7088 *need_to_vectorize = true;
7091 ok = true;
7092 if (!bb_vinfo
7093 && (STMT_VINFO_RELEVANT_P (stmt_info)
7094 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
7095 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, NULL)
7096 || vectorizable_conversion (stmt, NULL, NULL, NULL)
7097 || vectorizable_shift (stmt, NULL, NULL, NULL)
7098 || vectorizable_operation (stmt, NULL, NULL, NULL)
7099 || vectorizable_assignment (stmt, NULL, NULL, NULL)
7100 || vectorizable_load (stmt, NULL, NULL, NULL, NULL)
7101 || vectorizable_call (stmt, NULL, NULL, NULL)
7102 || vectorizable_store (stmt, NULL, NULL, NULL)
7103 || vectorizable_reduction (stmt, NULL, NULL, NULL)
7104 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, NULL));
7105 else
7107 if (bb_vinfo)
7108 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, node)
7109 || vectorizable_conversion (stmt, NULL, NULL, node)
7110 || vectorizable_shift (stmt, NULL, NULL, node)
7111 || vectorizable_operation (stmt, NULL, NULL, node)
7112 || vectorizable_assignment (stmt, NULL, NULL, node)
7113 || vectorizable_load (stmt, NULL, NULL, node, NULL)
7114 || vectorizable_call (stmt, NULL, NULL, node)
7115 || vectorizable_store (stmt, NULL, NULL, node)
7116 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node));
7119 if (!ok)
7121 if (dump_enabled_p ())
7123 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7124 "not vectorized: relevant stmt not ");
7125 dump_printf (MSG_MISSED_OPTIMIZATION, "supported: ");
7126 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
7127 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
7130 return false;
7133 if (bb_vinfo)
7134 return true;
7136 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7137 need extra handling, except for vectorizable reductions. */
7138 if (STMT_VINFO_LIVE_P (stmt_info)
7139 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
7140 ok = vectorizable_live_operation (stmt, NULL, NULL);
7142 if (!ok)
7144 if (dump_enabled_p ())
7146 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7147 "not vectorized: live stmt not ");
7148 dump_printf (MSG_MISSED_OPTIMIZATION, "supported: ");
7149 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
7150 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
7153 return false;
7156 return true;
7160 /* Function vect_transform_stmt.
7162 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7164 bool
7165 vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi,
7166 bool *grouped_store, slp_tree slp_node,
7167 slp_instance slp_node_instance)
7169 bool is_store = false;
7170 gimple vec_stmt = NULL;
7171 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7172 bool done;
7174 switch (STMT_VINFO_TYPE (stmt_info))
7176 case type_demotion_vec_info_type:
7177 case type_promotion_vec_info_type:
7178 case type_conversion_vec_info_type:
7179 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
7180 gcc_assert (done);
7181 break;
7183 case induc_vec_info_type:
7184 gcc_assert (!slp_node);
7185 done = vectorizable_induction (stmt, gsi, &vec_stmt);
7186 gcc_assert (done);
7187 break;
7189 case shift_vec_info_type:
7190 done = vectorizable_shift (stmt, gsi, &vec_stmt, slp_node);
7191 gcc_assert (done);
7192 break;
7194 case op_vec_info_type:
7195 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
7196 gcc_assert (done);
7197 break;
7199 case assignment_vec_info_type:
7200 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
7201 gcc_assert (done);
7202 break;
7204 case load_vec_info_type:
7205 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
7206 slp_node_instance);
7207 gcc_assert (done);
7208 break;
7210 case store_vec_info_type:
7211 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
7212 gcc_assert (done);
7213 if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && !slp_node)
7215 /* In case of interleaving, the whole chain is vectorized when the
7216 last store in the chain is reached. Store stmts before the last
7217 one are skipped, and there vec_stmt_info shouldn't be freed
7218 meanwhile. */
7219 *grouped_store = true;
7220 if (STMT_VINFO_VEC_STMT (stmt_info))
7221 is_store = true;
7223 else
7224 is_store = true;
7225 break;
7227 case condition_vec_info_type:
7228 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0, slp_node);
7229 gcc_assert (done);
7230 break;
7232 case call_vec_info_type:
7233 done = vectorizable_call (stmt, gsi, &vec_stmt, slp_node);
7234 stmt = gsi_stmt (*gsi);
7235 if (is_gimple_call (stmt)
7236 && gimple_call_internal_p (stmt)
7237 && gimple_call_internal_fn (stmt) == IFN_MASK_STORE)
7238 is_store = true;
7239 break;
7241 case call_simd_clone_vec_info_type:
7242 done = vectorizable_simd_clone_call (stmt, gsi, &vec_stmt, slp_node);
7243 stmt = gsi_stmt (*gsi);
7244 break;
7246 case reduc_vec_info_type:
7247 done = vectorizable_reduction (stmt, gsi, &vec_stmt, slp_node);
7248 gcc_assert (done);
7249 break;
7251 default:
7252 if (!STMT_VINFO_LIVE_P (stmt_info))
7254 if (dump_enabled_p ())
7255 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7256 "stmt not supported.\n");
7257 gcc_unreachable ();
7261 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
7262 is being vectorized, but outside the immediately enclosing loop. */
7263 if (vec_stmt
7264 && STMT_VINFO_LOOP_VINFO (stmt_info)
7265 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
7266 STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
7267 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
7268 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
7269 || STMT_VINFO_RELEVANT (stmt_info) ==
7270 vect_used_in_outer_by_reduction))
7272 struct loop *innerloop = LOOP_VINFO_LOOP (
7273 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
7274 imm_use_iterator imm_iter;
7275 use_operand_p use_p;
7276 tree scalar_dest;
7277 gimple exit_phi;
7279 if (dump_enabled_p ())
7280 dump_printf_loc (MSG_NOTE, vect_location,
7281 "Record the vdef for outer-loop vectorization.\n");
7283 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
7284 (to be used when vectorizing outer-loop stmts that use the DEF of
7285 STMT). */
7286 if (gimple_code (stmt) == GIMPLE_PHI)
7287 scalar_dest = PHI_RESULT (stmt);
7288 else
7289 scalar_dest = gimple_assign_lhs (stmt);
7291 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
7293 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
7295 exit_phi = USE_STMT (use_p);
7296 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
7301 /* Handle stmts whose DEF is used outside the loop-nest that is
7302 being vectorized. */
7303 if (STMT_VINFO_LIVE_P (stmt_info)
7304 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
7306 done = vectorizable_live_operation (stmt, gsi, &vec_stmt);
7307 gcc_assert (done);
7310 if (vec_stmt)
7311 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
7313 return is_store;
7317 /* Remove a group of stores (for SLP or interleaving), free their
7318 stmt_vec_info. */
7320 void
7321 vect_remove_stores (gimple first_stmt)
7323 gimple next = first_stmt;
7324 gimple tmp;
7325 gimple_stmt_iterator next_si;
7327 while (next)
7329 stmt_vec_info stmt_info = vinfo_for_stmt (next);
7331 tmp = GROUP_NEXT_ELEMENT (stmt_info);
7332 if (is_pattern_stmt_p (stmt_info))
7333 next = STMT_VINFO_RELATED_STMT (stmt_info);
7334 /* Free the attached stmt_vec_info and remove the stmt. */
7335 next_si = gsi_for_stmt (next);
7336 unlink_stmt_vdef (next);
7337 gsi_remove (&next_si, true);
7338 release_defs (next);
7339 free_stmt_vec_info (next);
7340 next = tmp;
7345 /* Function new_stmt_vec_info.
7347 Create and initialize a new stmt_vec_info struct for STMT. */
7349 stmt_vec_info
7350 new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo,
7351 bb_vec_info bb_vinfo)
7353 stmt_vec_info res;
7354 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
7356 STMT_VINFO_TYPE (res) = undef_vec_info_type;
7357 STMT_VINFO_STMT (res) = stmt;
7358 STMT_VINFO_LOOP_VINFO (res) = loop_vinfo;
7359 STMT_VINFO_BB_VINFO (res) = bb_vinfo;
7360 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
7361 STMT_VINFO_LIVE_P (res) = false;
7362 STMT_VINFO_VECTYPE (res) = NULL;
7363 STMT_VINFO_VEC_STMT (res) = NULL;
7364 STMT_VINFO_VECTORIZABLE (res) = true;
7365 STMT_VINFO_IN_PATTERN_P (res) = false;
7366 STMT_VINFO_RELATED_STMT (res) = NULL;
7367 STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
7368 STMT_VINFO_DATA_REF (res) = NULL;
7370 STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
7371 STMT_VINFO_DR_OFFSET (res) = NULL;
7372 STMT_VINFO_DR_INIT (res) = NULL;
7373 STMT_VINFO_DR_STEP (res) = NULL;
7374 STMT_VINFO_DR_ALIGNED_TO (res) = NULL;
7376 if (gimple_code (stmt) == GIMPLE_PHI
7377 && is_loop_header_bb_p (gimple_bb (stmt)))
7378 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
7379 else
7380 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
7382 STMT_VINFO_SAME_ALIGN_REFS (res).create (0);
7383 STMT_SLP_TYPE (res) = loop_vect;
7384 GROUP_FIRST_ELEMENT (res) = NULL;
7385 GROUP_NEXT_ELEMENT (res) = NULL;
7386 GROUP_SIZE (res) = 0;
7387 GROUP_STORE_COUNT (res) = 0;
7388 GROUP_GAP (res) = 0;
7389 GROUP_SAME_DR_STMT (res) = NULL;
7391 return res;
7395 /* Create a hash table for stmt_vec_info. */
7397 void
7398 init_stmt_vec_info_vec (void)
7400 gcc_assert (!stmt_vec_info_vec.exists ());
7401 stmt_vec_info_vec.create (50);
7405 /* Free hash table for stmt_vec_info. */
7407 void
7408 free_stmt_vec_info_vec (void)
7410 unsigned int i;
7411 vec_void_p info;
7412 FOR_EACH_VEC_ELT (stmt_vec_info_vec, i, info)
7413 if (info != NULL)
7414 free_stmt_vec_info (STMT_VINFO_STMT ((stmt_vec_info) info));
7415 gcc_assert (stmt_vec_info_vec.exists ());
7416 stmt_vec_info_vec.release ();
7420 /* Free stmt vectorization related info. */
7422 void
7423 free_stmt_vec_info (gimple stmt)
7425 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7427 if (!stmt_info)
7428 return;
7430 /* Check if this statement has a related "pattern stmt"
7431 (introduced by the vectorizer during the pattern recognition
7432 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
7433 too. */
7434 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
7436 stmt_vec_info patt_info
7437 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
7438 if (patt_info)
7440 gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (patt_info);
7441 gimple patt_stmt = STMT_VINFO_STMT (patt_info);
7442 gimple_set_bb (patt_stmt, NULL);
7443 tree lhs = gimple_get_lhs (patt_stmt);
7444 if (TREE_CODE (lhs) == SSA_NAME)
7445 release_ssa_name (lhs);
7446 if (seq)
7448 gimple_stmt_iterator si;
7449 for (si = gsi_start (seq); !gsi_end_p (si); gsi_next (&si))
7451 gimple seq_stmt = gsi_stmt (si);
7452 gimple_set_bb (seq_stmt, NULL);
7453 lhs = gimple_get_lhs (patt_stmt);
7454 if (TREE_CODE (lhs) == SSA_NAME)
7455 release_ssa_name (lhs);
7456 free_stmt_vec_info (seq_stmt);
7459 free_stmt_vec_info (patt_stmt);
7463 STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release ();
7464 set_vinfo_for_stmt (stmt, NULL);
7465 free (stmt_info);
7469 /* Function get_vectype_for_scalar_type_and_size.
7471 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
7472 by the target. */
7474 static tree
7475 get_vectype_for_scalar_type_and_size (tree scalar_type, unsigned size)
7477 enum machine_mode inner_mode = TYPE_MODE (scalar_type);
7478 enum machine_mode simd_mode;
7479 unsigned int nbytes = GET_MODE_SIZE (inner_mode);
7480 int nunits;
7481 tree vectype;
7483 if (nbytes == 0)
7484 return NULL_TREE;
7486 if (GET_MODE_CLASS (inner_mode) != MODE_INT
7487 && GET_MODE_CLASS (inner_mode) != MODE_FLOAT)
7488 return NULL_TREE;
7490 /* For vector types of elements whose mode precision doesn't
7491 match their types precision we use a element type of mode
7492 precision. The vectorization routines will have to make sure
7493 they support the proper result truncation/extension.
7494 We also make sure to build vector types with INTEGER_TYPE
7495 component type only. */
7496 if (INTEGRAL_TYPE_P (scalar_type)
7497 && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
7498 || TREE_CODE (scalar_type) != INTEGER_TYPE))
7499 scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
7500 TYPE_UNSIGNED (scalar_type));
7502 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
7503 When the component mode passes the above test simply use a type
7504 corresponding to that mode. The theory is that any use that
7505 would cause problems with this will disable vectorization anyway. */
7506 else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
7507 && !INTEGRAL_TYPE_P (scalar_type))
7508 scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
7510 /* We can't build a vector type of elements with alignment bigger than
7511 their size. */
7512 else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
7513 scalar_type = lang_hooks.types.type_for_mode (inner_mode,
7514 TYPE_UNSIGNED (scalar_type));
7516 /* If we felt back to using the mode fail if there was
7517 no scalar type for it. */
7518 if (scalar_type == NULL_TREE)
7519 return NULL_TREE;
7521 /* If no size was supplied use the mode the target prefers. Otherwise
7522 lookup a vector mode of the specified size. */
7523 if (size == 0)
7524 simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
7525 else
7526 simd_mode = mode_for_vector (inner_mode, size / nbytes);
7527 nunits = GET_MODE_SIZE (simd_mode) / nbytes;
7528 if (nunits <= 1)
7529 return NULL_TREE;
7531 vectype = build_vector_type (scalar_type, nunits);
7533 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
7534 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
7535 return NULL_TREE;
7537 return vectype;
7540 unsigned int current_vector_size;
7542 /* Function get_vectype_for_scalar_type.
7544 Returns the vector type corresponding to SCALAR_TYPE as supported
7545 by the target. */
7547 tree
7548 get_vectype_for_scalar_type (tree scalar_type)
7550 tree vectype;
7551 vectype = get_vectype_for_scalar_type_and_size (scalar_type,
7552 current_vector_size);
7553 if (vectype
7554 && current_vector_size == 0)
7555 current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
7556 return vectype;
7559 /* Function get_same_sized_vectype
7561 Returns a vector type corresponding to SCALAR_TYPE of size
7562 VECTOR_TYPE if supported by the target. */
7564 tree
7565 get_same_sized_vectype (tree scalar_type, tree vector_type)
7567 return get_vectype_for_scalar_type_and_size
7568 (scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
7571 /* Function vect_is_simple_use.
7573 Input:
7574 LOOP_VINFO - the vect info of the loop that is being vectorized.
7575 BB_VINFO - the vect info of the basic block that is being vectorized.
7576 OPERAND - operand of STMT in the loop or bb.
7577 DEF - the defining stmt in case OPERAND is an SSA_NAME.
7579 Returns whether a stmt with OPERAND can be vectorized.
7580 For loops, supportable operands are constants, loop invariants, and operands
7581 that are defined by the current iteration of the loop. Unsupportable
7582 operands are those that are defined by a previous iteration of the loop (as
7583 is the case in reduction/induction computations).
7584 For basic blocks, supportable operands are constants and bb invariants.
7585 For now, operands defined outside the basic block are not supported. */
7587 bool
7588 vect_is_simple_use (tree operand, gimple stmt, loop_vec_info loop_vinfo,
7589 bb_vec_info bb_vinfo, gimple *def_stmt,
7590 tree *def, enum vect_def_type *dt)
7592 basic_block bb;
7593 stmt_vec_info stmt_vinfo;
7594 struct loop *loop = NULL;
7596 if (loop_vinfo)
7597 loop = LOOP_VINFO_LOOP (loop_vinfo);
7599 *def_stmt = NULL;
7600 *def = NULL_TREE;
7602 if (dump_enabled_p ())
7604 dump_printf_loc (MSG_NOTE, vect_location,
7605 "vect_is_simple_use: operand ");
7606 dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
7607 dump_printf (MSG_NOTE, "\n");
7610 if (CONSTANT_CLASS_P (operand))
7612 *dt = vect_constant_def;
7613 return true;
7616 if (is_gimple_min_invariant (operand))
7618 *def = operand;
7619 *dt = vect_external_def;
7620 return true;
7623 if (TREE_CODE (operand) == PAREN_EXPR)
7625 if (dump_enabled_p ())
7626 dump_printf_loc (MSG_NOTE, vect_location, "non-associatable copy.\n");
7627 operand = TREE_OPERAND (operand, 0);
7630 if (TREE_CODE (operand) != SSA_NAME)
7632 if (dump_enabled_p ())
7633 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7634 "not ssa-name.\n");
7635 return false;
7638 *def_stmt = SSA_NAME_DEF_STMT (operand);
7639 if (*def_stmt == NULL)
7641 if (dump_enabled_p ())
7642 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7643 "no def_stmt.\n");
7644 return false;
7647 if (dump_enabled_p ())
7649 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt: ");
7650 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0);
7651 dump_printf (MSG_NOTE, "\n");
7654 /* Empty stmt is expected only in case of a function argument.
7655 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
7656 if (gimple_nop_p (*def_stmt))
7658 *def = operand;
7659 *dt = vect_external_def;
7660 return true;
7663 bb = gimple_bb (*def_stmt);
7665 if ((loop && !flow_bb_inside_loop_p (loop, bb))
7666 || (!loop && bb != BB_VINFO_BB (bb_vinfo))
7667 || (!loop && gimple_code (*def_stmt) == GIMPLE_PHI))
7668 *dt = vect_external_def;
7669 else
7671 stmt_vinfo = vinfo_for_stmt (*def_stmt);
7672 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
7675 if (*dt == vect_unknown_def_type
7676 || (stmt
7677 && *dt == vect_double_reduction_def
7678 && gimple_code (stmt) != GIMPLE_PHI))
7680 if (dump_enabled_p ())
7681 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7682 "Unsupported pattern.\n");
7683 return false;
7686 if (dump_enabled_p ())
7687 dump_printf_loc (MSG_NOTE, vect_location, "type of def: %d.\n", *dt);
7689 switch (gimple_code (*def_stmt))
7691 case GIMPLE_PHI:
7692 *def = gimple_phi_result (*def_stmt);
7693 break;
7695 case GIMPLE_ASSIGN:
7696 *def = gimple_assign_lhs (*def_stmt);
7697 break;
7699 case GIMPLE_CALL:
7700 *def = gimple_call_lhs (*def_stmt);
7701 if (*def != NULL)
7702 break;
7703 /* FALLTHRU */
7704 default:
7705 if (dump_enabled_p ())
7706 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7707 "unsupported defining stmt:\n");
7708 return false;
7711 return true;
7714 /* Function vect_is_simple_use_1.
7716 Same as vect_is_simple_use_1 but also determines the vector operand
7717 type of OPERAND and stores it to *VECTYPE. If the definition of
7718 OPERAND is vect_uninitialized_def, vect_constant_def or
7719 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
7720 is responsible to compute the best suited vector type for the
7721 scalar operand. */
7723 bool
7724 vect_is_simple_use_1 (tree operand, gimple stmt, loop_vec_info loop_vinfo,
7725 bb_vec_info bb_vinfo, gimple *def_stmt,
7726 tree *def, enum vect_def_type *dt, tree *vectype)
7728 if (!vect_is_simple_use (operand, stmt, loop_vinfo, bb_vinfo, def_stmt,
7729 def, dt))
7730 return false;
7732 /* Now get a vector type if the def is internal, otherwise supply
7733 NULL_TREE and leave it up to the caller to figure out a proper
7734 type for the use stmt. */
7735 if (*dt == vect_internal_def
7736 || *dt == vect_induction_def
7737 || *dt == vect_reduction_def
7738 || *dt == vect_double_reduction_def
7739 || *dt == vect_nested_cycle)
7741 stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
7743 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
7744 && !STMT_VINFO_RELEVANT (stmt_info)
7745 && !STMT_VINFO_LIVE_P (stmt_info))
7746 stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
7748 *vectype = STMT_VINFO_VECTYPE (stmt_info);
7749 gcc_assert (*vectype != NULL_TREE);
7751 else if (*dt == vect_uninitialized_def
7752 || *dt == vect_constant_def
7753 || *dt == vect_external_def)
7754 *vectype = NULL_TREE;
7755 else
7756 gcc_unreachable ();
7758 return true;
7762 /* Function supportable_widening_operation
7764 Check whether an operation represented by the code CODE is a
7765 widening operation that is supported by the target platform in
7766 vector form (i.e., when operating on arguments of type VECTYPE_IN
7767 producing a result of type VECTYPE_OUT).
7769 Widening operations we currently support are NOP (CONVERT), FLOAT
7770 and WIDEN_MULT. This function checks if these operations are supported
7771 by the target platform either directly (via vector tree-codes), or via
7772 target builtins.
7774 Output:
7775 - CODE1 and CODE2 are codes of vector operations to be used when
7776 vectorizing the operation, if available.
7777 - MULTI_STEP_CVT determines the number of required intermediate steps in
7778 case of multi-step conversion (like char->short->int - in that case
7779 MULTI_STEP_CVT will be 1).
7780 - INTERM_TYPES contains the intermediate type required to perform the
7781 widening operation (short in the above example). */
7783 bool
7784 supportable_widening_operation (enum tree_code code, gimple stmt,
7785 tree vectype_out, tree vectype_in,
7786 enum tree_code *code1, enum tree_code *code2,
7787 int *multi_step_cvt,
7788 vec<tree> *interm_types)
7790 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7791 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
7792 struct loop *vect_loop = NULL;
7793 enum machine_mode vec_mode;
7794 enum insn_code icode1, icode2;
7795 optab optab1, optab2;
7796 tree vectype = vectype_in;
7797 tree wide_vectype = vectype_out;
7798 enum tree_code c1, c2;
7799 int i;
7800 tree prev_type, intermediate_type;
7801 enum machine_mode intermediate_mode, prev_mode;
7802 optab optab3, optab4;
7804 *multi_step_cvt = 0;
7805 if (loop_info)
7806 vect_loop = LOOP_VINFO_LOOP (loop_info);
7808 switch (code)
7810 case WIDEN_MULT_EXPR:
7811 /* The result of a vectorized widening operation usually requires
7812 two vectors (because the widened results do not fit into one vector).
7813 The generated vector results would normally be expected to be
7814 generated in the same order as in the original scalar computation,
7815 i.e. if 8 results are generated in each vector iteration, they are
7816 to be organized as follows:
7817 vect1: [res1,res2,res3,res4],
7818 vect2: [res5,res6,res7,res8].
7820 However, in the special case that the result of the widening
7821 operation is used in a reduction computation only, the order doesn't
7822 matter (because when vectorizing a reduction we change the order of
7823 the computation). Some targets can take advantage of this and
7824 generate more efficient code. For example, targets like Altivec,
7825 that support widen_mult using a sequence of {mult_even,mult_odd}
7826 generate the following vectors:
7827 vect1: [res1,res3,res5,res7],
7828 vect2: [res2,res4,res6,res8].
7830 When vectorizing outer-loops, we execute the inner-loop sequentially
7831 (each vectorized inner-loop iteration contributes to VF outer-loop
7832 iterations in parallel). We therefore don't allow to change the
7833 order of the computation in the inner-loop during outer-loop
7834 vectorization. */
7835 /* TODO: Another case in which order doesn't *really* matter is when we
7836 widen and then contract again, e.g. (short)((int)x * y >> 8).
7837 Normally, pack_trunc performs an even/odd permute, whereas the
7838 repack from an even/odd expansion would be an interleave, which
7839 would be significantly simpler for e.g. AVX2. */
7840 /* In any case, in order to avoid duplicating the code below, recurse
7841 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
7842 are properly set up for the caller. If we fail, we'll continue with
7843 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
7844 if (vect_loop
7845 && STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
7846 && !nested_in_vect_loop_p (vect_loop, stmt)
7847 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
7848 stmt, vectype_out, vectype_in,
7849 code1, code2, multi_step_cvt,
7850 interm_types))
7852 /* Elements in a vector with vect_used_by_reduction property cannot
7853 be reordered if the use chain with this property does not have the
7854 same operation. One such an example is s += a * b, where elements
7855 in a and b cannot be reordered. Here we check if the vector defined
7856 by STMT is only directly used in the reduction statement. */
7857 tree lhs = gimple_assign_lhs (stmt);
7858 use_operand_p dummy;
7859 gimple use_stmt;
7860 stmt_vec_info use_stmt_info = NULL;
7861 if (single_imm_use (lhs, &dummy, &use_stmt)
7862 && (use_stmt_info = vinfo_for_stmt (use_stmt))
7863 && STMT_VINFO_DEF_TYPE (use_stmt_info) == vect_reduction_def)
7864 return true;
7866 c1 = VEC_WIDEN_MULT_LO_EXPR;
7867 c2 = VEC_WIDEN_MULT_HI_EXPR;
7868 break;
7870 case VEC_WIDEN_MULT_EVEN_EXPR:
7871 /* Support the recursion induced just above. */
7872 c1 = VEC_WIDEN_MULT_EVEN_EXPR;
7873 c2 = VEC_WIDEN_MULT_ODD_EXPR;
7874 break;
7876 case WIDEN_LSHIFT_EXPR:
7877 c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
7878 c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
7879 break;
7881 CASE_CONVERT:
7882 c1 = VEC_UNPACK_LO_EXPR;
7883 c2 = VEC_UNPACK_HI_EXPR;
7884 break;
7886 case FLOAT_EXPR:
7887 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
7888 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
7889 break;
7891 case FIX_TRUNC_EXPR:
7892 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
7893 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
7894 computing the operation. */
7895 return false;
7897 default:
7898 gcc_unreachable ();
7901 if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
7903 enum tree_code ctmp = c1;
7904 c1 = c2;
7905 c2 = ctmp;
7908 if (code == FIX_TRUNC_EXPR)
7910 /* The signedness is determined from output operand. */
7911 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
7912 optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
7914 else
7916 optab1 = optab_for_tree_code (c1, vectype, optab_default);
7917 optab2 = optab_for_tree_code (c2, vectype, optab_default);
7920 if (!optab1 || !optab2)
7921 return false;
7923 vec_mode = TYPE_MODE (vectype);
7924 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
7925 || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
7926 return false;
7928 *code1 = c1;
7929 *code2 = c2;
7931 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
7932 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
7933 return true;
7935 /* Check if it's a multi-step conversion that can be done using intermediate
7936 types. */
7938 prev_type = vectype;
7939 prev_mode = vec_mode;
7941 if (!CONVERT_EXPR_CODE_P (code))
7942 return false;
7944 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
7945 intermediate steps in promotion sequence. We try
7946 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
7947 not. */
7948 interm_types->create (MAX_INTERM_CVT_STEPS);
7949 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
7951 intermediate_mode = insn_data[icode1].operand[0].mode;
7952 intermediate_type
7953 = lang_hooks.types.type_for_mode (intermediate_mode,
7954 TYPE_UNSIGNED (prev_type));
7955 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
7956 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
7958 if (!optab3 || !optab4
7959 || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
7960 || insn_data[icode1].operand[0].mode != intermediate_mode
7961 || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
7962 || insn_data[icode2].operand[0].mode != intermediate_mode
7963 || ((icode1 = optab_handler (optab3, intermediate_mode))
7964 == CODE_FOR_nothing)
7965 || ((icode2 = optab_handler (optab4, intermediate_mode))
7966 == CODE_FOR_nothing))
7967 break;
7969 interm_types->quick_push (intermediate_type);
7970 (*multi_step_cvt)++;
7972 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
7973 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
7974 return true;
7976 prev_type = intermediate_type;
7977 prev_mode = intermediate_mode;
7980 interm_types->release ();
7981 return false;
7985 /* Function supportable_narrowing_operation
7987 Check whether an operation represented by the code CODE is a
7988 narrowing operation that is supported by the target platform in
7989 vector form (i.e., when operating on arguments of type VECTYPE_IN
7990 and producing a result of type VECTYPE_OUT).
7992 Narrowing operations we currently support are NOP (CONVERT) and
7993 FIX_TRUNC. This function checks if these operations are supported by
7994 the target platform directly via vector tree-codes.
7996 Output:
7997 - CODE1 is the code of a vector operation to be used when
7998 vectorizing the operation, if available.
7999 - MULTI_STEP_CVT determines the number of required intermediate steps in
8000 case of multi-step conversion (like int->short->char - in that case
8001 MULTI_STEP_CVT will be 1).
8002 - INTERM_TYPES contains the intermediate type required to perform the
8003 narrowing operation (short in the above example). */
8005 bool
8006 supportable_narrowing_operation (enum tree_code code,
8007 tree vectype_out, tree vectype_in,
8008 enum tree_code *code1, int *multi_step_cvt,
8009 vec<tree> *interm_types)
8011 enum machine_mode vec_mode;
8012 enum insn_code icode1;
8013 optab optab1, interm_optab;
8014 tree vectype = vectype_in;
8015 tree narrow_vectype = vectype_out;
8016 enum tree_code c1;
8017 tree intermediate_type;
8018 enum machine_mode intermediate_mode, prev_mode;
8019 int i;
8020 bool uns;
8022 *multi_step_cvt = 0;
8023 switch (code)
8025 CASE_CONVERT:
8026 c1 = VEC_PACK_TRUNC_EXPR;
8027 break;
8029 case FIX_TRUNC_EXPR:
8030 c1 = VEC_PACK_FIX_TRUNC_EXPR;
8031 break;
8033 case FLOAT_EXPR:
8034 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8035 tree code and optabs used for computing the operation. */
8036 return false;
8038 default:
8039 gcc_unreachable ();
8042 if (code == FIX_TRUNC_EXPR)
8043 /* The signedness is determined from output operand. */
8044 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
8045 else
8046 optab1 = optab_for_tree_code (c1, vectype, optab_default);
8048 if (!optab1)
8049 return false;
8051 vec_mode = TYPE_MODE (vectype);
8052 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
8053 return false;
8055 *code1 = c1;
8057 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
8058 return true;
8060 /* Check if it's a multi-step conversion that can be done using intermediate
8061 types. */
8062 prev_mode = vec_mode;
8063 if (code == FIX_TRUNC_EXPR)
8064 uns = TYPE_UNSIGNED (vectype_out);
8065 else
8066 uns = TYPE_UNSIGNED (vectype);
8068 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8069 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8070 costly than signed. */
8071 if (code == FIX_TRUNC_EXPR && uns)
8073 enum insn_code icode2;
8075 intermediate_type
8076 = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
8077 interm_optab
8078 = optab_for_tree_code (c1, intermediate_type, optab_default);
8079 if (interm_optab != unknown_optab
8080 && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
8081 && insn_data[icode1].operand[0].mode
8082 == insn_data[icode2].operand[0].mode)
8084 uns = false;
8085 optab1 = interm_optab;
8086 icode1 = icode2;
8090 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8091 intermediate steps in promotion sequence. We try
8092 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8093 interm_types->create (MAX_INTERM_CVT_STEPS);
8094 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
8096 intermediate_mode = insn_data[icode1].operand[0].mode;
8097 intermediate_type
8098 = lang_hooks.types.type_for_mode (intermediate_mode, uns);
8099 interm_optab
8100 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
8101 optab_default);
8102 if (!interm_optab
8103 || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
8104 || insn_data[icode1].operand[0].mode != intermediate_mode
8105 || ((icode1 = optab_handler (interm_optab, intermediate_mode))
8106 == CODE_FOR_nothing))
8107 break;
8109 interm_types->quick_push (intermediate_type);
8110 (*multi_step_cvt)++;
8112 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
8113 return true;
8115 prev_mode = intermediate_mode;
8116 optab1 = interm_optab;
8119 interm_types->release ();
8120 return false;