Minor reformatting.
[official-gcc.git] / gcc / tree-vect-stmts.c
blob9ab4af4f97eb9a651a6335c64a3574a058dad5f0
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2016 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #include "config.h"
23 #include "system.h"
24 #include "coretypes.h"
25 #include "backend.h"
26 #include "target.h"
27 #include "rtl.h"
28 #include "tree.h"
29 #include "gimple.h"
30 #include "ssa.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
34 #include "cgraph.h"
35 #include "dumpfile.h"
36 #include "alias.h"
37 #include "fold-const.h"
38 #include "stor-layout.h"
39 #include "tree-eh.h"
40 #include "gimplify.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "tree-cfg.h"
44 #include "tree-ssa-loop-manip.h"
45 #include "cfgloop.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
49 #include "builtins.h"
50 #include "internal-fn.h"
52 /* For lang_hooks.types.type_for_mode. */
53 #include "langhooks.h"
55 /* Return the vectorized type for the given statement. */
57 tree
58 stmt_vectype (struct _stmt_vec_info *stmt_info)
60 return STMT_VINFO_VECTYPE (stmt_info);
63 /* Return TRUE iff the given statement is in an inner loop relative to
64 the loop being vectorized. */
65 bool
66 stmt_in_inner_loop_p (struct _stmt_vec_info *stmt_info)
68 gimple *stmt = STMT_VINFO_STMT (stmt_info);
69 basic_block bb = gimple_bb (stmt);
70 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
71 struct loop* loop;
73 if (!loop_vinfo)
74 return false;
76 loop = LOOP_VINFO_LOOP (loop_vinfo);
78 return (bb->loop_father == loop->inner);
81 /* Record the cost of a statement, either by directly informing the
82 target model or by saving it in a vector for later processing.
83 Return a preliminary estimate of the statement's cost. */
85 unsigned
86 record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count,
87 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
88 int misalign, enum vect_cost_model_location where)
90 if (body_cost_vec)
92 tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
93 stmt_info_for_cost si = { count, kind,
94 stmt_info ? STMT_VINFO_STMT (stmt_info) : NULL,
95 misalign };
96 body_cost_vec->safe_push (si);
97 return (unsigned)
98 (builtin_vectorization_cost (kind, vectype, misalign) * count);
100 else
101 return add_stmt_cost (stmt_info->vinfo->target_cost_data,
102 count, kind, stmt_info, misalign, where);
105 /* Return a variable of type ELEM_TYPE[NELEMS]. */
107 static tree
108 create_vector_array (tree elem_type, unsigned HOST_WIDE_INT nelems)
110 return create_tmp_var (build_array_type_nelts (elem_type, nelems),
111 "vect_array");
114 /* ARRAY is an array of vectors created by create_vector_array.
115 Return an SSA_NAME for the vector in index N. The reference
116 is part of the vectorization of STMT and the vector is associated
117 with scalar destination SCALAR_DEST. */
119 static tree
120 read_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree scalar_dest,
121 tree array, unsigned HOST_WIDE_INT n)
123 tree vect_type, vect, vect_name, array_ref;
124 gimple *new_stmt;
126 gcc_assert (TREE_CODE (TREE_TYPE (array)) == ARRAY_TYPE);
127 vect_type = TREE_TYPE (TREE_TYPE (array));
128 vect = vect_create_destination_var (scalar_dest, vect_type);
129 array_ref = build4 (ARRAY_REF, vect_type, array,
130 build_int_cst (size_type_node, n),
131 NULL_TREE, NULL_TREE);
133 new_stmt = gimple_build_assign (vect, array_ref);
134 vect_name = make_ssa_name (vect, new_stmt);
135 gimple_assign_set_lhs (new_stmt, vect_name);
136 vect_finish_stmt_generation (stmt, new_stmt, gsi);
138 return vect_name;
141 /* ARRAY is an array of vectors created by create_vector_array.
142 Emit code to store SSA_NAME VECT in index N of the array.
143 The store is part of the vectorization of STMT. */
145 static void
146 write_vector_array (gimple *stmt, gimple_stmt_iterator *gsi, tree vect,
147 tree array, unsigned HOST_WIDE_INT n)
149 tree array_ref;
150 gimple *new_stmt;
152 array_ref = build4 (ARRAY_REF, TREE_TYPE (vect), array,
153 build_int_cst (size_type_node, n),
154 NULL_TREE, NULL_TREE);
156 new_stmt = gimple_build_assign (array_ref, vect);
157 vect_finish_stmt_generation (stmt, new_stmt, gsi);
160 /* PTR is a pointer to an array of type TYPE. Return a representation
161 of *PTR. The memory reference replaces those in FIRST_DR
162 (and its group). */
164 static tree
165 create_array_ref (tree type, tree ptr, struct data_reference *first_dr)
167 tree mem_ref, alias_ptr_type;
169 alias_ptr_type = reference_alias_ptr_type (DR_REF (first_dr));
170 mem_ref = build2 (MEM_REF, type, ptr, build_int_cst (alias_ptr_type, 0));
171 /* Arrays have the same alignment as their type. */
172 set_ptr_info_alignment (get_ptr_info (ptr), TYPE_ALIGN_UNIT (type), 0);
173 return mem_ref;
176 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
178 /* Function vect_mark_relevant.
180 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
182 static void
183 vect_mark_relevant (vec<gimple *> *worklist, gimple *stmt,
184 enum vect_relevant relevant, bool live_p)
186 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
187 enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info);
188 bool save_live_p = STMT_VINFO_LIVE_P (stmt_info);
189 gimple *pattern_stmt;
191 if (dump_enabled_p ())
193 dump_printf_loc (MSG_NOTE, vect_location,
194 "mark relevant %d, live %d: ", relevant, live_p);
195 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
198 /* If this stmt is an original stmt in a pattern, we might need to mark its
199 related pattern stmt instead of the original stmt. However, such stmts
200 may have their own uses that are not in any pattern, in such cases the
201 stmt itself should be marked. */
202 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
204 /* This is the last stmt in a sequence that was detected as a
205 pattern that can potentially be vectorized. Don't mark the stmt
206 as relevant/live because it's not going to be vectorized.
207 Instead mark the pattern-stmt that replaces it. */
209 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
211 if (dump_enabled_p ())
212 dump_printf_loc (MSG_NOTE, vect_location,
213 "last stmt in pattern. don't mark"
214 " relevant/live.\n");
215 stmt_info = vinfo_for_stmt (pattern_stmt);
216 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt);
217 save_relevant = STMT_VINFO_RELEVANT (stmt_info);
218 save_live_p = STMT_VINFO_LIVE_P (stmt_info);
219 stmt = pattern_stmt;
222 STMT_VINFO_LIVE_P (stmt_info) |= live_p;
223 if (relevant > STMT_VINFO_RELEVANT (stmt_info))
224 STMT_VINFO_RELEVANT (stmt_info) = relevant;
226 if (STMT_VINFO_RELEVANT (stmt_info) == save_relevant
227 && STMT_VINFO_LIVE_P (stmt_info) == save_live_p)
229 if (dump_enabled_p ())
230 dump_printf_loc (MSG_NOTE, vect_location,
231 "already marked relevant/live.\n");
232 return;
235 worklist->safe_push (stmt);
239 /* Function vect_stmt_relevant_p.
241 Return true if STMT in loop that is represented by LOOP_VINFO is
242 "relevant for vectorization".
244 A stmt is considered "relevant for vectorization" if:
245 - it has uses outside the loop.
246 - it has vdefs (it alters memory).
247 - control stmts in the loop (except for the exit condition).
249 CHECKME: what other side effects would the vectorizer allow? */
251 static bool
252 vect_stmt_relevant_p (gimple *stmt, loop_vec_info loop_vinfo,
253 enum vect_relevant *relevant, bool *live_p)
255 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
256 ssa_op_iter op_iter;
257 imm_use_iterator imm_iter;
258 use_operand_p use_p;
259 def_operand_p def_p;
261 *relevant = vect_unused_in_scope;
262 *live_p = false;
264 /* cond stmt other than loop exit cond. */
265 if (is_ctrl_stmt (stmt)
266 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt))
267 != loop_exit_ctrl_vec_info_type)
268 *relevant = vect_used_in_scope;
270 /* changing memory. */
271 if (gimple_code (stmt) != GIMPLE_PHI)
272 if (gimple_vdef (stmt)
273 && !gimple_clobber_p (stmt))
275 if (dump_enabled_p ())
276 dump_printf_loc (MSG_NOTE, vect_location,
277 "vec_stmt_relevant_p: stmt has vdefs.\n");
278 *relevant = vect_used_in_scope;
281 /* uses outside the loop. */
282 FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
284 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, DEF_FROM_PTR (def_p))
286 basic_block bb = gimple_bb (USE_STMT (use_p));
287 if (!flow_bb_inside_loop_p (loop, bb))
289 if (dump_enabled_p ())
290 dump_printf_loc (MSG_NOTE, vect_location,
291 "vec_stmt_relevant_p: used out of loop.\n");
293 if (is_gimple_debug (USE_STMT (use_p)))
294 continue;
296 /* We expect all such uses to be in the loop exit phis
297 (because of loop closed form) */
298 gcc_assert (gimple_code (USE_STMT (use_p)) == GIMPLE_PHI);
299 gcc_assert (bb == single_exit (loop)->dest);
301 *live_p = true;
306 return (*live_p || *relevant);
310 /* Function exist_non_indexing_operands_for_use_p
312 USE is one of the uses attached to STMT. Check if USE is
313 used in STMT for anything other than indexing an array. */
315 static bool
316 exist_non_indexing_operands_for_use_p (tree use, gimple *stmt)
318 tree operand;
319 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
321 /* USE corresponds to some operand in STMT. If there is no data
322 reference in STMT, then any operand that corresponds to USE
323 is not indexing an array. */
324 if (!STMT_VINFO_DATA_REF (stmt_info))
325 return true;
327 /* STMT has a data_ref. FORNOW this means that its of one of
328 the following forms:
329 -1- ARRAY_REF = var
330 -2- var = ARRAY_REF
331 (This should have been verified in analyze_data_refs).
333 'var' in the second case corresponds to a def, not a use,
334 so USE cannot correspond to any operands that are not used
335 for array indexing.
337 Therefore, all we need to check is if STMT falls into the
338 first case, and whether var corresponds to USE. */
340 if (!gimple_assign_copy_p (stmt))
342 if (is_gimple_call (stmt)
343 && gimple_call_internal_p (stmt))
344 switch (gimple_call_internal_fn (stmt))
346 case IFN_MASK_STORE:
347 operand = gimple_call_arg (stmt, 3);
348 if (operand == use)
349 return true;
350 /* FALLTHRU */
351 case IFN_MASK_LOAD:
352 operand = gimple_call_arg (stmt, 2);
353 if (operand == use)
354 return true;
355 break;
356 default:
357 break;
359 return false;
362 if (TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
363 return false;
364 operand = gimple_assign_rhs1 (stmt);
365 if (TREE_CODE (operand) != SSA_NAME)
366 return false;
368 if (operand == use)
369 return true;
371 return false;
376 Function process_use.
378 Inputs:
379 - a USE in STMT in a loop represented by LOOP_VINFO
380 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
381 that defined USE. This is done by calling mark_relevant and passing it
382 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
383 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
384 be performed.
386 Outputs:
387 Generally, LIVE_P and RELEVANT are used to define the liveness and
388 relevance info of the DEF_STMT of this USE:
389 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
390 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
391 Exceptions:
392 - case 1: If USE is used only for address computations (e.g. array indexing),
393 which does not need to be directly vectorized, then the liveness/relevance
394 of the respective DEF_STMT is left unchanged.
395 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
396 skip DEF_STMT cause it had already been processed.
397 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
398 be modified accordingly.
400 Return true if everything is as expected. Return false otherwise. */
402 static bool
403 process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, bool live_p,
404 enum vect_relevant relevant, vec<gimple *> *worklist,
405 bool force)
407 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
408 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
409 stmt_vec_info dstmt_vinfo;
410 basic_block bb, def_bb;
411 gimple *def_stmt;
412 enum vect_def_type dt;
414 /* case 1: we are only interested in uses that need to be vectorized. Uses
415 that are used for address computation are not considered relevant. */
416 if (!force && !exist_non_indexing_operands_for_use_p (use, stmt))
417 return true;
419 if (!vect_is_simple_use (use, loop_vinfo, &def_stmt, &dt))
421 if (dump_enabled_p ())
422 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
423 "not vectorized: unsupported use in stmt.\n");
424 return false;
427 if (!def_stmt || gimple_nop_p (def_stmt))
428 return true;
430 def_bb = gimple_bb (def_stmt);
431 if (!flow_bb_inside_loop_p (loop, def_bb))
433 if (dump_enabled_p ())
434 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt is out of loop.\n");
435 return true;
438 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
439 DEF_STMT must have already been processed, because this should be the
440 only way that STMT, which is a reduction-phi, was put in the worklist,
441 as there should be no other uses for DEF_STMT in the loop. So we just
442 check that everything is as expected, and we are done. */
443 dstmt_vinfo = vinfo_for_stmt (def_stmt);
444 bb = gimple_bb (stmt);
445 if (gimple_code (stmt) == GIMPLE_PHI
446 && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
447 && gimple_code (def_stmt) != GIMPLE_PHI
448 && STMT_VINFO_DEF_TYPE (dstmt_vinfo) == vect_reduction_def
449 && bb->loop_father == def_bb->loop_father)
451 if (dump_enabled_p ())
452 dump_printf_loc (MSG_NOTE, vect_location,
453 "reduc-stmt defining reduc-phi in the same nest.\n");
454 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo))
455 dstmt_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo));
456 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo) < vect_used_by_reduction);
457 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo)
458 || STMT_VINFO_RELEVANT (dstmt_vinfo) > vect_unused_in_scope);
459 return true;
462 /* case 3a: outer-loop stmt defining an inner-loop stmt:
463 outer-loop-header-bb:
464 d = def_stmt
465 inner-loop:
466 stmt # use (d)
467 outer-loop-tail-bb:
468 ... */
469 if (flow_loop_nested_p (def_bb->loop_father, bb->loop_father))
471 if (dump_enabled_p ())
472 dump_printf_loc (MSG_NOTE, vect_location,
473 "outer-loop def-stmt defining inner-loop stmt.\n");
475 switch (relevant)
477 case vect_unused_in_scope:
478 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_nested_cycle) ?
479 vect_used_in_scope : vect_unused_in_scope;
480 break;
482 case vect_used_in_outer_by_reduction:
483 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
484 relevant = vect_used_by_reduction;
485 break;
487 case vect_used_in_outer:
488 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def);
489 relevant = vect_used_in_scope;
490 break;
492 case vect_used_in_scope:
493 break;
495 default:
496 gcc_unreachable ();
500 /* case 3b: inner-loop stmt defining an outer-loop stmt:
501 outer-loop-header-bb:
503 inner-loop:
504 d = def_stmt
505 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
506 stmt # use (d) */
507 else if (flow_loop_nested_p (bb->loop_father, def_bb->loop_father))
509 if (dump_enabled_p ())
510 dump_printf_loc (MSG_NOTE, vect_location,
511 "inner-loop def-stmt defining outer-loop stmt.\n");
513 switch (relevant)
515 case vect_unused_in_scope:
516 relevant = (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_reduction_def
517 || STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_double_reduction_def) ?
518 vect_used_in_outer_by_reduction : vect_unused_in_scope;
519 break;
521 case vect_used_by_reduction:
522 relevant = vect_used_in_outer_by_reduction;
523 break;
525 case vect_used_in_scope:
526 relevant = vect_used_in_outer;
527 break;
529 default:
530 gcc_unreachable ();
534 vect_mark_relevant (worklist, def_stmt, relevant, live_p);
535 return true;
539 /* Function vect_mark_stmts_to_be_vectorized.
541 Not all stmts in the loop need to be vectorized. For example:
543 for i...
544 for j...
545 1. T0 = i + j
546 2. T1 = a[T0]
548 3. j = j + 1
550 Stmt 1 and 3 do not need to be vectorized, because loop control and
551 addressing of vectorized data-refs are handled differently.
553 This pass detects such stmts. */
555 bool
556 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo)
558 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
559 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
560 unsigned int nbbs = loop->num_nodes;
561 gimple_stmt_iterator si;
562 gimple *stmt;
563 unsigned int i;
564 stmt_vec_info stmt_vinfo;
565 basic_block bb;
566 gimple *phi;
567 bool live_p;
568 enum vect_relevant relevant, tmp_relevant;
569 enum vect_def_type def_type;
571 if (dump_enabled_p ())
572 dump_printf_loc (MSG_NOTE, vect_location,
573 "=== vect_mark_stmts_to_be_vectorized ===\n");
575 auto_vec<gimple *, 64> worklist;
577 /* 1. Init worklist. */
578 for (i = 0; i < nbbs; i++)
580 bb = bbs[i];
581 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
583 phi = gsi_stmt (si);
584 if (dump_enabled_p ())
586 dump_printf_loc (MSG_NOTE, vect_location, "init: phi relevant? ");
587 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
590 if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p))
591 vect_mark_relevant (&worklist, phi, relevant, live_p);
593 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
595 stmt = gsi_stmt (si);
596 if (dump_enabled_p ())
598 dump_printf_loc (MSG_NOTE, vect_location, "init: stmt relevant? ");
599 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
602 if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p))
603 vect_mark_relevant (&worklist, stmt, relevant, live_p);
607 /* 2. Process_worklist */
608 while (worklist.length () > 0)
610 use_operand_p use_p;
611 ssa_op_iter iter;
613 stmt = worklist.pop ();
614 if (dump_enabled_p ())
616 dump_printf_loc (MSG_NOTE, vect_location, "worklist: examine stmt: ");
617 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
620 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
621 (DEF_STMT) as relevant/irrelevant and live/dead according to the
622 liveness and relevance properties of STMT. */
623 stmt_vinfo = vinfo_for_stmt (stmt);
624 relevant = STMT_VINFO_RELEVANT (stmt_vinfo);
625 live_p = STMT_VINFO_LIVE_P (stmt_vinfo);
627 /* Generally, the liveness and relevance properties of STMT are
628 propagated as is to the DEF_STMTs of its USEs:
629 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
630 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
632 One exception is when STMT has been identified as defining a reduction
633 variable; in this case we set the liveness/relevance as follows:
634 live_p = false
635 relevant = vect_used_by_reduction
636 This is because we distinguish between two kinds of relevant stmts -
637 those that are used by a reduction computation, and those that are
638 (also) used by a regular computation. This allows us later on to
639 identify stmts that are used solely by a reduction, and therefore the
640 order of the results that they produce does not have to be kept. */
642 def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo);
643 tmp_relevant = relevant;
644 switch (def_type)
646 case vect_reduction_def:
647 switch (tmp_relevant)
649 case vect_unused_in_scope:
650 relevant = vect_used_by_reduction;
651 break;
653 case vect_used_by_reduction:
654 if (gimple_code (stmt) == GIMPLE_PHI)
655 break;
656 /* fall through */
658 default:
659 if (dump_enabled_p ())
660 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
661 "unsupported use of reduction.\n");
662 return false;
665 live_p = false;
666 break;
668 case vect_nested_cycle:
669 if (tmp_relevant != vect_unused_in_scope
670 && tmp_relevant != vect_used_in_outer_by_reduction
671 && tmp_relevant != vect_used_in_outer)
673 if (dump_enabled_p ())
674 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
675 "unsupported use of nested cycle.\n");
677 return false;
680 live_p = false;
681 break;
683 case vect_double_reduction_def:
684 if (tmp_relevant != vect_unused_in_scope
685 && tmp_relevant != vect_used_by_reduction)
687 if (dump_enabled_p ())
688 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
689 "unsupported use of double reduction.\n");
691 return false;
694 live_p = false;
695 break;
697 default:
698 break;
701 if (is_pattern_stmt_p (stmt_vinfo))
703 /* Pattern statements are not inserted into the code, so
704 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
705 have to scan the RHS or function arguments instead. */
706 if (is_gimple_assign (stmt))
708 enum tree_code rhs_code = gimple_assign_rhs_code (stmt);
709 tree op = gimple_assign_rhs1 (stmt);
711 i = 1;
712 if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op))
714 if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo,
715 live_p, relevant, &worklist, false)
716 || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo,
717 live_p, relevant, &worklist, false))
718 return false;
719 i = 2;
721 for (; i < gimple_num_ops (stmt); i++)
723 op = gimple_op (stmt, i);
724 if (TREE_CODE (op) == SSA_NAME
725 && !process_use (stmt, op, loop_vinfo, live_p, relevant,
726 &worklist, false))
727 return false;
730 else if (is_gimple_call (stmt))
732 for (i = 0; i < gimple_call_num_args (stmt); i++)
734 tree arg = gimple_call_arg (stmt, i);
735 if (!process_use (stmt, arg, loop_vinfo, live_p, relevant,
736 &worklist, false))
737 return false;
741 else
742 FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
744 tree op = USE_FROM_PTR (use_p);
745 if (!process_use (stmt, op, loop_vinfo, live_p, relevant,
746 &worklist, false))
747 return false;
750 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo))
752 tree off;
753 tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL);
754 gcc_assert (decl);
755 if (!process_use (stmt, off, loop_vinfo, live_p, relevant,
756 &worklist, true))
757 return false;
759 } /* while worklist */
761 return true;
765 /* Function vect_model_simple_cost.
767 Models cost for simple operations, i.e. those that only emit ncopies of a
768 single op. Right now, this does not account for multiple insns that could
769 be generated for the single vector op. We will handle that shortly. */
771 void
772 vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies,
773 enum vect_def_type *dt,
774 stmt_vector_for_cost *prologue_cost_vec,
775 stmt_vector_for_cost *body_cost_vec)
777 int i;
778 int inside_cost = 0, prologue_cost = 0;
780 /* The SLP costs were already calculated during SLP tree build. */
781 if (PURE_SLP_STMT (stmt_info))
782 return;
784 /* FORNOW: Assuming maximum 2 args per stmts. */
785 for (i = 0; i < 2; i++)
786 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
787 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, vector_stmt,
788 stmt_info, 0, vect_prologue);
790 /* Pass the inside-of-loop statements to the target-specific cost model. */
791 inside_cost = record_stmt_cost (body_cost_vec, ncopies, vector_stmt,
792 stmt_info, 0, vect_body);
794 if (dump_enabled_p ())
795 dump_printf_loc (MSG_NOTE, vect_location,
796 "vect_model_simple_cost: inside_cost = %d, "
797 "prologue_cost = %d .\n", inside_cost, prologue_cost);
801 /* Model cost for type demotion and promotion operations. PWR is normally
802 zero for single-step promotions and demotions. It will be one if
803 two-step promotion/demotion is required, and so on. Each additional
804 step doubles the number of instructions required. */
806 static void
807 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info,
808 enum vect_def_type *dt, int pwr)
810 int i, tmp;
811 int inside_cost = 0, prologue_cost = 0;
812 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
813 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
814 void *target_cost_data;
816 /* The SLP costs were already calculated during SLP tree build. */
817 if (PURE_SLP_STMT (stmt_info))
818 return;
820 if (loop_vinfo)
821 target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
822 else
823 target_cost_data = BB_VINFO_TARGET_COST_DATA (bb_vinfo);
825 for (i = 0; i < pwr + 1; i++)
827 tmp = (STMT_VINFO_TYPE (stmt_info) == type_promotion_vec_info_type) ?
828 (i + 1) : i;
829 inside_cost += add_stmt_cost (target_cost_data, vect_pow2 (tmp),
830 vec_promote_demote, stmt_info, 0,
831 vect_body);
834 /* FORNOW: Assuming maximum 2 args per stmts. */
835 for (i = 0; i < 2; i++)
836 if (dt[i] == vect_constant_def || dt[i] == vect_external_def)
837 prologue_cost += add_stmt_cost (target_cost_data, 1, vector_stmt,
838 stmt_info, 0, vect_prologue);
840 if (dump_enabled_p ())
841 dump_printf_loc (MSG_NOTE, vect_location,
842 "vect_model_promotion_demotion_cost: inside_cost = %d, "
843 "prologue_cost = %d .\n", inside_cost, prologue_cost);
846 /* Function vect_cost_group_size
848 For grouped load or store, return the group_size only if it is the first
849 load or store of a group, else return 1. This ensures that group size is
850 only returned once per group. */
852 static int
853 vect_cost_group_size (stmt_vec_info stmt_info)
855 gimple *first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
857 if (first_stmt == STMT_VINFO_STMT (stmt_info))
858 return GROUP_SIZE (stmt_info);
860 return 1;
864 /* Function vect_model_store_cost
866 Models cost for stores. In the case of grouped accesses, one access
867 has the overhead of the grouped access attributed to it. */
869 void
870 vect_model_store_cost (stmt_vec_info stmt_info, int ncopies,
871 bool store_lanes_p, enum vect_def_type dt,
872 slp_tree slp_node,
873 stmt_vector_for_cost *prologue_cost_vec,
874 stmt_vector_for_cost *body_cost_vec)
876 int group_size;
877 unsigned int inside_cost = 0, prologue_cost = 0;
878 struct data_reference *first_dr;
879 gimple *first_stmt;
881 if (dt == vect_constant_def || dt == vect_external_def)
882 prologue_cost += record_stmt_cost (prologue_cost_vec, 1, scalar_to_vec,
883 stmt_info, 0, vect_prologue);
885 /* Grouped access? */
886 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
888 if (slp_node)
890 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
891 group_size = 1;
893 else
895 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
896 group_size = vect_cost_group_size (stmt_info);
899 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
901 /* Not a grouped access. */
902 else
904 group_size = 1;
905 first_dr = STMT_VINFO_DATA_REF (stmt_info);
908 /* We assume that the cost of a single store-lanes instruction is
909 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
910 access is instead being provided by a permute-and-store operation,
911 include the cost of the permutes. */
912 if (!store_lanes_p && group_size > 1
913 && !STMT_VINFO_STRIDED_P (stmt_info))
915 /* Uses a high and low interleave or shuffle operations for each
916 needed permute. */
917 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
918 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
919 stmt_info, 0, vect_body);
921 if (dump_enabled_p ())
922 dump_printf_loc (MSG_NOTE, vect_location,
923 "vect_model_store_cost: strided group_size = %d .\n",
924 group_size);
927 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
928 /* Costs of the stores. */
929 if (STMT_VINFO_STRIDED_P (stmt_info)
930 && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
932 /* N scalar stores plus extracting the elements. */
933 inside_cost += record_stmt_cost (body_cost_vec,
934 ncopies * TYPE_VECTOR_SUBPARTS (vectype),
935 scalar_store, stmt_info, 0, vect_body);
937 else
938 vect_get_store_cost (first_dr, ncopies, &inside_cost, body_cost_vec);
940 if (STMT_VINFO_STRIDED_P (stmt_info))
941 inside_cost += record_stmt_cost (body_cost_vec,
942 ncopies * TYPE_VECTOR_SUBPARTS (vectype),
943 vec_to_scalar, stmt_info, 0, vect_body);
945 if (dump_enabled_p ())
946 dump_printf_loc (MSG_NOTE, vect_location,
947 "vect_model_store_cost: inside_cost = %d, "
948 "prologue_cost = %d .\n", inside_cost, prologue_cost);
952 /* Calculate cost of DR's memory access. */
953 void
954 vect_get_store_cost (struct data_reference *dr, int ncopies,
955 unsigned int *inside_cost,
956 stmt_vector_for_cost *body_cost_vec)
958 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
959 gimple *stmt = DR_STMT (dr);
960 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
962 switch (alignment_support_scheme)
964 case dr_aligned:
966 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
967 vector_store, stmt_info, 0,
968 vect_body);
970 if (dump_enabled_p ())
971 dump_printf_loc (MSG_NOTE, vect_location,
972 "vect_model_store_cost: aligned.\n");
973 break;
976 case dr_unaligned_supported:
978 /* Here, we assign an additional cost for the unaligned store. */
979 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
980 unaligned_store, stmt_info,
981 DR_MISALIGNMENT (dr), vect_body);
982 if (dump_enabled_p ())
983 dump_printf_loc (MSG_NOTE, vect_location,
984 "vect_model_store_cost: unaligned supported by "
985 "hardware.\n");
986 break;
989 case dr_unaligned_unsupported:
991 *inside_cost = VECT_MAX_COST;
993 if (dump_enabled_p ())
994 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
995 "vect_model_store_cost: unsupported access.\n");
996 break;
999 default:
1000 gcc_unreachable ();
1005 /* Function vect_model_load_cost
1007 Models cost for loads. In the case of grouped accesses, the last access
1008 has the overhead of the grouped access attributed to it. Since unaligned
1009 accesses are supported for loads, we also account for the costs of the
1010 access scheme chosen. */
1012 void
1013 vect_model_load_cost (stmt_vec_info stmt_info, int ncopies,
1014 bool load_lanes_p, slp_tree slp_node,
1015 stmt_vector_for_cost *prologue_cost_vec,
1016 stmt_vector_for_cost *body_cost_vec)
1018 int group_size;
1019 gimple *first_stmt;
1020 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr;
1021 unsigned int inside_cost = 0, prologue_cost = 0;
1023 /* Grouped accesses? */
1024 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
1025 if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && first_stmt && !slp_node)
1027 group_size = vect_cost_group_size (stmt_info);
1028 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
1030 /* Not a grouped access. */
1031 else
1033 group_size = 1;
1034 first_dr = dr;
1037 /* We assume that the cost of a single load-lanes instruction is
1038 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1039 access is instead being provided by a load-and-permute operation,
1040 include the cost of the permutes. */
1041 if (!load_lanes_p && group_size > 1
1042 && !STMT_VINFO_STRIDED_P (stmt_info))
1044 /* Uses an even and odd extract operations or shuffle operations
1045 for each needed permute. */
1046 int nstmts = ncopies * ceil_log2 (group_size) * group_size;
1047 inside_cost = record_stmt_cost (body_cost_vec, nstmts, vec_perm,
1048 stmt_info, 0, vect_body);
1050 if (dump_enabled_p ())
1051 dump_printf_loc (MSG_NOTE, vect_location,
1052 "vect_model_load_cost: strided group_size = %d .\n",
1053 group_size);
1056 /* The loads themselves. */
1057 if (STMT_VINFO_STRIDED_P (stmt_info)
1058 && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
1060 /* N scalar loads plus gathering them into a vector. */
1061 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1062 inside_cost += record_stmt_cost (body_cost_vec,
1063 ncopies * TYPE_VECTOR_SUBPARTS (vectype),
1064 scalar_load, stmt_info, 0, vect_body);
1066 else
1067 vect_get_load_cost (first_dr, ncopies,
1068 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info))
1069 || group_size > 1 || slp_node),
1070 &inside_cost, &prologue_cost,
1071 prologue_cost_vec, body_cost_vec, true);
1072 if (STMT_VINFO_STRIDED_P (stmt_info))
1073 inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_construct,
1074 stmt_info, 0, vect_body);
1076 if (dump_enabled_p ())
1077 dump_printf_loc (MSG_NOTE, vect_location,
1078 "vect_model_load_cost: inside_cost = %d, "
1079 "prologue_cost = %d .\n", inside_cost, prologue_cost);
1083 /* Calculate cost of DR's memory access. */
1084 void
1085 vect_get_load_cost (struct data_reference *dr, int ncopies,
1086 bool add_realign_cost, unsigned int *inside_cost,
1087 unsigned int *prologue_cost,
1088 stmt_vector_for_cost *prologue_cost_vec,
1089 stmt_vector_for_cost *body_cost_vec,
1090 bool record_prologue_costs)
1092 int alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
1093 gimple *stmt = DR_STMT (dr);
1094 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1096 switch (alignment_support_scheme)
1098 case dr_aligned:
1100 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1101 stmt_info, 0, vect_body);
1103 if (dump_enabled_p ())
1104 dump_printf_loc (MSG_NOTE, vect_location,
1105 "vect_model_load_cost: aligned.\n");
1107 break;
1109 case dr_unaligned_supported:
1111 /* Here, we assign an additional cost for the unaligned load. */
1112 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1113 unaligned_load, stmt_info,
1114 DR_MISALIGNMENT (dr), vect_body);
1116 if (dump_enabled_p ())
1117 dump_printf_loc (MSG_NOTE, vect_location,
1118 "vect_model_load_cost: unaligned supported by "
1119 "hardware.\n");
1121 break;
1123 case dr_explicit_realign:
1125 *inside_cost += record_stmt_cost (body_cost_vec, ncopies * 2,
1126 vector_load, stmt_info, 0, vect_body);
1127 *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
1128 vec_perm, stmt_info, 0, vect_body);
1130 /* FIXME: If the misalignment remains fixed across the iterations of
1131 the containing loop, the following cost should be added to the
1132 prologue costs. */
1133 if (targetm.vectorize.builtin_mask_for_load)
1134 *inside_cost += record_stmt_cost (body_cost_vec, 1, vector_stmt,
1135 stmt_info, 0, vect_body);
1137 if (dump_enabled_p ())
1138 dump_printf_loc (MSG_NOTE, vect_location,
1139 "vect_model_load_cost: explicit realign\n");
1141 break;
1143 case dr_explicit_realign_optimized:
1145 if (dump_enabled_p ())
1146 dump_printf_loc (MSG_NOTE, vect_location,
1147 "vect_model_load_cost: unaligned software "
1148 "pipelined.\n");
1150 /* Unaligned software pipeline has a load of an address, an initial
1151 load, and possibly a mask operation to "prime" the loop. However,
1152 if this is an access in a group of loads, which provide grouped
1153 access, then the above cost should only be considered for one
1154 access in the group. Inside the loop, there is a load op
1155 and a realignment op. */
1157 if (add_realign_cost && record_prologue_costs)
1159 *prologue_cost += record_stmt_cost (prologue_cost_vec, 2,
1160 vector_stmt, stmt_info,
1161 0, vect_prologue);
1162 if (targetm.vectorize.builtin_mask_for_load)
1163 *prologue_cost += record_stmt_cost (prologue_cost_vec, 1,
1164 vector_stmt, stmt_info,
1165 0, vect_prologue);
1168 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vector_load,
1169 stmt_info, 0, vect_body);
1170 *inside_cost += record_stmt_cost (body_cost_vec, ncopies, vec_perm,
1171 stmt_info, 0, vect_body);
1173 if (dump_enabled_p ())
1174 dump_printf_loc (MSG_NOTE, vect_location,
1175 "vect_model_load_cost: explicit realign optimized"
1176 "\n");
1178 break;
1181 case dr_unaligned_unsupported:
1183 *inside_cost = VECT_MAX_COST;
1185 if (dump_enabled_p ())
1186 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1187 "vect_model_load_cost: unsupported access.\n");
1188 break;
1191 default:
1192 gcc_unreachable ();
1196 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1197 the loop preheader for the vectorized stmt STMT. */
1199 static void
1200 vect_init_vector_1 (gimple *stmt, gimple *new_stmt, gimple_stmt_iterator *gsi)
1202 if (gsi)
1203 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1204 else
1206 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1207 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1209 if (loop_vinfo)
1211 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1212 basic_block new_bb;
1213 edge pe;
1215 if (nested_in_vect_loop_p (loop, stmt))
1216 loop = loop->inner;
1218 pe = loop_preheader_edge (loop);
1219 new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
1220 gcc_assert (!new_bb);
1222 else
1224 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
1225 basic_block bb;
1226 gimple_stmt_iterator gsi_bb_start;
1228 gcc_assert (bb_vinfo);
1229 bb = BB_VINFO_BB (bb_vinfo);
1230 gsi_bb_start = gsi_after_labels (bb);
1231 gsi_insert_before (&gsi_bb_start, new_stmt, GSI_SAME_STMT);
1235 if (dump_enabled_p ())
1237 dump_printf_loc (MSG_NOTE, vect_location,
1238 "created new init_stmt: ");
1239 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, new_stmt, 0);
1243 /* Function vect_init_vector.
1245 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1246 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1247 vector type a vector with all elements equal to VAL is created first.
1248 Place the initialization at BSI if it is not NULL. Otherwise, place the
1249 initialization at the loop preheader.
1250 Return the DEF of INIT_STMT.
1251 It will be used in the vectorization of STMT. */
1253 tree
1254 vect_init_vector (gimple *stmt, tree val, tree type, gimple_stmt_iterator *gsi)
1256 gimple *init_stmt;
1257 tree new_temp;
1259 if (TREE_CODE (type) == VECTOR_TYPE
1260 && TREE_CODE (TREE_TYPE (val)) != VECTOR_TYPE)
1262 if (!types_compatible_p (TREE_TYPE (type), TREE_TYPE (val)))
1264 /* Scalar boolean value should be transformed into
1265 all zeros or all ones value before building a vector. */
1266 if (VECTOR_BOOLEAN_TYPE_P (type))
1268 tree true_val = build_all_ones_cst (TREE_TYPE (type));
1269 tree false_val = build_zero_cst (TREE_TYPE (type));
1271 if (CONSTANT_CLASS_P (val))
1272 val = integer_zerop (val) ? false_val : true_val;
1273 else
1275 new_temp = make_ssa_name (TREE_TYPE (type));
1276 init_stmt = gimple_build_assign (new_temp, COND_EXPR,
1277 val, true_val, false_val);
1278 vect_init_vector_1 (stmt, init_stmt, gsi);
1279 val = new_temp;
1282 else if (CONSTANT_CLASS_P (val))
1283 val = fold_convert (TREE_TYPE (type), val);
1284 else
1286 new_temp = make_ssa_name (TREE_TYPE (type));
1287 init_stmt = gimple_build_assign (new_temp, NOP_EXPR, val);
1288 vect_init_vector_1 (stmt, init_stmt, gsi);
1289 val = new_temp;
1292 val = build_vector_from_val (type, val);
1295 new_temp = vect_get_new_ssa_name (type, vect_simple_var, "cst_");
1296 init_stmt = gimple_build_assign (new_temp, val);
1297 vect_init_vector_1 (stmt, init_stmt, gsi);
1298 return new_temp;
1302 /* Function vect_get_vec_def_for_operand.
1304 OP is an operand in STMT. This function returns a (vector) def that will be
1305 used in the vectorized stmt for STMT.
1307 In the case that OP is an SSA_NAME which is defined in the loop, then
1308 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1310 In case OP is an invariant or constant, a new stmt that creates a vector def
1311 needs to be introduced. VECTYPE may be used to specify a required type for
1312 vector invariant. */
1314 tree
1315 vect_get_vec_def_for_operand (tree op, gimple *stmt, tree vectype)
1317 tree vec_oprnd;
1318 gimple *vec_stmt;
1319 gimple *def_stmt;
1320 stmt_vec_info def_stmt_info = NULL;
1321 stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
1322 tree stmt_vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
1323 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
1324 enum vect_def_type dt;
1325 bool is_simple_use;
1326 tree vector_type;
1328 if (dump_enabled_p ())
1330 dump_printf_loc (MSG_NOTE, vect_location,
1331 "vect_get_vec_def_for_operand: ");
1332 dump_generic_expr (MSG_NOTE, TDF_SLIM, op);
1333 dump_printf (MSG_NOTE, "\n");
1336 is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt);
1337 gcc_assert (is_simple_use);
1338 if (dump_enabled_p ())
1340 int loc_printed = 0;
1341 if (def_stmt)
1343 if (loc_printed)
1344 dump_printf (MSG_NOTE, " def_stmt = ");
1345 else
1346 dump_printf_loc (MSG_NOTE, vect_location, " def_stmt = ");
1347 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0);
1351 switch (dt)
1353 /* operand is a constant or a loop invariant. */
1354 case vect_constant_def:
1355 case vect_external_def:
1357 if (vectype)
1358 vector_type = vectype;
1359 else if (TREE_CODE (TREE_TYPE (op)) == BOOLEAN_TYPE
1360 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype))
1361 vector_type = build_same_sized_truth_vector_type (stmt_vectype);
1362 else
1363 vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
1365 gcc_assert (vector_type);
1366 return vect_init_vector (stmt, op, vector_type, NULL);
1369 /* operand is defined inside the loop. */
1370 case vect_internal_def:
1372 /* Get the def from the vectorized stmt. */
1373 def_stmt_info = vinfo_for_stmt (def_stmt);
1375 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1376 /* Get vectorized pattern statement. */
1377 if (!vec_stmt
1378 && STMT_VINFO_IN_PATTERN_P (def_stmt_info)
1379 && !STMT_VINFO_RELEVANT (def_stmt_info))
1380 vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1381 STMT_VINFO_RELATED_STMT (def_stmt_info)));
1382 gcc_assert (vec_stmt);
1383 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1384 vec_oprnd = PHI_RESULT (vec_stmt);
1385 else if (is_gimple_call (vec_stmt))
1386 vec_oprnd = gimple_call_lhs (vec_stmt);
1387 else
1388 vec_oprnd = gimple_assign_lhs (vec_stmt);
1389 return vec_oprnd;
1392 /* operand is defined by a loop header phi - reduction */
1393 case vect_reduction_def:
1394 case vect_double_reduction_def:
1395 case vect_nested_cycle:
1396 /* Code should use get_initial_def_for_reduction. */
1397 gcc_unreachable ();
1399 /* operand is defined by loop-header phi - induction. */
1400 case vect_induction_def:
1402 gcc_assert (gimple_code (def_stmt) == GIMPLE_PHI);
1404 /* Get the def from the vectorized stmt. */
1405 def_stmt_info = vinfo_for_stmt (def_stmt);
1406 vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
1407 if (gimple_code (vec_stmt) == GIMPLE_PHI)
1408 vec_oprnd = PHI_RESULT (vec_stmt);
1409 else
1410 vec_oprnd = gimple_get_lhs (vec_stmt);
1411 return vec_oprnd;
1414 default:
1415 gcc_unreachable ();
1420 /* Function vect_get_vec_def_for_stmt_copy
1422 Return a vector-def for an operand. This function is used when the
1423 vectorized stmt to be created (by the caller to this function) is a "copy"
1424 created in case the vectorized result cannot fit in one vector, and several
1425 copies of the vector-stmt are required. In this case the vector-def is
1426 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1427 of the stmt that defines VEC_OPRND.
1428 DT is the type of the vector def VEC_OPRND.
1430 Context:
1431 In case the vectorization factor (VF) is bigger than the number
1432 of elements that can fit in a vectype (nunits), we have to generate
1433 more than one vector stmt to vectorize the scalar stmt. This situation
1434 arises when there are multiple data-types operated upon in the loop; the
1435 smallest data-type determines the VF, and as a result, when vectorizing
1436 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1437 vector stmt (each computing a vector of 'nunits' results, and together
1438 computing 'VF' results in each iteration). This function is called when
1439 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1440 which VF=16 and nunits=4, so the number of copies required is 4):
1442 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1444 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1445 VS1.1: vx.1 = memref1 VS1.2
1446 VS1.2: vx.2 = memref2 VS1.3
1447 VS1.3: vx.3 = memref3
1449 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1450 VSnew.1: vz1 = vx.1 + ... VSnew.2
1451 VSnew.2: vz2 = vx.2 + ... VSnew.3
1452 VSnew.3: vz3 = vx.3 + ...
1454 The vectorization of S1 is explained in vectorizable_load.
1455 The vectorization of S2:
1456 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1457 the function 'vect_get_vec_def_for_operand' is called to
1458 get the relevant vector-def for each operand of S2. For operand x it
1459 returns the vector-def 'vx.0'.
1461 To create the remaining copies of the vector-stmt (VSnew.j), this
1462 function is called to get the relevant vector-def for each operand. It is
1463 obtained from the respective VS1.j stmt, which is recorded in the
1464 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1466 For example, to obtain the vector-def 'vx.1' in order to create the
1467 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1468 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1469 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1470 and return its def ('vx.1').
1471 Overall, to create the above sequence this function will be called 3 times:
1472 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1473 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1474 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1476 tree
1477 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt, tree vec_oprnd)
1479 gimple *vec_stmt_for_operand;
1480 stmt_vec_info def_stmt_info;
1482 /* Do nothing; can reuse same def. */
1483 if (dt == vect_external_def || dt == vect_constant_def )
1484 return vec_oprnd;
1486 vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
1487 def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
1488 gcc_assert (def_stmt_info);
1489 vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
1490 gcc_assert (vec_stmt_for_operand);
1491 if (gimple_code (vec_stmt_for_operand) == GIMPLE_PHI)
1492 vec_oprnd = PHI_RESULT (vec_stmt_for_operand);
1493 else
1494 vec_oprnd = gimple_get_lhs (vec_stmt_for_operand);
1495 return vec_oprnd;
1499 /* Get vectorized definitions for the operands to create a copy of an original
1500 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1502 static void
1503 vect_get_vec_defs_for_stmt_copy (enum vect_def_type *dt,
1504 vec<tree> *vec_oprnds0,
1505 vec<tree> *vec_oprnds1)
1507 tree vec_oprnd = vec_oprnds0->pop ();
1509 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd);
1510 vec_oprnds0->quick_push (vec_oprnd);
1512 if (vec_oprnds1 && vec_oprnds1->length ())
1514 vec_oprnd = vec_oprnds1->pop ();
1515 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt[1], vec_oprnd);
1516 vec_oprnds1->quick_push (vec_oprnd);
1521 /* Get vectorized definitions for OP0 and OP1.
1522 REDUC_INDEX is the index of reduction operand in case of reduction,
1523 and -1 otherwise. */
1525 void
1526 vect_get_vec_defs (tree op0, tree op1, gimple *stmt,
1527 vec<tree> *vec_oprnds0,
1528 vec<tree> *vec_oprnds1,
1529 slp_tree slp_node, int reduc_index)
1531 if (slp_node)
1533 int nops = (op1 == NULL_TREE) ? 1 : 2;
1534 auto_vec<tree> ops (nops);
1535 auto_vec<vec<tree> > vec_defs (nops);
1537 ops.quick_push (op0);
1538 if (op1)
1539 ops.quick_push (op1);
1541 vect_get_slp_defs (ops, slp_node, &vec_defs, reduc_index);
1543 *vec_oprnds0 = vec_defs[0];
1544 if (op1)
1545 *vec_oprnds1 = vec_defs[1];
1547 else
1549 tree vec_oprnd;
1551 vec_oprnds0->create (1);
1552 vec_oprnd = vect_get_vec_def_for_operand (op0, stmt);
1553 vec_oprnds0->quick_push (vec_oprnd);
1555 if (op1)
1557 vec_oprnds1->create (1);
1558 vec_oprnd = vect_get_vec_def_for_operand (op1, stmt);
1559 vec_oprnds1->quick_push (vec_oprnd);
1565 /* Function vect_finish_stmt_generation.
1567 Insert a new stmt. */
1569 void
1570 vect_finish_stmt_generation (gimple *stmt, gimple *vec_stmt,
1571 gimple_stmt_iterator *gsi)
1573 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1574 vec_info *vinfo = stmt_info->vinfo;
1576 gcc_assert (gimple_code (stmt) != GIMPLE_LABEL);
1578 if (!gsi_end_p (*gsi)
1579 && gimple_has_mem_ops (vec_stmt))
1581 gimple *at_stmt = gsi_stmt (*gsi);
1582 tree vuse = gimple_vuse (at_stmt);
1583 if (vuse && TREE_CODE (vuse) == SSA_NAME)
1585 tree vdef = gimple_vdef (at_stmt);
1586 gimple_set_vuse (vec_stmt, gimple_vuse (at_stmt));
1587 /* If we have an SSA vuse and insert a store, update virtual
1588 SSA form to avoid triggering the renamer. Do so only
1589 if we can easily see all uses - which is what almost always
1590 happens with the way vectorized stmts are inserted. */
1591 if ((vdef && TREE_CODE (vdef) == SSA_NAME)
1592 && ((is_gimple_assign (vec_stmt)
1593 && !is_gimple_reg (gimple_assign_lhs (vec_stmt)))
1594 || (is_gimple_call (vec_stmt)
1595 && !(gimple_call_flags (vec_stmt)
1596 & (ECF_CONST|ECF_PURE|ECF_NOVOPS)))))
1598 tree new_vdef = copy_ssa_name (vuse, vec_stmt);
1599 gimple_set_vdef (vec_stmt, new_vdef);
1600 SET_USE (gimple_vuse_op (at_stmt), new_vdef);
1604 gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT);
1606 set_vinfo_for_stmt (vec_stmt, new_stmt_vec_info (vec_stmt, vinfo));
1608 if (dump_enabled_p ())
1610 dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: ");
1611 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, vec_stmt, 0);
1614 gimple_set_location (vec_stmt, gimple_location (stmt));
1616 /* While EH edges will generally prevent vectorization, stmt might
1617 e.g. be in a must-not-throw region. Ensure newly created stmts
1618 that could throw are part of the same region. */
1619 int lp_nr = lookup_stmt_eh_lp (stmt);
1620 if (lp_nr != 0 && stmt_could_throw_p (vec_stmt))
1621 add_stmt_to_eh_lp (vec_stmt, lp_nr);
1624 /* We want to vectorize a call to combined function CFN with function
1625 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1626 as the types of all inputs. Check whether this is possible using
1627 an internal function, returning its code if so or IFN_LAST if not. */
1629 static internal_fn
1630 vectorizable_internal_function (combined_fn cfn, tree fndecl,
1631 tree vectype_out, tree vectype_in)
1633 internal_fn ifn;
1634 if (internal_fn_p (cfn))
1635 ifn = as_internal_fn (cfn);
1636 else
1637 ifn = associated_internal_fn (fndecl);
1638 if (ifn != IFN_LAST && direct_internal_fn_p (ifn))
1640 const direct_internal_fn_info &info = direct_internal_fn (ifn);
1641 if (info.vectorizable)
1643 tree type0 = (info.type0 < 0 ? vectype_out : vectype_in);
1644 tree type1 = (info.type1 < 0 ? vectype_out : vectype_in);
1645 if (direct_internal_fn_supported_p (ifn, tree_pair (type0, type1),
1646 OPTIMIZE_FOR_SPEED))
1647 return ifn;
1650 return IFN_LAST;
1654 static tree permute_vec_elements (tree, tree, tree, gimple *,
1655 gimple_stmt_iterator *);
1658 /* Function vectorizable_mask_load_store.
1660 Check if STMT performs a conditional load or store that can be vectorized.
1661 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1662 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1663 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1665 static bool
1666 vectorizable_mask_load_store (gimple *stmt, gimple_stmt_iterator *gsi,
1667 gimple **vec_stmt, slp_tree slp_node)
1669 tree vec_dest = NULL;
1670 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
1671 stmt_vec_info prev_stmt_info;
1672 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
1673 struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
1674 bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
1675 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
1676 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1677 tree rhs_vectype = NULL_TREE;
1678 tree mask_vectype;
1679 tree elem_type;
1680 gimple *new_stmt;
1681 tree dummy;
1682 tree dataref_ptr = NULL_TREE;
1683 gimple *ptr_incr;
1684 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
1685 int ncopies;
1686 int i, j;
1687 bool inv_p;
1688 tree gather_base = NULL_TREE, gather_off = NULL_TREE;
1689 tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
1690 int gather_scale = 1;
1691 enum vect_def_type gather_dt = vect_unknown_def_type;
1692 bool is_store;
1693 tree mask;
1694 gimple *def_stmt;
1695 enum vect_def_type dt;
1697 if (slp_node != NULL)
1698 return false;
1700 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
1701 gcc_assert (ncopies >= 1);
1703 is_store = gimple_call_internal_fn (stmt) == IFN_MASK_STORE;
1704 mask = gimple_call_arg (stmt, 2);
1706 if (TREE_CODE (TREE_TYPE (mask)) != BOOLEAN_TYPE)
1707 return false;
1709 /* FORNOW. This restriction should be relaxed. */
1710 if (nested_in_vect_loop && ncopies > 1)
1712 if (dump_enabled_p ())
1713 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1714 "multiple types in nested loop.");
1715 return false;
1718 if (!STMT_VINFO_RELEVANT_P (stmt_info))
1719 return false;
1721 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
1722 && ! vec_stmt)
1723 return false;
1725 if (!STMT_VINFO_DATA_REF (stmt_info))
1726 return false;
1728 elem_type = TREE_TYPE (vectype);
1730 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1731 return false;
1733 if (STMT_VINFO_STRIDED_P (stmt_info))
1734 return false;
1736 if (TREE_CODE (mask) != SSA_NAME)
1737 return false;
1739 if (!vect_is_simple_use (mask, loop_vinfo, &def_stmt, &dt, &mask_vectype))
1740 return false;
1742 if (!mask_vectype)
1743 mask_vectype = get_mask_type_for_scalar_type (TREE_TYPE (vectype));
1745 if (!mask_vectype || !VECTOR_BOOLEAN_TYPE_P (mask_vectype)
1746 || TYPE_VECTOR_SUBPARTS (mask_vectype) != TYPE_VECTOR_SUBPARTS (vectype))
1747 return false;
1749 if (is_store)
1751 tree rhs = gimple_call_arg (stmt, 3);
1752 if (!vect_is_simple_use (rhs, loop_vinfo, &def_stmt, &dt, &rhs_vectype))
1753 return false;
1756 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1758 gimple *def_stmt;
1759 gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base,
1760 &gather_off, &gather_scale);
1761 gcc_assert (gather_decl);
1762 if (!vect_is_simple_use (gather_off, loop_vinfo, &def_stmt, &gather_dt,
1763 &gather_off_vectype))
1765 if (dump_enabled_p ())
1766 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1767 "gather index use not simple.");
1768 return false;
1771 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
1772 tree masktype
1773 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist))));
1774 if (TREE_CODE (masktype) == INTEGER_TYPE)
1776 if (dump_enabled_p ())
1777 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1778 "masked gather with integer mask not supported.");
1779 return false;
1782 else if (tree_int_cst_compare (nested_in_vect_loop
1783 ? STMT_VINFO_DR_STEP (stmt_info)
1784 : DR_STEP (dr), size_zero_node) <= 0)
1785 return false;
1786 else if (!VECTOR_MODE_P (TYPE_MODE (vectype))
1787 || !can_vec_mask_load_store_p (TYPE_MODE (vectype),
1788 TYPE_MODE (mask_vectype),
1789 !is_store)
1790 || (rhs_vectype
1791 && !useless_type_conversion_p (vectype, rhs_vectype)))
1792 return false;
1794 if (!vec_stmt) /* transformation not required. */
1796 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
1797 if (is_store)
1798 vect_model_store_cost (stmt_info, ncopies, false, dt,
1799 NULL, NULL, NULL);
1800 else
1801 vect_model_load_cost (stmt_info, ncopies, false, NULL, NULL, NULL);
1802 return true;
1805 /** Transform. **/
1807 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1809 tree vec_oprnd0 = NULL_TREE, op;
1810 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
1811 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
1812 tree ptr, vec_mask = NULL_TREE, mask_op = NULL_TREE, var, scale;
1813 tree perm_mask = NULL_TREE, prev_res = NULL_TREE;
1814 tree mask_perm_mask = NULL_TREE;
1815 edge pe = loop_preheader_edge (loop);
1816 gimple_seq seq;
1817 basic_block new_bb;
1818 enum { NARROW, NONE, WIDEN } modifier;
1819 int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
1821 rettype = TREE_TYPE (TREE_TYPE (gather_decl));
1822 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1823 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1824 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1825 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
1826 scaletype = TREE_VALUE (arglist);
1827 gcc_checking_assert (types_compatible_p (srctype, rettype)
1828 && types_compatible_p (srctype, masktype));
1830 if (nunits == gather_off_nunits)
1831 modifier = NONE;
1832 else if (nunits == gather_off_nunits / 2)
1834 unsigned char *sel = XALLOCAVEC (unsigned char, gather_off_nunits);
1835 modifier = WIDEN;
1837 for (i = 0; i < gather_off_nunits; ++i)
1838 sel[i] = i | nunits;
1840 perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel);
1842 else if (nunits == gather_off_nunits * 2)
1844 unsigned char *sel = XALLOCAVEC (unsigned char, nunits);
1845 modifier = NARROW;
1847 for (i = 0; i < nunits; ++i)
1848 sel[i] = i < gather_off_nunits
1849 ? i : i + nunits - gather_off_nunits;
1851 perm_mask = vect_gen_perm_mask_checked (vectype, sel);
1852 ncopies *= 2;
1853 for (i = 0; i < nunits; ++i)
1854 sel[i] = i | gather_off_nunits;
1855 mask_perm_mask = vect_gen_perm_mask_checked (masktype, sel);
1857 else
1858 gcc_unreachable ();
1860 vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype);
1862 ptr = fold_convert (ptrtype, gather_base);
1863 if (!is_gimple_min_invariant (ptr))
1865 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
1866 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
1867 gcc_assert (!new_bb);
1870 scale = build_int_cst (scaletype, gather_scale);
1872 prev_stmt_info = NULL;
1873 for (j = 0; j < ncopies; ++j)
1875 if (modifier == WIDEN && (j & 1))
1876 op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
1877 perm_mask, stmt, gsi);
1878 else if (j == 0)
1879 op = vec_oprnd0
1880 = vect_get_vec_def_for_operand (gather_off, stmt);
1881 else
1882 op = vec_oprnd0
1883 = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
1885 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
1887 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
1888 == TYPE_VECTOR_SUBPARTS (idxtype));
1889 var = vect_get_new_ssa_name (idxtype, vect_simple_var);
1890 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
1891 new_stmt
1892 = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
1893 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1894 op = var;
1897 if (mask_perm_mask && (j & 1))
1898 mask_op = permute_vec_elements (mask_op, mask_op,
1899 mask_perm_mask, stmt, gsi);
1900 else
1902 if (j == 0)
1903 vec_mask = vect_get_vec_def_for_operand (mask, stmt);
1904 else
1906 vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
1907 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
1910 mask_op = vec_mask;
1911 if (!useless_type_conversion_p (masktype, TREE_TYPE (vec_mask)))
1913 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op))
1914 == TYPE_VECTOR_SUBPARTS (masktype));
1915 var = vect_get_new_ssa_name (masktype, vect_simple_var);
1916 mask_op = build1 (VIEW_CONVERT_EXPR, masktype, mask_op);
1917 new_stmt
1918 = gimple_build_assign (var, VIEW_CONVERT_EXPR, mask_op);
1919 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1920 mask_op = var;
1924 new_stmt
1925 = gimple_build_call (gather_decl, 5, mask_op, ptr, op, mask_op,
1926 scale);
1928 if (!useless_type_conversion_p (vectype, rettype))
1930 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
1931 == TYPE_VECTOR_SUBPARTS (rettype));
1932 op = vect_get_new_ssa_name (rettype, vect_simple_var);
1933 gimple_call_set_lhs (new_stmt, op);
1934 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1935 var = make_ssa_name (vec_dest);
1936 op = build1 (VIEW_CONVERT_EXPR, vectype, op);
1937 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
1939 else
1941 var = make_ssa_name (vec_dest, new_stmt);
1942 gimple_call_set_lhs (new_stmt, var);
1945 vect_finish_stmt_generation (stmt, new_stmt, gsi);
1947 if (modifier == NARROW)
1949 if ((j & 1) == 0)
1951 prev_res = var;
1952 continue;
1954 var = permute_vec_elements (prev_res, var,
1955 perm_mask, stmt, gsi);
1956 new_stmt = SSA_NAME_DEF_STMT (var);
1959 if (prev_stmt_info == NULL)
1960 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
1961 else
1962 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
1963 prev_stmt_info = vinfo_for_stmt (new_stmt);
1966 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
1967 from the IL. */
1968 if (STMT_VINFO_RELATED_STMT (stmt_info))
1970 stmt = STMT_VINFO_RELATED_STMT (stmt_info);
1971 stmt_info = vinfo_for_stmt (stmt);
1973 tree lhs = gimple_call_lhs (stmt);
1974 new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
1975 set_vinfo_for_stmt (new_stmt, stmt_info);
1976 set_vinfo_for_stmt (stmt, NULL);
1977 STMT_VINFO_STMT (stmt_info) = new_stmt;
1978 gsi_replace (gsi, new_stmt, true);
1979 return true;
1981 else if (is_store)
1983 tree vec_rhs = NULL_TREE, vec_mask = NULL_TREE;
1984 prev_stmt_info = NULL;
1985 LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
1986 for (i = 0; i < ncopies; i++)
1988 unsigned align, misalign;
1990 if (i == 0)
1992 tree rhs = gimple_call_arg (stmt, 3);
1993 vec_rhs = vect_get_vec_def_for_operand (rhs, stmt);
1994 vec_mask = vect_get_vec_def_for_operand (mask, stmt);
1995 /* We should have catched mismatched types earlier. */
1996 gcc_assert (useless_type_conversion_p (vectype,
1997 TREE_TYPE (vec_rhs)));
1998 dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL,
1999 NULL_TREE, &dummy, gsi,
2000 &ptr_incr, false, &inv_p);
2001 gcc_assert (!inv_p);
2003 else
2005 vect_is_simple_use (vec_rhs, loop_vinfo, &def_stmt, &dt);
2006 vec_rhs = vect_get_vec_def_for_stmt_copy (dt, vec_rhs);
2007 vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
2008 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
2009 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
2010 TYPE_SIZE_UNIT (vectype));
2013 align = TYPE_ALIGN_UNIT (vectype);
2014 if (aligned_access_p (dr))
2015 misalign = 0;
2016 else if (DR_MISALIGNMENT (dr) == -1)
2018 align = TYPE_ALIGN_UNIT (elem_type);
2019 misalign = 0;
2021 else
2022 misalign = DR_MISALIGNMENT (dr);
2023 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
2024 misalign);
2025 tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)),
2026 misalign ? misalign & -misalign : align);
2027 new_stmt
2028 = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
2029 ptr, vec_mask, vec_rhs);
2030 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2031 if (i == 0)
2032 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2033 else
2034 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2035 prev_stmt_info = vinfo_for_stmt (new_stmt);
2038 else
2040 tree vec_mask = NULL_TREE;
2041 prev_stmt_info = NULL;
2042 vec_dest = vect_create_destination_var (gimple_call_lhs (stmt), vectype);
2043 for (i = 0; i < ncopies; i++)
2045 unsigned align, misalign;
2047 if (i == 0)
2049 vec_mask = vect_get_vec_def_for_operand (mask, stmt);
2050 dataref_ptr = vect_create_data_ref_ptr (stmt, vectype, NULL,
2051 NULL_TREE, &dummy, gsi,
2052 &ptr_incr, false, &inv_p);
2053 gcc_assert (!inv_p);
2055 else
2057 vect_is_simple_use (vec_mask, loop_vinfo, &def_stmt, &dt);
2058 vec_mask = vect_get_vec_def_for_stmt_copy (dt, vec_mask);
2059 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
2060 TYPE_SIZE_UNIT (vectype));
2063 align = TYPE_ALIGN_UNIT (vectype);
2064 if (aligned_access_p (dr))
2065 misalign = 0;
2066 else if (DR_MISALIGNMENT (dr) == -1)
2068 align = TYPE_ALIGN_UNIT (elem_type);
2069 misalign = 0;
2071 else
2072 misalign = DR_MISALIGNMENT (dr);
2073 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
2074 misalign);
2075 tree ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)),
2076 misalign ? misalign & -misalign : align);
2077 new_stmt
2078 = gimple_build_call_internal (IFN_MASK_LOAD, 3, dataref_ptr,
2079 ptr, vec_mask);
2080 gimple_call_set_lhs (new_stmt, make_ssa_name (vec_dest));
2081 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2082 if (i == 0)
2083 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2084 else
2085 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2086 prev_stmt_info = vinfo_for_stmt (new_stmt);
2090 if (!is_store)
2092 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2093 from the IL. */
2094 if (STMT_VINFO_RELATED_STMT (stmt_info))
2096 stmt = STMT_VINFO_RELATED_STMT (stmt_info);
2097 stmt_info = vinfo_for_stmt (stmt);
2099 tree lhs = gimple_call_lhs (stmt);
2100 new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
2101 set_vinfo_for_stmt (new_stmt, stmt_info);
2102 set_vinfo_for_stmt (stmt, NULL);
2103 STMT_VINFO_STMT (stmt_info) = new_stmt;
2104 gsi_replace (gsi, new_stmt, true);
2107 return true;
2110 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
2111 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
2112 in a single step. On success, store the binary pack code in
2113 *CONVERT_CODE. */
2115 static bool
2116 simple_integer_narrowing (tree vectype_out, tree vectype_in,
2117 tree_code *convert_code)
2119 if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype_out))
2120 || !INTEGRAL_TYPE_P (TREE_TYPE (vectype_in)))
2121 return false;
2123 tree_code code;
2124 int multi_step_cvt = 0;
2125 auto_vec <tree, 8> interm_types;
2126 if (!supportable_narrowing_operation (NOP_EXPR, vectype_out, vectype_in,
2127 &code, &multi_step_cvt,
2128 &interm_types)
2129 || multi_step_cvt)
2130 return false;
2132 *convert_code = code;
2133 return true;
2136 /* Function vectorizable_call.
2138 Check if GS performs a function call that can be vectorized.
2139 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2140 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2141 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2143 static bool
2144 vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, gimple **vec_stmt,
2145 slp_tree slp_node)
2147 gcall *stmt;
2148 tree vec_dest;
2149 tree scalar_dest;
2150 tree op, type;
2151 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
2152 stmt_vec_info stmt_info = vinfo_for_stmt (gs), prev_stmt_info;
2153 tree vectype_out, vectype_in;
2154 int nunits_in;
2155 int nunits_out;
2156 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2157 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2158 vec_info *vinfo = stmt_info->vinfo;
2159 tree fndecl, new_temp, rhs_type;
2160 gimple *def_stmt;
2161 enum vect_def_type dt[3]
2162 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
2163 gimple *new_stmt = NULL;
2164 int ncopies, j;
2165 vec<tree> vargs = vNULL;
2166 enum { NARROW, NONE, WIDEN } modifier;
2167 size_t i, nargs;
2168 tree lhs;
2170 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2171 return false;
2173 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
2174 && ! vec_stmt)
2175 return false;
2177 /* Is GS a vectorizable call? */
2178 stmt = dyn_cast <gcall *> (gs);
2179 if (!stmt)
2180 return false;
2182 if (gimple_call_internal_p (stmt)
2183 && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
2184 || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
2185 return vectorizable_mask_load_store (stmt, gsi, vec_stmt,
2186 slp_node);
2188 if (gimple_call_lhs (stmt) == NULL_TREE
2189 || TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
2190 return false;
2192 gcc_checking_assert (!stmt_can_throw_internal (stmt));
2194 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
2196 /* Process function arguments. */
2197 rhs_type = NULL_TREE;
2198 vectype_in = NULL_TREE;
2199 nargs = gimple_call_num_args (stmt);
2201 /* Bail out if the function has more than three arguments, we do not have
2202 interesting builtin functions to vectorize with more than two arguments
2203 except for fma. No arguments is also not good. */
2204 if (nargs == 0 || nargs > 3)
2205 return false;
2207 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2208 if (gimple_call_internal_p (stmt)
2209 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
2211 nargs = 0;
2212 rhs_type = unsigned_type_node;
2215 for (i = 0; i < nargs; i++)
2217 tree opvectype;
2219 op = gimple_call_arg (stmt, i);
2221 /* We can only handle calls with arguments of the same type. */
2222 if (rhs_type
2223 && !types_compatible_p (rhs_type, TREE_TYPE (op)))
2225 if (dump_enabled_p ())
2226 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2227 "argument types differ.\n");
2228 return false;
2230 if (!rhs_type)
2231 rhs_type = TREE_TYPE (op);
2233 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[i], &opvectype))
2235 if (dump_enabled_p ())
2236 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2237 "use not simple.\n");
2238 return false;
2241 if (!vectype_in)
2242 vectype_in = opvectype;
2243 else if (opvectype
2244 && opvectype != vectype_in)
2246 if (dump_enabled_p ())
2247 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2248 "argument vector types differ.\n");
2249 return false;
2252 /* If all arguments are external or constant defs use a vector type with
2253 the same size as the output vector type. */
2254 if (!vectype_in)
2255 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
2256 if (vec_stmt)
2257 gcc_assert (vectype_in);
2258 if (!vectype_in)
2260 if (dump_enabled_p ())
2262 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2263 "no vectype for scalar type ");
2264 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
2265 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
2268 return false;
2271 /* FORNOW */
2272 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
2273 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
2274 if (nunits_in == nunits_out / 2)
2275 modifier = NARROW;
2276 else if (nunits_out == nunits_in)
2277 modifier = NONE;
2278 else if (nunits_out == nunits_in / 2)
2279 modifier = WIDEN;
2280 else
2281 return false;
2283 /* We only handle functions that do not read or clobber memory. */
2284 if (gimple_vuse (stmt))
2286 if (dump_enabled_p ())
2287 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2288 "function reads from or writes to memory.\n");
2289 return false;
2292 /* For now, we only vectorize functions if a target specific builtin
2293 is available. TODO -- in some cases, it might be profitable to
2294 insert the calls for pieces of the vector, in order to be able
2295 to vectorize other operations in the loop. */
2296 fndecl = NULL_TREE;
2297 internal_fn ifn = IFN_LAST;
2298 combined_fn cfn = gimple_call_combined_fn (stmt);
2299 tree callee = gimple_call_fndecl (stmt);
2301 /* First try using an internal function. */
2302 tree_code convert_code = ERROR_MARK;
2303 if (cfn != CFN_LAST
2304 && (modifier == NONE
2305 || (modifier == NARROW
2306 && simple_integer_narrowing (vectype_out, vectype_in,
2307 &convert_code))))
2308 ifn = vectorizable_internal_function (cfn, callee, vectype_out,
2309 vectype_in);
2311 /* If that fails, try asking for a target-specific built-in function. */
2312 if (ifn == IFN_LAST)
2314 if (cfn != CFN_LAST)
2315 fndecl = targetm.vectorize.builtin_vectorized_function
2316 (cfn, vectype_out, vectype_in);
2317 else
2318 fndecl = targetm.vectorize.builtin_md_vectorized_function
2319 (callee, vectype_out, vectype_in);
2322 if (ifn == IFN_LAST && !fndecl)
2324 if (cfn == CFN_GOMP_SIMD_LANE
2325 && !slp_node
2326 && loop_vinfo
2327 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
2328 && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME
2329 && LOOP_VINFO_LOOP (loop_vinfo)->simduid
2330 == SSA_NAME_VAR (gimple_call_arg (stmt, 0)))
2332 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2333 { 0, 1, 2, ... vf - 1 } vector. */
2334 gcc_assert (nargs == 0);
2336 else
2338 if (dump_enabled_p ())
2339 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2340 "function is not vectorizable.\n");
2341 return false;
2345 if (slp_node || PURE_SLP_STMT (stmt_info))
2346 ncopies = 1;
2347 else if (modifier == NARROW && ifn == IFN_LAST)
2348 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
2349 else
2350 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
2352 /* Sanity check: make sure that at least one copy of the vectorized stmt
2353 needs to be generated. */
2354 gcc_assert (ncopies >= 1);
2356 if (!vec_stmt) /* transformation not required. */
2358 STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
2359 if (dump_enabled_p ())
2360 dump_printf_loc (MSG_NOTE, vect_location, "=== vectorizable_call ==="
2361 "\n");
2362 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
2363 if (ifn != IFN_LAST && modifier == NARROW && !slp_node)
2364 add_stmt_cost (stmt_info->vinfo->target_cost_data, ncopies / 2,
2365 vec_promote_demote, stmt_info, 0, vect_body);
2367 return true;
2370 /** Transform. **/
2372 if (dump_enabled_p ())
2373 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
2375 /* Handle def. */
2376 scalar_dest = gimple_call_lhs (stmt);
2377 vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
2379 prev_stmt_info = NULL;
2380 if (modifier == NONE || ifn != IFN_LAST)
2382 tree prev_res = NULL_TREE;
2383 for (j = 0; j < ncopies; ++j)
2385 /* Build argument list for the vectorized call. */
2386 if (j == 0)
2387 vargs.create (nargs);
2388 else
2389 vargs.truncate (0);
2391 if (slp_node)
2393 auto_vec<vec<tree> > vec_defs (nargs);
2394 vec<tree> vec_oprnds0;
2396 for (i = 0; i < nargs; i++)
2397 vargs.quick_push (gimple_call_arg (stmt, i));
2398 vect_get_slp_defs (vargs, slp_node, &vec_defs, -1);
2399 vec_oprnds0 = vec_defs[0];
2401 /* Arguments are ready. Create the new vector stmt. */
2402 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_oprnd0)
2404 size_t k;
2405 for (k = 0; k < nargs; k++)
2407 vec<tree> vec_oprndsk = vec_defs[k];
2408 vargs[k] = vec_oprndsk[i];
2410 if (modifier == NARROW)
2412 tree half_res = make_ssa_name (vectype_in);
2413 new_stmt = gimple_build_call_internal_vec (ifn, vargs);
2414 gimple_call_set_lhs (new_stmt, half_res);
2415 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2416 if ((i & 1) == 0)
2418 prev_res = half_res;
2419 continue;
2421 new_temp = make_ssa_name (vec_dest);
2422 new_stmt = gimple_build_assign (new_temp, convert_code,
2423 prev_res, half_res);
2425 else
2427 if (ifn != IFN_LAST)
2428 new_stmt = gimple_build_call_internal_vec (ifn, vargs);
2429 else
2430 new_stmt = gimple_build_call_vec (fndecl, vargs);
2431 new_temp = make_ssa_name (vec_dest, new_stmt);
2432 gimple_call_set_lhs (new_stmt, new_temp);
2434 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2435 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
2438 for (i = 0; i < nargs; i++)
2440 vec<tree> vec_oprndsi = vec_defs[i];
2441 vec_oprndsi.release ();
2443 continue;
2446 for (i = 0; i < nargs; i++)
2448 op = gimple_call_arg (stmt, i);
2449 if (j == 0)
2450 vec_oprnd0
2451 = vect_get_vec_def_for_operand (op, stmt);
2452 else
2454 vec_oprnd0 = gimple_call_arg (new_stmt, i);
2455 vec_oprnd0
2456 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2459 vargs.quick_push (vec_oprnd0);
2462 if (gimple_call_internal_p (stmt)
2463 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
2465 tree *v = XALLOCAVEC (tree, nunits_out);
2466 int k;
2467 for (k = 0; k < nunits_out; ++k)
2468 v[k] = build_int_cst (unsigned_type_node, j * nunits_out + k);
2469 tree cst = build_vector (vectype_out, v);
2470 tree new_var
2471 = vect_get_new_ssa_name (vectype_out, vect_simple_var, "cst_");
2472 gimple *init_stmt = gimple_build_assign (new_var, cst);
2473 vect_init_vector_1 (stmt, init_stmt, NULL);
2474 new_temp = make_ssa_name (vec_dest);
2475 new_stmt = gimple_build_assign (new_temp, new_var);
2477 else if (modifier == NARROW)
2479 tree half_res = make_ssa_name (vectype_in);
2480 new_stmt = gimple_build_call_internal_vec (ifn, vargs);
2481 gimple_call_set_lhs (new_stmt, half_res);
2482 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2483 if ((j & 1) == 0)
2485 prev_res = half_res;
2486 continue;
2488 new_temp = make_ssa_name (vec_dest);
2489 new_stmt = gimple_build_assign (new_temp, convert_code,
2490 prev_res, half_res);
2492 else
2494 if (ifn != IFN_LAST)
2495 new_stmt = gimple_build_call_internal_vec (ifn, vargs);
2496 else
2497 new_stmt = gimple_build_call_vec (fndecl, vargs);
2498 new_temp = make_ssa_name (vec_dest, new_stmt);
2499 gimple_call_set_lhs (new_stmt, new_temp);
2501 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2503 if (j == (modifier == NARROW ? 1 : 0))
2504 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
2505 else
2506 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2508 prev_stmt_info = vinfo_for_stmt (new_stmt);
2511 else if (modifier == NARROW)
2513 for (j = 0; j < ncopies; ++j)
2515 /* Build argument list for the vectorized call. */
2516 if (j == 0)
2517 vargs.create (nargs * 2);
2518 else
2519 vargs.truncate (0);
2521 if (slp_node)
2523 auto_vec<vec<tree> > vec_defs (nargs);
2524 vec<tree> vec_oprnds0;
2526 for (i = 0; i < nargs; i++)
2527 vargs.quick_push (gimple_call_arg (stmt, i));
2528 vect_get_slp_defs (vargs, slp_node, &vec_defs, -1);
2529 vec_oprnds0 = vec_defs[0];
2531 /* Arguments are ready. Create the new vector stmt. */
2532 for (i = 0; vec_oprnds0.iterate (i, &vec_oprnd0); i += 2)
2534 size_t k;
2535 vargs.truncate (0);
2536 for (k = 0; k < nargs; k++)
2538 vec<tree> vec_oprndsk = vec_defs[k];
2539 vargs.quick_push (vec_oprndsk[i]);
2540 vargs.quick_push (vec_oprndsk[i + 1]);
2542 if (ifn != IFN_LAST)
2543 new_stmt = gimple_build_call_internal_vec (ifn, vargs);
2544 else
2545 new_stmt = gimple_build_call_vec (fndecl, vargs);
2546 new_temp = make_ssa_name (vec_dest, new_stmt);
2547 gimple_call_set_lhs (new_stmt, new_temp);
2548 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2549 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
2552 for (i = 0; i < nargs; i++)
2554 vec<tree> vec_oprndsi = vec_defs[i];
2555 vec_oprndsi.release ();
2557 continue;
2560 for (i = 0; i < nargs; i++)
2562 op = gimple_call_arg (stmt, i);
2563 if (j == 0)
2565 vec_oprnd0
2566 = vect_get_vec_def_for_operand (op, stmt);
2567 vec_oprnd1
2568 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2570 else
2572 vec_oprnd1 = gimple_call_arg (new_stmt, 2*i + 1);
2573 vec_oprnd0
2574 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd1);
2575 vec_oprnd1
2576 = vect_get_vec_def_for_stmt_copy (dt[i], vec_oprnd0);
2579 vargs.quick_push (vec_oprnd0);
2580 vargs.quick_push (vec_oprnd1);
2583 new_stmt = gimple_build_call_vec (fndecl, vargs);
2584 new_temp = make_ssa_name (vec_dest, new_stmt);
2585 gimple_call_set_lhs (new_stmt, new_temp);
2586 vect_finish_stmt_generation (stmt, new_stmt, gsi);
2588 if (j == 0)
2589 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
2590 else
2591 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
2593 prev_stmt_info = vinfo_for_stmt (new_stmt);
2596 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
2598 else
2599 /* No current target implements this case. */
2600 return false;
2602 vargs.release ();
2604 /* The call in STMT might prevent it from being removed in dce.
2605 We however cannot remove it here, due to the way the ssa name
2606 it defines is mapped to the new definition. So just replace
2607 rhs of the statement with something harmless. */
2609 if (slp_node)
2610 return true;
2612 type = TREE_TYPE (scalar_dest);
2613 if (is_pattern_stmt_p (stmt_info))
2614 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
2615 else
2616 lhs = gimple_call_lhs (stmt);
2618 if (gimple_call_internal_p (stmt)
2619 && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE)
2621 /* Replace uses of the lhs of GOMP_SIMD_LANE call outside the loop
2622 with vf - 1 rather than 0, that is the last iteration of the
2623 vectorized loop. */
2624 imm_use_iterator iter;
2625 use_operand_p use_p;
2626 gimple *use_stmt;
2627 FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
2629 basic_block use_bb = gimple_bb (use_stmt);
2630 if (use_bb
2631 && !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo), use_bb))
2633 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
2634 SET_USE (use_p, build_int_cst (TREE_TYPE (lhs),
2635 ncopies * nunits_out - 1));
2636 update_stmt (use_stmt);
2641 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
2642 set_vinfo_for_stmt (new_stmt, stmt_info);
2643 set_vinfo_for_stmt (stmt, NULL);
2644 STMT_VINFO_STMT (stmt_info) = new_stmt;
2645 gsi_replace (gsi, new_stmt, false);
2647 return true;
2651 struct simd_call_arg_info
2653 tree vectype;
2654 tree op;
2655 enum vect_def_type dt;
2656 HOST_WIDE_INT linear_step;
2657 unsigned int align;
2658 bool simd_lane_linear;
2661 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
2662 is linear within simd lane (but not within whole loop), note it in
2663 *ARGINFO. */
2665 static void
2666 vect_simd_lane_linear (tree op, struct loop *loop,
2667 struct simd_call_arg_info *arginfo)
2669 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
2671 if (!is_gimple_assign (def_stmt)
2672 || gimple_assign_rhs_code (def_stmt) != POINTER_PLUS_EXPR
2673 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt)))
2674 return;
2676 tree base = gimple_assign_rhs1 (def_stmt);
2677 HOST_WIDE_INT linear_step = 0;
2678 tree v = gimple_assign_rhs2 (def_stmt);
2679 while (TREE_CODE (v) == SSA_NAME)
2681 tree t;
2682 def_stmt = SSA_NAME_DEF_STMT (v);
2683 if (is_gimple_assign (def_stmt))
2684 switch (gimple_assign_rhs_code (def_stmt))
2686 case PLUS_EXPR:
2687 t = gimple_assign_rhs2 (def_stmt);
2688 if (linear_step || TREE_CODE (t) != INTEGER_CST)
2689 return;
2690 base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (base), base, t);
2691 v = gimple_assign_rhs1 (def_stmt);
2692 continue;
2693 case MULT_EXPR:
2694 t = gimple_assign_rhs2 (def_stmt);
2695 if (linear_step || !tree_fits_shwi_p (t) || integer_zerop (t))
2696 return;
2697 linear_step = tree_to_shwi (t);
2698 v = gimple_assign_rhs1 (def_stmt);
2699 continue;
2700 CASE_CONVERT:
2701 t = gimple_assign_rhs1 (def_stmt);
2702 if (TREE_CODE (TREE_TYPE (t)) != INTEGER_TYPE
2703 || (TYPE_PRECISION (TREE_TYPE (v))
2704 < TYPE_PRECISION (TREE_TYPE (t))))
2705 return;
2706 if (!linear_step)
2707 linear_step = 1;
2708 v = t;
2709 continue;
2710 default:
2711 return;
2713 else if (is_gimple_call (def_stmt)
2714 && gimple_call_internal_p (def_stmt)
2715 && gimple_call_internal_fn (def_stmt) == IFN_GOMP_SIMD_LANE
2716 && loop->simduid
2717 && TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME
2718 && (SSA_NAME_VAR (gimple_call_arg (def_stmt, 0))
2719 == loop->simduid))
2721 if (!linear_step)
2722 linear_step = 1;
2723 arginfo->linear_step = linear_step;
2724 arginfo->op = base;
2725 arginfo->simd_lane_linear = true;
2726 return;
2731 /* Function vectorizable_simd_clone_call.
2733 Check if STMT performs a function call that can be vectorized
2734 by calling a simd clone of the function.
2735 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2736 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2737 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2739 static bool
2740 vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi,
2741 gimple **vec_stmt, slp_tree slp_node)
2743 tree vec_dest;
2744 tree scalar_dest;
2745 tree op, type;
2746 tree vec_oprnd0 = NULL_TREE;
2747 stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
2748 tree vectype;
2749 unsigned int nunits;
2750 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
2751 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
2752 vec_info *vinfo = stmt_info->vinfo;
2753 struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
2754 tree fndecl, new_temp;
2755 gimple *def_stmt;
2756 gimple *new_stmt = NULL;
2757 int ncopies, j;
2758 vec<simd_call_arg_info> arginfo = vNULL;
2759 vec<tree> vargs = vNULL;
2760 size_t i, nargs;
2761 tree lhs, rtype, ratype;
2762 vec<constructor_elt, va_gc> *ret_ctor_elts;
2764 /* Is STMT a vectorizable call? */
2765 if (!is_gimple_call (stmt))
2766 return false;
2768 fndecl = gimple_call_fndecl (stmt);
2769 if (fndecl == NULL_TREE)
2770 return false;
2772 struct cgraph_node *node = cgraph_node::get (fndecl);
2773 if (node == NULL || node->simd_clones == NULL)
2774 return false;
2776 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
2777 return false;
2779 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
2780 && ! vec_stmt)
2781 return false;
2783 if (gimple_call_lhs (stmt)
2784 && TREE_CODE (gimple_call_lhs (stmt)) != SSA_NAME)
2785 return false;
2787 gcc_checking_assert (!stmt_can_throw_internal (stmt));
2789 vectype = STMT_VINFO_VECTYPE (stmt_info);
2791 if (loop_vinfo && nested_in_vect_loop_p (loop, stmt))
2792 return false;
2794 /* FORNOW */
2795 if (slp_node || PURE_SLP_STMT (stmt_info))
2796 return false;
2798 /* Process function arguments. */
2799 nargs = gimple_call_num_args (stmt);
2801 /* Bail out if the function has zero arguments. */
2802 if (nargs == 0)
2803 return false;
2805 arginfo.create (nargs);
2807 for (i = 0; i < nargs; i++)
2809 simd_call_arg_info thisarginfo;
2810 affine_iv iv;
2812 thisarginfo.linear_step = 0;
2813 thisarginfo.align = 0;
2814 thisarginfo.op = NULL_TREE;
2815 thisarginfo.simd_lane_linear = false;
2817 op = gimple_call_arg (stmt, i);
2818 if (!vect_is_simple_use (op, vinfo, &def_stmt, &thisarginfo.dt,
2819 &thisarginfo.vectype)
2820 || thisarginfo.dt == vect_uninitialized_def)
2822 if (dump_enabled_p ())
2823 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2824 "use not simple.\n");
2825 arginfo.release ();
2826 return false;
2829 if (thisarginfo.dt == vect_constant_def
2830 || thisarginfo.dt == vect_external_def)
2831 gcc_assert (thisarginfo.vectype == NULL_TREE);
2832 else
2833 gcc_assert (thisarginfo.vectype != NULL_TREE);
2835 /* For linear arguments, the analyze phase should have saved
2836 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2837 if (i * 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info).length ()
2838 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2])
2840 gcc_assert (vec_stmt);
2841 thisarginfo.linear_step
2842 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2]);
2843 thisarginfo.op
2844 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 1];
2845 thisarginfo.simd_lane_linear
2846 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 3]
2847 == boolean_true_node);
2848 /* If loop has been peeled for alignment, we need to adjust it. */
2849 tree n1 = LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo);
2850 tree n2 = LOOP_VINFO_NITERS (loop_vinfo);
2851 if (n1 != n2 && !thisarginfo.simd_lane_linear)
2853 tree bias = fold_build2 (MINUS_EXPR, TREE_TYPE (n1), n1, n2);
2854 tree step = STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[i * 3 + 2];
2855 tree opt = TREE_TYPE (thisarginfo.op);
2856 bias = fold_convert (TREE_TYPE (step), bias);
2857 bias = fold_build2 (MULT_EXPR, TREE_TYPE (step), bias, step);
2858 thisarginfo.op
2859 = fold_build2 (POINTER_TYPE_P (opt)
2860 ? POINTER_PLUS_EXPR : PLUS_EXPR, opt,
2861 thisarginfo.op, bias);
2864 else if (!vec_stmt
2865 && thisarginfo.dt != vect_constant_def
2866 && thisarginfo.dt != vect_external_def
2867 && loop_vinfo
2868 && TREE_CODE (op) == SSA_NAME
2869 && simple_iv (loop, loop_containing_stmt (stmt), op,
2870 &iv, false)
2871 && tree_fits_shwi_p (iv.step))
2873 thisarginfo.linear_step = tree_to_shwi (iv.step);
2874 thisarginfo.op = iv.base;
2876 else if ((thisarginfo.dt == vect_constant_def
2877 || thisarginfo.dt == vect_external_def)
2878 && POINTER_TYPE_P (TREE_TYPE (op)))
2879 thisarginfo.align = get_pointer_alignment (op) / BITS_PER_UNIT;
2880 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
2881 linear too. */
2882 if (POINTER_TYPE_P (TREE_TYPE (op))
2883 && !thisarginfo.linear_step
2884 && !vec_stmt
2885 && thisarginfo.dt != vect_constant_def
2886 && thisarginfo.dt != vect_external_def
2887 && loop_vinfo
2888 && !slp_node
2889 && TREE_CODE (op) == SSA_NAME)
2890 vect_simd_lane_linear (op, loop, &thisarginfo);
2892 arginfo.quick_push (thisarginfo);
2895 unsigned int badness = 0;
2896 struct cgraph_node *bestn = NULL;
2897 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info).exists ())
2898 bestn = cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info)[0]);
2899 else
2900 for (struct cgraph_node *n = node->simd_clones; n != NULL;
2901 n = n->simdclone->next_clone)
2903 unsigned int this_badness = 0;
2904 if (n->simdclone->simdlen
2905 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo)
2906 || n->simdclone->nargs != nargs)
2907 continue;
2908 if (n->simdclone->simdlen
2909 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo))
2910 this_badness += (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo))
2911 - exact_log2 (n->simdclone->simdlen)) * 1024;
2912 if (n->simdclone->inbranch)
2913 this_badness += 2048;
2914 int target_badness = targetm.simd_clone.usable (n);
2915 if (target_badness < 0)
2916 continue;
2917 this_badness += target_badness * 512;
2918 /* FORNOW: Have to add code to add the mask argument. */
2919 if (n->simdclone->inbranch)
2920 continue;
2921 for (i = 0; i < nargs; i++)
2923 switch (n->simdclone->args[i].arg_type)
2925 case SIMD_CLONE_ARG_TYPE_VECTOR:
2926 if (!useless_type_conversion_p
2927 (n->simdclone->args[i].orig_type,
2928 TREE_TYPE (gimple_call_arg (stmt, i))))
2929 i = -1;
2930 else if (arginfo[i].dt == vect_constant_def
2931 || arginfo[i].dt == vect_external_def
2932 || arginfo[i].linear_step)
2933 this_badness += 64;
2934 break;
2935 case SIMD_CLONE_ARG_TYPE_UNIFORM:
2936 if (arginfo[i].dt != vect_constant_def
2937 && arginfo[i].dt != vect_external_def)
2938 i = -1;
2939 break;
2940 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
2941 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
2942 if (arginfo[i].dt == vect_constant_def
2943 || arginfo[i].dt == vect_external_def
2944 || (arginfo[i].linear_step
2945 != n->simdclone->args[i].linear_step))
2946 i = -1;
2947 break;
2948 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
2949 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
2950 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
2951 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
2952 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
2953 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
2954 /* FORNOW */
2955 i = -1;
2956 break;
2957 case SIMD_CLONE_ARG_TYPE_MASK:
2958 gcc_unreachable ();
2960 if (i == (size_t) -1)
2961 break;
2962 if (n->simdclone->args[i].alignment > arginfo[i].align)
2964 i = -1;
2965 break;
2967 if (arginfo[i].align)
2968 this_badness += (exact_log2 (arginfo[i].align)
2969 - exact_log2 (n->simdclone->args[i].alignment));
2971 if (i == (size_t) -1)
2972 continue;
2973 if (bestn == NULL || this_badness < badness)
2975 bestn = n;
2976 badness = this_badness;
2980 if (bestn == NULL)
2982 arginfo.release ();
2983 return false;
2986 for (i = 0; i < nargs; i++)
2987 if ((arginfo[i].dt == vect_constant_def
2988 || arginfo[i].dt == vect_external_def)
2989 && bestn->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
2991 arginfo[i].vectype
2992 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt,
2993 i)));
2994 if (arginfo[i].vectype == NULL
2995 || (TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)
2996 > bestn->simdclone->simdlen))
2998 arginfo.release ();
2999 return false;
3003 fndecl = bestn->decl;
3004 nunits = bestn->simdclone->simdlen;
3005 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
3007 /* If the function isn't const, only allow it in simd loops where user
3008 has asserted that at least nunits consecutive iterations can be
3009 performed using SIMD instructions. */
3010 if ((loop == NULL || (unsigned) loop->safelen < nunits)
3011 && gimple_vuse (stmt))
3013 arginfo.release ();
3014 return false;
3017 /* Sanity check: make sure that at least one copy of the vectorized stmt
3018 needs to be generated. */
3019 gcc_assert (ncopies >= 1);
3021 if (!vec_stmt) /* transformation not required. */
3023 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (bestn->decl);
3024 for (i = 0; i < nargs; i++)
3025 if (bestn->simdclone->args[i].arg_type
3026 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
3028 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_grow_cleared (i * 3
3029 + 1);
3030 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (arginfo[i].op);
3031 tree lst = POINTER_TYPE_P (TREE_TYPE (arginfo[i].op))
3032 ? size_type_node : TREE_TYPE (arginfo[i].op);
3033 tree ls = build_int_cst (lst, arginfo[i].linear_step);
3034 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (ls);
3035 tree sll = arginfo[i].simd_lane_linear
3036 ? boolean_true_node : boolean_false_node;
3037 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).safe_push (sll);
3039 STMT_VINFO_TYPE (stmt_info) = call_simd_clone_vec_info_type;
3040 if (dump_enabled_p ())
3041 dump_printf_loc (MSG_NOTE, vect_location,
3042 "=== vectorizable_simd_clone_call ===\n");
3043 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
3044 arginfo.release ();
3045 return true;
3048 /** Transform. **/
3050 if (dump_enabled_p ())
3051 dump_printf_loc (MSG_NOTE, vect_location, "transform call.\n");
3053 /* Handle def. */
3054 scalar_dest = gimple_call_lhs (stmt);
3055 vec_dest = NULL_TREE;
3056 rtype = NULL_TREE;
3057 ratype = NULL_TREE;
3058 if (scalar_dest)
3060 vec_dest = vect_create_destination_var (scalar_dest, vectype);
3061 rtype = TREE_TYPE (TREE_TYPE (fndecl));
3062 if (TREE_CODE (rtype) == ARRAY_TYPE)
3064 ratype = rtype;
3065 rtype = TREE_TYPE (ratype);
3069 prev_stmt_info = NULL;
3070 for (j = 0; j < ncopies; ++j)
3072 /* Build argument list for the vectorized call. */
3073 if (j == 0)
3074 vargs.create (nargs);
3075 else
3076 vargs.truncate (0);
3078 for (i = 0; i < nargs; i++)
3080 unsigned int k, l, m, o;
3081 tree atype;
3082 op = gimple_call_arg (stmt, i);
3083 switch (bestn->simdclone->args[i].arg_type)
3085 case SIMD_CLONE_ARG_TYPE_VECTOR:
3086 atype = bestn->simdclone->args[i].vector_type;
3087 o = nunits / TYPE_VECTOR_SUBPARTS (atype);
3088 for (m = j * o; m < (j + 1) * o; m++)
3090 if (TYPE_VECTOR_SUBPARTS (atype)
3091 < TYPE_VECTOR_SUBPARTS (arginfo[i].vectype))
3093 unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (atype));
3094 k = (TYPE_VECTOR_SUBPARTS (arginfo[i].vectype)
3095 / TYPE_VECTOR_SUBPARTS (atype));
3096 gcc_assert ((k & (k - 1)) == 0);
3097 if (m == 0)
3098 vec_oprnd0
3099 = vect_get_vec_def_for_operand (op, stmt);
3100 else
3102 vec_oprnd0 = arginfo[i].op;
3103 if ((m & (k - 1)) == 0)
3104 vec_oprnd0
3105 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
3106 vec_oprnd0);
3108 arginfo[i].op = vec_oprnd0;
3109 vec_oprnd0
3110 = build3 (BIT_FIELD_REF, atype, vec_oprnd0,
3111 size_int (prec),
3112 bitsize_int ((m & (k - 1)) * prec));
3113 new_stmt
3114 = gimple_build_assign (make_ssa_name (atype),
3115 vec_oprnd0);
3116 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3117 vargs.safe_push (gimple_assign_lhs (new_stmt));
3119 else
3121 k = (TYPE_VECTOR_SUBPARTS (atype)
3122 / TYPE_VECTOR_SUBPARTS (arginfo[i].vectype));
3123 gcc_assert ((k & (k - 1)) == 0);
3124 vec<constructor_elt, va_gc> *ctor_elts;
3125 if (k != 1)
3126 vec_alloc (ctor_elts, k);
3127 else
3128 ctor_elts = NULL;
3129 for (l = 0; l < k; l++)
3131 if (m == 0 && l == 0)
3132 vec_oprnd0
3133 = vect_get_vec_def_for_operand (op, stmt);
3134 else
3135 vec_oprnd0
3136 = vect_get_vec_def_for_stmt_copy (arginfo[i].dt,
3137 arginfo[i].op);
3138 arginfo[i].op = vec_oprnd0;
3139 if (k == 1)
3140 break;
3141 CONSTRUCTOR_APPEND_ELT (ctor_elts, NULL_TREE,
3142 vec_oprnd0);
3144 if (k == 1)
3145 vargs.safe_push (vec_oprnd0);
3146 else
3148 vec_oprnd0 = build_constructor (atype, ctor_elts);
3149 new_stmt
3150 = gimple_build_assign (make_ssa_name (atype),
3151 vec_oprnd0);
3152 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3153 vargs.safe_push (gimple_assign_lhs (new_stmt));
3157 break;
3158 case SIMD_CLONE_ARG_TYPE_UNIFORM:
3159 vargs.safe_push (op);
3160 break;
3161 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
3162 if (j == 0)
3164 gimple_seq stmts;
3165 arginfo[i].op
3166 = force_gimple_operand (arginfo[i].op, &stmts, true,
3167 NULL_TREE);
3168 if (stmts != NULL)
3170 basic_block new_bb;
3171 edge pe = loop_preheader_edge (loop);
3172 new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
3173 gcc_assert (!new_bb);
3175 if (arginfo[i].simd_lane_linear)
3177 vargs.safe_push (arginfo[i].op);
3178 break;
3180 tree phi_res = copy_ssa_name (op);
3181 gphi *new_phi = create_phi_node (phi_res, loop->header);
3182 set_vinfo_for_stmt (new_phi,
3183 new_stmt_vec_info (new_phi, loop_vinfo));
3184 add_phi_arg (new_phi, arginfo[i].op,
3185 loop_preheader_edge (loop), UNKNOWN_LOCATION);
3186 enum tree_code code
3187 = POINTER_TYPE_P (TREE_TYPE (op))
3188 ? POINTER_PLUS_EXPR : PLUS_EXPR;
3189 tree type = POINTER_TYPE_P (TREE_TYPE (op))
3190 ? sizetype : TREE_TYPE (op);
3191 widest_int cst
3192 = wi::mul (bestn->simdclone->args[i].linear_step,
3193 ncopies * nunits);
3194 tree tcst = wide_int_to_tree (type, cst);
3195 tree phi_arg = copy_ssa_name (op);
3196 new_stmt
3197 = gimple_build_assign (phi_arg, code, phi_res, tcst);
3198 gimple_stmt_iterator si = gsi_after_labels (loop->header);
3199 gsi_insert_after (&si, new_stmt, GSI_NEW_STMT);
3200 set_vinfo_for_stmt (new_stmt,
3201 new_stmt_vec_info (new_stmt, loop_vinfo));
3202 add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop),
3203 UNKNOWN_LOCATION);
3204 arginfo[i].op = phi_res;
3205 vargs.safe_push (phi_res);
3207 else
3209 enum tree_code code
3210 = POINTER_TYPE_P (TREE_TYPE (op))
3211 ? POINTER_PLUS_EXPR : PLUS_EXPR;
3212 tree type = POINTER_TYPE_P (TREE_TYPE (op))
3213 ? sizetype : TREE_TYPE (op);
3214 widest_int cst
3215 = wi::mul (bestn->simdclone->args[i].linear_step,
3216 j * nunits);
3217 tree tcst = wide_int_to_tree (type, cst);
3218 new_temp = make_ssa_name (TREE_TYPE (op));
3219 new_stmt = gimple_build_assign (new_temp, code,
3220 arginfo[i].op, tcst);
3221 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3222 vargs.safe_push (new_temp);
3224 break;
3225 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
3226 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
3227 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
3228 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
3229 default:
3230 gcc_unreachable ();
3234 new_stmt = gimple_build_call_vec (fndecl, vargs);
3235 if (vec_dest)
3237 gcc_assert (ratype || TYPE_VECTOR_SUBPARTS (rtype) == nunits);
3238 if (ratype)
3239 new_temp = create_tmp_var (ratype);
3240 else if (TYPE_VECTOR_SUBPARTS (vectype)
3241 == TYPE_VECTOR_SUBPARTS (rtype))
3242 new_temp = make_ssa_name (vec_dest, new_stmt);
3243 else
3244 new_temp = make_ssa_name (rtype, new_stmt);
3245 gimple_call_set_lhs (new_stmt, new_temp);
3247 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3249 if (vec_dest)
3251 if (TYPE_VECTOR_SUBPARTS (vectype) < nunits)
3253 unsigned int k, l;
3254 unsigned int prec = GET_MODE_BITSIZE (TYPE_MODE (vectype));
3255 k = nunits / TYPE_VECTOR_SUBPARTS (vectype);
3256 gcc_assert ((k & (k - 1)) == 0);
3257 for (l = 0; l < k; l++)
3259 tree t;
3260 if (ratype)
3262 t = build_fold_addr_expr (new_temp);
3263 t = build2 (MEM_REF, vectype, t,
3264 build_int_cst (TREE_TYPE (t),
3265 l * prec / BITS_PER_UNIT));
3267 else
3268 t = build3 (BIT_FIELD_REF, vectype, new_temp,
3269 size_int (prec), bitsize_int (l * prec));
3270 new_stmt
3271 = gimple_build_assign (make_ssa_name (vectype), t);
3272 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3273 if (j == 0 && l == 0)
3274 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3275 else
3276 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3278 prev_stmt_info = vinfo_for_stmt (new_stmt);
3281 if (ratype)
3283 tree clobber = build_constructor (ratype, NULL);
3284 TREE_THIS_VOLATILE (clobber) = 1;
3285 new_stmt = gimple_build_assign (new_temp, clobber);
3286 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3288 continue;
3290 else if (TYPE_VECTOR_SUBPARTS (vectype) > nunits)
3292 unsigned int k = (TYPE_VECTOR_SUBPARTS (vectype)
3293 / TYPE_VECTOR_SUBPARTS (rtype));
3294 gcc_assert ((k & (k - 1)) == 0);
3295 if ((j & (k - 1)) == 0)
3296 vec_alloc (ret_ctor_elts, k);
3297 if (ratype)
3299 unsigned int m, o = nunits / TYPE_VECTOR_SUBPARTS (rtype);
3300 for (m = 0; m < o; m++)
3302 tree tem = build4 (ARRAY_REF, rtype, new_temp,
3303 size_int (m), NULL_TREE, NULL_TREE);
3304 new_stmt
3305 = gimple_build_assign (make_ssa_name (rtype), tem);
3306 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3307 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE,
3308 gimple_assign_lhs (new_stmt));
3310 tree clobber = build_constructor (ratype, NULL);
3311 TREE_THIS_VOLATILE (clobber) = 1;
3312 new_stmt = gimple_build_assign (new_temp, clobber);
3313 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3315 else
3316 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, new_temp);
3317 if ((j & (k - 1)) != k - 1)
3318 continue;
3319 vec_oprnd0 = build_constructor (vectype, ret_ctor_elts);
3320 new_stmt
3321 = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0);
3322 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3324 if ((unsigned) j == k - 1)
3325 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3326 else
3327 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3329 prev_stmt_info = vinfo_for_stmt (new_stmt);
3330 continue;
3332 else if (ratype)
3334 tree t = build_fold_addr_expr (new_temp);
3335 t = build2 (MEM_REF, vectype, t,
3336 build_int_cst (TREE_TYPE (t), 0));
3337 new_stmt
3338 = gimple_build_assign (make_ssa_name (vec_dest), t);
3339 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3340 tree clobber = build_constructor (ratype, NULL);
3341 TREE_THIS_VOLATILE (clobber) = 1;
3342 vect_finish_stmt_generation (stmt,
3343 gimple_build_assign (new_temp,
3344 clobber), gsi);
3348 if (j == 0)
3349 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3350 else
3351 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
3353 prev_stmt_info = vinfo_for_stmt (new_stmt);
3356 vargs.release ();
3358 /* The call in STMT might prevent it from being removed in dce.
3359 We however cannot remove it here, due to the way the ssa name
3360 it defines is mapped to the new definition. So just replace
3361 rhs of the statement with something harmless. */
3363 if (slp_node)
3364 return true;
3366 if (scalar_dest)
3368 type = TREE_TYPE (scalar_dest);
3369 if (is_pattern_stmt_p (stmt_info))
3370 lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info));
3371 else
3372 lhs = gimple_call_lhs (stmt);
3373 new_stmt = gimple_build_assign (lhs, build_zero_cst (type));
3375 else
3376 new_stmt = gimple_build_nop ();
3377 set_vinfo_for_stmt (new_stmt, stmt_info);
3378 set_vinfo_for_stmt (stmt, NULL);
3379 STMT_VINFO_STMT (stmt_info) = new_stmt;
3380 gsi_replace (gsi, new_stmt, true);
3381 unlink_stmt_vdef (stmt);
3383 return true;
3387 /* Function vect_gen_widened_results_half
3389 Create a vector stmt whose code, type, number of arguments, and result
3390 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3391 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3392 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3393 needs to be created (DECL is a function-decl of a target-builtin).
3394 STMT is the original scalar stmt that we are vectorizing. */
3396 static gimple *
3397 vect_gen_widened_results_half (enum tree_code code,
3398 tree decl,
3399 tree vec_oprnd0, tree vec_oprnd1, int op_type,
3400 tree vec_dest, gimple_stmt_iterator *gsi,
3401 gimple *stmt)
3403 gimple *new_stmt;
3404 tree new_temp;
3406 /* Generate half of the widened result: */
3407 if (code == CALL_EXPR)
3409 /* Target specific support */
3410 if (op_type == binary_op)
3411 new_stmt = gimple_build_call (decl, 2, vec_oprnd0, vec_oprnd1);
3412 else
3413 new_stmt = gimple_build_call (decl, 1, vec_oprnd0);
3414 new_temp = make_ssa_name (vec_dest, new_stmt);
3415 gimple_call_set_lhs (new_stmt, new_temp);
3417 else
3419 /* Generic support */
3420 gcc_assert (op_type == TREE_CODE_LENGTH (code));
3421 if (op_type != binary_op)
3422 vec_oprnd1 = NULL;
3423 new_stmt = gimple_build_assign (vec_dest, code, vec_oprnd0, vec_oprnd1);
3424 new_temp = make_ssa_name (vec_dest, new_stmt);
3425 gimple_assign_set_lhs (new_stmt, new_temp);
3427 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3429 return new_stmt;
3433 /* Get vectorized definitions for loop-based vectorization. For the first
3434 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3435 scalar operand), and for the rest we get a copy with
3436 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3437 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3438 The vectors are collected into VEC_OPRNDS. */
3440 static void
3441 vect_get_loop_based_defs (tree *oprnd, gimple *stmt, enum vect_def_type dt,
3442 vec<tree> *vec_oprnds, int multi_step_cvt)
3444 tree vec_oprnd;
3446 /* Get first vector operand. */
3447 /* All the vector operands except the very first one (that is scalar oprnd)
3448 are stmt copies. */
3449 if (TREE_CODE (TREE_TYPE (*oprnd)) != VECTOR_TYPE)
3450 vec_oprnd = vect_get_vec_def_for_operand (*oprnd, stmt);
3451 else
3452 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, *oprnd);
3454 vec_oprnds->quick_push (vec_oprnd);
3456 /* Get second vector operand. */
3457 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
3458 vec_oprnds->quick_push (vec_oprnd);
3460 *oprnd = vec_oprnd;
3462 /* For conversion in multiple steps, continue to get operands
3463 recursively. */
3464 if (multi_step_cvt)
3465 vect_get_loop_based_defs (oprnd, stmt, dt, vec_oprnds, multi_step_cvt - 1);
3469 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3470 For multi-step conversions store the resulting vectors and call the function
3471 recursively. */
3473 static void
3474 vect_create_vectorized_demotion_stmts (vec<tree> *vec_oprnds,
3475 int multi_step_cvt, gimple *stmt,
3476 vec<tree> vec_dsts,
3477 gimple_stmt_iterator *gsi,
3478 slp_tree slp_node, enum tree_code code,
3479 stmt_vec_info *prev_stmt_info)
3481 unsigned int i;
3482 tree vop0, vop1, new_tmp, vec_dest;
3483 gimple *new_stmt;
3484 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3486 vec_dest = vec_dsts.pop ();
3488 for (i = 0; i < vec_oprnds->length (); i += 2)
3490 /* Create demotion operation. */
3491 vop0 = (*vec_oprnds)[i];
3492 vop1 = (*vec_oprnds)[i + 1];
3493 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
3494 new_tmp = make_ssa_name (vec_dest, new_stmt);
3495 gimple_assign_set_lhs (new_stmt, new_tmp);
3496 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3498 if (multi_step_cvt)
3499 /* Store the resulting vector for next recursive call. */
3500 (*vec_oprnds)[i/2] = new_tmp;
3501 else
3503 /* This is the last step of the conversion sequence. Store the
3504 vectors in SLP_NODE or in vector info of the scalar statement
3505 (or in STMT_VINFO_RELATED_STMT chain). */
3506 if (slp_node)
3507 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3508 else
3510 if (!*prev_stmt_info)
3511 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
3512 else
3513 STMT_VINFO_RELATED_STMT (*prev_stmt_info) = new_stmt;
3515 *prev_stmt_info = vinfo_for_stmt (new_stmt);
3520 /* For multi-step demotion operations we first generate demotion operations
3521 from the source type to the intermediate types, and then combine the
3522 results (stored in VEC_OPRNDS) in demotion operation to the destination
3523 type. */
3524 if (multi_step_cvt)
3526 /* At each level of recursion we have half of the operands we had at the
3527 previous level. */
3528 vec_oprnds->truncate ((i+1)/2);
3529 vect_create_vectorized_demotion_stmts (vec_oprnds, multi_step_cvt - 1,
3530 stmt, vec_dsts, gsi, slp_node,
3531 VEC_PACK_TRUNC_EXPR,
3532 prev_stmt_info);
3535 vec_dsts.quick_push (vec_dest);
3539 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3540 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3541 the resulting vectors and call the function recursively. */
3543 static void
3544 vect_create_vectorized_promotion_stmts (vec<tree> *vec_oprnds0,
3545 vec<tree> *vec_oprnds1,
3546 gimple *stmt, tree vec_dest,
3547 gimple_stmt_iterator *gsi,
3548 enum tree_code code1,
3549 enum tree_code code2, tree decl1,
3550 tree decl2, int op_type)
3552 int i;
3553 tree vop0, vop1, new_tmp1, new_tmp2;
3554 gimple *new_stmt1, *new_stmt2;
3555 vec<tree> vec_tmp = vNULL;
3557 vec_tmp.create (vec_oprnds0->length () * 2);
3558 FOR_EACH_VEC_ELT (*vec_oprnds0, i, vop0)
3560 if (op_type == binary_op)
3561 vop1 = (*vec_oprnds1)[i];
3562 else
3563 vop1 = NULL_TREE;
3565 /* Generate the two halves of promotion operation. */
3566 new_stmt1 = vect_gen_widened_results_half (code1, decl1, vop0, vop1,
3567 op_type, vec_dest, gsi, stmt);
3568 new_stmt2 = vect_gen_widened_results_half (code2, decl2, vop0, vop1,
3569 op_type, vec_dest, gsi, stmt);
3570 if (is_gimple_call (new_stmt1))
3572 new_tmp1 = gimple_call_lhs (new_stmt1);
3573 new_tmp2 = gimple_call_lhs (new_stmt2);
3575 else
3577 new_tmp1 = gimple_assign_lhs (new_stmt1);
3578 new_tmp2 = gimple_assign_lhs (new_stmt2);
3581 /* Store the results for the next step. */
3582 vec_tmp.quick_push (new_tmp1);
3583 vec_tmp.quick_push (new_tmp2);
3586 vec_oprnds0->release ();
3587 *vec_oprnds0 = vec_tmp;
3591 /* Check if STMT performs a conversion operation, that can be vectorized.
3592 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3593 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3594 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3596 static bool
3597 vectorizable_conversion (gimple *stmt, gimple_stmt_iterator *gsi,
3598 gimple **vec_stmt, slp_tree slp_node)
3600 tree vec_dest;
3601 tree scalar_dest;
3602 tree op0, op1 = NULL_TREE;
3603 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE;
3604 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
3605 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
3606 enum tree_code code, code1 = ERROR_MARK, code2 = ERROR_MARK;
3607 enum tree_code codecvt1 = ERROR_MARK, codecvt2 = ERROR_MARK;
3608 tree decl1 = NULL_TREE, decl2 = NULL_TREE;
3609 tree new_temp;
3610 gimple *def_stmt;
3611 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
3612 gimple *new_stmt = NULL;
3613 stmt_vec_info prev_stmt_info;
3614 int nunits_in;
3615 int nunits_out;
3616 tree vectype_out, vectype_in;
3617 int ncopies, i, j;
3618 tree lhs_type, rhs_type;
3619 enum { NARROW, NONE, WIDEN } modifier;
3620 vec<tree> vec_oprnds0 = vNULL;
3621 vec<tree> vec_oprnds1 = vNULL;
3622 tree vop0;
3623 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
3624 vec_info *vinfo = stmt_info->vinfo;
3625 int multi_step_cvt = 0;
3626 vec<tree> vec_dsts = vNULL;
3627 vec<tree> interm_types = vNULL;
3628 tree last_oprnd, intermediate_type, cvt_type = NULL_TREE;
3629 int op_type;
3630 machine_mode rhs_mode;
3631 unsigned short fltsz;
3633 /* Is STMT a vectorizable conversion? */
3635 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
3636 return false;
3638 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
3639 && ! vec_stmt)
3640 return false;
3642 if (!is_gimple_assign (stmt))
3643 return false;
3645 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
3646 return false;
3648 code = gimple_assign_rhs_code (stmt);
3649 if (!CONVERT_EXPR_CODE_P (code)
3650 && code != FIX_TRUNC_EXPR
3651 && code != FLOAT_EXPR
3652 && code != WIDEN_MULT_EXPR
3653 && code != WIDEN_LSHIFT_EXPR)
3654 return false;
3656 op_type = TREE_CODE_LENGTH (code);
3658 /* Check types of lhs and rhs. */
3659 scalar_dest = gimple_assign_lhs (stmt);
3660 lhs_type = TREE_TYPE (scalar_dest);
3661 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
3663 op0 = gimple_assign_rhs1 (stmt);
3664 rhs_type = TREE_TYPE (op0);
3666 if ((code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
3667 && !((INTEGRAL_TYPE_P (lhs_type)
3668 && INTEGRAL_TYPE_P (rhs_type))
3669 || (SCALAR_FLOAT_TYPE_P (lhs_type)
3670 && SCALAR_FLOAT_TYPE_P (rhs_type))))
3671 return false;
3673 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
3674 && ((INTEGRAL_TYPE_P (lhs_type)
3675 && (TYPE_PRECISION (lhs_type)
3676 != GET_MODE_PRECISION (TYPE_MODE (lhs_type))))
3677 || (INTEGRAL_TYPE_P (rhs_type)
3678 && (TYPE_PRECISION (rhs_type)
3679 != GET_MODE_PRECISION (TYPE_MODE (rhs_type))))))
3681 if (dump_enabled_p ())
3682 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3683 "type conversion to/from bit-precision unsupported."
3684 "\n");
3685 return false;
3688 /* Check the operands of the operation. */
3689 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype_in))
3691 if (dump_enabled_p ())
3692 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3693 "use not simple.\n");
3694 return false;
3696 if (op_type == binary_op)
3698 bool ok;
3700 op1 = gimple_assign_rhs2 (stmt);
3701 gcc_assert (code == WIDEN_MULT_EXPR || code == WIDEN_LSHIFT_EXPR);
3702 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3703 OP1. */
3704 if (CONSTANT_CLASS_P (op0))
3705 ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &vectype_in);
3706 else
3707 ok = vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]);
3709 if (!ok)
3711 if (dump_enabled_p ())
3712 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3713 "use not simple.\n");
3714 return false;
3718 /* If op0 is an external or constant defs use a vector type of
3719 the same size as the output vector type. */
3720 if (!vectype_in)
3721 vectype_in = get_same_sized_vectype (rhs_type, vectype_out);
3722 if (vec_stmt)
3723 gcc_assert (vectype_in);
3724 if (!vectype_in)
3726 if (dump_enabled_p ())
3728 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3729 "no vectype for scalar type ");
3730 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
3731 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
3734 return false;
3737 if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
3738 && !VECTOR_BOOLEAN_TYPE_P (vectype_in))
3740 if (dump_enabled_p ())
3742 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3743 "can't convert between boolean and non "
3744 "boolean vectors");
3745 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, rhs_type);
3746 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
3749 return false;
3752 nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
3753 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
3754 if (nunits_in < nunits_out)
3755 modifier = NARROW;
3756 else if (nunits_out == nunits_in)
3757 modifier = NONE;
3758 else
3759 modifier = WIDEN;
3761 /* Multiple types in SLP are handled by creating the appropriate number of
3762 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3763 case of SLP. */
3764 if (slp_node || PURE_SLP_STMT (stmt_info))
3765 ncopies = 1;
3766 else if (modifier == NARROW)
3767 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
3768 else
3769 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
3771 /* Sanity check: make sure that at least one copy of the vectorized stmt
3772 needs to be generated. */
3773 gcc_assert (ncopies >= 1);
3775 /* Supportable by target? */
3776 switch (modifier)
3778 case NONE:
3779 if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
3780 return false;
3781 if (supportable_convert_operation (code, vectype_out, vectype_in,
3782 &decl1, &code1))
3783 break;
3784 /* FALLTHRU */
3785 unsupported:
3786 if (dump_enabled_p ())
3787 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3788 "conversion not supported by target.\n");
3789 return false;
3791 case WIDEN:
3792 if (supportable_widening_operation (code, stmt, vectype_out, vectype_in,
3793 &code1, &code2, &multi_step_cvt,
3794 &interm_types))
3796 /* Binary widening operation can only be supported directly by the
3797 architecture. */
3798 gcc_assert (!(multi_step_cvt && op_type == binary_op));
3799 break;
3802 if (code != FLOAT_EXPR
3803 || (GET_MODE_SIZE (TYPE_MODE (lhs_type))
3804 <= GET_MODE_SIZE (TYPE_MODE (rhs_type))))
3805 goto unsupported;
3807 rhs_mode = TYPE_MODE (rhs_type);
3808 fltsz = GET_MODE_SIZE (TYPE_MODE (lhs_type));
3809 for (rhs_mode = GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type));
3810 rhs_mode != VOIDmode && GET_MODE_SIZE (rhs_mode) <= fltsz;
3811 rhs_mode = GET_MODE_2XWIDER_MODE (rhs_mode))
3813 cvt_type
3814 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
3815 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
3816 if (cvt_type == NULL_TREE)
3817 goto unsupported;
3819 if (GET_MODE_SIZE (rhs_mode) == fltsz)
3821 if (!supportable_convert_operation (code, vectype_out,
3822 cvt_type, &decl1, &codecvt1))
3823 goto unsupported;
3825 else if (!supportable_widening_operation (code, stmt, vectype_out,
3826 cvt_type, &codecvt1,
3827 &codecvt2, &multi_step_cvt,
3828 &interm_types))
3829 continue;
3830 else
3831 gcc_assert (multi_step_cvt == 0);
3833 if (supportable_widening_operation (NOP_EXPR, stmt, cvt_type,
3834 vectype_in, &code1, &code2,
3835 &multi_step_cvt, &interm_types))
3836 break;
3839 if (rhs_mode == VOIDmode || GET_MODE_SIZE (rhs_mode) > fltsz)
3840 goto unsupported;
3842 if (GET_MODE_SIZE (rhs_mode) == fltsz)
3843 codecvt2 = ERROR_MARK;
3844 else
3846 multi_step_cvt++;
3847 interm_types.safe_push (cvt_type);
3848 cvt_type = NULL_TREE;
3850 break;
3852 case NARROW:
3853 gcc_assert (op_type == unary_op);
3854 if (supportable_narrowing_operation (code, vectype_out, vectype_in,
3855 &code1, &multi_step_cvt,
3856 &interm_types))
3857 break;
3859 if (code != FIX_TRUNC_EXPR
3860 || (GET_MODE_SIZE (TYPE_MODE (lhs_type))
3861 >= GET_MODE_SIZE (TYPE_MODE (rhs_type))))
3862 goto unsupported;
3864 rhs_mode = TYPE_MODE (rhs_type);
3865 cvt_type
3866 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode), 0);
3867 cvt_type = get_same_sized_vectype (cvt_type, vectype_in);
3868 if (cvt_type == NULL_TREE)
3869 goto unsupported;
3870 if (!supportable_convert_operation (code, cvt_type, vectype_in,
3871 &decl1, &codecvt1))
3872 goto unsupported;
3873 if (supportable_narrowing_operation (NOP_EXPR, vectype_out, cvt_type,
3874 &code1, &multi_step_cvt,
3875 &interm_types))
3876 break;
3877 goto unsupported;
3879 default:
3880 gcc_unreachable ();
3883 if (!vec_stmt) /* transformation not required. */
3885 if (dump_enabled_p ())
3886 dump_printf_loc (MSG_NOTE, vect_location,
3887 "=== vectorizable_conversion ===\n");
3888 if (code == FIX_TRUNC_EXPR || code == FLOAT_EXPR)
3890 STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
3891 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
3893 else if (modifier == NARROW)
3895 STMT_VINFO_TYPE (stmt_info) = type_demotion_vec_info_type;
3896 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
3898 else
3900 STMT_VINFO_TYPE (stmt_info) = type_promotion_vec_info_type;
3901 vect_model_promotion_demotion_cost (stmt_info, dt, multi_step_cvt);
3903 interm_types.release ();
3904 return true;
3907 /** Transform. **/
3908 if (dump_enabled_p ())
3909 dump_printf_loc (MSG_NOTE, vect_location,
3910 "transform conversion. ncopies = %d.\n", ncopies);
3912 if (op_type == binary_op)
3914 if (CONSTANT_CLASS_P (op0))
3915 op0 = fold_convert (TREE_TYPE (op1), op0);
3916 else if (CONSTANT_CLASS_P (op1))
3917 op1 = fold_convert (TREE_TYPE (op0), op1);
3920 /* In case of multi-step conversion, we first generate conversion operations
3921 to the intermediate types, and then from that types to the final one.
3922 We create vector destinations for the intermediate type (TYPES) received
3923 from supportable_*_operation, and store them in the correct order
3924 for future use in vect_create_vectorized_*_stmts (). */
3925 vec_dsts.create (multi_step_cvt + 1);
3926 vec_dest = vect_create_destination_var (scalar_dest,
3927 (cvt_type && modifier == WIDEN)
3928 ? cvt_type : vectype_out);
3929 vec_dsts.quick_push (vec_dest);
3931 if (multi_step_cvt)
3933 for (i = interm_types.length () - 1;
3934 interm_types.iterate (i, &intermediate_type); i--)
3936 vec_dest = vect_create_destination_var (scalar_dest,
3937 intermediate_type);
3938 vec_dsts.quick_push (vec_dest);
3942 if (cvt_type)
3943 vec_dest = vect_create_destination_var (scalar_dest,
3944 modifier == WIDEN
3945 ? vectype_out : cvt_type);
3947 if (!slp_node)
3949 if (modifier == WIDEN)
3951 vec_oprnds0.create (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1);
3952 if (op_type == binary_op)
3953 vec_oprnds1.create (1);
3955 else if (modifier == NARROW)
3956 vec_oprnds0.create (
3957 2 * (multi_step_cvt ? vect_pow2 (multi_step_cvt) : 1));
3959 else if (code == WIDEN_LSHIFT_EXPR)
3960 vec_oprnds1.create (slp_node->vec_stmts_size);
3962 last_oprnd = op0;
3963 prev_stmt_info = NULL;
3964 switch (modifier)
3966 case NONE:
3967 for (j = 0; j < ncopies; j++)
3969 if (j == 0)
3970 vect_get_vec_defs (op0, NULL, stmt, &vec_oprnds0, NULL, slp_node,
3971 -1);
3972 else
3973 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, NULL);
3975 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
3977 /* Arguments are ready, create the new vector stmt. */
3978 if (code1 == CALL_EXPR)
3980 new_stmt = gimple_build_call (decl1, 1, vop0);
3981 new_temp = make_ssa_name (vec_dest, new_stmt);
3982 gimple_call_set_lhs (new_stmt, new_temp);
3984 else
3986 gcc_assert (TREE_CODE_LENGTH (code1) == unary_op);
3987 new_stmt = gimple_build_assign (vec_dest, code1, vop0);
3988 new_temp = make_ssa_name (vec_dest, new_stmt);
3989 gimple_assign_set_lhs (new_stmt, new_temp);
3992 vect_finish_stmt_generation (stmt, new_stmt, gsi);
3993 if (slp_node)
3994 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
3995 else
3997 if (!prev_stmt_info)
3998 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
3999 else
4000 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4001 prev_stmt_info = vinfo_for_stmt (new_stmt);
4005 break;
4007 case WIDEN:
4008 /* In case the vectorization factor (VF) is bigger than the number
4009 of elements that we can fit in a vectype (nunits), we have to
4010 generate more than one vector stmt - i.e - we need to "unroll"
4011 the vector stmt by a factor VF/nunits. */
4012 for (j = 0; j < ncopies; j++)
4014 /* Handle uses. */
4015 if (j == 0)
4017 if (slp_node)
4019 if (code == WIDEN_LSHIFT_EXPR)
4021 unsigned int k;
4023 vec_oprnd1 = op1;
4024 /* Store vec_oprnd1 for every vector stmt to be created
4025 for SLP_NODE. We check during the analysis that all
4026 the shift arguments are the same. */
4027 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
4028 vec_oprnds1.quick_push (vec_oprnd1);
4030 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4031 slp_node, -1);
4033 else
4034 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0,
4035 &vec_oprnds1, slp_node, -1);
4037 else
4039 vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt);
4040 vec_oprnds0.quick_push (vec_oprnd0);
4041 if (op_type == binary_op)
4043 if (code == WIDEN_LSHIFT_EXPR)
4044 vec_oprnd1 = op1;
4045 else
4046 vec_oprnd1 = vect_get_vec_def_for_operand (op1, stmt);
4047 vec_oprnds1.quick_push (vec_oprnd1);
4051 else
4053 vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt[0], vec_oprnd0);
4054 vec_oprnds0.truncate (0);
4055 vec_oprnds0.quick_push (vec_oprnd0);
4056 if (op_type == binary_op)
4058 if (code == WIDEN_LSHIFT_EXPR)
4059 vec_oprnd1 = op1;
4060 else
4061 vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt[1],
4062 vec_oprnd1);
4063 vec_oprnds1.truncate (0);
4064 vec_oprnds1.quick_push (vec_oprnd1);
4068 /* Arguments are ready. Create the new vector stmts. */
4069 for (i = multi_step_cvt; i >= 0; i--)
4071 tree this_dest = vec_dsts[i];
4072 enum tree_code c1 = code1, c2 = code2;
4073 if (i == 0 && codecvt2 != ERROR_MARK)
4075 c1 = codecvt1;
4076 c2 = codecvt2;
4078 vect_create_vectorized_promotion_stmts (&vec_oprnds0,
4079 &vec_oprnds1,
4080 stmt, this_dest, gsi,
4081 c1, c2, decl1, decl2,
4082 op_type);
4085 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4087 if (cvt_type)
4089 if (codecvt1 == CALL_EXPR)
4091 new_stmt = gimple_build_call (decl1, 1, vop0);
4092 new_temp = make_ssa_name (vec_dest, new_stmt);
4093 gimple_call_set_lhs (new_stmt, new_temp);
4095 else
4097 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
4098 new_temp = make_ssa_name (vec_dest);
4099 new_stmt = gimple_build_assign (new_temp, codecvt1,
4100 vop0);
4103 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4105 else
4106 new_stmt = SSA_NAME_DEF_STMT (vop0);
4108 if (slp_node)
4109 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4110 else
4112 if (!prev_stmt_info)
4113 STMT_VINFO_VEC_STMT (stmt_info) = new_stmt;
4114 else
4115 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4116 prev_stmt_info = vinfo_for_stmt (new_stmt);
4121 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
4122 break;
4124 case NARROW:
4125 /* In case the vectorization factor (VF) is bigger than the number
4126 of elements that we can fit in a vectype (nunits), we have to
4127 generate more than one vector stmt - i.e - we need to "unroll"
4128 the vector stmt by a factor VF/nunits. */
4129 for (j = 0; j < ncopies; j++)
4131 /* Handle uses. */
4132 if (slp_node)
4133 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4134 slp_node, -1);
4135 else
4137 vec_oprnds0.truncate (0);
4138 vect_get_loop_based_defs (&last_oprnd, stmt, dt[0], &vec_oprnds0,
4139 vect_pow2 (multi_step_cvt) - 1);
4142 /* Arguments are ready. Create the new vector stmts. */
4143 if (cvt_type)
4144 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4146 if (codecvt1 == CALL_EXPR)
4148 new_stmt = gimple_build_call (decl1, 1, vop0);
4149 new_temp = make_ssa_name (vec_dest, new_stmt);
4150 gimple_call_set_lhs (new_stmt, new_temp);
4152 else
4154 gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op);
4155 new_temp = make_ssa_name (vec_dest);
4156 new_stmt = gimple_build_assign (new_temp, codecvt1,
4157 vop0);
4160 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4161 vec_oprnds0[i] = new_temp;
4164 vect_create_vectorized_demotion_stmts (&vec_oprnds0, multi_step_cvt,
4165 stmt, vec_dsts, gsi,
4166 slp_node, code1,
4167 &prev_stmt_info);
4170 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
4171 break;
4174 vec_oprnds0.release ();
4175 vec_oprnds1.release ();
4176 vec_dsts.release ();
4177 interm_types.release ();
4179 return true;
4183 /* Function vectorizable_assignment.
4185 Check if STMT performs an assignment (copy) that can be vectorized.
4186 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4187 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4188 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4190 static bool
4191 vectorizable_assignment (gimple *stmt, gimple_stmt_iterator *gsi,
4192 gimple **vec_stmt, slp_tree slp_node)
4194 tree vec_dest;
4195 tree scalar_dest;
4196 tree op;
4197 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4198 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4199 tree new_temp;
4200 gimple *def_stmt;
4201 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
4202 int ncopies;
4203 int i, j;
4204 vec<tree> vec_oprnds = vNULL;
4205 tree vop;
4206 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4207 vec_info *vinfo = stmt_info->vinfo;
4208 gimple *new_stmt = NULL;
4209 stmt_vec_info prev_stmt_info = NULL;
4210 enum tree_code code;
4211 tree vectype_in;
4213 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4214 return false;
4216 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4217 && ! vec_stmt)
4218 return false;
4220 /* Is vectorizable assignment? */
4221 if (!is_gimple_assign (stmt))
4222 return false;
4224 scalar_dest = gimple_assign_lhs (stmt);
4225 if (TREE_CODE (scalar_dest) != SSA_NAME)
4226 return false;
4228 code = gimple_assign_rhs_code (stmt);
4229 if (gimple_assign_single_p (stmt)
4230 || code == PAREN_EXPR
4231 || CONVERT_EXPR_CODE_P (code))
4232 op = gimple_assign_rhs1 (stmt);
4233 else
4234 return false;
4236 if (code == VIEW_CONVERT_EXPR)
4237 op = TREE_OPERAND (op, 0);
4239 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4240 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
4242 /* Multiple types in SLP are handled by creating the appropriate number of
4243 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4244 case of SLP. */
4245 if (slp_node || PURE_SLP_STMT (stmt_info))
4246 ncopies = 1;
4247 else
4248 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
4250 gcc_assert (ncopies >= 1);
4252 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt[0], &vectype_in))
4254 if (dump_enabled_p ())
4255 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4256 "use not simple.\n");
4257 return false;
4260 /* We can handle NOP_EXPR conversions that do not change the number
4261 of elements or the vector size. */
4262 if ((CONVERT_EXPR_CODE_P (code)
4263 || code == VIEW_CONVERT_EXPR)
4264 && (!vectype_in
4265 || TYPE_VECTOR_SUBPARTS (vectype_in) != nunits
4266 || (GET_MODE_SIZE (TYPE_MODE (vectype))
4267 != GET_MODE_SIZE (TYPE_MODE (vectype_in)))))
4268 return false;
4270 /* We do not handle bit-precision changes. */
4271 if ((CONVERT_EXPR_CODE_P (code)
4272 || code == VIEW_CONVERT_EXPR)
4273 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
4274 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest))
4275 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4276 || ((TYPE_PRECISION (TREE_TYPE (op))
4277 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op))))))
4278 /* But a conversion that does not change the bit-pattern is ok. */
4279 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest))
4280 > TYPE_PRECISION (TREE_TYPE (op)))
4281 && TYPE_UNSIGNED (TREE_TYPE (op)))
4282 /* Conversion between boolean types of different sizes is
4283 a simple assignment in case their vectypes are same
4284 boolean vectors. */
4285 && (!VECTOR_BOOLEAN_TYPE_P (vectype)
4286 || !VECTOR_BOOLEAN_TYPE_P (vectype_in)))
4288 if (dump_enabled_p ())
4289 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4290 "type conversion to/from bit-precision "
4291 "unsupported.\n");
4292 return false;
4295 if (!vec_stmt) /* transformation not required. */
4297 STMT_VINFO_TYPE (stmt_info) = assignment_vec_info_type;
4298 if (dump_enabled_p ())
4299 dump_printf_loc (MSG_NOTE, vect_location,
4300 "=== vectorizable_assignment ===\n");
4301 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4302 return true;
4305 /** Transform. **/
4306 if (dump_enabled_p ())
4307 dump_printf_loc (MSG_NOTE, vect_location, "transform assignment.\n");
4309 /* Handle def. */
4310 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4312 /* Handle use. */
4313 for (j = 0; j < ncopies; j++)
4315 /* Handle uses. */
4316 if (j == 0)
4317 vect_get_vec_defs (op, NULL, stmt, &vec_oprnds, NULL, slp_node, -1);
4318 else
4319 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL);
4321 /* Arguments are ready. create the new vector stmt. */
4322 FOR_EACH_VEC_ELT (vec_oprnds, i, vop)
4324 if (CONVERT_EXPR_CODE_P (code)
4325 || code == VIEW_CONVERT_EXPR)
4326 vop = build1 (VIEW_CONVERT_EXPR, vectype, vop);
4327 new_stmt = gimple_build_assign (vec_dest, vop);
4328 new_temp = make_ssa_name (vec_dest, new_stmt);
4329 gimple_assign_set_lhs (new_stmt, new_temp);
4330 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4331 if (slp_node)
4332 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4335 if (slp_node)
4336 continue;
4338 if (j == 0)
4339 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4340 else
4341 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4343 prev_stmt_info = vinfo_for_stmt (new_stmt);
4346 vec_oprnds.release ();
4347 return true;
4351 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4352 either as shift by a scalar or by a vector. */
4354 bool
4355 vect_supportable_shift (enum tree_code code, tree scalar_type)
4358 machine_mode vec_mode;
4359 optab optab;
4360 int icode;
4361 tree vectype;
4363 vectype = get_vectype_for_scalar_type (scalar_type);
4364 if (!vectype)
4365 return false;
4367 optab = optab_for_tree_code (code, vectype, optab_scalar);
4368 if (!optab
4369 || optab_handler (optab, TYPE_MODE (vectype)) == CODE_FOR_nothing)
4371 optab = optab_for_tree_code (code, vectype, optab_vector);
4372 if (!optab
4373 || (optab_handler (optab, TYPE_MODE (vectype))
4374 == CODE_FOR_nothing))
4375 return false;
4378 vec_mode = TYPE_MODE (vectype);
4379 icode = (int) optab_handler (optab, vec_mode);
4380 if (icode == CODE_FOR_nothing)
4381 return false;
4383 return true;
4387 /* Function vectorizable_shift.
4389 Check if STMT performs a shift operation that can be vectorized.
4390 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4391 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4392 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4394 static bool
4395 vectorizable_shift (gimple *stmt, gimple_stmt_iterator *gsi,
4396 gimple **vec_stmt, slp_tree slp_node)
4398 tree vec_dest;
4399 tree scalar_dest;
4400 tree op0, op1 = NULL;
4401 tree vec_oprnd1 = NULL_TREE;
4402 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4403 tree vectype;
4404 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4405 enum tree_code code;
4406 machine_mode vec_mode;
4407 tree new_temp;
4408 optab optab;
4409 int icode;
4410 machine_mode optab_op2_mode;
4411 gimple *def_stmt;
4412 enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
4413 gimple *new_stmt = NULL;
4414 stmt_vec_info prev_stmt_info;
4415 int nunits_in;
4416 int nunits_out;
4417 tree vectype_out;
4418 tree op1_vectype;
4419 int ncopies;
4420 int j, i;
4421 vec<tree> vec_oprnds0 = vNULL;
4422 vec<tree> vec_oprnds1 = vNULL;
4423 tree vop0, vop1;
4424 unsigned int k;
4425 bool scalar_shift_arg = true;
4426 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4427 vec_info *vinfo = stmt_info->vinfo;
4428 int vf;
4430 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4431 return false;
4433 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4434 && ! vec_stmt)
4435 return false;
4437 /* Is STMT a vectorizable binary/unary operation? */
4438 if (!is_gimple_assign (stmt))
4439 return false;
4441 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
4442 return false;
4444 code = gimple_assign_rhs_code (stmt);
4446 if (!(code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
4447 || code == RROTATE_EXPR))
4448 return false;
4450 scalar_dest = gimple_assign_lhs (stmt);
4451 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
4452 if (TYPE_PRECISION (TREE_TYPE (scalar_dest))
4453 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4455 if (dump_enabled_p ())
4456 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4457 "bit-precision shifts not supported.\n");
4458 return false;
4461 op0 = gimple_assign_rhs1 (stmt);
4462 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
4464 if (dump_enabled_p ())
4465 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4466 "use not simple.\n");
4467 return false;
4469 /* If op0 is an external or constant def use a vector type with
4470 the same size as the output vector type. */
4471 if (!vectype)
4472 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
4473 if (vec_stmt)
4474 gcc_assert (vectype);
4475 if (!vectype)
4477 if (dump_enabled_p ())
4478 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4479 "no vectype for scalar type\n");
4480 return false;
4483 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
4484 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
4485 if (nunits_out != nunits_in)
4486 return false;
4488 op1 = gimple_assign_rhs2 (stmt);
4489 if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1], &op1_vectype))
4491 if (dump_enabled_p ())
4492 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4493 "use not simple.\n");
4494 return false;
4497 if (loop_vinfo)
4498 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4499 else
4500 vf = 1;
4502 /* Multiple types in SLP are handled by creating the appropriate number of
4503 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4504 case of SLP. */
4505 if (slp_node || PURE_SLP_STMT (stmt_info))
4506 ncopies = 1;
4507 else
4508 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
4510 gcc_assert (ncopies >= 1);
4512 /* Determine whether the shift amount is a vector, or scalar. If the
4513 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4515 if ((dt[1] == vect_internal_def
4516 || dt[1] == vect_induction_def)
4517 && !slp_node)
4518 scalar_shift_arg = false;
4519 else if (dt[1] == vect_constant_def
4520 || dt[1] == vect_external_def
4521 || dt[1] == vect_internal_def)
4523 /* In SLP, need to check whether the shift count is the same,
4524 in loops if it is a constant or invariant, it is always
4525 a scalar shift. */
4526 if (slp_node)
4528 vec<gimple *> stmts = SLP_TREE_SCALAR_STMTS (slp_node);
4529 gimple *slpstmt;
4531 FOR_EACH_VEC_ELT (stmts, k, slpstmt)
4532 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt), op1, 0))
4533 scalar_shift_arg = false;
4536 /* If the shift amount is computed by a pattern stmt we cannot
4537 use the scalar amount directly thus give up and use a vector
4538 shift. */
4539 if (dt[1] == vect_internal_def)
4541 gimple *def = SSA_NAME_DEF_STMT (op1);
4542 if (is_pattern_stmt_p (vinfo_for_stmt (def)))
4543 scalar_shift_arg = false;
4546 else
4548 if (dump_enabled_p ())
4549 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4550 "operand mode requires invariant argument.\n");
4551 return false;
4554 /* Vector shifted by vector. */
4555 if (!scalar_shift_arg)
4557 optab = optab_for_tree_code (code, vectype, optab_vector);
4558 if (dump_enabled_p ())
4559 dump_printf_loc (MSG_NOTE, vect_location,
4560 "vector/vector shift/rotate found.\n");
4562 if (!op1_vectype)
4563 op1_vectype = get_same_sized_vectype (TREE_TYPE (op1), vectype_out);
4564 if (op1_vectype == NULL_TREE
4565 || TYPE_MODE (op1_vectype) != TYPE_MODE (vectype))
4567 if (dump_enabled_p ())
4568 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4569 "unusable type for last operand in"
4570 " vector/vector shift/rotate.\n");
4571 return false;
4574 /* See if the machine has a vector shifted by scalar insn and if not
4575 then see if it has a vector shifted by vector insn. */
4576 else
4578 optab = optab_for_tree_code (code, vectype, optab_scalar);
4579 if (optab
4580 && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)
4582 if (dump_enabled_p ())
4583 dump_printf_loc (MSG_NOTE, vect_location,
4584 "vector/scalar shift/rotate found.\n");
4586 else
4588 optab = optab_for_tree_code (code, vectype, optab_vector);
4589 if (optab
4590 && (optab_handler (optab, TYPE_MODE (vectype))
4591 != CODE_FOR_nothing))
4593 scalar_shift_arg = false;
4595 if (dump_enabled_p ())
4596 dump_printf_loc (MSG_NOTE, vect_location,
4597 "vector/vector shift/rotate found.\n");
4599 /* Unlike the other binary operators, shifts/rotates have
4600 the rhs being int, instead of the same type as the lhs,
4601 so make sure the scalar is the right type if we are
4602 dealing with vectors of long long/long/short/char. */
4603 if (dt[1] == vect_constant_def)
4604 op1 = fold_convert (TREE_TYPE (vectype), op1);
4605 else if (!useless_type_conversion_p (TREE_TYPE (vectype),
4606 TREE_TYPE (op1)))
4608 if (slp_node
4609 && TYPE_MODE (TREE_TYPE (vectype))
4610 != TYPE_MODE (TREE_TYPE (op1)))
4612 if (dump_enabled_p ())
4613 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4614 "unusable type for last operand in"
4615 " vector/vector shift/rotate.\n");
4616 return false;
4618 if (vec_stmt && !slp_node)
4620 op1 = fold_convert (TREE_TYPE (vectype), op1);
4621 op1 = vect_init_vector (stmt, op1,
4622 TREE_TYPE (vectype), NULL);
4629 /* Supportable by target? */
4630 if (!optab)
4632 if (dump_enabled_p ())
4633 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4634 "no optab.\n");
4635 return false;
4637 vec_mode = TYPE_MODE (vectype);
4638 icode = (int) optab_handler (optab, vec_mode);
4639 if (icode == CODE_FOR_nothing)
4641 if (dump_enabled_p ())
4642 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4643 "op not supported by target.\n");
4644 /* Check only during analysis. */
4645 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
4646 || (vf < vect_min_worthwhile_factor (code)
4647 && !vec_stmt))
4648 return false;
4649 if (dump_enabled_p ())
4650 dump_printf_loc (MSG_NOTE, vect_location,
4651 "proceeding using word mode.\n");
4654 /* Worthwhile without SIMD support? Check only during analysis. */
4655 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
4656 && vf < vect_min_worthwhile_factor (code)
4657 && !vec_stmt)
4659 if (dump_enabled_p ())
4660 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4661 "not worthwhile without SIMD support.\n");
4662 return false;
4665 if (!vec_stmt) /* transformation not required. */
4667 STMT_VINFO_TYPE (stmt_info) = shift_vec_info_type;
4668 if (dump_enabled_p ())
4669 dump_printf_loc (MSG_NOTE, vect_location,
4670 "=== vectorizable_shift ===\n");
4671 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4672 return true;
4675 /** Transform. **/
4677 if (dump_enabled_p ())
4678 dump_printf_loc (MSG_NOTE, vect_location,
4679 "transform binary/unary operation.\n");
4681 /* Handle def. */
4682 vec_dest = vect_create_destination_var (scalar_dest, vectype);
4684 prev_stmt_info = NULL;
4685 for (j = 0; j < ncopies; j++)
4687 /* Handle uses. */
4688 if (j == 0)
4690 if (scalar_shift_arg)
4692 /* Vector shl and shr insn patterns can be defined with scalar
4693 operand 2 (shift operand). In this case, use constant or loop
4694 invariant op1 directly, without extending it to vector mode
4695 first. */
4696 optab_op2_mode = insn_data[icode].operand[2].mode;
4697 if (!VECTOR_MODE_P (optab_op2_mode))
4699 if (dump_enabled_p ())
4700 dump_printf_loc (MSG_NOTE, vect_location,
4701 "operand 1 using scalar mode.\n");
4702 vec_oprnd1 = op1;
4703 vec_oprnds1.create (slp_node ? slp_node->vec_stmts_size : 1);
4704 vec_oprnds1.quick_push (vec_oprnd1);
4705 if (slp_node)
4707 /* Store vec_oprnd1 for every vector stmt to be created
4708 for SLP_NODE. We check during the analysis that all
4709 the shift arguments are the same.
4710 TODO: Allow different constants for different vector
4711 stmts generated for an SLP instance. */
4712 for (k = 0; k < slp_node->vec_stmts_size - 1; k++)
4713 vec_oprnds1.quick_push (vec_oprnd1);
4718 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4719 (a special case for certain kind of vector shifts); otherwise,
4720 operand 1 should be of a vector type (the usual case). */
4721 if (vec_oprnd1)
4722 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
4723 slp_node, -1);
4724 else
4725 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
4726 slp_node, -1);
4728 else
4729 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
4731 /* Arguments are ready. Create the new vector stmt. */
4732 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
4734 vop1 = vec_oprnds1[i];
4735 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1);
4736 new_temp = make_ssa_name (vec_dest, new_stmt);
4737 gimple_assign_set_lhs (new_stmt, new_temp);
4738 vect_finish_stmt_generation (stmt, new_stmt, gsi);
4739 if (slp_node)
4740 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
4743 if (slp_node)
4744 continue;
4746 if (j == 0)
4747 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
4748 else
4749 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
4750 prev_stmt_info = vinfo_for_stmt (new_stmt);
4753 vec_oprnds0.release ();
4754 vec_oprnds1.release ();
4756 return true;
4760 /* Function vectorizable_operation.
4762 Check if STMT performs a binary, unary or ternary operation that can
4763 be vectorized.
4764 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4765 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4766 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4768 static bool
4769 vectorizable_operation (gimple *stmt, gimple_stmt_iterator *gsi,
4770 gimple **vec_stmt, slp_tree slp_node)
4772 tree vec_dest;
4773 tree scalar_dest;
4774 tree op0, op1 = NULL_TREE, op2 = NULL_TREE;
4775 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
4776 tree vectype;
4777 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
4778 enum tree_code code;
4779 machine_mode vec_mode;
4780 tree new_temp;
4781 int op_type;
4782 optab optab;
4783 bool target_support_p;
4784 gimple *def_stmt;
4785 enum vect_def_type dt[3]
4786 = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type};
4787 gimple *new_stmt = NULL;
4788 stmt_vec_info prev_stmt_info;
4789 int nunits_in;
4790 int nunits_out;
4791 tree vectype_out;
4792 int ncopies;
4793 int j, i;
4794 vec<tree> vec_oprnds0 = vNULL;
4795 vec<tree> vec_oprnds1 = vNULL;
4796 vec<tree> vec_oprnds2 = vNULL;
4797 tree vop0, vop1, vop2;
4798 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
4799 vec_info *vinfo = stmt_info->vinfo;
4800 int vf;
4802 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
4803 return false;
4805 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
4806 && ! vec_stmt)
4807 return false;
4809 /* Is STMT a vectorizable binary/unary operation? */
4810 if (!is_gimple_assign (stmt))
4811 return false;
4813 if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
4814 return false;
4816 code = gimple_assign_rhs_code (stmt);
4818 /* For pointer addition, we should use the normal plus for
4819 the vector addition. */
4820 if (code == POINTER_PLUS_EXPR)
4821 code = PLUS_EXPR;
4823 /* Support only unary or binary operations. */
4824 op_type = TREE_CODE_LENGTH (code);
4825 if (op_type != unary_op && op_type != binary_op && op_type != ternary_op)
4827 if (dump_enabled_p ())
4828 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4829 "num. args = %d (not unary/binary/ternary op).\n",
4830 op_type);
4831 return false;
4834 scalar_dest = gimple_assign_lhs (stmt);
4835 vectype_out = STMT_VINFO_VECTYPE (stmt_info);
4837 /* Most operations cannot handle bit-precision types without extra
4838 truncations. */
4839 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out)
4840 && (TYPE_PRECISION (TREE_TYPE (scalar_dest))
4841 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest))))
4842 /* Exception are bitwise binary operations. */
4843 && code != BIT_IOR_EXPR
4844 && code != BIT_XOR_EXPR
4845 && code != BIT_AND_EXPR)
4847 if (dump_enabled_p ())
4848 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4849 "bit-precision arithmetic not supported.\n");
4850 return false;
4853 op0 = gimple_assign_rhs1 (stmt);
4854 if (!vect_is_simple_use (op0, vinfo, &def_stmt, &dt[0], &vectype))
4856 if (dump_enabled_p ())
4857 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4858 "use not simple.\n");
4859 return false;
4861 /* If op0 is an external or constant def use a vector type with
4862 the same size as the output vector type. */
4863 if (!vectype)
4865 /* For boolean type we cannot determine vectype by
4866 invariant value (don't know whether it is a vector
4867 of booleans or vector of integers). We use output
4868 vectype because operations on boolean don't change
4869 type. */
4870 if (TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE)
4872 if (TREE_CODE (TREE_TYPE (scalar_dest)) != BOOLEAN_TYPE)
4874 if (dump_enabled_p ())
4875 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4876 "not supported operation on bool value.\n");
4877 return false;
4879 vectype = vectype_out;
4881 else
4882 vectype = get_same_sized_vectype (TREE_TYPE (op0), vectype_out);
4884 if (vec_stmt)
4885 gcc_assert (vectype);
4886 if (!vectype)
4888 if (dump_enabled_p ())
4890 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4891 "no vectype for scalar type ");
4892 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
4893 TREE_TYPE (op0));
4894 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
4897 return false;
4900 nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
4901 nunits_in = TYPE_VECTOR_SUBPARTS (vectype);
4902 if (nunits_out != nunits_in)
4903 return false;
4905 if (op_type == binary_op || op_type == ternary_op)
4907 op1 = gimple_assign_rhs2 (stmt);
4908 if (!vect_is_simple_use (op1, vinfo, &def_stmt, &dt[1]))
4910 if (dump_enabled_p ())
4911 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4912 "use not simple.\n");
4913 return false;
4916 if (op_type == ternary_op)
4918 op2 = gimple_assign_rhs3 (stmt);
4919 if (!vect_is_simple_use (op2, vinfo, &def_stmt, &dt[2]))
4921 if (dump_enabled_p ())
4922 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4923 "use not simple.\n");
4924 return false;
4928 if (loop_vinfo)
4929 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4930 else
4931 vf = 1;
4933 /* Multiple types in SLP are handled by creating the appropriate number of
4934 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4935 case of SLP. */
4936 if (slp_node || PURE_SLP_STMT (stmt_info))
4937 ncopies = 1;
4938 else
4939 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
4941 gcc_assert (ncopies >= 1);
4943 /* Shifts are handled in vectorizable_shift (). */
4944 if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
4945 || code == RROTATE_EXPR)
4946 return false;
4948 /* Supportable by target? */
4950 vec_mode = TYPE_MODE (vectype);
4951 if (code == MULT_HIGHPART_EXPR)
4952 target_support_p = can_mult_highpart_p (vec_mode, TYPE_UNSIGNED (vectype));
4953 else
4955 optab = optab_for_tree_code (code, vectype, optab_default);
4956 if (!optab)
4958 if (dump_enabled_p ())
4959 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4960 "no optab.\n");
4961 return false;
4963 target_support_p = (optab_handler (optab, vec_mode)
4964 != CODE_FOR_nothing);
4967 if (!target_support_p)
4969 if (dump_enabled_p ())
4970 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4971 "op not supported by target.\n");
4972 /* Check only during analysis. */
4973 if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
4974 || (!vec_stmt && vf < vect_min_worthwhile_factor (code)))
4975 return false;
4976 if (dump_enabled_p ())
4977 dump_printf_loc (MSG_NOTE, vect_location,
4978 "proceeding using word mode.\n");
4981 /* Worthwhile without SIMD support? Check only during analysis. */
4982 if (!VECTOR_MODE_P (vec_mode)
4983 && !vec_stmt
4984 && vf < vect_min_worthwhile_factor (code))
4986 if (dump_enabled_p ())
4987 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4988 "not worthwhile without SIMD support.\n");
4989 return false;
4992 if (!vec_stmt) /* transformation not required. */
4994 STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
4995 if (dump_enabled_p ())
4996 dump_printf_loc (MSG_NOTE, vect_location,
4997 "=== vectorizable_operation ===\n");
4998 vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL);
4999 return true;
5002 /** Transform. **/
5004 if (dump_enabled_p ())
5005 dump_printf_loc (MSG_NOTE, vect_location,
5006 "transform binary/unary operation.\n");
5008 /* Handle def. */
5009 vec_dest = vect_create_destination_var (scalar_dest, vectype);
5011 /* In case the vectorization factor (VF) is bigger than the number
5012 of elements that we can fit in a vectype (nunits), we have to generate
5013 more than one vector stmt - i.e - we need to "unroll" the
5014 vector stmt by a factor VF/nunits. In doing so, we record a pointer
5015 from one copy of the vector stmt to the next, in the field
5016 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
5017 stages to find the correct vector defs to be used when vectorizing
5018 stmts that use the defs of the current stmt. The example below
5019 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
5020 we need to create 4 vectorized stmts):
5022 before vectorization:
5023 RELATED_STMT VEC_STMT
5024 S1: x = memref - -
5025 S2: z = x + 1 - -
5027 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
5028 there):
5029 RELATED_STMT VEC_STMT
5030 VS1_0: vx0 = memref0 VS1_1 -
5031 VS1_1: vx1 = memref1 VS1_2 -
5032 VS1_2: vx2 = memref2 VS1_3 -
5033 VS1_3: vx3 = memref3 - -
5034 S1: x = load - VS1_0
5035 S2: z = x + 1 - -
5037 step2: vectorize stmt S2 (done here):
5038 To vectorize stmt S2 we first need to find the relevant vector
5039 def for the first operand 'x'. This is, as usual, obtained from
5040 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
5041 that defines 'x' (S1). This way we find the stmt VS1_0, and the
5042 relevant vector def 'vx0'. Having found 'vx0' we can generate
5043 the vector stmt VS2_0, and as usual, record it in the
5044 STMT_VINFO_VEC_STMT of stmt S2.
5045 When creating the second copy (VS2_1), we obtain the relevant vector
5046 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
5047 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
5048 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
5049 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
5050 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
5051 chain of stmts and pointers:
5052 RELATED_STMT VEC_STMT
5053 VS1_0: vx0 = memref0 VS1_1 -
5054 VS1_1: vx1 = memref1 VS1_2 -
5055 VS1_2: vx2 = memref2 VS1_3 -
5056 VS1_3: vx3 = memref3 - -
5057 S1: x = load - VS1_0
5058 VS2_0: vz0 = vx0 + v1 VS2_1 -
5059 VS2_1: vz1 = vx1 + v1 VS2_2 -
5060 VS2_2: vz2 = vx2 + v1 VS2_3 -
5061 VS2_3: vz3 = vx3 + v1 - -
5062 S2: z = x + 1 - VS2_0 */
5064 prev_stmt_info = NULL;
5065 for (j = 0; j < ncopies; j++)
5067 /* Handle uses. */
5068 if (j == 0)
5070 if (op_type == binary_op || op_type == ternary_op)
5071 vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1,
5072 slp_node, -1);
5073 else
5074 vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL,
5075 slp_node, -1);
5076 if (op_type == ternary_op)
5078 vec_oprnds2.create (1);
5079 vec_oprnds2.quick_push (vect_get_vec_def_for_operand (op2,
5080 stmt));
5083 else
5085 vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1);
5086 if (op_type == ternary_op)
5088 tree vec_oprnd = vec_oprnds2.pop ();
5089 vec_oprnds2.quick_push (vect_get_vec_def_for_stmt_copy (dt[2],
5090 vec_oprnd));
5094 /* Arguments are ready. Create the new vector stmt. */
5095 FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0)
5097 vop1 = ((op_type == binary_op || op_type == ternary_op)
5098 ? vec_oprnds1[i] : NULL_TREE);
5099 vop2 = ((op_type == ternary_op)
5100 ? vec_oprnds2[i] : NULL_TREE);
5101 new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2);
5102 new_temp = make_ssa_name (vec_dest, new_stmt);
5103 gimple_assign_set_lhs (new_stmt, new_temp);
5104 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5105 if (slp_node)
5106 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
5109 if (slp_node)
5110 continue;
5112 if (j == 0)
5113 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5114 else
5115 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5116 prev_stmt_info = vinfo_for_stmt (new_stmt);
5119 vec_oprnds0.release ();
5120 vec_oprnds1.release ();
5121 vec_oprnds2.release ();
5123 return true;
5126 /* A helper function to ensure data reference DR's base alignment
5127 for STMT_INFO. */
5129 static void
5130 ensure_base_align (stmt_vec_info stmt_info, struct data_reference *dr)
5132 if (!dr->aux)
5133 return;
5135 if (DR_VECT_AUX (dr)->base_misaligned)
5137 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
5138 tree base_decl = DR_VECT_AUX (dr)->base_decl;
5140 if (decl_in_symtab_p (base_decl))
5141 symtab_node::get (base_decl)->increase_alignment (TYPE_ALIGN (vectype));
5142 else
5144 SET_DECL_ALIGN (base_decl, TYPE_ALIGN (vectype));
5145 DECL_USER_ALIGN (base_decl) = 1;
5147 DR_VECT_AUX (dr)->base_misaligned = false;
5152 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
5153 reversal of the vector elements. If that is impossible to do,
5154 returns NULL. */
5156 static tree
5157 perm_mask_for_reverse (tree vectype)
5159 int i, nunits;
5160 unsigned char *sel;
5162 nunits = TYPE_VECTOR_SUBPARTS (vectype);
5163 sel = XALLOCAVEC (unsigned char, nunits);
5165 for (i = 0; i < nunits; ++i)
5166 sel[i] = nunits - 1 - i;
5168 if (!can_vec_perm_p (TYPE_MODE (vectype), false, sel))
5169 return NULL_TREE;
5170 return vect_gen_perm_mask_checked (vectype, sel);
5173 /* Function vectorizable_store.
5175 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
5176 can be vectorized.
5177 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5178 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5179 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5181 static bool
5182 vectorizable_store (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
5183 slp_tree slp_node)
5185 tree scalar_dest;
5186 tree data_ref;
5187 tree op;
5188 tree vec_oprnd = NULL_TREE;
5189 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
5190 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
5191 tree elem_type;
5192 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
5193 struct loop *loop = NULL;
5194 machine_mode vec_mode;
5195 tree dummy;
5196 enum dr_alignment_support alignment_support_scheme;
5197 gimple *def_stmt;
5198 enum vect_def_type dt;
5199 stmt_vec_info prev_stmt_info = NULL;
5200 tree dataref_ptr = NULL_TREE;
5201 tree dataref_offset = NULL_TREE;
5202 gimple *ptr_incr = NULL;
5203 int ncopies;
5204 int j;
5205 gimple *next_stmt, *first_stmt = NULL;
5206 bool grouped_store = false;
5207 bool store_lanes_p = false;
5208 unsigned int group_size, i;
5209 vec<tree> dr_chain = vNULL;
5210 vec<tree> oprnds = vNULL;
5211 vec<tree> result_chain = vNULL;
5212 bool inv_p;
5213 bool negative = false;
5214 tree offset = NULL_TREE;
5215 vec<tree> vec_oprnds = vNULL;
5216 bool slp = (slp_node != NULL);
5217 unsigned int vec_num;
5218 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
5219 vec_info *vinfo = stmt_info->vinfo;
5220 tree aggr_type;
5221 tree scatter_base = NULL_TREE, scatter_off = NULL_TREE;
5222 tree scatter_off_vectype = NULL_TREE, scatter_decl = NULL_TREE;
5223 int scatter_scale = 1;
5224 enum vect_def_type scatter_idx_dt = vect_unknown_def_type;
5225 enum vect_def_type scatter_src_dt = vect_unknown_def_type;
5226 gimple *new_stmt;
5228 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
5229 return false;
5231 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
5232 && ! vec_stmt)
5233 return false;
5235 /* Is vectorizable store? */
5237 if (!is_gimple_assign (stmt))
5238 return false;
5240 scalar_dest = gimple_assign_lhs (stmt);
5241 if (TREE_CODE (scalar_dest) == VIEW_CONVERT_EXPR
5242 && is_pattern_stmt_p (stmt_info))
5243 scalar_dest = TREE_OPERAND (scalar_dest, 0);
5244 if (TREE_CODE (scalar_dest) != ARRAY_REF
5245 && TREE_CODE (scalar_dest) != BIT_FIELD_REF
5246 && TREE_CODE (scalar_dest) != INDIRECT_REF
5247 && TREE_CODE (scalar_dest) != COMPONENT_REF
5248 && TREE_CODE (scalar_dest) != IMAGPART_EXPR
5249 && TREE_CODE (scalar_dest) != REALPART_EXPR
5250 && TREE_CODE (scalar_dest) != MEM_REF)
5251 return false;
5253 gcc_assert (gimple_assign_single_p (stmt));
5255 tree vectype = STMT_VINFO_VECTYPE (stmt_info), rhs_vectype = NULL_TREE;
5256 unsigned int nunits = TYPE_VECTOR_SUBPARTS (vectype);
5258 if (loop_vinfo)
5259 loop = LOOP_VINFO_LOOP (loop_vinfo);
5261 /* Multiple types in SLP are handled by creating the appropriate number of
5262 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5263 case of SLP. */
5264 if (slp || PURE_SLP_STMT (stmt_info))
5265 ncopies = 1;
5266 else
5267 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
5269 gcc_assert (ncopies >= 1);
5271 /* FORNOW. This restriction should be relaxed. */
5272 if (loop && nested_in_vect_loop_p (loop, stmt) && ncopies > 1)
5274 if (dump_enabled_p ())
5275 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5276 "multiple types in nested loop.\n");
5277 return false;
5280 op = gimple_assign_rhs1 (stmt);
5282 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt, &rhs_vectype))
5284 if (dump_enabled_p ())
5285 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5286 "use not simple.\n");
5287 return false;
5290 if (rhs_vectype && !useless_type_conversion_p (vectype, rhs_vectype))
5291 return false;
5293 elem_type = TREE_TYPE (vectype);
5294 vec_mode = TYPE_MODE (vectype);
5296 /* FORNOW. In some cases can vectorize even if data-type not supported
5297 (e.g. - array initialization with 0). */
5298 if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing)
5299 return false;
5301 if (!STMT_VINFO_DATA_REF (stmt_info))
5302 return false;
5304 if (!STMT_VINFO_STRIDED_P (stmt_info))
5306 negative =
5307 tree_int_cst_compare (loop && nested_in_vect_loop_p (loop, stmt)
5308 ? STMT_VINFO_DR_STEP (stmt_info) : DR_STEP (dr),
5309 size_zero_node) < 0;
5310 if (negative && ncopies > 1)
5312 if (dump_enabled_p ())
5313 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5314 "multiple types with negative step.\n");
5315 return false;
5317 if (negative)
5319 gcc_assert (!grouped_store);
5320 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
5321 if (alignment_support_scheme != dr_aligned
5322 && alignment_support_scheme != dr_unaligned_supported)
5324 if (dump_enabled_p ())
5325 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5326 "negative step but alignment required.\n");
5327 return false;
5329 if (dt != vect_constant_def
5330 && dt != vect_external_def
5331 && !perm_mask_for_reverse (vectype))
5333 if (dump_enabled_p ())
5334 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5335 "negative step and reversing not supported.\n");
5336 return false;
5341 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5343 grouped_store = true;
5344 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
5345 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
5346 if (!slp
5347 && !PURE_SLP_STMT (stmt_info)
5348 && !STMT_VINFO_STRIDED_P (stmt_info))
5350 if (vect_store_lanes_supported (vectype, group_size))
5351 store_lanes_p = true;
5352 else if (!vect_grouped_store_supported (vectype, group_size))
5353 return false;
5356 if (STMT_VINFO_STRIDED_P (stmt_info)
5357 && (slp || PURE_SLP_STMT (stmt_info))
5358 && (group_size > nunits
5359 || nunits % group_size != 0))
5361 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5362 "unhandled strided group store\n");
5363 return false;
5366 if (first_stmt == stmt)
5368 /* STMT is the leader of the group. Check the operands of all the
5369 stmts of the group. */
5370 next_stmt = GROUP_NEXT_ELEMENT (stmt_info);
5371 while (next_stmt)
5373 gcc_assert (gimple_assign_single_p (next_stmt));
5374 op = gimple_assign_rhs1 (next_stmt);
5375 if (!vect_is_simple_use (op, vinfo, &def_stmt, &dt))
5377 if (dump_enabled_p ())
5378 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5379 "use not simple.\n");
5380 return false;
5382 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5387 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
5389 gimple *def_stmt;
5390 scatter_decl = vect_check_gather_scatter (stmt, loop_vinfo, &scatter_base,
5391 &scatter_off, &scatter_scale);
5392 gcc_assert (scatter_decl);
5393 if (!vect_is_simple_use (scatter_off, vinfo, &def_stmt, &scatter_idx_dt,
5394 &scatter_off_vectype))
5396 if (dump_enabled_p ())
5397 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5398 "scatter index use not simple.");
5399 return false;
5403 if (!vec_stmt) /* transformation not required. */
5405 STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
5406 /* The SLP costs are calculated during SLP analysis. */
5407 if (!PURE_SLP_STMT (stmt_info))
5408 vect_model_store_cost (stmt_info, ncopies, store_lanes_p, dt,
5409 NULL, NULL, NULL);
5410 return true;
5413 /** Transform. **/
5415 ensure_base_align (stmt_info, dr);
5417 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
5419 tree vec_oprnd0 = NULL_TREE, vec_oprnd1 = NULL_TREE, op, src;
5420 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (scatter_decl));
5421 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
5422 tree ptr, mask, var, scale, perm_mask = NULL_TREE;
5423 edge pe = loop_preheader_edge (loop);
5424 gimple_seq seq;
5425 basic_block new_bb;
5426 enum { NARROW, NONE, WIDEN } modifier;
5427 int scatter_off_nunits = TYPE_VECTOR_SUBPARTS (scatter_off_vectype);
5429 if (nunits == (unsigned int) scatter_off_nunits)
5430 modifier = NONE;
5431 else if (nunits == (unsigned int) scatter_off_nunits / 2)
5433 unsigned char *sel = XALLOCAVEC (unsigned char, scatter_off_nunits);
5434 modifier = WIDEN;
5436 for (i = 0; i < (unsigned int) scatter_off_nunits; ++i)
5437 sel[i] = i | nunits;
5439 perm_mask = vect_gen_perm_mask_checked (scatter_off_vectype, sel);
5440 gcc_assert (perm_mask != NULL_TREE);
5442 else if (nunits == (unsigned int) scatter_off_nunits * 2)
5444 unsigned char *sel = XALLOCAVEC (unsigned char, nunits);
5445 modifier = NARROW;
5447 for (i = 0; i < (unsigned int) nunits; ++i)
5448 sel[i] = i | scatter_off_nunits;
5450 perm_mask = vect_gen_perm_mask_checked (vectype, sel);
5451 gcc_assert (perm_mask != NULL_TREE);
5452 ncopies *= 2;
5454 else
5455 gcc_unreachable ();
5457 rettype = TREE_TYPE (TREE_TYPE (scatter_decl));
5458 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5459 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5460 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5461 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
5462 scaletype = TREE_VALUE (arglist);
5464 gcc_checking_assert (TREE_CODE (masktype) == INTEGER_TYPE
5465 && TREE_CODE (rettype) == VOID_TYPE);
5467 ptr = fold_convert (ptrtype, scatter_base);
5468 if (!is_gimple_min_invariant (ptr))
5470 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
5471 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
5472 gcc_assert (!new_bb);
5475 /* Currently we support only unconditional scatter stores,
5476 so mask should be all ones. */
5477 mask = build_int_cst (masktype, -1);
5478 mask = vect_init_vector (stmt, mask, masktype, NULL);
5480 scale = build_int_cst (scaletype, scatter_scale);
5482 prev_stmt_info = NULL;
5483 for (j = 0; j < ncopies; ++j)
5485 if (j == 0)
5487 src = vec_oprnd1
5488 = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt), stmt);
5489 op = vec_oprnd0
5490 = vect_get_vec_def_for_operand (scatter_off, stmt);
5492 else if (modifier != NONE && (j & 1))
5494 if (modifier == WIDEN)
5496 src = vec_oprnd1
5497 = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1);
5498 op = permute_vec_elements (vec_oprnd0, vec_oprnd0, perm_mask,
5499 stmt, gsi);
5501 else if (modifier == NARROW)
5503 src = permute_vec_elements (vec_oprnd1, vec_oprnd1, perm_mask,
5504 stmt, gsi);
5505 op = vec_oprnd0
5506 = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0);
5508 else
5509 gcc_unreachable ();
5511 else
5513 src = vec_oprnd1
5514 = vect_get_vec_def_for_stmt_copy (scatter_src_dt, vec_oprnd1);
5515 op = vec_oprnd0
5516 = vect_get_vec_def_for_stmt_copy (scatter_idx_dt, vec_oprnd0);
5519 if (!useless_type_conversion_p (srctype, TREE_TYPE (src)))
5521 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src))
5522 == TYPE_VECTOR_SUBPARTS (srctype));
5523 var = vect_get_new_ssa_name (srctype, vect_simple_var);
5524 src = build1 (VIEW_CONVERT_EXPR, srctype, src);
5525 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, src);
5526 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5527 src = var;
5530 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
5532 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
5533 == TYPE_VECTOR_SUBPARTS (idxtype));
5534 var = vect_get_new_ssa_name (idxtype, vect_simple_var);
5535 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
5536 new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
5537 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5538 op = var;
5541 new_stmt
5542 = gimple_build_call (scatter_decl, 5, ptr, mask, op, src, scale);
5544 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5546 if (prev_stmt_info == NULL)
5547 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
5548 else
5549 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
5550 prev_stmt_info = vinfo_for_stmt (new_stmt);
5552 return true;
5555 if (grouped_store)
5557 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
5558 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
5560 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++;
5562 /* FORNOW */
5563 gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt));
5565 /* We vectorize all the stmts of the interleaving group when we
5566 reach the last stmt in the group. */
5567 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))
5568 < GROUP_SIZE (vinfo_for_stmt (first_stmt))
5569 && !slp)
5571 *vec_stmt = NULL;
5572 return true;
5575 if (slp)
5577 grouped_store = false;
5578 /* VEC_NUM is the number of vect stmts to be created for this
5579 group. */
5580 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
5581 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
5582 gcc_assert (GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_stmt)) == first_stmt);
5583 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
5584 op = gimple_assign_rhs1 (first_stmt);
5586 else
5587 /* VEC_NUM is the number of vect stmts to be created for this
5588 group. */
5589 vec_num = group_size;
5591 else
5593 first_stmt = stmt;
5594 first_dr = dr;
5595 group_size = vec_num = 1;
5598 if (dump_enabled_p ())
5599 dump_printf_loc (MSG_NOTE, vect_location,
5600 "transform store. ncopies = %d\n", ncopies);
5602 if (STMT_VINFO_STRIDED_P (stmt_info))
5604 gimple_stmt_iterator incr_gsi;
5605 bool insert_after;
5606 gimple *incr;
5607 tree offvar;
5608 tree ivstep;
5609 tree running_off;
5610 gimple_seq stmts = NULL;
5611 tree stride_base, stride_step, alias_off;
5612 tree vec_oprnd;
5613 unsigned int g;
5615 gcc_assert (!nested_in_vect_loop_p (loop, stmt));
5617 stride_base
5618 = fold_build_pointer_plus
5619 (unshare_expr (DR_BASE_ADDRESS (first_dr)),
5620 size_binop (PLUS_EXPR,
5621 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr))),
5622 convert_to_ptrofftype (DR_INIT(first_dr))));
5623 stride_step = fold_convert (sizetype, unshare_expr (DR_STEP (first_dr)));
5625 /* For a store with loop-invariant (but other than power-of-2)
5626 stride (i.e. not a grouped access) like so:
5628 for (i = 0; i < n; i += stride)
5629 array[i] = ...;
5631 we generate a new induction variable and new stores from
5632 the components of the (vectorized) rhs:
5634 for (j = 0; ; j += VF*stride)
5635 vectemp = ...;
5636 tmp1 = vectemp[0];
5637 array[j] = tmp1;
5638 tmp2 = vectemp[1];
5639 array[j + stride] = tmp2;
5643 unsigned nstores = nunits;
5644 tree ltype = elem_type;
5645 if (slp)
5647 nstores = nunits / group_size;
5648 if (group_size < nunits)
5649 ltype = build_vector_type (elem_type, group_size);
5650 else
5651 ltype = vectype;
5652 ltype = build_aligned_type (ltype, TYPE_ALIGN (elem_type));
5653 ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
5654 group_size = 1;
5657 ivstep = stride_step;
5658 ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (ivstep), ivstep,
5659 build_int_cst (TREE_TYPE (ivstep),
5660 ncopies * nstores));
5662 standard_iv_increment_position (loop, &incr_gsi, &insert_after);
5664 create_iv (stride_base, ivstep, NULL,
5665 loop, &incr_gsi, insert_after,
5666 &offvar, NULL);
5667 incr = gsi_stmt (incr_gsi);
5668 set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
5670 stride_step = force_gimple_operand (stride_step, &stmts, true, NULL_TREE);
5671 if (stmts)
5672 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
5674 prev_stmt_info = NULL;
5675 alias_off = build_int_cst (reference_alias_ptr_type (DR_REF (first_dr)), 0);
5676 next_stmt = first_stmt;
5677 for (g = 0; g < group_size; g++)
5679 running_off = offvar;
5680 if (g)
5682 tree size = TYPE_SIZE_UNIT (ltype);
5683 tree pos = fold_build2 (MULT_EXPR, sizetype, size_int (g),
5684 size);
5685 tree newoff = copy_ssa_name (running_off, NULL);
5686 incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
5687 running_off, pos);
5688 vect_finish_stmt_generation (stmt, incr, gsi);
5689 running_off = newoff;
5691 for (j = 0; j < ncopies; j++)
5693 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5694 and first_stmt == stmt. */
5695 if (j == 0)
5697 if (slp)
5699 vect_get_vec_defs (op, NULL_TREE, stmt, &vec_oprnds, NULL,
5700 slp_node, -1);
5701 vec_oprnd = vec_oprnds[0];
5703 else
5705 gcc_assert (gimple_assign_single_p (next_stmt));
5706 op = gimple_assign_rhs1 (next_stmt);
5707 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
5710 else
5712 if (slp)
5713 vec_oprnd = vec_oprnds[j];
5714 else
5716 vect_is_simple_use (vec_oprnd, vinfo, &def_stmt, &dt);
5717 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd);
5721 for (i = 0; i < nstores; i++)
5723 tree newref, newoff;
5724 gimple *incr, *assign;
5725 tree size = TYPE_SIZE (ltype);
5726 /* Extract the i'th component. */
5727 tree pos = fold_build2 (MULT_EXPR, bitsizetype,
5728 bitsize_int (i), size);
5729 tree elem = fold_build3 (BIT_FIELD_REF, ltype, vec_oprnd,
5730 size, pos);
5732 elem = force_gimple_operand_gsi (gsi, elem, true,
5733 NULL_TREE, true,
5734 GSI_SAME_STMT);
5736 newref = build2 (MEM_REF, ltype,
5737 running_off, alias_off);
5739 /* And store it to *running_off. */
5740 assign = gimple_build_assign (newref, elem);
5741 vect_finish_stmt_generation (stmt, assign, gsi);
5743 newoff = copy_ssa_name (running_off, NULL);
5744 incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
5745 running_off, stride_step);
5746 vect_finish_stmt_generation (stmt, incr, gsi);
5748 running_off = newoff;
5749 if (g == group_size - 1
5750 && !slp)
5752 if (j == 0 && i == 0)
5753 STMT_VINFO_VEC_STMT (stmt_info)
5754 = *vec_stmt = assign;
5755 else
5756 STMT_VINFO_RELATED_STMT (prev_stmt_info) = assign;
5757 prev_stmt_info = vinfo_for_stmt (assign);
5761 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5763 return true;
5766 dr_chain.create (group_size);
5767 oprnds.create (group_size);
5769 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
5770 gcc_assert (alignment_support_scheme);
5771 /* Targets with store-lane instructions must not require explicit
5772 realignment. */
5773 gcc_assert (!store_lanes_p
5774 || alignment_support_scheme == dr_aligned
5775 || alignment_support_scheme == dr_unaligned_supported);
5777 if (negative)
5778 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
5780 if (store_lanes_p)
5781 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
5782 else
5783 aggr_type = vectype;
5785 /* In case the vectorization factor (VF) is bigger than the number
5786 of elements that we can fit in a vectype (nunits), we have to generate
5787 more than one vector stmt - i.e - we need to "unroll" the
5788 vector stmt by a factor VF/nunits. For more details see documentation in
5789 vect_get_vec_def_for_copy_stmt. */
5791 /* In case of interleaving (non-unit grouped access):
5793 S1: &base + 2 = x2
5794 S2: &base = x0
5795 S3: &base + 1 = x1
5796 S4: &base + 3 = x3
5798 We create vectorized stores starting from base address (the access of the
5799 first stmt in the chain (S2 in the above example), when the last store stmt
5800 of the chain (S4) is reached:
5802 VS1: &base = vx2
5803 VS2: &base + vec_size*1 = vx0
5804 VS3: &base + vec_size*2 = vx1
5805 VS4: &base + vec_size*3 = vx3
5807 Then permutation statements are generated:
5809 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5810 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5813 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5814 (the order of the data-refs in the output of vect_permute_store_chain
5815 corresponds to the order of scalar stmts in the interleaving chain - see
5816 the documentation of vect_permute_store_chain()).
5818 In case of both multiple types and interleaving, above vector stores and
5819 permutation stmts are created for every copy. The result vector stmts are
5820 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5821 STMT_VINFO_RELATED_STMT for the next copies.
5824 prev_stmt_info = NULL;
5825 for (j = 0; j < ncopies; j++)
5828 if (j == 0)
5830 if (slp)
5832 /* Get vectorized arguments for SLP_NODE. */
5833 vect_get_vec_defs (op, NULL_TREE, stmt, &vec_oprnds,
5834 NULL, slp_node, -1);
5836 vec_oprnd = vec_oprnds[0];
5838 else
5840 /* For interleaved stores we collect vectorized defs for all the
5841 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5842 used as an input to vect_permute_store_chain(), and OPRNDS as
5843 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5845 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5846 OPRNDS are of size 1. */
5847 next_stmt = first_stmt;
5848 for (i = 0; i < group_size; i++)
5850 /* Since gaps are not supported for interleaved stores,
5851 GROUP_SIZE is the exact number of stmts in the chain.
5852 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5853 there is no interleaving, GROUP_SIZE is 1, and only one
5854 iteration of the loop will be executed. */
5855 gcc_assert (next_stmt
5856 && gimple_assign_single_p (next_stmt));
5857 op = gimple_assign_rhs1 (next_stmt);
5859 vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt);
5860 dr_chain.quick_push (vec_oprnd);
5861 oprnds.quick_push (vec_oprnd);
5862 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
5866 /* We should have catched mismatched types earlier. */
5867 gcc_assert (useless_type_conversion_p (vectype,
5868 TREE_TYPE (vec_oprnd)));
5869 bool simd_lane_access_p
5870 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
5871 if (simd_lane_access_p
5872 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
5873 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
5874 && integer_zerop (DR_OFFSET (first_dr))
5875 && integer_zerop (DR_INIT (first_dr))
5876 && alias_sets_conflict_p (get_alias_set (aggr_type),
5877 get_alias_set (DR_REF (first_dr))))
5879 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
5880 dataref_offset = build_int_cst (reference_alias_ptr_type
5881 (DR_REF (first_dr)), 0);
5882 inv_p = false;
5884 else
5885 dataref_ptr
5886 = vect_create_data_ref_ptr (first_stmt, aggr_type,
5887 simd_lane_access_p ? loop : NULL,
5888 offset, &dummy, gsi, &ptr_incr,
5889 simd_lane_access_p, &inv_p);
5890 gcc_assert (bb_vinfo || !inv_p);
5892 else
5894 /* For interleaved stores we created vectorized defs for all the
5895 defs stored in OPRNDS in the previous iteration (previous copy).
5896 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5897 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5898 next copy.
5899 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5900 OPRNDS are of size 1. */
5901 for (i = 0; i < group_size; i++)
5903 op = oprnds[i];
5904 vect_is_simple_use (op, vinfo, &def_stmt, &dt);
5905 vec_oprnd = vect_get_vec_def_for_stmt_copy (dt, op);
5906 dr_chain[i] = vec_oprnd;
5907 oprnds[i] = vec_oprnd;
5909 if (dataref_offset)
5910 dataref_offset
5911 = int_const_binop (PLUS_EXPR, dataref_offset,
5912 TYPE_SIZE_UNIT (aggr_type));
5913 else
5914 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
5915 TYPE_SIZE_UNIT (aggr_type));
5918 if (store_lanes_p)
5920 tree vec_array;
5922 /* Combine all the vectors into an array. */
5923 vec_array = create_vector_array (vectype, vec_num);
5924 for (i = 0; i < vec_num; i++)
5926 vec_oprnd = dr_chain[i];
5927 write_vector_array (stmt, gsi, vec_oprnd, vec_array, i);
5930 /* Emit:
5931 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5932 data_ref = create_array_ref (aggr_type, dataref_ptr, first_dr);
5933 new_stmt = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
5934 gimple_call_set_lhs (new_stmt, data_ref);
5935 vect_finish_stmt_generation (stmt, new_stmt, gsi);
5937 else
5939 new_stmt = NULL;
5940 if (grouped_store)
5942 if (j == 0)
5943 result_chain.create (group_size);
5944 /* Permute. */
5945 vect_permute_store_chain (dr_chain, group_size, stmt, gsi,
5946 &result_chain);
5949 next_stmt = first_stmt;
5950 for (i = 0; i < vec_num; i++)
5952 unsigned align, misalign;
5954 if (i > 0)
5955 /* Bump the vector pointer. */
5956 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
5957 stmt, NULL_TREE);
5959 if (slp)
5960 vec_oprnd = vec_oprnds[i];
5961 else if (grouped_store)
5962 /* For grouped stores vectorized defs are interleaved in
5963 vect_permute_store_chain(). */
5964 vec_oprnd = result_chain[i];
5966 data_ref = fold_build2 (MEM_REF, TREE_TYPE (vec_oprnd),
5967 dataref_ptr,
5968 dataref_offset
5969 ? dataref_offset
5970 : build_int_cst (reference_alias_ptr_type
5971 (DR_REF (first_dr)), 0));
5972 align = TYPE_ALIGN_UNIT (vectype);
5973 if (aligned_access_p (first_dr))
5974 misalign = 0;
5975 else if (DR_MISALIGNMENT (first_dr) == -1)
5977 if (DR_VECT_AUX (first_dr)->base_element_aligned)
5978 align = TYPE_ALIGN_UNIT (elem_type);
5979 else
5980 align = get_object_alignment (DR_REF (first_dr))
5981 / BITS_PER_UNIT;
5982 misalign = 0;
5983 TREE_TYPE (data_ref)
5984 = build_aligned_type (TREE_TYPE (data_ref),
5985 align * BITS_PER_UNIT);
5987 else
5989 TREE_TYPE (data_ref)
5990 = build_aligned_type (TREE_TYPE (data_ref),
5991 TYPE_ALIGN (elem_type));
5992 misalign = DR_MISALIGNMENT (first_dr);
5994 if (dataref_offset == NULL_TREE
5995 && TREE_CODE (dataref_ptr) == SSA_NAME)
5996 set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
5997 misalign);
5999 if (negative
6000 && dt != vect_constant_def
6001 && dt != vect_external_def)
6003 tree perm_mask = perm_mask_for_reverse (vectype);
6004 tree perm_dest
6005 = vect_create_destination_var (gimple_assign_rhs1 (stmt),
6006 vectype);
6007 tree new_temp = make_ssa_name (perm_dest);
6009 /* Generate the permute statement. */
6010 gimple *perm_stmt
6011 = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
6012 vec_oprnd, perm_mask);
6013 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
6015 perm_stmt = SSA_NAME_DEF_STMT (new_temp);
6016 vec_oprnd = new_temp;
6019 /* Arguments are ready. Create the new vector stmt. */
6020 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
6021 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6023 if (slp)
6024 continue;
6026 next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
6027 if (!next_stmt)
6028 break;
6031 if (!slp)
6033 if (j == 0)
6034 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6035 else
6036 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6037 prev_stmt_info = vinfo_for_stmt (new_stmt);
6041 dr_chain.release ();
6042 oprnds.release ();
6043 result_chain.release ();
6044 vec_oprnds.release ();
6046 return true;
6049 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
6050 VECTOR_CST mask. No checks are made that the target platform supports the
6051 mask, so callers may wish to test can_vec_perm_p separately, or use
6052 vect_gen_perm_mask_checked. */
6054 tree
6055 vect_gen_perm_mask_any (tree vectype, const unsigned char *sel)
6057 tree mask_elt_type, mask_type, mask_vec, *mask_elts;
6058 int i, nunits;
6060 nunits = TYPE_VECTOR_SUBPARTS (vectype);
6062 mask_elt_type = lang_hooks.types.type_for_mode
6063 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype))), 1);
6064 mask_type = get_vectype_for_scalar_type (mask_elt_type);
6066 mask_elts = XALLOCAVEC (tree, nunits);
6067 for (i = nunits - 1; i >= 0; i--)
6068 mask_elts[i] = build_int_cst (mask_elt_type, sel[i]);
6069 mask_vec = build_vector (mask_type, mask_elts);
6071 return mask_vec;
6074 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
6075 i.e. that the target supports the pattern _for arbitrary input vectors_. */
6077 tree
6078 vect_gen_perm_mask_checked (tree vectype, const unsigned char *sel)
6080 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype), false, sel));
6081 return vect_gen_perm_mask_any (vectype, sel);
6084 /* Given a vector variable X and Y, that was generated for the scalar
6085 STMT, generate instructions to permute the vector elements of X and Y
6086 using permutation mask MASK_VEC, insert them at *GSI and return the
6087 permuted vector variable. */
6089 static tree
6090 permute_vec_elements (tree x, tree y, tree mask_vec, gimple *stmt,
6091 gimple_stmt_iterator *gsi)
6093 tree vectype = TREE_TYPE (x);
6094 tree perm_dest, data_ref;
6095 gimple *perm_stmt;
6097 perm_dest = vect_create_destination_var (gimple_get_lhs (stmt), vectype);
6098 data_ref = make_ssa_name (perm_dest);
6100 /* Generate the permute statement. */
6101 perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, x, y, mask_vec);
6102 vect_finish_stmt_generation (stmt, perm_stmt, gsi);
6104 return data_ref;
6107 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
6108 inserting them on the loops preheader edge. Returns true if we
6109 were successful in doing so (and thus STMT can be moved then),
6110 otherwise returns false. */
6112 static bool
6113 hoist_defs_of_uses (gimple *stmt, struct loop *loop)
6115 ssa_op_iter i;
6116 tree op;
6117 bool any = false;
6119 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
6121 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
6122 if (!gimple_nop_p (def_stmt)
6123 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
6125 /* Make sure we don't need to recurse. While we could do
6126 so in simple cases when there are more complex use webs
6127 we don't have an easy way to preserve stmt order to fulfil
6128 dependencies within them. */
6129 tree op2;
6130 ssa_op_iter i2;
6131 if (gimple_code (def_stmt) == GIMPLE_PHI)
6132 return false;
6133 FOR_EACH_SSA_TREE_OPERAND (op2, def_stmt, i2, SSA_OP_USE)
6135 gimple *def_stmt2 = SSA_NAME_DEF_STMT (op2);
6136 if (!gimple_nop_p (def_stmt2)
6137 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt2)))
6138 return false;
6140 any = true;
6144 if (!any)
6145 return true;
6147 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_USE)
6149 gimple *def_stmt = SSA_NAME_DEF_STMT (op);
6150 if (!gimple_nop_p (def_stmt)
6151 && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
6153 gimple_stmt_iterator gsi = gsi_for_stmt (def_stmt);
6154 gsi_remove (&gsi, false);
6155 gsi_insert_on_edge_immediate (loop_preheader_edge (loop), def_stmt);
6159 return true;
6162 /* vectorizable_load.
6164 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
6165 can be vectorized.
6166 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6167 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
6168 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6170 static bool
6171 vectorizable_load (gimple *stmt, gimple_stmt_iterator *gsi, gimple **vec_stmt,
6172 slp_tree slp_node, slp_instance slp_node_instance)
6174 tree scalar_dest;
6175 tree vec_dest = NULL;
6176 tree data_ref = NULL;
6177 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
6178 stmt_vec_info prev_stmt_info;
6179 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
6180 struct loop *loop = NULL;
6181 struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
6182 bool nested_in_vect_loop = false;
6183 struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr = NULL;
6184 tree elem_type;
6185 tree new_temp;
6186 machine_mode mode;
6187 gimple *new_stmt = NULL;
6188 tree dummy;
6189 enum dr_alignment_support alignment_support_scheme;
6190 tree dataref_ptr = NULL_TREE;
6191 tree dataref_offset = NULL_TREE;
6192 gimple *ptr_incr = NULL;
6193 int ncopies;
6194 int i, j, group_size = -1, group_gap_adj;
6195 tree msq = NULL_TREE, lsq;
6196 tree offset = NULL_TREE;
6197 tree byte_offset = NULL_TREE;
6198 tree realignment_token = NULL_TREE;
6199 gphi *phi = NULL;
6200 vec<tree> dr_chain = vNULL;
6201 bool grouped_load = false;
6202 bool load_lanes_p = false;
6203 gimple *first_stmt;
6204 gimple *first_stmt_for_drptr = NULL;
6205 bool inv_p;
6206 bool negative = false;
6207 bool compute_in_loop = false;
6208 struct loop *at_loop;
6209 int vec_num;
6210 bool slp = (slp_node != NULL);
6211 bool slp_perm = false;
6212 enum tree_code code;
6213 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
6214 int vf;
6215 tree aggr_type;
6216 tree gather_base = NULL_TREE, gather_off = NULL_TREE;
6217 tree gather_off_vectype = NULL_TREE, gather_decl = NULL_TREE;
6218 int gather_scale = 1;
6219 enum vect_def_type gather_dt = vect_unknown_def_type;
6220 vec_info *vinfo = stmt_info->vinfo;
6222 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
6223 return false;
6225 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
6226 && ! vec_stmt)
6227 return false;
6229 /* Is vectorizable load? */
6230 if (!is_gimple_assign (stmt))
6231 return false;
6233 scalar_dest = gimple_assign_lhs (stmt);
6234 if (TREE_CODE (scalar_dest) != SSA_NAME)
6235 return false;
6237 code = gimple_assign_rhs_code (stmt);
6238 if (code != ARRAY_REF
6239 && code != BIT_FIELD_REF
6240 && code != INDIRECT_REF
6241 && code != COMPONENT_REF
6242 && code != IMAGPART_EXPR
6243 && code != REALPART_EXPR
6244 && code != MEM_REF
6245 && TREE_CODE_CLASS (code) != tcc_declaration)
6246 return false;
6248 if (!STMT_VINFO_DATA_REF (stmt_info))
6249 return false;
6251 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6252 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
6254 if (loop_vinfo)
6256 loop = LOOP_VINFO_LOOP (loop_vinfo);
6257 nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
6258 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
6260 else
6261 vf = 1;
6263 /* Multiple types in SLP are handled by creating the appropriate number of
6264 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6265 case of SLP. */
6266 if (slp || PURE_SLP_STMT (stmt_info))
6267 ncopies = 1;
6268 else
6269 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
6271 gcc_assert (ncopies >= 1);
6273 /* FORNOW. This restriction should be relaxed. */
6274 if (nested_in_vect_loop && ncopies > 1)
6276 if (dump_enabled_p ())
6277 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6278 "multiple types in nested loop.\n");
6279 return false;
6282 /* Invalidate assumptions made by dependence analysis when vectorization
6283 on the unrolled body effectively re-orders stmts. */
6284 if (ncopies > 1
6285 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
6286 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo)
6287 > STMT_VINFO_MIN_NEG_DIST (stmt_info)))
6289 if (dump_enabled_p ())
6290 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6291 "cannot perform implicit CSE when unrolling "
6292 "with negative dependence distance\n");
6293 return false;
6296 elem_type = TREE_TYPE (vectype);
6297 mode = TYPE_MODE (vectype);
6299 /* FORNOW. In some cases can vectorize even if data-type not supported
6300 (e.g. - data copies). */
6301 if (optab_handler (mov_optab, mode) == CODE_FOR_nothing)
6303 if (dump_enabled_p ())
6304 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6305 "Aligned load, but unsupported type.\n");
6306 return false;
6309 /* Check if the load is a part of an interleaving chain. */
6310 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
6312 grouped_load = true;
6313 /* FORNOW */
6314 gcc_assert (!nested_in_vect_loop && !STMT_VINFO_GATHER_SCATTER_P (stmt_info));
6316 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
6318 /* If this is single-element interleaving with an element distance
6319 that leaves unused vector loads around punt - we at least create
6320 very sub-optimal code in that case (and blow up memory,
6321 see PR65518). */
6322 bool force_peeling = false;
6323 if (first_stmt == stmt
6324 && !GROUP_NEXT_ELEMENT (stmt_info))
6326 if (GROUP_SIZE (stmt_info) > TYPE_VECTOR_SUBPARTS (vectype))
6328 if (dump_enabled_p ())
6329 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6330 "single-element interleaving not supported "
6331 "for not adjacent vector loads\n");
6332 return false;
6335 /* Single-element interleaving requires peeling for gaps. */
6336 force_peeling = true;
6339 /* If there is a gap in the end of the group or the group size cannot
6340 be made a multiple of the vector element count then we access excess
6341 elements in the last iteration and thus need to peel that off. */
6342 if (loop_vinfo
6343 && ! STMT_VINFO_STRIDED_P (stmt_info)
6344 && (force_peeling
6345 || GROUP_GAP (vinfo_for_stmt (first_stmt)) != 0
6346 || (!slp && vf % GROUP_SIZE (vinfo_for_stmt (first_stmt)) != 0)))
6348 if (dump_enabled_p ())
6349 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6350 "Data access with gaps requires scalar "
6351 "epilogue loop\n");
6352 if (loop->inner)
6354 if (dump_enabled_p ())
6355 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6356 "Peeling for outer loop is not supported\n");
6357 return false;
6360 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
6363 if (slp && SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
6364 slp_perm = true;
6366 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
6368 /* ??? The following is overly pessimistic (as well as the loop
6369 case above) in the case we can statically determine the excess
6370 elements loaded are within the bounds of a decl that is accessed.
6371 Likewise for BB vectorizations using masked loads is a possibility. */
6372 if (bb_vinfo && slp_perm && group_size % nunits != 0)
6374 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6375 "BB vectorization with gaps at the end of a load "
6376 "is not supported\n");
6377 return false;
6380 if (!slp
6381 && !PURE_SLP_STMT (stmt_info)
6382 && !STMT_VINFO_STRIDED_P (stmt_info))
6384 if (vect_load_lanes_supported (vectype, group_size))
6385 load_lanes_p = true;
6386 else if (!vect_grouped_load_supported (vectype, group_size))
6387 return false;
6390 /* Invalidate assumptions made by dependence analysis when vectorization
6391 on the unrolled body effectively re-orders stmts. */
6392 if (!PURE_SLP_STMT (stmt_info)
6393 && STMT_VINFO_MIN_NEG_DIST (stmt_info) != 0
6394 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo)
6395 > STMT_VINFO_MIN_NEG_DIST (stmt_info)))
6397 if (dump_enabled_p ())
6398 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6399 "cannot perform implicit CSE when performing "
6400 "group loads with negative dependence distance\n");
6401 return false;
6404 /* Similarly when the stmt is a load that is both part of a SLP
6405 instance and a loop vectorized stmt via the same-dr mechanism
6406 we have to give up. */
6407 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info)
6408 && (STMT_SLP_TYPE (stmt_info)
6409 != STMT_SLP_TYPE (vinfo_for_stmt
6410 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info)))))
6412 if (dump_enabled_p ())
6413 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6414 "conflicting SLP types for CSEd load\n");
6415 return false;
6420 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6422 gimple *def_stmt;
6423 gather_decl = vect_check_gather_scatter (stmt, loop_vinfo, &gather_base,
6424 &gather_off, &gather_scale);
6425 gcc_assert (gather_decl);
6426 if (!vect_is_simple_use (gather_off, vinfo, &def_stmt, &gather_dt,
6427 &gather_off_vectype))
6429 if (dump_enabled_p ())
6430 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6431 "gather index use not simple.\n");
6432 return false;
6435 else if (STMT_VINFO_STRIDED_P (stmt_info))
6437 if ((grouped_load
6438 && (slp || PURE_SLP_STMT (stmt_info)))
6439 && (group_size > nunits
6440 || nunits % group_size != 0))
6442 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6443 "unhandled strided group load\n");
6444 return false;
6447 else
6449 negative = tree_int_cst_compare (nested_in_vect_loop
6450 ? STMT_VINFO_DR_STEP (stmt_info)
6451 : DR_STEP (dr),
6452 size_zero_node) < 0;
6453 if (negative && ncopies > 1)
6455 if (dump_enabled_p ())
6456 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6457 "multiple types with negative step.\n");
6458 return false;
6461 if (negative)
6463 if (grouped_load)
6465 if (dump_enabled_p ())
6466 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6467 "negative step for group load not supported"
6468 "\n");
6469 return false;
6471 alignment_support_scheme = vect_supportable_dr_alignment (dr, false);
6472 if (alignment_support_scheme != dr_aligned
6473 && alignment_support_scheme != dr_unaligned_supported)
6475 if (dump_enabled_p ())
6476 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6477 "negative step but alignment required.\n");
6478 return false;
6480 if (!perm_mask_for_reverse (vectype))
6482 if (dump_enabled_p ())
6483 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6484 "negative step and reversing not supported."
6485 "\n");
6486 return false;
6491 if (!vec_stmt) /* transformation not required. */
6493 STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
6494 /* The SLP costs are calculated during SLP analysis. */
6495 if (!PURE_SLP_STMT (stmt_info))
6496 vect_model_load_cost (stmt_info, ncopies, load_lanes_p,
6497 NULL, NULL, NULL);
6498 return true;
6501 if (dump_enabled_p ())
6502 dump_printf_loc (MSG_NOTE, vect_location,
6503 "transform load. ncopies = %d\n", ncopies);
6505 /** Transform. **/
6507 ensure_base_align (stmt_info, dr);
6509 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
6511 tree vec_oprnd0 = NULL_TREE, op;
6512 tree arglist = TYPE_ARG_TYPES (TREE_TYPE (gather_decl));
6513 tree rettype, srctype, ptrtype, idxtype, masktype, scaletype;
6514 tree ptr, mask, var, scale, merge, perm_mask = NULL_TREE, prev_res = NULL_TREE;
6515 edge pe = loop_preheader_edge (loop);
6516 gimple_seq seq;
6517 basic_block new_bb;
6518 enum { NARROW, NONE, WIDEN } modifier;
6519 int gather_off_nunits = TYPE_VECTOR_SUBPARTS (gather_off_vectype);
6521 if (nunits == gather_off_nunits)
6522 modifier = NONE;
6523 else if (nunits == gather_off_nunits / 2)
6525 unsigned char *sel = XALLOCAVEC (unsigned char, gather_off_nunits);
6526 modifier = WIDEN;
6528 for (i = 0; i < gather_off_nunits; ++i)
6529 sel[i] = i | nunits;
6531 perm_mask = vect_gen_perm_mask_checked (gather_off_vectype, sel);
6533 else if (nunits == gather_off_nunits * 2)
6535 unsigned char *sel = XALLOCAVEC (unsigned char, nunits);
6536 modifier = NARROW;
6538 for (i = 0; i < nunits; ++i)
6539 sel[i] = i < gather_off_nunits
6540 ? i : i + nunits - gather_off_nunits;
6542 perm_mask = vect_gen_perm_mask_checked (vectype, sel);
6543 ncopies *= 2;
6545 else
6546 gcc_unreachable ();
6548 rettype = TREE_TYPE (TREE_TYPE (gather_decl));
6549 srctype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6550 ptrtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6551 idxtype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6552 masktype = TREE_VALUE (arglist); arglist = TREE_CHAIN (arglist);
6553 scaletype = TREE_VALUE (arglist);
6554 gcc_checking_assert (types_compatible_p (srctype, rettype));
6556 vec_dest = vect_create_destination_var (scalar_dest, vectype);
6558 ptr = fold_convert (ptrtype, gather_base);
6559 if (!is_gimple_min_invariant (ptr))
6561 ptr = force_gimple_operand (ptr, &seq, true, NULL_TREE);
6562 new_bb = gsi_insert_seq_on_edge_immediate (pe, seq);
6563 gcc_assert (!new_bb);
6566 /* Currently we support only unconditional gather loads,
6567 so mask should be all ones. */
6568 if (TREE_CODE (masktype) == INTEGER_TYPE)
6569 mask = build_int_cst (masktype, -1);
6570 else if (TREE_CODE (TREE_TYPE (masktype)) == INTEGER_TYPE)
6572 mask = build_int_cst (TREE_TYPE (masktype), -1);
6573 mask = build_vector_from_val (masktype, mask);
6574 mask = vect_init_vector (stmt, mask, masktype, NULL);
6576 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype)))
6578 REAL_VALUE_TYPE r;
6579 long tmp[6];
6580 for (j = 0; j < 6; ++j)
6581 tmp[j] = -1;
6582 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (masktype)));
6583 mask = build_real (TREE_TYPE (masktype), r);
6584 mask = build_vector_from_val (masktype, mask);
6585 mask = vect_init_vector (stmt, mask, masktype, NULL);
6587 else
6588 gcc_unreachable ();
6590 scale = build_int_cst (scaletype, gather_scale);
6592 if (TREE_CODE (TREE_TYPE (rettype)) == INTEGER_TYPE)
6593 merge = build_int_cst (TREE_TYPE (rettype), 0);
6594 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype)))
6596 REAL_VALUE_TYPE r;
6597 long tmp[6];
6598 for (j = 0; j < 6; ++j)
6599 tmp[j] = 0;
6600 real_from_target (&r, tmp, TYPE_MODE (TREE_TYPE (rettype)));
6601 merge = build_real (TREE_TYPE (rettype), r);
6603 else
6604 gcc_unreachable ();
6605 merge = build_vector_from_val (rettype, merge);
6606 merge = vect_init_vector (stmt, merge, rettype, NULL);
6608 prev_stmt_info = NULL;
6609 for (j = 0; j < ncopies; ++j)
6611 if (modifier == WIDEN && (j & 1))
6612 op = permute_vec_elements (vec_oprnd0, vec_oprnd0,
6613 perm_mask, stmt, gsi);
6614 else if (j == 0)
6615 op = vec_oprnd0
6616 = vect_get_vec_def_for_operand (gather_off, stmt);
6617 else
6618 op = vec_oprnd0
6619 = vect_get_vec_def_for_stmt_copy (gather_dt, vec_oprnd0);
6621 if (!useless_type_conversion_p (idxtype, TREE_TYPE (op)))
6623 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op))
6624 == TYPE_VECTOR_SUBPARTS (idxtype));
6625 var = vect_get_new_ssa_name (idxtype, vect_simple_var);
6626 op = build1 (VIEW_CONVERT_EXPR, idxtype, op);
6627 new_stmt
6628 = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
6629 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6630 op = var;
6633 new_stmt
6634 = gimple_build_call (gather_decl, 5, merge, ptr, op, mask, scale);
6636 if (!useless_type_conversion_p (vectype, rettype))
6638 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype)
6639 == TYPE_VECTOR_SUBPARTS (rettype));
6640 op = vect_get_new_ssa_name (rettype, vect_simple_var);
6641 gimple_call_set_lhs (new_stmt, op);
6642 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6643 var = make_ssa_name (vec_dest);
6644 op = build1 (VIEW_CONVERT_EXPR, vectype, op);
6645 new_stmt
6646 = gimple_build_assign (var, VIEW_CONVERT_EXPR, op);
6648 else
6650 var = make_ssa_name (vec_dest, new_stmt);
6651 gimple_call_set_lhs (new_stmt, var);
6654 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6656 if (modifier == NARROW)
6658 if ((j & 1) == 0)
6660 prev_res = var;
6661 continue;
6663 var = permute_vec_elements (prev_res, var,
6664 perm_mask, stmt, gsi);
6665 new_stmt = SSA_NAME_DEF_STMT (var);
6668 if (prev_stmt_info == NULL)
6669 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6670 else
6671 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6672 prev_stmt_info = vinfo_for_stmt (new_stmt);
6674 return true;
6676 else if (STMT_VINFO_STRIDED_P (stmt_info))
6678 gimple_stmt_iterator incr_gsi;
6679 bool insert_after;
6680 gimple *incr;
6681 tree offvar;
6682 tree ivstep;
6683 tree running_off;
6684 vec<constructor_elt, va_gc> *v = NULL;
6685 gimple_seq stmts = NULL;
6686 tree stride_base, stride_step, alias_off;
6688 gcc_assert (!nested_in_vect_loop);
6690 if (slp && grouped_load)
6691 first_dr = STMT_VINFO_DATA_REF
6692 (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info)));
6693 else
6694 first_dr = dr;
6696 stride_base
6697 = fold_build_pointer_plus
6698 (DR_BASE_ADDRESS (first_dr),
6699 size_binop (PLUS_EXPR,
6700 convert_to_ptrofftype (DR_OFFSET (first_dr)),
6701 convert_to_ptrofftype (DR_INIT (first_dr))));
6702 stride_step = fold_convert (sizetype, DR_STEP (first_dr));
6704 /* For a load with loop-invariant (but other than power-of-2)
6705 stride (i.e. not a grouped access) like so:
6707 for (i = 0; i < n; i += stride)
6708 ... = array[i];
6710 we generate a new induction variable and new accesses to
6711 form a new vector (or vectors, depending on ncopies):
6713 for (j = 0; ; j += VF*stride)
6714 tmp1 = array[j];
6715 tmp2 = array[j + stride];
6717 vectemp = {tmp1, tmp2, ...}
6720 ivstep = fold_build2 (MULT_EXPR, TREE_TYPE (stride_step), stride_step,
6721 build_int_cst (TREE_TYPE (stride_step), vf));
6723 standard_iv_increment_position (loop, &incr_gsi, &insert_after);
6725 create_iv (unshare_expr (stride_base), unshare_expr (ivstep), NULL,
6726 loop, &incr_gsi, insert_after,
6727 &offvar, NULL);
6728 incr = gsi_stmt (incr_gsi);
6729 set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
6731 stride_step = force_gimple_operand (unshare_expr (stride_step),
6732 &stmts, true, NULL_TREE);
6733 if (stmts)
6734 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
6736 prev_stmt_info = NULL;
6737 running_off = offvar;
6738 alias_off = build_int_cst (reference_alias_ptr_type (DR_REF (first_dr)), 0);
6739 int nloads = nunits;
6740 tree ltype = TREE_TYPE (vectype);
6741 auto_vec<tree> dr_chain;
6742 if (slp)
6744 nloads = nunits / group_size;
6745 if (group_size < nunits)
6746 ltype = build_vector_type (TREE_TYPE (vectype), group_size);
6747 else
6748 ltype = vectype;
6749 ltype = build_aligned_type (ltype, TYPE_ALIGN (TREE_TYPE (vectype)));
6750 /* For SLP permutation support we need to load the whole group,
6751 not only the number of vector stmts the permutation result
6752 fits in. */
6753 if (slp_perm)
6755 ncopies = (group_size * vf + nunits - 1) / nunits;
6756 dr_chain.create (ncopies);
6758 else
6759 ncopies = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
6761 for (j = 0; j < ncopies; j++)
6763 tree vec_inv;
6765 if (nloads > 1)
6767 vec_alloc (v, nloads);
6768 for (i = 0; i < nloads; i++)
6770 tree newref, newoff;
6771 gimple *incr;
6772 newref = build2 (MEM_REF, ltype, running_off, alias_off);
6774 newref = force_gimple_operand_gsi (gsi, newref, true,
6775 NULL_TREE, true,
6776 GSI_SAME_STMT);
6777 CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, newref);
6778 newoff = copy_ssa_name (running_off);
6779 incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
6780 running_off, stride_step);
6781 vect_finish_stmt_generation (stmt, incr, gsi);
6783 running_off = newoff;
6786 vec_inv = build_constructor (vectype, v);
6787 new_temp = vect_init_vector (stmt, vec_inv, vectype, gsi);
6788 new_stmt = SSA_NAME_DEF_STMT (new_temp);
6790 else
6792 new_stmt = gimple_build_assign (make_ssa_name (ltype),
6793 build2 (MEM_REF, ltype,
6794 running_off, alias_off));
6795 vect_finish_stmt_generation (stmt, new_stmt, gsi);
6797 tree newoff = copy_ssa_name (running_off);
6798 gimple *incr = gimple_build_assign (newoff, POINTER_PLUS_EXPR,
6799 running_off, stride_step);
6800 vect_finish_stmt_generation (stmt, incr, gsi);
6802 running_off = newoff;
6805 if (slp)
6807 if (slp_perm)
6808 dr_chain.quick_push (gimple_assign_lhs (new_stmt));
6809 else
6810 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
6812 else
6814 if (j == 0)
6815 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
6816 else
6817 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
6818 prev_stmt_info = vinfo_for_stmt (new_stmt);
6821 if (slp_perm)
6822 vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
6823 slp_node_instance, false);
6824 return true;
6827 if (grouped_load)
6829 first_stmt = GROUP_FIRST_ELEMENT (stmt_info);
6830 /* For SLP vectorization we directly vectorize a subchain
6831 without permutation. */
6832 if (slp && ! SLP_TREE_LOAD_PERMUTATION (slp_node).exists ())
6833 first_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[0];
6834 /* For BB vectorization always use the first stmt to base
6835 the data ref pointer on. */
6836 if (bb_vinfo)
6837 first_stmt_for_drptr = SLP_TREE_SCALAR_STMTS (slp_node)[0];
6839 /* Check if the chain of loads is already vectorized. */
6840 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt))
6841 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6842 ??? But we can only do so if there is exactly one
6843 as we have no way to get at the rest. Leave the CSE
6844 opportunity alone.
6845 ??? With the group load eventually participating
6846 in multiple different permutations (having multiple
6847 slp nodes which refer to the same group) the CSE
6848 is even wrong code. See PR56270. */
6849 && !slp)
6851 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
6852 return true;
6854 first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt));
6855 group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt));
6856 group_gap_adj = 0;
6858 /* VEC_NUM is the number of vect stmts to be created for this group. */
6859 if (slp)
6861 grouped_load = false;
6862 /* For SLP permutation support we need to load the whole group,
6863 not only the number of vector stmts the permutation result
6864 fits in. */
6865 if (slp_perm)
6866 vec_num = (group_size * vf + nunits - 1) / nunits;
6867 else
6868 vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
6869 group_gap_adj = vf * group_size - nunits * vec_num;
6871 else
6872 vec_num = group_size;
6874 else
6876 first_stmt = stmt;
6877 first_dr = dr;
6878 group_size = vec_num = 1;
6879 group_gap_adj = 0;
6882 alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false);
6883 gcc_assert (alignment_support_scheme);
6884 /* Targets with load-lane instructions must not require explicit
6885 realignment. */
6886 gcc_assert (!load_lanes_p
6887 || alignment_support_scheme == dr_aligned
6888 || alignment_support_scheme == dr_unaligned_supported);
6890 /* In case the vectorization factor (VF) is bigger than the number
6891 of elements that we can fit in a vectype (nunits), we have to generate
6892 more than one vector stmt - i.e - we need to "unroll" the
6893 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6894 from one copy of the vector stmt to the next, in the field
6895 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6896 stages to find the correct vector defs to be used when vectorizing
6897 stmts that use the defs of the current stmt. The example below
6898 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6899 need to create 4 vectorized stmts):
6901 before vectorization:
6902 RELATED_STMT VEC_STMT
6903 S1: x = memref - -
6904 S2: z = x + 1 - -
6906 step 1: vectorize stmt S1:
6907 We first create the vector stmt VS1_0, and, as usual, record a
6908 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6909 Next, we create the vector stmt VS1_1, and record a pointer to
6910 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6911 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6912 stmts and pointers:
6913 RELATED_STMT VEC_STMT
6914 VS1_0: vx0 = memref0 VS1_1 -
6915 VS1_1: vx1 = memref1 VS1_2 -
6916 VS1_2: vx2 = memref2 VS1_3 -
6917 VS1_3: vx3 = memref3 - -
6918 S1: x = load - VS1_0
6919 S2: z = x + 1 - -
6921 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6922 information we recorded in RELATED_STMT field is used to vectorize
6923 stmt S2. */
6925 /* In case of interleaving (non-unit grouped access):
6927 S1: x2 = &base + 2
6928 S2: x0 = &base
6929 S3: x1 = &base + 1
6930 S4: x3 = &base + 3
6932 Vectorized loads are created in the order of memory accesses
6933 starting from the access of the first stmt of the chain:
6935 VS1: vx0 = &base
6936 VS2: vx1 = &base + vec_size*1
6937 VS3: vx3 = &base + vec_size*2
6938 VS4: vx4 = &base + vec_size*3
6940 Then permutation statements are generated:
6942 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6943 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6946 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6947 (the order of the data-refs in the output of vect_permute_load_chain
6948 corresponds to the order of scalar stmts in the interleaving chain - see
6949 the documentation of vect_permute_load_chain()).
6950 The generation of permutation stmts and recording them in
6951 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6953 In case of both multiple types and interleaving, the vector loads and
6954 permutation stmts above are created for every copy. The result vector
6955 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6956 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6958 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6959 on a target that supports unaligned accesses (dr_unaligned_supported)
6960 we generate the following code:
6961 p = initial_addr;
6962 indx = 0;
6963 loop {
6964 p = p + indx * vectype_size;
6965 vec_dest = *(p);
6966 indx = indx + 1;
6969 Otherwise, the data reference is potentially unaligned on a target that
6970 does not support unaligned accesses (dr_explicit_realign_optimized) -
6971 then generate the following code, in which the data in each iteration is
6972 obtained by two vector loads, one from the previous iteration, and one
6973 from the current iteration:
6974 p1 = initial_addr;
6975 msq_init = *(floor(p1))
6976 p2 = initial_addr + VS - 1;
6977 realignment_token = call target_builtin;
6978 indx = 0;
6979 loop {
6980 p2 = p2 + indx * vectype_size
6981 lsq = *(floor(p2))
6982 vec_dest = realign_load (msq, lsq, realignment_token)
6983 indx = indx + 1;
6984 msq = lsq;
6985 } */
6987 /* If the misalignment remains the same throughout the execution of the
6988 loop, we can create the init_addr and permutation mask at the loop
6989 preheader. Otherwise, it needs to be created inside the loop.
6990 This can only occur when vectorizing memory accesses in the inner-loop
6991 nested within an outer-loop that is being vectorized. */
6993 if (nested_in_vect_loop
6994 && (TREE_INT_CST_LOW (DR_STEP (dr))
6995 % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
6997 gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
6998 compute_in_loop = true;
7001 if ((alignment_support_scheme == dr_explicit_realign_optimized
7002 || alignment_support_scheme == dr_explicit_realign)
7003 && !compute_in_loop)
7005 msq = vect_setup_realignment (first_stmt, gsi, &realignment_token,
7006 alignment_support_scheme, NULL_TREE,
7007 &at_loop);
7008 if (alignment_support_scheme == dr_explicit_realign_optimized)
7010 phi = as_a <gphi *> (SSA_NAME_DEF_STMT (msq));
7011 byte_offset = size_binop (MINUS_EXPR, TYPE_SIZE_UNIT (vectype),
7012 size_one_node);
7015 else
7016 at_loop = loop;
7018 if (negative)
7019 offset = size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1);
7021 if (load_lanes_p)
7022 aggr_type = build_array_type_nelts (elem_type, vec_num * nunits);
7023 else
7024 aggr_type = vectype;
7026 prev_stmt_info = NULL;
7027 for (j = 0; j < ncopies; j++)
7029 /* 1. Create the vector or array pointer update chain. */
7030 if (j == 0)
7032 bool simd_lane_access_p
7033 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info);
7034 if (simd_lane_access_p
7035 && TREE_CODE (DR_BASE_ADDRESS (first_dr)) == ADDR_EXPR
7036 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr), 0))
7037 && integer_zerop (DR_OFFSET (first_dr))
7038 && integer_zerop (DR_INIT (first_dr))
7039 && alias_sets_conflict_p (get_alias_set (aggr_type),
7040 get_alias_set (DR_REF (first_dr)))
7041 && (alignment_support_scheme == dr_aligned
7042 || alignment_support_scheme == dr_unaligned_supported))
7044 dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr));
7045 dataref_offset = build_int_cst (reference_alias_ptr_type
7046 (DR_REF (first_dr)), 0);
7047 inv_p = false;
7049 else if (first_stmt_for_drptr
7050 && first_stmt != first_stmt_for_drptr)
7052 dataref_ptr
7053 = vect_create_data_ref_ptr (first_stmt_for_drptr, aggr_type,
7054 at_loop, offset, &dummy, gsi,
7055 &ptr_incr, simd_lane_access_p,
7056 &inv_p, byte_offset);
7057 /* Adjust the pointer by the difference to first_stmt. */
7058 data_reference_p ptrdr
7059 = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt_for_drptr));
7060 tree diff = fold_convert (sizetype,
7061 size_binop (MINUS_EXPR,
7062 DR_INIT (first_dr),
7063 DR_INIT (ptrdr)));
7064 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
7065 stmt, diff);
7067 else
7068 dataref_ptr
7069 = vect_create_data_ref_ptr (first_stmt, aggr_type, at_loop,
7070 offset, &dummy, gsi, &ptr_incr,
7071 simd_lane_access_p, &inv_p,
7072 byte_offset);
7074 else if (dataref_offset)
7075 dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset,
7076 TYPE_SIZE_UNIT (aggr_type));
7077 else
7078 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt,
7079 TYPE_SIZE_UNIT (aggr_type));
7081 if (grouped_load || slp_perm)
7082 dr_chain.create (vec_num);
7084 if (load_lanes_p)
7086 tree vec_array;
7088 vec_array = create_vector_array (vectype, vec_num);
7090 /* Emit:
7091 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
7092 data_ref = create_array_ref (aggr_type, dataref_ptr, first_dr);
7093 new_stmt = gimple_build_call_internal (IFN_LOAD_LANES, 1, data_ref);
7094 gimple_call_set_lhs (new_stmt, vec_array);
7095 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7097 /* Extract each vector into an SSA_NAME. */
7098 for (i = 0; i < vec_num; i++)
7100 new_temp = read_vector_array (stmt, gsi, scalar_dest,
7101 vec_array, i);
7102 dr_chain.quick_push (new_temp);
7105 /* Record the mapping between SSA_NAMEs and statements. */
7106 vect_record_grouped_load_vectors (stmt, dr_chain);
7108 else
7110 for (i = 0; i < vec_num; i++)
7112 if (i > 0)
7113 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
7114 stmt, NULL_TREE);
7116 /* 2. Create the vector-load in the loop. */
7117 switch (alignment_support_scheme)
7119 case dr_aligned:
7120 case dr_unaligned_supported:
7122 unsigned int align, misalign;
7124 data_ref
7125 = fold_build2 (MEM_REF, vectype, dataref_ptr,
7126 dataref_offset
7127 ? dataref_offset
7128 : build_int_cst (reference_alias_ptr_type
7129 (DR_REF (first_dr)), 0));
7130 align = TYPE_ALIGN_UNIT (vectype);
7131 if (alignment_support_scheme == dr_aligned)
7133 gcc_assert (aligned_access_p (first_dr));
7134 misalign = 0;
7136 else if (DR_MISALIGNMENT (first_dr) == -1)
7138 if (DR_VECT_AUX (first_dr)->base_element_aligned)
7139 align = TYPE_ALIGN_UNIT (elem_type);
7140 else
7141 align = (get_object_alignment (DR_REF (first_dr))
7142 / BITS_PER_UNIT);
7143 misalign = 0;
7144 TREE_TYPE (data_ref)
7145 = build_aligned_type (TREE_TYPE (data_ref),
7146 align * BITS_PER_UNIT);
7148 else
7150 TREE_TYPE (data_ref)
7151 = build_aligned_type (TREE_TYPE (data_ref),
7152 TYPE_ALIGN (elem_type));
7153 misalign = DR_MISALIGNMENT (first_dr);
7155 if (dataref_offset == NULL_TREE
7156 && TREE_CODE (dataref_ptr) == SSA_NAME)
7157 set_ptr_info_alignment (get_ptr_info (dataref_ptr),
7158 align, misalign);
7159 break;
7161 case dr_explicit_realign:
7163 tree ptr, bump;
7165 tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
7167 if (compute_in_loop)
7168 msq = vect_setup_realignment (first_stmt, gsi,
7169 &realignment_token,
7170 dr_explicit_realign,
7171 dataref_ptr, NULL);
7173 if (TREE_CODE (dataref_ptr) == SSA_NAME)
7174 ptr = copy_ssa_name (dataref_ptr);
7175 else
7176 ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
7177 new_stmt = gimple_build_assign
7178 (ptr, BIT_AND_EXPR, dataref_ptr,
7179 build_int_cst
7180 (TREE_TYPE (dataref_ptr),
7181 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
7182 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7183 data_ref
7184 = build2 (MEM_REF, vectype, ptr,
7185 build_int_cst (reference_alias_ptr_type
7186 (DR_REF (first_dr)), 0));
7187 vec_dest = vect_create_destination_var (scalar_dest,
7188 vectype);
7189 new_stmt = gimple_build_assign (vec_dest, data_ref);
7190 new_temp = make_ssa_name (vec_dest, new_stmt);
7191 gimple_assign_set_lhs (new_stmt, new_temp);
7192 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
7193 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
7194 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7195 msq = new_temp;
7197 bump = size_binop (MULT_EXPR, vs,
7198 TYPE_SIZE_UNIT (elem_type));
7199 bump = size_binop (MINUS_EXPR, bump, size_one_node);
7200 ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump);
7201 new_stmt = gimple_build_assign
7202 (NULL_TREE, BIT_AND_EXPR, ptr,
7203 build_int_cst
7204 (TREE_TYPE (ptr),
7205 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
7206 ptr = copy_ssa_name (ptr, new_stmt);
7207 gimple_assign_set_lhs (new_stmt, ptr);
7208 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7209 data_ref
7210 = build2 (MEM_REF, vectype, ptr,
7211 build_int_cst (reference_alias_ptr_type
7212 (DR_REF (first_dr)), 0));
7213 break;
7215 case dr_explicit_realign_optimized:
7216 if (TREE_CODE (dataref_ptr) == SSA_NAME)
7217 new_temp = copy_ssa_name (dataref_ptr);
7218 else
7219 new_temp = make_ssa_name (TREE_TYPE (dataref_ptr));
7220 new_stmt = gimple_build_assign
7221 (new_temp, BIT_AND_EXPR, dataref_ptr,
7222 build_int_cst
7223 (TREE_TYPE (dataref_ptr),
7224 -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype)));
7225 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7226 data_ref
7227 = build2 (MEM_REF, vectype, new_temp,
7228 build_int_cst (reference_alias_ptr_type
7229 (DR_REF (first_dr)), 0));
7230 break;
7231 default:
7232 gcc_unreachable ();
7234 vec_dest = vect_create_destination_var (scalar_dest, vectype);
7235 new_stmt = gimple_build_assign (vec_dest, data_ref);
7236 new_temp = make_ssa_name (vec_dest, new_stmt);
7237 gimple_assign_set_lhs (new_stmt, new_temp);
7238 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7240 /* 3. Handle explicit realignment if necessary/supported.
7241 Create in loop:
7242 vec_dest = realign_load (msq, lsq, realignment_token) */
7243 if (alignment_support_scheme == dr_explicit_realign_optimized
7244 || alignment_support_scheme == dr_explicit_realign)
7246 lsq = gimple_assign_lhs (new_stmt);
7247 if (!realignment_token)
7248 realignment_token = dataref_ptr;
7249 vec_dest = vect_create_destination_var (scalar_dest, vectype);
7250 new_stmt = gimple_build_assign (vec_dest, REALIGN_LOAD_EXPR,
7251 msq, lsq, realignment_token);
7252 new_temp = make_ssa_name (vec_dest, new_stmt);
7253 gimple_assign_set_lhs (new_stmt, new_temp);
7254 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7256 if (alignment_support_scheme == dr_explicit_realign_optimized)
7258 gcc_assert (phi);
7259 if (i == vec_num - 1 && j == ncopies - 1)
7260 add_phi_arg (phi, lsq,
7261 loop_latch_edge (containing_loop),
7262 UNKNOWN_LOCATION);
7263 msq = lsq;
7267 /* 4. Handle invariant-load. */
7268 if (inv_p && !bb_vinfo)
7270 gcc_assert (!grouped_load);
7271 /* If we have versioned for aliasing or the loop doesn't
7272 have any data dependencies that would preclude this,
7273 then we are sure this is a loop invariant load and
7274 thus we can insert it on the preheader edge. */
7275 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo)
7276 && !nested_in_vect_loop
7277 && hoist_defs_of_uses (stmt, loop))
7279 if (dump_enabled_p ())
7281 dump_printf_loc (MSG_NOTE, vect_location,
7282 "hoisting out of the vectorized "
7283 "loop: ");
7284 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
7286 tree tem = copy_ssa_name (scalar_dest);
7287 gsi_insert_on_edge_immediate
7288 (loop_preheader_edge (loop),
7289 gimple_build_assign (tem,
7290 unshare_expr
7291 (gimple_assign_rhs1 (stmt))));
7292 new_temp = vect_init_vector (stmt, tem, vectype, NULL);
7293 new_stmt = SSA_NAME_DEF_STMT (new_temp);
7294 set_vinfo_for_stmt (new_stmt,
7295 new_stmt_vec_info (new_stmt, vinfo));
7297 else
7299 gimple_stmt_iterator gsi2 = *gsi;
7300 gsi_next (&gsi2);
7301 new_temp = vect_init_vector (stmt, scalar_dest,
7302 vectype, &gsi2);
7303 new_stmt = SSA_NAME_DEF_STMT (new_temp);
7307 if (negative)
7309 tree perm_mask = perm_mask_for_reverse (vectype);
7310 new_temp = permute_vec_elements (new_temp, new_temp,
7311 perm_mask, stmt, gsi);
7312 new_stmt = SSA_NAME_DEF_STMT (new_temp);
7315 /* Collect vector loads and later create their permutation in
7316 vect_transform_grouped_load (). */
7317 if (grouped_load || slp_perm)
7318 dr_chain.quick_push (new_temp);
7320 /* Store vector loads in the corresponding SLP_NODE. */
7321 if (slp && !slp_perm)
7322 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
7324 /* Bump the vector pointer to account for a gap or for excess
7325 elements loaded for a permuted SLP load. */
7326 if (group_gap_adj != 0)
7328 bool ovf;
7329 tree bump
7330 = wide_int_to_tree (sizetype,
7331 wi::smul (TYPE_SIZE_UNIT (elem_type),
7332 group_gap_adj, &ovf));
7333 dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi,
7334 stmt, bump);
7338 if (slp && !slp_perm)
7339 continue;
7341 if (slp_perm)
7343 if (!vect_transform_slp_perm_load (slp_node, dr_chain, gsi, vf,
7344 slp_node_instance, false))
7346 dr_chain.release ();
7347 return false;
7350 else
7352 if (grouped_load)
7354 if (!load_lanes_p)
7355 vect_transform_grouped_load (stmt, dr_chain, group_size, gsi);
7356 *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
7358 else
7360 if (j == 0)
7361 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
7362 else
7363 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
7364 prev_stmt_info = vinfo_for_stmt (new_stmt);
7367 dr_chain.release ();
7370 return true;
7373 /* Function vect_is_simple_cond.
7375 Input:
7376 LOOP - the loop that is being vectorized.
7377 COND - Condition that is checked for simple use.
7379 Output:
7380 *COMP_VECTYPE - the vector type for the comparison.
7382 Returns whether a COND can be vectorized. Checks whether
7383 condition operands are supportable using vec_is_simple_use. */
7385 static bool
7386 vect_is_simple_cond (tree cond, vec_info *vinfo, tree *comp_vectype)
7388 tree lhs, rhs;
7389 enum vect_def_type dt;
7390 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
7392 /* Mask case. */
7393 if (TREE_CODE (cond) == SSA_NAME
7394 && TREE_CODE (TREE_TYPE (cond)) == BOOLEAN_TYPE)
7396 gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (cond);
7397 if (!vect_is_simple_use (cond, vinfo, &lhs_def_stmt,
7398 &dt, comp_vectype)
7399 || !*comp_vectype
7400 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype))
7401 return false;
7402 return true;
7405 if (!COMPARISON_CLASS_P (cond))
7406 return false;
7408 lhs = TREE_OPERAND (cond, 0);
7409 rhs = TREE_OPERAND (cond, 1);
7411 if (TREE_CODE (lhs) == SSA_NAME)
7413 gimple *lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
7414 if (!vect_is_simple_use (lhs, vinfo, &lhs_def_stmt, &dt, &vectype1))
7415 return false;
7417 else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST
7418 && TREE_CODE (lhs) != FIXED_CST)
7419 return false;
7421 if (TREE_CODE (rhs) == SSA_NAME)
7423 gimple *rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
7424 if (!vect_is_simple_use (rhs, vinfo, &rhs_def_stmt, &dt, &vectype2))
7425 return false;
7427 else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST
7428 && TREE_CODE (rhs) != FIXED_CST)
7429 return false;
7431 if (vectype1 && vectype2
7432 && TYPE_VECTOR_SUBPARTS (vectype1) != TYPE_VECTOR_SUBPARTS (vectype2))
7433 return false;
7435 *comp_vectype = vectype1 ? vectype1 : vectype2;
7436 return true;
7439 /* vectorizable_condition.
7441 Check if STMT is conditional modify expression that can be vectorized.
7442 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7443 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
7444 at GSI.
7446 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
7447 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
7448 else clause if it is 2).
7450 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7452 bool
7453 vectorizable_condition (gimple *stmt, gimple_stmt_iterator *gsi,
7454 gimple **vec_stmt, tree reduc_def, int reduc_index,
7455 slp_tree slp_node)
7457 tree scalar_dest = NULL_TREE;
7458 tree vec_dest = NULL_TREE;
7459 tree cond_expr, then_clause, else_clause;
7460 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7461 tree comp_vectype = NULL_TREE;
7462 tree vec_cond_lhs = NULL_TREE, vec_cond_rhs = NULL_TREE;
7463 tree vec_then_clause = NULL_TREE, vec_else_clause = NULL_TREE;
7464 tree vec_compare;
7465 tree new_temp;
7466 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
7467 enum vect_def_type dt, dts[4];
7468 int ncopies;
7469 enum tree_code code;
7470 stmt_vec_info prev_stmt_info = NULL;
7471 int i, j;
7472 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
7473 vec<tree> vec_oprnds0 = vNULL;
7474 vec<tree> vec_oprnds1 = vNULL;
7475 vec<tree> vec_oprnds2 = vNULL;
7476 vec<tree> vec_oprnds3 = vNULL;
7477 tree vec_cmp_type;
7478 bool masked = false;
7480 if (reduc_index && STMT_SLP_TYPE (stmt_info))
7481 return false;
7483 if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == TREE_CODE_REDUCTION)
7485 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
7486 return false;
7488 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
7489 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
7490 && reduc_def))
7491 return false;
7493 /* FORNOW: not yet supported. */
7494 if (STMT_VINFO_LIVE_P (stmt_info))
7496 if (dump_enabled_p ())
7497 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7498 "value used after loop.\n");
7499 return false;
7503 /* Is vectorizable conditional operation? */
7504 if (!is_gimple_assign (stmt))
7505 return false;
7507 code = gimple_assign_rhs_code (stmt);
7509 if (code != COND_EXPR)
7510 return false;
7512 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
7513 int nunits = TYPE_VECTOR_SUBPARTS (vectype);
7514 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
7516 if (slp_node || PURE_SLP_STMT (stmt_info))
7517 ncopies = 1;
7518 else
7519 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
7521 gcc_assert (ncopies >= 1);
7522 if (reduc_index && ncopies > 1)
7523 return false; /* FORNOW */
7525 cond_expr = gimple_assign_rhs1 (stmt);
7526 then_clause = gimple_assign_rhs2 (stmt);
7527 else_clause = gimple_assign_rhs3 (stmt);
7529 if (!vect_is_simple_cond (cond_expr, stmt_info->vinfo, &comp_vectype)
7530 || !comp_vectype)
7531 return false;
7533 gimple *def_stmt;
7534 if (!vect_is_simple_use (then_clause, stmt_info->vinfo, &def_stmt, &dt,
7535 &vectype1))
7536 return false;
7537 if (!vect_is_simple_use (else_clause, stmt_info->vinfo, &def_stmt, &dt,
7538 &vectype2))
7539 return false;
7541 if (vectype1 && !useless_type_conversion_p (vectype, vectype1))
7542 return false;
7544 if (vectype2 && !useless_type_conversion_p (vectype, vectype2))
7545 return false;
7547 masked = !COMPARISON_CLASS_P (cond_expr);
7548 vec_cmp_type = build_same_sized_truth_vector_type (comp_vectype);
7550 if (vec_cmp_type == NULL_TREE)
7551 return false;
7553 if (!vec_stmt)
7555 STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
7556 return expand_vec_cond_expr_p (vectype, comp_vectype);
7559 /* Transform. */
7561 if (!slp_node)
7563 vec_oprnds0.create (1);
7564 vec_oprnds1.create (1);
7565 vec_oprnds2.create (1);
7566 vec_oprnds3.create (1);
7569 /* Handle def. */
7570 scalar_dest = gimple_assign_lhs (stmt);
7571 vec_dest = vect_create_destination_var (scalar_dest, vectype);
7573 /* Handle cond expr. */
7574 for (j = 0; j < ncopies; j++)
7576 gassign *new_stmt = NULL;
7577 if (j == 0)
7579 if (slp_node)
7581 auto_vec<tree, 4> ops;
7582 auto_vec<vec<tree>, 4> vec_defs;
7584 if (masked)
7585 ops.safe_push (cond_expr);
7586 else
7588 ops.safe_push (TREE_OPERAND (cond_expr, 0));
7589 ops.safe_push (TREE_OPERAND (cond_expr, 1));
7591 ops.safe_push (then_clause);
7592 ops.safe_push (else_clause);
7593 vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
7594 vec_oprnds3 = vec_defs.pop ();
7595 vec_oprnds2 = vec_defs.pop ();
7596 if (!masked)
7597 vec_oprnds1 = vec_defs.pop ();
7598 vec_oprnds0 = vec_defs.pop ();
7600 ops.release ();
7601 vec_defs.release ();
7603 else
7605 gimple *gtemp;
7606 if (masked)
7608 vec_cond_lhs
7609 = vect_get_vec_def_for_operand (cond_expr, stmt,
7610 comp_vectype);
7611 vect_is_simple_use (cond_expr, stmt_info->vinfo,
7612 &gtemp, &dts[0]);
7614 else
7616 vec_cond_lhs =
7617 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0),
7618 stmt, comp_vectype);
7619 vect_is_simple_use (TREE_OPERAND (cond_expr, 0),
7620 loop_vinfo, &gtemp, &dts[0]);
7622 vec_cond_rhs =
7623 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1),
7624 stmt, comp_vectype);
7625 vect_is_simple_use (TREE_OPERAND (cond_expr, 1),
7626 loop_vinfo, &gtemp, &dts[1]);
7628 if (reduc_index == 1)
7629 vec_then_clause = reduc_def;
7630 else
7632 vec_then_clause = vect_get_vec_def_for_operand (then_clause,
7633 stmt);
7634 vect_is_simple_use (then_clause, loop_vinfo,
7635 &gtemp, &dts[2]);
7637 if (reduc_index == 2)
7638 vec_else_clause = reduc_def;
7639 else
7641 vec_else_clause = vect_get_vec_def_for_operand (else_clause,
7642 stmt);
7643 vect_is_simple_use (else_clause, loop_vinfo, &gtemp, &dts[3]);
7647 else
7649 vec_cond_lhs
7650 = vect_get_vec_def_for_stmt_copy (dts[0],
7651 vec_oprnds0.pop ());
7652 if (!masked)
7653 vec_cond_rhs
7654 = vect_get_vec_def_for_stmt_copy (dts[1],
7655 vec_oprnds1.pop ());
7657 vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2],
7658 vec_oprnds2.pop ());
7659 vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3],
7660 vec_oprnds3.pop ());
7663 if (!slp_node)
7665 vec_oprnds0.quick_push (vec_cond_lhs);
7666 if (!masked)
7667 vec_oprnds1.quick_push (vec_cond_rhs);
7668 vec_oprnds2.quick_push (vec_then_clause);
7669 vec_oprnds3.quick_push (vec_else_clause);
7672 /* Arguments are ready. Create the new vector stmt. */
7673 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_cond_lhs)
7675 vec_then_clause = vec_oprnds2[i];
7676 vec_else_clause = vec_oprnds3[i];
7678 if (masked)
7679 vec_compare = vec_cond_lhs;
7680 else
7682 vec_cond_rhs = vec_oprnds1[i];
7683 vec_compare = build2 (TREE_CODE (cond_expr), vec_cmp_type,
7684 vec_cond_lhs, vec_cond_rhs);
7686 new_temp = make_ssa_name (vec_dest);
7687 new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR,
7688 vec_compare, vec_then_clause,
7689 vec_else_clause);
7690 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7691 if (slp_node)
7692 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
7695 if (slp_node)
7696 continue;
7698 if (j == 0)
7699 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
7700 else
7701 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
7703 prev_stmt_info = vinfo_for_stmt (new_stmt);
7706 vec_oprnds0.release ();
7707 vec_oprnds1.release ();
7708 vec_oprnds2.release ();
7709 vec_oprnds3.release ();
7711 return true;
7714 /* vectorizable_comparison.
7716 Check if STMT is comparison expression that can be vectorized.
7717 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7718 comparison, put it in VEC_STMT, and insert it at GSI.
7720 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7722 bool
7723 vectorizable_comparison (gimple *stmt, gimple_stmt_iterator *gsi,
7724 gimple **vec_stmt, tree reduc_def,
7725 slp_tree slp_node)
7727 tree lhs, rhs1, rhs2;
7728 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7729 tree vectype1 = NULL_TREE, vectype2 = NULL_TREE;
7730 tree vectype = STMT_VINFO_VECTYPE (stmt_info);
7731 tree vec_rhs1 = NULL_TREE, vec_rhs2 = NULL_TREE;
7732 tree new_temp;
7733 loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
7734 enum vect_def_type dts[2] = {vect_unknown_def_type, vect_unknown_def_type};
7735 unsigned nunits;
7736 int ncopies;
7737 enum tree_code code;
7738 stmt_vec_info prev_stmt_info = NULL;
7739 int i, j;
7740 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
7741 vec<tree> vec_oprnds0 = vNULL;
7742 vec<tree> vec_oprnds1 = vNULL;
7743 gimple *def_stmt;
7744 tree mask_type;
7745 tree mask;
7747 if (!STMT_VINFO_RELEVANT_P (stmt_info) && !bb_vinfo)
7748 return false;
7750 if (!vectype || !VECTOR_BOOLEAN_TYPE_P (vectype))
7751 return false;
7753 mask_type = vectype;
7754 nunits = TYPE_VECTOR_SUBPARTS (vectype);
7756 if (slp_node || PURE_SLP_STMT (stmt_info))
7757 ncopies = 1;
7758 else
7759 ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
7761 gcc_assert (ncopies >= 1);
7762 if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_internal_def
7763 && !(STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
7764 && reduc_def))
7765 return false;
7767 if (STMT_VINFO_LIVE_P (stmt_info))
7769 if (dump_enabled_p ())
7770 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7771 "value used after loop.\n");
7772 return false;
7775 if (!is_gimple_assign (stmt))
7776 return false;
7778 code = gimple_assign_rhs_code (stmt);
7780 if (TREE_CODE_CLASS (code) != tcc_comparison)
7781 return false;
7783 rhs1 = gimple_assign_rhs1 (stmt);
7784 rhs2 = gimple_assign_rhs2 (stmt);
7786 if (!vect_is_simple_use (rhs1, stmt_info->vinfo, &def_stmt,
7787 &dts[0], &vectype1))
7788 return false;
7790 if (!vect_is_simple_use (rhs2, stmt_info->vinfo, &def_stmt,
7791 &dts[1], &vectype2))
7792 return false;
7794 if (vectype1 && vectype2
7795 && TYPE_VECTOR_SUBPARTS (vectype1) != TYPE_VECTOR_SUBPARTS (vectype2))
7796 return false;
7798 vectype = vectype1 ? vectype1 : vectype2;
7800 /* Invariant comparison. */
7801 if (!vectype)
7803 vectype = get_vectype_for_scalar_type (TREE_TYPE (rhs1));
7804 if (TYPE_VECTOR_SUBPARTS (vectype) != nunits)
7805 return false;
7807 else if (nunits != TYPE_VECTOR_SUBPARTS (vectype))
7808 return false;
7810 if (!vec_stmt)
7812 STMT_VINFO_TYPE (stmt_info) = comparison_vec_info_type;
7813 vect_model_simple_cost (stmt_info, ncopies, dts, NULL, NULL);
7814 return expand_vec_cmp_expr_p (vectype, mask_type);
7817 /* Transform. */
7818 if (!slp_node)
7820 vec_oprnds0.create (1);
7821 vec_oprnds1.create (1);
7824 /* Handle def. */
7825 lhs = gimple_assign_lhs (stmt);
7826 mask = vect_create_destination_var (lhs, mask_type);
7828 /* Handle cmp expr. */
7829 for (j = 0; j < ncopies; j++)
7831 gassign *new_stmt = NULL;
7832 if (j == 0)
7834 if (slp_node)
7836 auto_vec<tree, 2> ops;
7837 auto_vec<vec<tree>, 2> vec_defs;
7839 ops.safe_push (rhs1);
7840 ops.safe_push (rhs2);
7841 vect_get_slp_defs (ops, slp_node, &vec_defs, -1);
7842 vec_oprnds1 = vec_defs.pop ();
7843 vec_oprnds0 = vec_defs.pop ();
7845 else
7847 vec_rhs1 = vect_get_vec_def_for_operand (rhs1, stmt, vectype);
7848 vec_rhs2 = vect_get_vec_def_for_operand (rhs2, stmt, vectype);
7851 else
7853 vec_rhs1 = vect_get_vec_def_for_stmt_copy (dts[0],
7854 vec_oprnds0.pop ());
7855 vec_rhs2 = vect_get_vec_def_for_stmt_copy (dts[1],
7856 vec_oprnds1.pop ());
7859 if (!slp_node)
7861 vec_oprnds0.quick_push (vec_rhs1);
7862 vec_oprnds1.quick_push (vec_rhs2);
7865 /* Arguments are ready. Create the new vector stmt. */
7866 FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1)
7868 vec_rhs2 = vec_oprnds1[i];
7870 new_temp = make_ssa_name (mask);
7871 new_stmt = gimple_build_assign (new_temp, code, vec_rhs1, vec_rhs2);
7872 vect_finish_stmt_generation (stmt, new_stmt, gsi);
7873 if (slp_node)
7874 SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
7877 if (slp_node)
7878 continue;
7880 if (j == 0)
7881 STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
7882 else
7883 STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
7885 prev_stmt_info = vinfo_for_stmt (new_stmt);
7888 vec_oprnds0.release ();
7889 vec_oprnds1.release ();
7891 return true;
7894 /* Make sure the statement is vectorizable. */
7896 bool
7897 vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node)
7899 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
7900 bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
7901 enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
7902 bool ok;
7903 tree scalar_type, vectype;
7904 gimple *pattern_stmt;
7905 gimple_seq pattern_def_seq;
7907 if (dump_enabled_p ())
7909 dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
7910 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
7913 if (gimple_has_volatile_ops (stmt))
7915 if (dump_enabled_p ())
7916 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7917 "not vectorized: stmt has volatile operands\n");
7919 return false;
7922 /* Skip stmts that do not need to be vectorized. In loops this is expected
7923 to include:
7924 - the COND_EXPR which is the loop exit condition
7925 - any LABEL_EXPRs in the loop
7926 - computations that are used only for array indexing or loop control.
7927 In basic blocks we only analyze statements that are a part of some SLP
7928 instance, therefore, all the statements are relevant.
7930 Pattern statement needs to be analyzed instead of the original statement
7931 if the original statement is not relevant. Otherwise, we analyze both
7932 statements. In basic blocks we are called from some SLP instance
7933 traversal, don't analyze pattern stmts instead, the pattern stmts
7934 already will be part of SLP instance. */
7936 pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
7937 if (!STMT_VINFO_RELEVANT_P (stmt_info)
7938 && !STMT_VINFO_LIVE_P (stmt_info))
7940 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
7941 && pattern_stmt
7942 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
7943 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
7945 /* Analyze PATTERN_STMT instead of the original stmt. */
7946 stmt = pattern_stmt;
7947 stmt_info = vinfo_for_stmt (pattern_stmt);
7948 if (dump_enabled_p ())
7950 dump_printf_loc (MSG_NOTE, vect_location,
7951 "==> examining pattern statement: ");
7952 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
7955 else
7957 if (dump_enabled_p ())
7958 dump_printf_loc (MSG_NOTE, vect_location, "irrelevant.\n");
7960 return true;
7963 else if (STMT_VINFO_IN_PATTERN_P (stmt_info)
7964 && node == NULL
7965 && pattern_stmt
7966 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt))
7967 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt))))
7969 /* Analyze PATTERN_STMT too. */
7970 if (dump_enabled_p ())
7972 dump_printf_loc (MSG_NOTE, vect_location,
7973 "==> examining pattern statement: ");
7974 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0);
7977 if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node))
7978 return false;
7981 if (is_pattern_stmt_p (stmt_info)
7982 && node == NULL
7983 && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info)))
7985 gimple_stmt_iterator si;
7987 for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
7989 gimple *pattern_def_stmt = gsi_stmt (si);
7990 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
7991 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
7993 /* Analyze def stmt of STMT if it's a pattern stmt. */
7994 if (dump_enabled_p ())
7996 dump_printf_loc (MSG_NOTE, vect_location,
7997 "==> examining pattern def statement: ");
7998 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0);
8001 if (!vect_analyze_stmt (pattern_def_stmt,
8002 need_to_vectorize, node))
8003 return false;
8008 switch (STMT_VINFO_DEF_TYPE (stmt_info))
8010 case vect_internal_def:
8011 break;
8013 case vect_reduction_def:
8014 case vect_nested_cycle:
8015 gcc_assert (!bb_vinfo
8016 && (relevance == vect_used_in_outer
8017 || relevance == vect_used_in_outer_by_reduction
8018 || relevance == vect_used_by_reduction
8019 || relevance == vect_unused_in_scope));
8020 break;
8022 case vect_induction_def:
8023 case vect_constant_def:
8024 case vect_external_def:
8025 case vect_unknown_def_type:
8026 default:
8027 gcc_unreachable ();
8030 if (bb_vinfo)
8032 gcc_assert (PURE_SLP_STMT (stmt_info));
8034 scalar_type = TREE_TYPE (gimple_get_lhs (stmt));
8035 if (dump_enabled_p ())
8037 dump_printf_loc (MSG_NOTE, vect_location,
8038 "get vectype for scalar type: ");
8039 dump_generic_expr (MSG_NOTE, TDF_SLIM, scalar_type);
8040 dump_printf (MSG_NOTE, "\n");
8043 vectype = get_vectype_for_scalar_type (scalar_type);
8044 if (!vectype)
8046 if (dump_enabled_p ())
8048 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8049 "not SLPed: unsupported data-type ");
8050 dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
8051 scalar_type);
8052 dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
8054 return false;
8057 if (dump_enabled_p ())
8059 dump_printf_loc (MSG_NOTE, vect_location, "vectype: ");
8060 dump_generic_expr (MSG_NOTE, TDF_SLIM, vectype);
8061 dump_printf (MSG_NOTE, "\n");
8064 STMT_VINFO_VECTYPE (stmt_info) = vectype;
8067 if (STMT_VINFO_RELEVANT_P (stmt_info))
8069 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))));
8070 gcc_assert (STMT_VINFO_VECTYPE (stmt_info)
8071 || (is_gimple_call (stmt)
8072 && gimple_call_lhs (stmt) == NULL_TREE));
8073 *need_to_vectorize = true;
8076 if (PURE_SLP_STMT (stmt_info) && !node)
8078 dump_printf_loc (MSG_NOTE, vect_location,
8079 "handled only by SLP analysis\n");
8080 return true;
8083 ok = true;
8084 if (!bb_vinfo
8085 && (STMT_VINFO_RELEVANT_P (stmt_info)
8086 || STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def))
8087 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, node)
8088 || vectorizable_conversion (stmt, NULL, NULL, node)
8089 || vectorizable_shift (stmt, NULL, NULL, node)
8090 || vectorizable_operation (stmt, NULL, NULL, node)
8091 || vectorizable_assignment (stmt, NULL, NULL, node)
8092 || vectorizable_load (stmt, NULL, NULL, node, NULL)
8093 || vectorizable_call (stmt, NULL, NULL, node)
8094 || vectorizable_store (stmt, NULL, NULL, node)
8095 || vectorizable_reduction (stmt, NULL, NULL, node)
8096 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
8097 || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
8098 else
8100 if (bb_vinfo)
8101 ok = (vectorizable_simd_clone_call (stmt, NULL, NULL, node)
8102 || vectorizable_conversion (stmt, NULL, NULL, node)
8103 || vectorizable_shift (stmt, NULL, NULL, node)
8104 || vectorizable_operation (stmt, NULL, NULL, node)
8105 || vectorizable_assignment (stmt, NULL, NULL, node)
8106 || vectorizable_load (stmt, NULL, NULL, node, NULL)
8107 || vectorizable_call (stmt, NULL, NULL, node)
8108 || vectorizable_store (stmt, NULL, NULL, node)
8109 || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node)
8110 || vectorizable_comparison (stmt, NULL, NULL, NULL, node));
8113 if (!ok)
8115 if (dump_enabled_p ())
8117 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8118 "not vectorized: relevant stmt not ");
8119 dump_printf (MSG_MISSED_OPTIMIZATION, "supported: ");
8120 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
8123 return false;
8126 if (bb_vinfo)
8127 return true;
8129 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
8130 need extra handling, except for vectorizable reductions. */
8131 if (STMT_VINFO_LIVE_P (stmt_info)
8132 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
8133 ok = vectorizable_live_operation (stmt, NULL, NULL);
8135 if (!ok)
8137 if (dump_enabled_p ())
8139 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8140 "not vectorized: live stmt not ");
8141 dump_printf (MSG_MISSED_OPTIMIZATION, "supported: ");
8142 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
8145 return false;
8148 return true;
8152 /* Function vect_transform_stmt.
8154 Create a vectorized stmt to replace STMT, and insert it at BSI. */
8156 bool
8157 vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi,
8158 bool *grouped_store, slp_tree slp_node,
8159 slp_instance slp_node_instance)
8161 bool is_store = false;
8162 gimple *vec_stmt = NULL;
8163 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
8164 bool done;
8166 gimple *old_vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
8168 switch (STMT_VINFO_TYPE (stmt_info))
8170 case type_demotion_vec_info_type:
8171 case type_promotion_vec_info_type:
8172 case type_conversion_vec_info_type:
8173 done = vectorizable_conversion (stmt, gsi, &vec_stmt, slp_node);
8174 gcc_assert (done);
8175 break;
8177 case induc_vec_info_type:
8178 gcc_assert (!slp_node);
8179 done = vectorizable_induction (stmt, gsi, &vec_stmt);
8180 gcc_assert (done);
8181 break;
8183 case shift_vec_info_type:
8184 done = vectorizable_shift (stmt, gsi, &vec_stmt, slp_node);
8185 gcc_assert (done);
8186 break;
8188 case op_vec_info_type:
8189 done = vectorizable_operation (stmt, gsi, &vec_stmt, slp_node);
8190 gcc_assert (done);
8191 break;
8193 case assignment_vec_info_type:
8194 done = vectorizable_assignment (stmt, gsi, &vec_stmt, slp_node);
8195 gcc_assert (done);
8196 break;
8198 case load_vec_info_type:
8199 done = vectorizable_load (stmt, gsi, &vec_stmt, slp_node,
8200 slp_node_instance);
8201 gcc_assert (done);
8202 break;
8204 case store_vec_info_type:
8205 done = vectorizable_store (stmt, gsi, &vec_stmt, slp_node);
8206 gcc_assert (done);
8207 if (STMT_VINFO_GROUPED_ACCESS (stmt_info) && !slp_node)
8209 /* In case of interleaving, the whole chain is vectorized when the
8210 last store in the chain is reached. Store stmts before the last
8211 one are skipped, and there vec_stmt_info shouldn't be freed
8212 meanwhile. */
8213 *grouped_store = true;
8214 if (STMT_VINFO_VEC_STMT (stmt_info))
8215 is_store = true;
8217 else
8218 is_store = true;
8219 break;
8221 case condition_vec_info_type:
8222 done = vectorizable_condition (stmt, gsi, &vec_stmt, NULL, 0, slp_node);
8223 gcc_assert (done);
8224 break;
8226 case comparison_vec_info_type:
8227 done = vectorizable_comparison (stmt, gsi, &vec_stmt, NULL, slp_node);
8228 gcc_assert (done);
8229 break;
8231 case call_vec_info_type:
8232 done = vectorizable_call (stmt, gsi, &vec_stmt, slp_node);
8233 stmt = gsi_stmt (*gsi);
8234 if (is_gimple_call (stmt)
8235 && gimple_call_internal_p (stmt)
8236 && gimple_call_internal_fn (stmt) == IFN_MASK_STORE)
8237 is_store = true;
8238 break;
8240 case call_simd_clone_vec_info_type:
8241 done = vectorizable_simd_clone_call (stmt, gsi, &vec_stmt, slp_node);
8242 stmt = gsi_stmt (*gsi);
8243 break;
8245 case reduc_vec_info_type:
8246 done = vectorizable_reduction (stmt, gsi, &vec_stmt, slp_node);
8247 gcc_assert (done);
8248 break;
8250 default:
8251 if (!STMT_VINFO_LIVE_P (stmt_info))
8253 if (dump_enabled_p ())
8254 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8255 "stmt not supported.\n");
8256 gcc_unreachable ();
8260 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
8261 This would break hybrid SLP vectorization. */
8262 if (slp_node)
8263 gcc_assert (!vec_stmt
8264 && STMT_VINFO_VEC_STMT (stmt_info) == old_vec_stmt);
8266 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
8267 is being vectorized, but outside the immediately enclosing loop. */
8268 if (vec_stmt
8269 && STMT_VINFO_LOOP_VINFO (stmt_info)
8270 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
8271 STMT_VINFO_LOOP_VINFO (stmt_info)), stmt)
8272 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type
8273 && (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_outer
8274 || STMT_VINFO_RELEVANT (stmt_info) ==
8275 vect_used_in_outer_by_reduction))
8277 struct loop *innerloop = LOOP_VINFO_LOOP (
8278 STMT_VINFO_LOOP_VINFO (stmt_info))->inner;
8279 imm_use_iterator imm_iter;
8280 use_operand_p use_p;
8281 tree scalar_dest;
8282 gimple *exit_phi;
8284 if (dump_enabled_p ())
8285 dump_printf_loc (MSG_NOTE, vect_location,
8286 "Record the vdef for outer-loop vectorization.\n");
8288 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
8289 (to be used when vectorizing outer-loop stmts that use the DEF of
8290 STMT). */
8291 if (gimple_code (stmt) == GIMPLE_PHI)
8292 scalar_dest = PHI_RESULT (stmt);
8293 else
8294 scalar_dest = gimple_assign_lhs (stmt);
8296 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
8298 if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
8300 exit_phi = USE_STMT (use_p);
8301 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
8306 /* Handle stmts whose DEF is used outside the loop-nest that is
8307 being vectorized. */
8308 if (STMT_VINFO_LIVE_P (stmt_info)
8309 && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type)
8311 done = vectorizable_live_operation (stmt, gsi, &vec_stmt);
8312 gcc_assert (done);
8315 if (vec_stmt)
8316 STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
8318 return is_store;
8322 /* Remove a group of stores (for SLP or interleaving), free their
8323 stmt_vec_info. */
8325 void
8326 vect_remove_stores (gimple *first_stmt)
8328 gimple *next = first_stmt;
8329 gimple *tmp;
8330 gimple_stmt_iterator next_si;
8332 while (next)
8334 stmt_vec_info stmt_info = vinfo_for_stmt (next);
8336 tmp = GROUP_NEXT_ELEMENT (stmt_info);
8337 if (is_pattern_stmt_p (stmt_info))
8338 next = STMT_VINFO_RELATED_STMT (stmt_info);
8339 /* Free the attached stmt_vec_info and remove the stmt. */
8340 next_si = gsi_for_stmt (next);
8341 unlink_stmt_vdef (next);
8342 gsi_remove (&next_si, true);
8343 release_defs (next);
8344 free_stmt_vec_info (next);
8345 next = tmp;
8350 /* Function new_stmt_vec_info.
8352 Create and initialize a new stmt_vec_info struct for STMT. */
8354 stmt_vec_info
8355 new_stmt_vec_info (gimple *stmt, vec_info *vinfo)
8357 stmt_vec_info res;
8358 res = (stmt_vec_info) xcalloc (1, sizeof (struct _stmt_vec_info));
8360 STMT_VINFO_TYPE (res) = undef_vec_info_type;
8361 STMT_VINFO_STMT (res) = stmt;
8362 res->vinfo = vinfo;
8363 STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
8364 STMT_VINFO_LIVE_P (res) = false;
8365 STMT_VINFO_VECTYPE (res) = NULL;
8366 STMT_VINFO_VEC_STMT (res) = NULL;
8367 STMT_VINFO_VECTORIZABLE (res) = true;
8368 STMT_VINFO_IN_PATTERN_P (res) = false;
8369 STMT_VINFO_RELATED_STMT (res) = NULL;
8370 STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL;
8371 STMT_VINFO_DATA_REF (res) = NULL;
8372 STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
8374 STMT_VINFO_DR_BASE_ADDRESS (res) = NULL;
8375 STMT_VINFO_DR_OFFSET (res) = NULL;
8376 STMT_VINFO_DR_INIT (res) = NULL;
8377 STMT_VINFO_DR_STEP (res) = NULL;
8378 STMT_VINFO_DR_ALIGNED_TO (res) = NULL;
8380 if (gimple_code (stmt) == GIMPLE_PHI
8381 && is_loop_header_bb_p (gimple_bb (stmt)))
8382 STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
8383 else
8384 STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
8386 STMT_VINFO_SAME_ALIGN_REFS (res).create (0);
8387 STMT_SLP_TYPE (res) = loop_vect;
8388 STMT_VINFO_NUM_SLP_USES (res) = 0;
8390 GROUP_FIRST_ELEMENT (res) = NULL;
8391 GROUP_NEXT_ELEMENT (res) = NULL;
8392 GROUP_SIZE (res) = 0;
8393 GROUP_STORE_COUNT (res) = 0;
8394 GROUP_GAP (res) = 0;
8395 GROUP_SAME_DR_STMT (res) = NULL;
8397 return res;
8401 /* Create a hash table for stmt_vec_info. */
8403 void
8404 init_stmt_vec_info_vec (void)
8406 gcc_assert (!stmt_vec_info_vec.exists ());
8407 stmt_vec_info_vec.create (50);
8411 /* Free hash table for stmt_vec_info. */
8413 void
8414 free_stmt_vec_info_vec (void)
8416 unsigned int i;
8417 stmt_vec_info info;
8418 FOR_EACH_VEC_ELT (stmt_vec_info_vec, i, info)
8419 if (info != NULL)
8420 free_stmt_vec_info (STMT_VINFO_STMT (info));
8421 gcc_assert (stmt_vec_info_vec.exists ());
8422 stmt_vec_info_vec.release ();
8426 /* Free stmt vectorization related info. */
8428 void
8429 free_stmt_vec_info (gimple *stmt)
8431 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
8433 if (!stmt_info)
8434 return;
8436 /* Check if this statement has a related "pattern stmt"
8437 (introduced by the vectorizer during the pattern recognition
8438 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
8439 too. */
8440 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
8442 stmt_vec_info patt_info
8443 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
8444 if (patt_info)
8446 gimple_seq seq = STMT_VINFO_PATTERN_DEF_SEQ (patt_info);
8447 gimple *patt_stmt = STMT_VINFO_STMT (patt_info);
8448 gimple_set_bb (patt_stmt, NULL);
8449 tree lhs = gimple_get_lhs (patt_stmt);
8450 if (lhs && TREE_CODE (lhs) == SSA_NAME)
8451 release_ssa_name (lhs);
8452 if (seq)
8454 gimple_stmt_iterator si;
8455 for (si = gsi_start (seq); !gsi_end_p (si); gsi_next (&si))
8457 gimple *seq_stmt = gsi_stmt (si);
8458 gimple_set_bb (seq_stmt, NULL);
8459 lhs = gimple_get_lhs (seq_stmt);
8460 if (lhs && TREE_CODE (lhs) == SSA_NAME)
8461 release_ssa_name (lhs);
8462 free_stmt_vec_info (seq_stmt);
8465 free_stmt_vec_info (patt_stmt);
8469 STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release ();
8470 STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
8471 set_vinfo_for_stmt (stmt, NULL);
8472 free (stmt_info);
8476 /* Function get_vectype_for_scalar_type_and_size.
8478 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
8479 by the target. */
8481 static tree
8482 get_vectype_for_scalar_type_and_size (tree scalar_type, unsigned size)
8484 machine_mode inner_mode = TYPE_MODE (scalar_type);
8485 machine_mode simd_mode;
8486 unsigned int nbytes = GET_MODE_SIZE (inner_mode);
8487 int nunits;
8488 tree vectype;
8490 if (nbytes == 0)
8491 return NULL_TREE;
8493 if (GET_MODE_CLASS (inner_mode) != MODE_INT
8494 && GET_MODE_CLASS (inner_mode) != MODE_FLOAT)
8495 return NULL_TREE;
8497 /* For vector types of elements whose mode precision doesn't
8498 match their types precision we use a element type of mode
8499 precision. The vectorization routines will have to make sure
8500 they support the proper result truncation/extension.
8501 We also make sure to build vector types with INTEGER_TYPE
8502 component type only. */
8503 if (INTEGRAL_TYPE_P (scalar_type)
8504 && (GET_MODE_BITSIZE (inner_mode) != TYPE_PRECISION (scalar_type)
8505 || TREE_CODE (scalar_type) != INTEGER_TYPE))
8506 scalar_type = build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode),
8507 TYPE_UNSIGNED (scalar_type));
8509 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
8510 When the component mode passes the above test simply use a type
8511 corresponding to that mode. The theory is that any use that
8512 would cause problems with this will disable vectorization anyway. */
8513 else if (!SCALAR_FLOAT_TYPE_P (scalar_type)
8514 && !INTEGRAL_TYPE_P (scalar_type))
8515 scalar_type = lang_hooks.types.type_for_mode (inner_mode, 1);
8517 /* We can't build a vector type of elements with alignment bigger than
8518 their size. */
8519 else if (nbytes < TYPE_ALIGN_UNIT (scalar_type))
8520 scalar_type = lang_hooks.types.type_for_mode (inner_mode,
8521 TYPE_UNSIGNED (scalar_type));
8523 /* If we felt back to using the mode fail if there was
8524 no scalar type for it. */
8525 if (scalar_type == NULL_TREE)
8526 return NULL_TREE;
8528 /* If no size was supplied use the mode the target prefers. Otherwise
8529 lookup a vector mode of the specified size. */
8530 if (size == 0)
8531 simd_mode = targetm.vectorize.preferred_simd_mode (inner_mode);
8532 else
8533 simd_mode = mode_for_vector (inner_mode, size / nbytes);
8534 nunits = GET_MODE_SIZE (simd_mode) / nbytes;
8535 if (nunits <= 1)
8536 return NULL_TREE;
8538 vectype = build_vector_type (scalar_type, nunits);
8540 if (!VECTOR_MODE_P (TYPE_MODE (vectype))
8541 && !INTEGRAL_MODE_P (TYPE_MODE (vectype)))
8542 return NULL_TREE;
8544 return vectype;
8547 unsigned int current_vector_size;
8549 /* Function get_vectype_for_scalar_type.
8551 Returns the vector type corresponding to SCALAR_TYPE as supported
8552 by the target. */
8554 tree
8555 get_vectype_for_scalar_type (tree scalar_type)
8557 tree vectype;
8558 vectype = get_vectype_for_scalar_type_and_size (scalar_type,
8559 current_vector_size);
8560 if (vectype
8561 && current_vector_size == 0)
8562 current_vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
8563 return vectype;
8566 /* Function get_mask_type_for_scalar_type.
8568 Returns the mask type corresponding to a result of comparison
8569 of vectors of specified SCALAR_TYPE as supported by target. */
8571 tree
8572 get_mask_type_for_scalar_type (tree scalar_type)
8574 tree vectype = get_vectype_for_scalar_type (scalar_type);
8576 if (!vectype)
8577 return NULL;
8579 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype),
8580 current_vector_size);
8583 /* Function get_same_sized_vectype
8585 Returns a vector type corresponding to SCALAR_TYPE of size
8586 VECTOR_TYPE if supported by the target. */
8588 tree
8589 get_same_sized_vectype (tree scalar_type, tree vector_type)
8591 if (TREE_CODE (scalar_type) == BOOLEAN_TYPE)
8592 return build_same_sized_truth_vector_type (vector_type);
8594 return get_vectype_for_scalar_type_and_size
8595 (scalar_type, GET_MODE_SIZE (TYPE_MODE (vector_type)));
8598 /* Function vect_is_simple_use.
8600 Input:
8601 VINFO - the vect info of the loop or basic block that is being vectorized.
8602 OPERAND - operand in the loop or bb.
8603 Output:
8604 DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
8605 DT - the type of definition
8607 Returns whether a stmt with OPERAND can be vectorized.
8608 For loops, supportable operands are constants, loop invariants, and operands
8609 that are defined by the current iteration of the loop. Unsupportable
8610 operands are those that are defined by a previous iteration of the loop (as
8611 is the case in reduction/induction computations).
8612 For basic blocks, supportable operands are constants and bb invariants.
8613 For now, operands defined outside the basic block are not supported. */
8615 bool
8616 vect_is_simple_use (tree operand, vec_info *vinfo,
8617 gimple **def_stmt, enum vect_def_type *dt)
8619 *def_stmt = NULL;
8620 *dt = vect_unknown_def_type;
8622 if (dump_enabled_p ())
8624 dump_printf_loc (MSG_NOTE, vect_location,
8625 "vect_is_simple_use: operand ");
8626 dump_generic_expr (MSG_NOTE, TDF_SLIM, operand);
8627 dump_printf (MSG_NOTE, "\n");
8630 if (CONSTANT_CLASS_P (operand))
8632 *dt = vect_constant_def;
8633 return true;
8636 if (is_gimple_min_invariant (operand))
8638 *dt = vect_external_def;
8639 return true;
8642 if (TREE_CODE (operand) != SSA_NAME)
8644 if (dump_enabled_p ())
8645 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8646 "not ssa-name.\n");
8647 return false;
8650 if (SSA_NAME_IS_DEFAULT_DEF (operand))
8652 *dt = vect_external_def;
8653 return true;
8656 *def_stmt = SSA_NAME_DEF_STMT (operand);
8657 if (dump_enabled_p ())
8659 dump_printf_loc (MSG_NOTE, vect_location, "def_stmt: ");
8660 dump_gimple_stmt (MSG_NOTE, TDF_SLIM, *def_stmt, 0);
8663 if (! vect_stmt_in_region_p (vinfo, *def_stmt))
8664 *dt = vect_external_def;
8665 else
8667 stmt_vec_info stmt_vinfo = vinfo_for_stmt (*def_stmt);
8668 *dt = STMT_VINFO_DEF_TYPE (stmt_vinfo);
8671 if (dump_enabled_p ())
8673 dump_printf_loc (MSG_NOTE, vect_location, "type of def: ");
8674 switch (*dt)
8676 case vect_uninitialized_def:
8677 dump_printf (MSG_NOTE, "uninitialized\n");
8678 break;
8679 case vect_constant_def:
8680 dump_printf (MSG_NOTE, "constant\n");
8681 break;
8682 case vect_external_def:
8683 dump_printf (MSG_NOTE, "external\n");
8684 break;
8685 case vect_internal_def:
8686 dump_printf (MSG_NOTE, "internal\n");
8687 break;
8688 case vect_induction_def:
8689 dump_printf (MSG_NOTE, "induction\n");
8690 break;
8691 case vect_reduction_def:
8692 dump_printf (MSG_NOTE, "reduction\n");
8693 break;
8694 case vect_double_reduction_def:
8695 dump_printf (MSG_NOTE, "double reduction\n");
8696 break;
8697 case vect_nested_cycle:
8698 dump_printf (MSG_NOTE, "nested cycle\n");
8699 break;
8700 case vect_unknown_def_type:
8701 dump_printf (MSG_NOTE, "unknown\n");
8702 break;
8706 if (*dt == vect_unknown_def_type)
8708 if (dump_enabled_p ())
8709 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8710 "Unsupported pattern.\n");
8711 return false;
8714 switch (gimple_code (*def_stmt))
8716 case GIMPLE_PHI:
8717 case GIMPLE_ASSIGN:
8718 case GIMPLE_CALL:
8719 break;
8720 default:
8721 if (dump_enabled_p ())
8722 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
8723 "unsupported defining stmt:\n");
8724 return false;
8727 return true;
8730 /* Function vect_is_simple_use.
8732 Same as vect_is_simple_use but also determines the vector operand
8733 type of OPERAND and stores it to *VECTYPE. If the definition of
8734 OPERAND is vect_uninitialized_def, vect_constant_def or
8735 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8736 is responsible to compute the best suited vector type for the
8737 scalar operand. */
8739 bool
8740 vect_is_simple_use (tree operand, vec_info *vinfo,
8741 gimple **def_stmt, enum vect_def_type *dt, tree *vectype)
8743 if (!vect_is_simple_use (operand, vinfo, def_stmt, dt))
8744 return false;
8746 /* Now get a vector type if the def is internal, otherwise supply
8747 NULL_TREE and leave it up to the caller to figure out a proper
8748 type for the use stmt. */
8749 if (*dt == vect_internal_def
8750 || *dt == vect_induction_def
8751 || *dt == vect_reduction_def
8752 || *dt == vect_double_reduction_def
8753 || *dt == vect_nested_cycle)
8755 stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt);
8757 if (STMT_VINFO_IN_PATTERN_P (stmt_info)
8758 && !STMT_VINFO_RELEVANT (stmt_info)
8759 && !STMT_VINFO_LIVE_P (stmt_info))
8760 stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
8762 *vectype = STMT_VINFO_VECTYPE (stmt_info);
8763 gcc_assert (*vectype != NULL_TREE);
8765 else if (*dt == vect_uninitialized_def
8766 || *dt == vect_constant_def
8767 || *dt == vect_external_def)
8768 *vectype = NULL_TREE;
8769 else
8770 gcc_unreachable ();
8772 return true;
8776 /* Function supportable_widening_operation
8778 Check whether an operation represented by the code CODE is a
8779 widening operation that is supported by the target platform in
8780 vector form (i.e., when operating on arguments of type VECTYPE_IN
8781 producing a result of type VECTYPE_OUT).
8783 Widening operations we currently support are NOP (CONVERT), FLOAT
8784 and WIDEN_MULT. This function checks if these operations are supported
8785 by the target platform either directly (via vector tree-codes), or via
8786 target builtins.
8788 Output:
8789 - CODE1 and CODE2 are codes of vector operations to be used when
8790 vectorizing the operation, if available.
8791 - MULTI_STEP_CVT determines the number of required intermediate steps in
8792 case of multi-step conversion (like char->short->int - in that case
8793 MULTI_STEP_CVT will be 1).
8794 - INTERM_TYPES contains the intermediate type required to perform the
8795 widening operation (short in the above example). */
8797 bool
8798 supportable_widening_operation (enum tree_code code, gimple *stmt,
8799 tree vectype_out, tree vectype_in,
8800 enum tree_code *code1, enum tree_code *code2,
8801 int *multi_step_cvt,
8802 vec<tree> *interm_types)
8804 stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
8805 loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_info);
8806 struct loop *vect_loop = NULL;
8807 machine_mode vec_mode;
8808 enum insn_code icode1, icode2;
8809 optab optab1, optab2;
8810 tree vectype = vectype_in;
8811 tree wide_vectype = vectype_out;
8812 enum tree_code c1, c2;
8813 int i;
8814 tree prev_type, intermediate_type;
8815 machine_mode intermediate_mode, prev_mode;
8816 optab optab3, optab4;
8818 *multi_step_cvt = 0;
8819 if (loop_info)
8820 vect_loop = LOOP_VINFO_LOOP (loop_info);
8822 switch (code)
8824 case WIDEN_MULT_EXPR:
8825 /* The result of a vectorized widening operation usually requires
8826 two vectors (because the widened results do not fit into one vector).
8827 The generated vector results would normally be expected to be
8828 generated in the same order as in the original scalar computation,
8829 i.e. if 8 results are generated in each vector iteration, they are
8830 to be organized as follows:
8831 vect1: [res1,res2,res3,res4],
8832 vect2: [res5,res6,res7,res8].
8834 However, in the special case that the result of the widening
8835 operation is used in a reduction computation only, the order doesn't
8836 matter (because when vectorizing a reduction we change the order of
8837 the computation). Some targets can take advantage of this and
8838 generate more efficient code. For example, targets like Altivec,
8839 that support widen_mult using a sequence of {mult_even,mult_odd}
8840 generate the following vectors:
8841 vect1: [res1,res3,res5,res7],
8842 vect2: [res2,res4,res6,res8].
8844 When vectorizing outer-loops, we execute the inner-loop sequentially
8845 (each vectorized inner-loop iteration contributes to VF outer-loop
8846 iterations in parallel). We therefore don't allow to change the
8847 order of the computation in the inner-loop during outer-loop
8848 vectorization. */
8849 /* TODO: Another case in which order doesn't *really* matter is when we
8850 widen and then contract again, e.g. (short)((int)x * y >> 8).
8851 Normally, pack_trunc performs an even/odd permute, whereas the
8852 repack from an even/odd expansion would be an interleave, which
8853 would be significantly simpler for e.g. AVX2. */
8854 /* In any case, in order to avoid duplicating the code below, recurse
8855 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8856 are properly set up for the caller. If we fail, we'll continue with
8857 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8858 if (vect_loop
8859 && STMT_VINFO_RELEVANT (stmt_info) == vect_used_by_reduction
8860 && !nested_in_vect_loop_p (vect_loop, stmt)
8861 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR,
8862 stmt, vectype_out, vectype_in,
8863 code1, code2, multi_step_cvt,
8864 interm_types))
8866 /* Elements in a vector with vect_used_by_reduction property cannot
8867 be reordered if the use chain with this property does not have the
8868 same operation. One such an example is s += a * b, where elements
8869 in a and b cannot be reordered. Here we check if the vector defined
8870 by STMT is only directly used in the reduction statement. */
8871 tree lhs = gimple_assign_lhs (stmt);
8872 use_operand_p dummy;
8873 gimple *use_stmt;
8874 stmt_vec_info use_stmt_info = NULL;
8875 if (single_imm_use (lhs, &dummy, &use_stmt)
8876 && (use_stmt_info = vinfo_for_stmt (use_stmt))
8877 && STMT_VINFO_DEF_TYPE (use_stmt_info) == vect_reduction_def)
8878 return true;
8880 c1 = VEC_WIDEN_MULT_LO_EXPR;
8881 c2 = VEC_WIDEN_MULT_HI_EXPR;
8882 break;
8884 case DOT_PROD_EXPR:
8885 c1 = DOT_PROD_EXPR;
8886 c2 = DOT_PROD_EXPR;
8887 break;
8889 case SAD_EXPR:
8890 c1 = SAD_EXPR;
8891 c2 = SAD_EXPR;
8892 break;
8894 case VEC_WIDEN_MULT_EVEN_EXPR:
8895 /* Support the recursion induced just above. */
8896 c1 = VEC_WIDEN_MULT_EVEN_EXPR;
8897 c2 = VEC_WIDEN_MULT_ODD_EXPR;
8898 break;
8900 case WIDEN_LSHIFT_EXPR:
8901 c1 = VEC_WIDEN_LSHIFT_LO_EXPR;
8902 c2 = VEC_WIDEN_LSHIFT_HI_EXPR;
8903 break;
8905 CASE_CONVERT:
8906 c1 = VEC_UNPACK_LO_EXPR;
8907 c2 = VEC_UNPACK_HI_EXPR;
8908 break;
8910 case FLOAT_EXPR:
8911 c1 = VEC_UNPACK_FLOAT_LO_EXPR;
8912 c2 = VEC_UNPACK_FLOAT_HI_EXPR;
8913 break;
8915 case FIX_TRUNC_EXPR:
8916 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
8917 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
8918 computing the operation. */
8919 return false;
8921 default:
8922 gcc_unreachable ();
8925 if (BYTES_BIG_ENDIAN && c1 != VEC_WIDEN_MULT_EVEN_EXPR)
8926 std::swap (c1, c2);
8928 if (code == FIX_TRUNC_EXPR)
8930 /* The signedness is determined from output operand. */
8931 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
8932 optab2 = optab_for_tree_code (c2, vectype_out, optab_default);
8934 else
8936 optab1 = optab_for_tree_code (c1, vectype, optab_default);
8937 optab2 = optab_for_tree_code (c2, vectype, optab_default);
8940 if (!optab1 || !optab2)
8941 return false;
8943 vec_mode = TYPE_MODE (vectype);
8944 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing
8945 || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing)
8946 return false;
8948 *code1 = c1;
8949 *code2 = c2;
8951 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
8952 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
8953 /* For scalar masks we may have different boolean
8954 vector types having the same QImode. Thus we
8955 add additional check for elements number. */
8956 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
8957 || (TYPE_VECTOR_SUBPARTS (vectype) / 2
8958 == TYPE_VECTOR_SUBPARTS (wide_vectype)));
8960 /* Check if it's a multi-step conversion that can be done using intermediate
8961 types. */
8963 prev_type = vectype;
8964 prev_mode = vec_mode;
8966 if (!CONVERT_EXPR_CODE_P (code))
8967 return false;
8969 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8970 intermediate steps in promotion sequence. We try
8971 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
8972 not. */
8973 interm_types->create (MAX_INTERM_CVT_STEPS);
8974 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
8976 intermediate_mode = insn_data[icode1].operand[0].mode;
8977 if (VECTOR_BOOLEAN_TYPE_P (prev_type))
8979 intermediate_type
8980 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type) / 2,
8981 current_vector_size);
8982 if (intermediate_mode != TYPE_MODE (intermediate_type))
8983 return false;
8985 else
8986 intermediate_type
8987 = lang_hooks.types.type_for_mode (intermediate_mode,
8988 TYPE_UNSIGNED (prev_type));
8990 optab3 = optab_for_tree_code (c1, intermediate_type, optab_default);
8991 optab4 = optab_for_tree_code (c2, intermediate_type, optab_default);
8993 if (!optab3 || !optab4
8994 || (icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing
8995 || insn_data[icode1].operand[0].mode != intermediate_mode
8996 || (icode2 = optab_handler (optab2, prev_mode)) == CODE_FOR_nothing
8997 || insn_data[icode2].operand[0].mode != intermediate_mode
8998 || ((icode1 = optab_handler (optab3, intermediate_mode))
8999 == CODE_FOR_nothing)
9000 || ((icode2 = optab_handler (optab4, intermediate_mode))
9001 == CODE_FOR_nothing))
9002 break;
9004 interm_types->quick_push (intermediate_type);
9005 (*multi_step_cvt)++;
9007 if (insn_data[icode1].operand[0].mode == TYPE_MODE (wide_vectype)
9008 && insn_data[icode2].operand[0].mode == TYPE_MODE (wide_vectype))
9009 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
9010 || (TYPE_VECTOR_SUBPARTS (intermediate_type) / 2
9011 == TYPE_VECTOR_SUBPARTS (wide_vectype)));
9013 prev_type = intermediate_type;
9014 prev_mode = intermediate_mode;
9017 interm_types->release ();
9018 return false;
9022 /* Function supportable_narrowing_operation
9024 Check whether an operation represented by the code CODE is a
9025 narrowing operation that is supported by the target platform in
9026 vector form (i.e., when operating on arguments of type VECTYPE_IN
9027 and producing a result of type VECTYPE_OUT).
9029 Narrowing operations we currently support are NOP (CONVERT) and
9030 FIX_TRUNC. This function checks if these operations are supported by
9031 the target platform directly via vector tree-codes.
9033 Output:
9034 - CODE1 is the code of a vector operation to be used when
9035 vectorizing the operation, if available.
9036 - MULTI_STEP_CVT determines the number of required intermediate steps in
9037 case of multi-step conversion (like int->short->char - in that case
9038 MULTI_STEP_CVT will be 1).
9039 - INTERM_TYPES contains the intermediate type required to perform the
9040 narrowing operation (short in the above example). */
9042 bool
9043 supportable_narrowing_operation (enum tree_code code,
9044 tree vectype_out, tree vectype_in,
9045 enum tree_code *code1, int *multi_step_cvt,
9046 vec<tree> *interm_types)
9048 machine_mode vec_mode;
9049 enum insn_code icode1;
9050 optab optab1, interm_optab;
9051 tree vectype = vectype_in;
9052 tree narrow_vectype = vectype_out;
9053 enum tree_code c1;
9054 tree intermediate_type, prev_type;
9055 machine_mode intermediate_mode, prev_mode;
9056 int i;
9057 bool uns;
9059 *multi_step_cvt = 0;
9060 switch (code)
9062 CASE_CONVERT:
9063 c1 = VEC_PACK_TRUNC_EXPR;
9064 break;
9066 case FIX_TRUNC_EXPR:
9067 c1 = VEC_PACK_FIX_TRUNC_EXPR;
9068 break;
9070 case FLOAT_EXPR:
9071 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
9072 tree code and optabs used for computing the operation. */
9073 return false;
9075 default:
9076 gcc_unreachable ();
9079 if (code == FIX_TRUNC_EXPR)
9080 /* The signedness is determined from output operand. */
9081 optab1 = optab_for_tree_code (c1, vectype_out, optab_default);
9082 else
9083 optab1 = optab_for_tree_code (c1, vectype, optab_default);
9085 if (!optab1)
9086 return false;
9088 vec_mode = TYPE_MODE (vectype);
9089 if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing)
9090 return false;
9092 *code1 = c1;
9094 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
9095 /* For scalar masks we may have different boolean
9096 vector types having the same QImode. Thus we
9097 add additional check for elements number. */
9098 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
9099 || (TYPE_VECTOR_SUBPARTS (vectype) * 2
9100 == TYPE_VECTOR_SUBPARTS (narrow_vectype)));
9102 /* Check if it's a multi-step conversion that can be done using intermediate
9103 types. */
9104 prev_mode = vec_mode;
9105 prev_type = vectype;
9106 if (code == FIX_TRUNC_EXPR)
9107 uns = TYPE_UNSIGNED (vectype_out);
9108 else
9109 uns = TYPE_UNSIGNED (vectype);
9111 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
9112 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
9113 costly than signed. */
9114 if (code == FIX_TRUNC_EXPR && uns)
9116 enum insn_code icode2;
9118 intermediate_type
9119 = lang_hooks.types.type_for_mode (TYPE_MODE (vectype_out), 0);
9120 interm_optab
9121 = optab_for_tree_code (c1, intermediate_type, optab_default);
9122 if (interm_optab != unknown_optab
9123 && (icode2 = optab_handler (optab1, vec_mode)) != CODE_FOR_nothing
9124 && insn_data[icode1].operand[0].mode
9125 == insn_data[icode2].operand[0].mode)
9127 uns = false;
9128 optab1 = interm_optab;
9129 icode1 = icode2;
9133 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
9134 intermediate steps in promotion sequence. We try
9135 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
9136 interm_types->create (MAX_INTERM_CVT_STEPS);
9137 for (i = 0; i < MAX_INTERM_CVT_STEPS; i++)
9139 intermediate_mode = insn_data[icode1].operand[0].mode;
9140 if (VECTOR_BOOLEAN_TYPE_P (prev_type))
9142 intermediate_type
9143 = build_truth_vector_type (TYPE_VECTOR_SUBPARTS (prev_type) * 2,
9144 current_vector_size);
9145 if (intermediate_mode != TYPE_MODE (intermediate_type))
9146 return false;
9148 else
9149 intermediate_type
9150 = lang_hooks.types.type_for_mode (intermediate_mode, uns);
9151 interm_optab
9152 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR, intermediate_type,
9153 optab_default);
9154 if (!interm_optab
9155 || ((icode1 = optab_handler (optab1, prev_mode)) == CODE_FOR_nothing)
9156 || insn_data[icode1].operand[0].mode != intermediate_mode
9157 || ((icode1 = optab_handler (interm_optab, intermediate_mode))
9158 == CODE_FOR_nothing))
9159 break;
9161 interm_types->quick_push (intermediate_type);
9162 (*multi_step_cvt)++;
9164 if (insn_data[icode1].operand[0].mode == TYPE_MODE (narrow_vectype))
9165 return (!VECTOR_BOOLEAN_TYPE_P (vectype)
9166 || (TYPE_VECTOR_SUBPARTS (intermediate_type) * 2
9167 == TYPE_VECTOR_SUBPARTS (narrow_vectype)));
9169 prev_mode = intermediate_mode;
9170 prev_type = intermediate_type;
9171 optab1 = interm_optab;
9174 interm_types->release ();
9175 return false;