2 Copyright (C) 2003-2016 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Loop and basic block vectorizer.
23 This file contains drivers for the three vectorizers:
24 (1) loop vectorizer (inter-iteration parallelism),
25 (2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
27 (3) BB vectorizer (out-of-loops), aka SLP
29 The rest of the vectorizer's code is organized as follows:
30 - tree-vect-loop.c - loop specific parts such as reductions, etc. These are
31 used by drivers (1) and (2).
32 - tree-vect-loop-manip.c - vectorizer's loop control-flow utilities, used by
34 - tree-vect-slp.c - BB vectorization specific analysis and transformation,
35 used by drivers (2) and (3).
36 - tree-vect-stmts.c - statements analysis and transformation (used by all).
37 - tree-vect-data-refs.c - vectorizer specific data-refs analysis and
38 manipulations (used by all).
39 - tree-vect-patterns.c - vectorizable code patterns detector (used by all)
41 Here's a poor attempt at illustrating that:
44 loop_vect() loop_aware_slp() slp_vect()
47 tree-vect-loop.c tree-vect-slp.c
52 tree-vect-stmts.c tree-vect-data-refs.c
59 #include "coretypes.h"
64 #include "tree-pass.h"
67 #include "fold-const.h"
68 #include "stor-layout.h"
69 #include "gimple-iterator.h"
70 #include "gimple-walk.h"
71 #include "tree-ssa-loop-manip.h"
72 #include "tree-ssa-loop-niter.h"
75 #include "tree-vectorizer.h"
76 #include "tree-ssa-propagate.h"
78 #include "tree-scalar-evolution.h"
81 /* Loop or bb location. */
82 source_location vect_location
;
84 /* Vector mapping GIMPLE stmt to stmt_vec_info. */
85 vec
<stmt_vec_info
> stmt_vec_info_vec
;
87 /* For mapping simduid to vectorization factor. */
89 struct simduid_to_vf
: free_ptr_hash
<simduid_to_vf
>
94 /* hash_table support. */
95 static inline hashval_t
hash (const simduid_to_vf
*);
96 static inline int equal (const simduid_to_vf
*, const simduid_to_vf
*);
100 simduid_to_vf::hash (const simduid_to_vf
*p
)
106 simduid_to_vf::equal (const simduid_to_vf
*p1
, const simduid_to_vf
*p2
)
108 return p1
->simduid
== p2
->simduid
;
111 /* This hash maps the OMP simd array to the corresponding simduid used
112 to index into it. Like thus,
114 _7 = GOMP_SIMD_LANE (simduid.0)
120 This hash maps from the OMP simd array (D.1737[]) to DECL_UID of
123 struct simd_array_to_simduid
: free_ptr_hash
<simd_array_to_simduid
>
126 unsigned int simduid
;
128 /* hash_table support. */
129 static inline hashval_t
hash (const simd_array_to_simduid
*);
130 static inline int equal (const simd_array_to_simduid
*,
131 const simd_array_to_simduid
*);
135 simd_array_to_simduid::hash (const simd_array_to_simduid
*p
)
137 return DECL_UID (p
->decl
);
141 simd_array_to_simduid::equal (const simd_array_to_simduid
*p1
,
142 const simd_array_to_simduid
*p2
)
144 return p1
->decl
== p2
->decl
;
147 /* Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
148 into their corresponding constants and remove
149 IFN_GOMP_SIMD_ORDERED_{START,END}. */
152 adjust_simduid_builtins (hash_table
<simduid_to_vf
> *htab
)
156 FOR_EACH_BB_FN (bb
, cfun
)
158 gimple_stmt_iterator i
;
160 for (i
= gsi_start_bb (bb
); !gsi_end_p (i
); )
163 enum internal_fn ifn
;
164 gimple
*stmt
= gsi_stmt (i
);
166 if (!is_gimple_call (stmt
)
167 || !gimple_call_internal_p (stmt
))
172 ifn
= gimple_call_internal_fn (stmt
);
175 case IFN_GOMP_SIMD_LANE
:
176 case IFN_GOMP_SIMD_VF
:
177 case IFN_GOMP_SIMD_LAST_LANE
:
179 case IFN_GOMP_SIMD_ORDERED_START
:
180 case IFN_GOMP_SIMD_ORDERED_END
:
181 if (integer_onep (gimple_call_arg (stmt
, 0)))
183 enum built_in_function bcode
184 = (ifn
== IFN_GOMP_SIMD_ORDERED_START
185 ? BUILT_IN_GOMP_ORDERED_START
186 : BUILT_IN_GOMP_ORDERED_END
);
188 = gimple_build_call (builtin_decl_explicit (bcode
), 0);
189 tree vdef
= gimple_vdef (stmt
);
190 gimple_set_vdef (g
, vdef
);
191 SSA_NAME_DEF_STMT (vdef
) = g
;
192 gimple_set_vuse (g
, gimple_vuse (stmt
));
193 gsi_replace (&i
, g
, true);
196 gsi_remove (&i
, true);
197 unlink_stmt_vdef (stmt
);
203 tree arg
= gimple_call_arg (stmt
, 0);
204 gcc_assert (arg
!= NULL_TREE
);
205 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
206 simduid_to_vf
*p
= NULL
, data
;
207 data
.simduid
= DECL_UID (SSA_NAME_VAR (arg
));
208 /* Need to nullify loop safelen field since it's value is not
209 valid after transformation. */
210 if (bb
->loop_father
&& bb
->loop_father
->safelen
> 0)
211 bb
->loop_father
->safelen
= 0;
214 p
= htab
->find (&data
);
220 case IFN_GOMP_SIMD_VF
:
221 t
= build_int_cst (unsigned_type_node
, vf
);
223 case IFN_GOMP_SIMD_LANE
:
224 t
= build_int_cst (unsigned_type_node
, 0);
226 case IFN_GOMP_SIMD_LAST_LANE
:
227 t
= gimple_call_arg (stmt
, 1);
232 update_call_from_tree (&i
, t
);
238 /* Helper structure for note_simd_array_uses. */
240 struct note_simd_array_uses_struct
242 hash_table
<simd_array_to_simduid
> **htab
;
243 unsigned int simduid
;
246 /* Callback for note_simd_array_uses, called through walk_gimple_op. */
249 note_simd_array_uses_cb (tree
*tp
, int *walk_subtrees
, void *data
)
251 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
252 struct note_simd_array_uses_struct
*ns
253 = (struct note_simd_array_uses_struct
*) wi
->info
;
258 && lookup_attribute ("omp simd array", DECL_ATTRIBUTES (*tp
))
259 && DECL_CONTEXT (*tp
) == current_function_decl
)
261 simd_array_to_simduid data
;
263 *ns
->htab
= new hash_table
<simd_array_to_simduid
> (15);
265 data
.simduid
= ns
->simduid
;
266 simd_array_to_simduid
**slot
= (*ns
->htab
)->find_slot (&data
, INSERT
);
269 simd_array_to_simduid
*p
= XNEW (simd_array_to_simduid
);
273 else if ((*slot
)->simduid
!= ns
->simduid
)
274 (*slot
)->simduid
= -1U;
280 /* Find "omp simd array" temporaries and map them to corresponding
284 note_simd_array_uses (hash_table
<simd_array_to_simduid
> **htab
)
287 gimple_stmt_iterator gsi
;
288 struct walk_stmt_info wi
;
289 struct note_simd_array_uses_struct ns
;
291 memset (&wi
, 0, sizeof (wi
));
295 FOR_EACH_BB_FN (bb
, cfun
)
296 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
298 gimple
*stmt
= gsi_stmt (gsi
);
299 if (!is_gimple_call (stmt
) || !gimple_call_internal_p (stmt
))
301 switch (gimple_call_internal_fn (stmt
))
303 case IFN_GOMP_SIMD_LANE
:
304 case IFN_GOMP_SIMD_VF
:
305 case IFN_GOMP_SIMD_LAST_LANE
:
310 tree lhs
= gimple_call_lhs (stmt
);
311 if (lhs
== NULL_TREE
)
313 imm_use_iterator use_iter
;
315 ns
.simduid
= DECL_UID (SSA_NAME_VAR (gimple_call_arg (stmt
, 0)));
316 FOR_EACH_IMM_USE_STMT (use_stmt
, use_iter
, lhs
)
317 if (!is_gimple_debug (use_stmt
))
318 walk_gimple_op (use_stmt
, note_simd_array_uses_cb
, &wi
);
322 /* Shrink arrays with "omp simd array" attribute to the corresponding
323 vectorization factor. */
327 (hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
,
328 hash_table
<simduid_to_vf
> *simduid_to_vf_htab
)
330 for (hash_table
<simd_array_to_simduid
>::iterator iter
331 = simd_array_to_simduid_htab
->begin ();
332 iter
!= simd_array_to_simduid_htab
->end (); ++iter
)
333 if ((*iter
)->simduid
!= -1U)
335 tree decl
= (*iter
)->decl
;
337 if (simduid_to_vf_htab
)
339 simduid_to_vf
*p
= NULL
, data
;
340 data
.simduid
= (*iter
)->simduid
;
341 p
= simduid_to_vf_htab
->find (&data
);
346 = build_array_type_nelts (TREE_TYPE (TREE_TYPE (decl
)), vf
);
347 TREE_TYPE (decl
) = atype
;
348 relayout_decl (decl
);
351 delete simd_array_to_simduid_htab
;
354 /* A helper function to free data refs. */
357 vect_destroy_datarefs (vec_info
*vinfo
)
359 struct data_reference
*dr
;
362 FOR_EACH_VEC_ELT (vinfo
->datarefs
, i
, dr
)
369 free_data_refs (vinfo
->datarefs
);
372 /* A helper function to free scev and LOOP niter information, as well as
373 clear loop constraint LOOP_C_FINITE. */
376 vect_free_loop_info_assumptions (struct loop
*loop
)
379 /* We need to explicitly reset upper bound information since they are
380 used even after free_numbers_of_iterations_estimates_loop. */
381 loop
->any_upper_bound
= false;
382 loop
->any_likely_upper_bound
= false;
383 free_numbers_of_iterations_estimates_loop (loop
);
384 loop_constraint_clear (loop
, LOOP_C_FINITE
);
387 /* Return whether STMT is inside the region we try to vectorize. */
390 vect_stmt_in_region_p (vec_info
*vinfo
, gimple
*stmt
)
392 if (!gimple_bb (stmt
))
395 if (loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (vinfo
))
397 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
398 if (!flow_bb_inside_loop_p (loop
, gimple_bb (stmt
)))
403 bb_vec_info bb_vinfo
= as_a
<bb_vec_info
> (vinfo
);
404 if (gimple_bb (stmt
) != BB_VINFO_BB (bb_vinfo
)
405 || gimple_uid (stmt
) == -1U
406 || gimple_code (stmt
) == GIMPLE_PHI
)
414 /* If LOOP has been versioned during ifcvt, return the internal call
418 vect_loop_vectorized_call (struct loop
*loop
)
420 basic_block bb
= loop_preheader_edge (loop
)->src
;
427 if (!single_pred_p (bb
))
429 bb
= single_pred (bb
);
432 if (g
&& gimple_code (g
) == GIMPLE_COND
)
434 gimple_stmt_iterator gsi
= gsi_for_stmt (g
);
436 if (!gsi_end_p (gsi
))
439 if (is_gimple_call (g
)
440 && gimple_call_internal_p (g
)
441 && gimple_call_internal_fn (g
) == IFN_LOOP_VECTORIZED
442 && (tree_to_shwi (gimple_call_arg (g
, 0)) == loop
->num
443 || tree_to_shwi (gimple_call_arg (g
, 1)) == loop
->num
))
450 /* Fold LOOP_VECTORIZED internal call G to VALUE and
451 update any immediate uses of it's LHS. */
454 fold_loop_vectorized_call (gimple
*g
, tree value
)
456 tree lhs
= gimple_call_lhs (g
);
458 imm_use_iterator iter
;
460 gimple_stmt_iterator gsi
= gsi_for_stmt (g
);
462 update_call_from_tree (&gsi
, value
);
463 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, lhs
)
465 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
466 SET_USE (use_p
, value
);
467 update_stmt (use_stmt
);
470 /* Set the uids of all the statements in basic blocks inside loop
471 represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
472 call guarding the loop which has been if converted. */
474 set_uid_loop_bbs (loop_vec_info loop_vinfo
, gimple
*loop_vectorized_call
)
476 tree arg
= gimple_call_arg (loop_vectorized_call
, 1);
479 struct loop
*scalar_loop
= get_loop (cfun
, tree_to_shwi (arg
));
481 LOOP_VINFO_SCALAR_LOOP (loop_vinfo
) = scalar_loop
;
482 gcc_checking_assert (vect_loop_vectorized_call
483 (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
))
484 == loop_vectorized_call
);
485 bbs
= get_loop_body (scalar_loop
);
486 for (i
= 0; i
< scalar_loop
->num_nodes
; i
++)
488 basic_block bb
= bbs
[i
];
489 gimple_stmt_iterator gsi
;
490 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
492 gimple
*phi
= gsi_stmt (gsi
);
493 gimple_set_uid (phi
, 0);
495 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
497 gimple
*stmt
= gsi_stmt (gsi
);
498 gimple_set_uid (stmt
, 0);
504 /* Function vectorize_loops.
506 Entry point to loop vectorization phase. */
509 vectorize_loops (void)
512 unsigned int num_vectorized_loops
= 0;
513 unsigned int vect_loops_num
;
515 hash_table
<simduid_to_vf
> *simduid_to_vf_htab
= NULL
;
516 hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
= NULL
;
517 bool any_ifcvt_loops
= false;
520 vect_loops_num
= number_of_loops (cfun
);
522 /* Bail out if there are no loops. */
523 if (vect_loops_num
<= 1)
526 if (cfun
->has_simduid_loops
)
527 note_simd_array_uses (&simd_array_to_simduid_htab
);
529 init_stmt_vec_info_vec ();
531 /* ----------- Analyze loops. ----------- */
533 /* If some loop was duplicated, it gets bigger number
534 than all previously defined loops. This fact allows us to run
535 only over initial loops skipping newly generated ones. */
536 FOR_EACH_LOOP (loop
, 0)
537 if (loop
->dont_vectorize
)
538 any_ifcvt_loops
= true;
539 else if ((flag_tree_loop_vectorize
540 && optimize_loop_nest_for_speed_p (loop
))
541 || loop
->force_vectorize
)
543 loop_vec_info loop_vinfo
;
544 vect_location
= find_loop_location (loop
);
545 if (LOCATION_LOCUS (vect_location
) != UNKNOWN_LOCATION
546 && dump_enabled_p ())
547 dump_printf (MSG_NOTE
, "\nAnalyzing loop at %s:%d\n",
548 LOCATION_FILE (vect_location
),
549 LOCATION_LINE (vect_location
));
551 loop_vinfo
= vect_analyze_loop (loop
);
552 loop
->aux
= loop_vinfo
;
554 if (!loop_vinfo
|| !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
))
556 /* Free existing information if loop is analyzed with some
558 if (loop_constraint_set_p (loop
, LOOP_C_FINITE
))
559 vect_free_loop_info_assumptions (loop
);
564 if (!dbg_cnt (vect_loop
))
566 /* We may miss some if-converted loops due to
567 debug counter. Set any_ifcvt_loops to visit
568 them at finalization. */
569 any_ifcvt_loops
= true;
570 /* Free existing information if loop is analyzed with some
572 if (loop_constraint_set_p (loop
, LOOP_C_FINITE
))
573 vect_free_loop_info_assumptions (loop
);
578 gimple
*loop_vectorized_call
= vect_loop_vectorized_call (loop
);
579 if (loop_vectorized_call
)
580 set_uid_loop_bbs (loop_vinfo
, loop_vectorized_call
);
581 if (LOCATION_LOCUS (vect_location
) != UNKNOWN_LOCATION
582 && dump_enabled_p ())
583 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
,
584 "loop vectorized\n");
585 vect_transform_loop (loop_vinfo
);
586 num_vectorized_loops
++;
587 /* Now that the loop has been vectorized, allow it to be unrolled
589 loop
->force_vectorize
= false;
593 simduid_to_vf
*simduid_to_vf_data
= XNEW (simduid_to_vf
);
594 if (!simduid_to_vf_htab
)
595 simduid_to_vf_htab
= new hash_table
<simduid_to_vf
> (15);
596 simduid_to_vf_data
->simduid
= DECL_UID (loop
->simduid
);
597 simduid_to_vf_data
->vf
= loop_vinfo
->vectorization_factor
;
598 *simduid_to_vf_htab
->find_slot (simduid_to_vf_data
, INSERT
)
599 = simduid_to_vf_data
;
602 if (loop_vectorized_call
)
604 fold_loop_vectorized_call (loop_vectorized_call
, boolean_true_node
);
605 ret
|= TODO_cleanup_cfg
;
609 vect_location
= UNKNOWN_LOCATION
;
611 statistics_counter_event (cfun
, "Vectorized loops", num_vectorized_loops
);
612 if (dump_enabled_p ()
613 || (num_vectorized_loops
> 0 && dump_enabled_p ()))
614 dump_printf_loc (MSG_NOTE
, vect_location
,
615 "vectorized %u loops in function.\n",
616 num_vectorized_loops
);
618 /* ----------- Finalize. ----------- */
621 for (i
= 1; i
< vect_loops_num
; i
++)
623 loop
= get_loop (cfun
, i
);
624 if (loop
&& loop
->dont_vectorize
)
626 gimple
*g
= vect_loop_vectorized_call (loop
);
629 fold_loop_vectorized_call (g
, boolean_false_node
);
630 ret
|= TODO_cleanup_cfg
;
635 for (i
= 1; i
< vect_loops_num
; i
++)
637 loop_vec_info loop_vinfo
;
640 loop
= get_loop (cfun
, i
);
643 loop_vinfo
= (loop_vec_info
) loop
->aux
;
644 has_mask_store
= false;
646 has_mask_store
= LOOP_VINFO_HAS_MASK_STORE (loop_vinfo
);
647 destroy_loop_vec_info (loop_vinfo
, true);
649 optimize_mask_stores (loop
);
653 free_stmt_vec_info_vec ();
655 /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
656 if (cfun
->has_simduid_loops
)
657 adjust_simduid_builtins (simduid_to_vf_htab
);
659 /* Shrink any "omp array simd" temporary arrays to the
660 actual vectorization factors. */
661 if (simd_array_to_simduid_htab
)
662 shrink_simd_arrays (simd_array_to_simduid_htab
, simduid_to_vf_htab
);
663 delete simduid_to_vf_htab
;
664 cfun
->has_simduid_loops
= false;
666 if (num_vectorized_loops
> 0)
668 /* If we vectorized any loop only virtual SSA form needs to be updated.
669 ??? Also while we try hard to update loop-closed SSA form we fail
670 to properly do this in some corner-cases (see PR56286). */
671 rewrite_into_loop_closed_ssa (NULL
, TODO_update_ssa_only_virtuals
);
672 return TODO_cleanup_cfg
;
679 /* Entry point to the simduid cleanup pass. */
683 const pass_data pass_data_simduid_cleanup
=
685 GIMPLE_PASS
, /* type */
686 "simduid", /* name */
687 OPTGROUP_NONE
, /* optinfo_flags */
689 ( PROP_ssa
| PROP_cfg
), /* properties_required */
690 0, /* properties_provided */
691 0, /* properties_destroyed */
692 0, /* todo_flags_start */
693 0, /* todo_flags_finish */
696 class pass_simduid_cleanup
: public gimple_opt_pass
699 pass_simduid_cleanup (gcc::context
*ctxt
)
700 : gimple_opt_pass (pass_data_simduid_cleanup
, ctxt
)
703 /* opt_pass methods: */
704 opt_pass
* clone () { return new pass_simduid_cleanup (m_ctxt
); }
705 virtual bool gate (function
*fun
) { return fun
->has_simduid_loops
; }
706 virtual unsigned int execute (function
*);
708 }; // class pass_simduid_cleanup
711 pass_simduid_cleanup::execute (function
*fun
)
713 hash_table
<simd_array_to_simduid
> *simd_array_to_simduid_htab
= NULL
;
715 note_simd_array_uses (&simd_array_to_simduid_htab
);
717 /* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
718 adjust_simduid_builtins (NULL
);
720 /* Shrink any "omp array simd" temporary arrays to the
721 actual vectorization factors. */
722 if (simd_array_to_simduid_htab
)
723 shrink_simd_arrays (simd_array_to_simduid_htab
, NULL
);
724 fun
->has_simduid_loops
= false;
731 make_pass_simduid_cleanup (gcc::context
*ctxt
)
733 return new pass_simduid_cleanup (ctxt
);
737 /* Entry point to basic block SLP phase. */
741 const pass_data pass_data_slp_vectorize
=
743 GIMPLE_PASS
, /* type */
745 OPTGROUP_LOOP
| OPTGROUP_VEC
, /* optinfo_flags */
746 TV_TREE_SLP_VECTORIZATION
, /* tv_id */
747 ( PROP_ssa
| PROP_cfg
), /* properties_required */
748 0, /* properties_provided */
749 0, /* properties_destroyed */
750 0, /* todo_flags_start */
751 TODO_update_ssa
, /* todo_flags_finish */
754 class pass_slp_vectorize
: public gimple_opt_pass
757 pass_slp_vectorize (gcc::context
*ctxt
)
758 : gimple_opt_pass (pass_data_slp_vectorize
, ctxt
)
761 /* opt_pass methods: */
762 opt_pass
* clone () { return new pass_slp_vectorize (m_ctxt
); }
763 virtual bool gate (function
*) { return flag_tree_slp_vectorize
!= 0; }
764 virtual unsigned int execute (function
*);
766 }; // class pass_slp_vectorize
769 pass_slp_vectorize::execute (function
*fun
)
773 bool in_loop_pipeline
= scev_initialized_p ();
774 if (!in_loop_pipeline
)
776 loop_optimizer_init (LOOPS_NORMAL
);
780 /* Mark all stmts as not belonging to the current region and unvisited. */
781 FOR_EACH_BB_FN (bb
, fun
)
783 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
786 gimple
*stmt
= gsi_stmt (gsi
);
787 gimple_set_uid (stmt
, -1);
788 gimple_set_visited (stmt
, false);
792 init_stmt_vec_info_vec ();
794 FOR_EACH_BB_FN (bb
, fun
)
796 if (vect_slp_bb (bb
))
797 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, vect_location
,
798 "basic block vectorized\n");
801 free_stmt_vec_info_vec ();
803 if (!in_loop_pipeline
)
806 loop_optimizer_finalize ();
815 make_pass_slp_vectorize (gcc::context
*ctxt
)
817 return new pass_slp_vectorize (ctxt
);
821 /* Increase alignment of global arrays to improve vectorization potential.
823 - Consider also structs that have an array field.
824 - Use ipa analysis to prune arrays that can't be vectorized?
825 This should involve global alignment analysis and in the future also
828 static unsigned get_vec_alignment_for_type (tree
);
829 static hash_map
<tree
, unsigned> *type_align_map
;
831 /* Return alignment of array's vector type corresponding to scalar type.
832 0 if no vector type exists. */
834 get_vec_alignment_for_array_type (tree type
)
836 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
838 tree vectype
= get_vectype_for_scalar_type (strip_array_types (type
));
841 || TREE_CODE (TYPE_SIZE (type
)) != INTEGER_CST
842 || tree_int_cst_lt (TYPE_SIZE (type
), TYPE_SIZE (vectype
)))
845 return TYPE_ALIGN (vectype
);
848 /* Return alignment of field having maximum alignment of vector type
849 corresponding to it's scalar type. For now, we only consider fields whose
850 offset is a multiple of it's vector alignment.
851 0 if no suitable field is found. */
853 get_vec_alignment_for_record_type (tree type
)
855 gcc_assert (TREE_CODE (type
) == RECORD_TYPE
);
857 unsigned max_align
= 0, alignment
;
858 HOST_WIDE_INT offset
;
861 if (TYPE_PACKED (type
))
864 unsigned *slot
= type_align_map
->get (type
);
868 for (tree field
= first_field (type
);
870 field
= DECL_CHAIN (field
))
872 /* Skip if not FIELD_DECL or if alignment is set by user. */
873 if (TREE_CODE (field
) != FIELD_DECL
874 || DECL_USER_ALIGN (field
)
875 || DECL_ARTIFICIAL (field
))
878 /* We don't need to process the type further if offset is variable,
879 since the offsets of remaining members will also be variable. */
880 if (TREE_CODE (DECL_FIELD_OFFSET (field
)) != INTEGER_CST
881 || TREE_CODE (DECL_FIELD_BIT_OFFSET (field
)) != INTEGER_CST
)
884 /* Similarly stop processing the type if offset_tree
885 does not fit in unsigned HOST_WIDE_INT. */
886 offset_tree
= bit_position (field
);
887 if (!tree_fits_uhwi_p (offset_tree
))
890 offset
= tree_to_uhwi (offset_tree
);
891 alignment
= get_vec_alignment_for_type (TREE_TYPE (field
));
893 /* Get maximum alignment of vectorized field/array among those members
894 whose offset is multiple of the vector alignment. */
896 && (offset
% alignment
== 0)
897 && (alignment
> max_align
))
898 max_align
= alignment
;
901 type_align_map
->put (type
, max_align
);
905 /* Return alignment of vector type corresponding to decl's scalar type
906 or 0 if it doesn't exist or the vector alignment is lesser than
909 get_vec_alignment_for_type (tree type
)
911 if (type
== NULL_TREE
)
914 gcc_assert (TYPE_P (type
));
916 static unsigned alignment
= 0;
917 switch (TREE_CODE (type
))
920 alignment
= get_vec_alignment_for_array_type (type
);
923 alignment
= get_vec_alignment_for_record_type (type
);
930 return (alignment
> TYPE_ALIGN (type
)) ? alignment
: 0;
933 /* Entry point to increase_alignment pass. */
935 increase_alignment (void)
939 vect_location
= UNKNOWN_LOCATION
;
940 type_align_map
= new hash_map
<tree
, unsigned>;
942 /* Increase the alignment of all global arrays for vectorization. */
943 FOR_EACH_DEFINED_VARIABLE (vnode
)
945 tree decl
= vnode
->decl
;
946 unsigned int alignment
;
948 if ((decl_in_symtab_p (decl
)
949 && !symtab_node::get (decl
)->can_increase_alignment_p ())
950 || DECL_USER_ALIGN (decl
) || DECL_ARTIFICIAL (decl
))
953 alignment
= get_vec_alignment_for_type (TREE_TYPE (decl
));
954 if (alignment
&& vect_can_force_dr_alignment_p (decl
, alignment
))
956 vnode
->increase_alignment (alignment
);
957 dump_printf (MSG_NOTE
, "Increasing alignment of decl: ");
958 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, decl
);
959 dump_printf (MSG_NOTE
, "\n");
963 delete type_align_map
;
970 const pass_data pass_data_ipa_increase_alignment
=
972 SIMPLE_IPA_PASS
, /* type */
973 "increase_alignment", /* name */
974 OPTGROUP_LOOP
| OPTGROUP_VEC
, /* optinfo_flags */
975 TV_IPA_OPT
, /* tv_id */
976 0, /* properties_required */
977 0, /* properties_provided */
978 0, /* properties_destroyed */
979 0, /* todo_flags_start */
980 0, /* todo_flags_finish */
983 class pass_ipa_increase_alignment
: public simple_ipa_opt_pass
986 pass_ipa_increase_alignment (gcc::context
*ctxt
)
987 : simple_ipa_opt_pass (pass_data_ipa_increase_alignment
, ctxt
)
990 /* opt_pass methods: */
991 virtual bool gate (function
*)
993 return flag_section_anchors
&& flag_tree_loop_vectorize
;
996 virtual unsigned int execute (function
*) { return increase_alignment (); }
998 }; // class pass_ipa_increase_alignment
1002 simple_ipa_opt_pass
*
1003 make_pass_ipa_increase_alignment (gcc::context
*ctxt
)
1005 return new pass_ipa_increase_alignment (ctxt
);