2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #ifndef GCC_TREE_VECTORIZER_H
23 #define GCC_TREE_VECTORIZER_H
25 #include "tree-data-ref.h"
28 typedef source_location LOC
;
29 #define UNKNOWN_LOC UNKNOWN_LOCATION
30 #define EXPR_LOC(e) EXPR_LOCATION(e)
31 #define LOC_FILE(l) LOCATION_FILE (l)
32 #define LOC_LINE(l) LOCATION_LINE (l)
34 /* Used for naming of new temporaries. */
41 /* Defines type of operation. */
48 /* Define type of available alignment support. */
49 enum dr_alignment_support
{
50 dr_unaligned_unsupported
,
51 dr_unaligned_supported
,
53 dr_explicit_realign_optimized
,
57 /* Define type of def-use cross-iteration cycle. */
59 vect_uninitialized_def
= 0,
60 vect_constant_def
= 1,
65 vect_double_reduction_def
,
70 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
71 || ((D) == vect_double_reduction_def) \
72 || ((D) == vect_nested_cycle))
74 /* Structure to encapsulate information about a group of like
75 instructions to be presented to the target cost model. */
76 typedef struct _stmt_info_for_cost
{
78 enum vect_cost_for_stmt kind
;
83 DEF_VEC_O (stmt_info_for_cost
);
84 DEF_VEC_ALLOC_O (stmt_info_for_cost
, heap
);
86 typedef VEC(stmt_info_for_cost
, heap
) *stmt_vector_for_cost
;
89 add_stmt_info_to_vec (stmt_vector_for_cost
*stmt_cost_vec
, int count
,
90 enum vect_cost_for_stmt kind
, gimple stmt
, int misalign
)
92 stmt_info_for_cost si
;
96 si
.misalign
= misalign
;
97 VEC_safe_push (stmt_info_for_cost
, heap
, *stmt_cost_vec
, &si
);
100 /************************************************************************
102 ************************************************************************/
103 typedef void *slp_void_p
;
104 DEF_VEC_P (slp_void_p
);
105 DEF_VEC_ALLOC_P (slp_void_p
, heap
);
107 /* A computation tree of an SLP instance. Each node corresponds to a group of
108 stmts to be packed in a SIMD stmt. */
109 typedef struct _slp_tree
{
110 /* Nodes that contain def-stmts of this node statements operands. */
111 VEC (slp_void_p
, heap
) *children
;
112 /* A group of scalar stmts to be vectorized together. */
113 VEC (gimple
, heap
) *stmts
;
114 /* Vectorized stmt/s. */
115 VEC (gimple
, heap
) *vec_stmts
;
116 /* Number of vector stmts that are created to replace the group of scalar
117 stmts. It is calculated during the transformation phase as the number of
118 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
119 divided by vector size. */
120 unsigned int vec_stmts_size
;
121 /* Vectorization costs associated with SLP node. */
124 int outside_of_loop
; /* Statements generated outside loop. */
129 DEF_VEC_ALLOC_P(slp_tree
, heap
);
131 /* SLP instance is a sequence of stmts in a loop that can be packed into
133 typedef struct _slp_instance
{
134 /* The root of SLP tree. */
137 /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
138 unsigned int group_size
;
140 /* The unrolling factor required to vectorized this SLP instance. */
141 unsigned int unrolling_factor
;
143 /* Vectorization costs associated with SLP instance. */
146 int outside_of_loop
; /* Statements generated outside loop. */
149 /* Inside-loop costs. */
150 stmt_vector_for_cost stmt_cost_vec
;
152 /* Loads permutation relatively to the stores, NULL if there is no
154 VEC (int, heap
) *load_permutation
;
156 /* The group of nodes that contain loads of this SLP instance. */
157 VEC (slp_tree
, heap
) *loads
;
159 /* The first scalar load of the instance. The created vector loads will be
160 inserted before this statement. */
164 DEF_VEC_P(slp_instance
);
165 DEF_VEC_ALLOC_P(slp_instance
, heap
);
167 /* Access Functions. */
168 #define SLP_INSTANCE_TREE(S) (S)->root
169 #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
170 #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
171 #define SLP_INSTANCE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
172 #define SLP_INSTANCE_STMT_COST_VEC(S) (S)->stmt_cost_vec
173 #define SLP_INSTANCE_LOAD_PERMUTATION(S) (S)->load_permutation
174 #define SLP_INSTANCE_LOADS(S) (S)->loads
175 #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load
177 #define SLP_TREE_CHILDREN(S) (S)->children
178 #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
179 #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
180 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
181 #define SLP_TREE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
183 /* This structure is used in creation of an SLP tree. Each instance
184 corresponds to the same operand in a group of scalar stmts in an SLP
186 typedef struct _slp_oprnd_info
188 /* Def-stmts for the operands. */
189 VEC (gimple
, heap
) *def_stmts
;
190 /* Information about the first statement, its vector def-type, type, the
191 operand itself in case it's constant, and an indication if it's a pattern
193 enum vect_def_type first_dt
;
195 tree first_const_oprnd
;
199 DEF_VEC_P(slp_oprnd_info
);
200 DEF_VEC_ALLOC_P(slp_oprnd_info
, heap
);
203 typedef struct _vect_peel_info
206 struct data_reference
*dr
;
210 typedef struct _vect_peel_extended_info
212 struct _vect_peel_info peel_info
;
213 unsigned int inside_cost
;
214 unsigned int outside_cost
;
215 stmt_vector_for_cost stmt_cost_vec
;
216 } *vect_peel_extended_info
;
218 /*-----------------------------------------------------------------*/
219 /* Info on vectorized loops. */
220 /*-----------------------------------------------------------------*/
221 typedef struct _loop_vec_info
{
223 /* The loop to which this info struct refers to. */
226 /* The loop basic blocks. */
229 /* Number of iterations. */
231 tree num_iters_unchanged
;
233 /* Minimum number of iterations below which vectorization is expected to
234 not be profitable (as estimated by the cost model).
235 -1 indicates that vectorization will not be profitable.
236 FORNOW: This field is an int. Will be a tree in the future, to represent
237 values unknown at compile time. */
238 int min_profitable_iters
;
240 /* Is the loop vectorizable? */
243 /* Unrolling factor */
244 int vectorization_factor
;
246 /* The loop location in the source. */
247 LOC loop_line_number
;
249 /* Unknown DRs according to which loop was peeled. */
250 struct data_reference
*unaligned_dr
;
252 /* peeling_for_alignment indicates whether peeling for alignment will take
253 place, and what the peeling factor should be:
254 peeling_for_alignment = X means:
255 If X=0: Peeling for alignment will not be applied.
256 If X>0: Peel first X iterations.
257 If X=-1: Generate a runtime test to calculate the number of iterations
258 to be peeled, using the dataref recorded in the field
260 int peeling_for_alignment
;
262 /* The mask used to check the alignment of pointers or arrays. */
265 /* The loop nest in which the data dependences are computed. */
266 VEC (loop_p
, heap
) *loop_nest
;
268 /* All data references in the loop. */
269 VEC (data_reference_p
, heap
) *datarefs
;
271 /* All data dependences in the loop. */
272 VEC (ddr_p
, heap
) *ddrs
;
274 /* Data Dependence Relations defining address ranges that are candidates
275 for a run-time aliasing check. */
276 VEC (ddr_p
, heap
) *may_alias_ddrs
;
278 /* Statements in the loop that have data references that are candidates for a
279 runtime (loop versioning) misalignment check. */
280 VEC(gimple
,heap
) *may_misalign_stmts
;
282 /* All interleaving chains of stores in the loop, represented by the first
283 stmt in the chain. */
284 VEC(gimple
, heap
) *grouped_stores
;
286 /* All SLP instances in the loop. This is a subset of the set of GROUP_STORES
288 VEC(slp_instance
, heap
) *slp_instances
;
290 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
291 applied to the loop, i.e., no unrolling is needed, this is 1. */
292 unsigned slp_unrolling_factor
;
294 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
295 VEC (gimple
, heap
) *reductions
;
297 /* All reduction chains in the loop, represented by the first
298 stmt in the chain. */
299 VEC (gimple
, heap
) *reduction_chains
;
301 /* Hash table used to choose the best peeling option. */
304 /* Cost data used by the target cost model. */
305 void *target_cost_data
;
307 /* When we have grouped data accesses with gaps, we may introduce invalid
308 memory accesses. We peel the last iteration of the loop to prevent
310 bool peeling_for_gaps
;
314 /* Access Functions. */
315 #define LOOP_VINFO_LOOP(L) (L)->loop
316 #define LOOP_VINFO_BBS(L) (L)->bbs
317 #define LOOP_VINFO_NITERS(L) (L)->num_iters
318 /* Since LOOP_VINFO_NITERS can change after prologue peeling
319 retain total unchanged scalar loop iterations for cost model. */
320 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
321 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
322 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
323 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
324 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
325 #define LOOP_VINFO_LOOP_NEST(L) (L)->loop_nest
326 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs
327 #define LOOP_VINFO_DDRS(L) (L)->ddrs
328 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
329 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
330 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
331 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
332 #define LOOP_VINFO_LOC(L) (L)->loop_line_number
333 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
334 #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
335 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
336 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
337 #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
338 #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
339 #define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab
340 #define LOOP_VINFO_TARGET_COST_DATA(L) (L)->target_cost_data
341 #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
343 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
344 VEC_length (gimple, (L)->may_misalign_stmts) > 0
345 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
346 VEC_length (ddr_p, (L)->may_alias_ddrs) > 0
348 #define NITERS_KNOWN_P(n) \
349 (host_integerp ((n),0) \
350 && TREE_INT_CST_LOW ((n)) > 0)
352 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
353 NITERS_KNOWN_P((L)->num_iters)
355 static inline loop_vec_info
356 loop_vec_info_for_loop (struct loop
*loop
)
358 return (loop_vec_info
) loop
->aux
;
362 nested_in_vect_loop_p (struct loop
*loop
, gimple stmt
)
365 && (loop
->inner
== (gimple_bb (stmt
))->loop_father
));
368 typedef struct _bb_vec_info
{
371 /* All interleaving chains of stores in the basic block, represented by the
372 first stmt in the chain. */
373 VEC(gimple
, heap
) *grouped_stores
;
375 /* All SLP instances in the basic block. This is a subset of the set of
376 GROUP_STORES of the basic block. */
377 VEC(slp_instance
, heap
) *slp_instances
;
379 /* All data references in the basic block. */
380 VEC (data_reference_p
, heap
) *datarefs
;
382 /* All data dependences in the basic block. */
383 VEC (ddr_p
, heap
) *ddrs
;
385 /* Cost data used by the target cost model. */
386 void *target_cost_data
;
390 #define BB_VINFO_BB(B) (B)->bb
391 #define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
392 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
393 #define BB_VINFO_DATAREFS(B) (B)->datarefs
394 #define BB_VINFO_DDRS(B) (B)->ddrs
395 #define BB_VINFO_TARGET_COST_DATA(B) (B)->target_cost_data
397 static inline bb_vec_info
398 vec_info_for_bb (basic_block bb
)
400 return (bb_vec_info
) bb
->aux
;
403 /*-----------------------------------------------------------------*/
404 /* Info on vectorized defs. */
405 /*-----------------------------------------------------------------*/
406 enum stmt_vec_info_type
{
407 undef_vec_info_type
= 0,
413 assignment_vec_info_type
,
414 condition_vec_info_type
,
417 type_promotion_vec_info_type
,
418 type_demotion_vec_info_type
,
419 type_conversion_vec_info_type
,
420 loop_exit_ctrl_vec_info_type
423 /* Indicates whether/how a variable is used in the scope of loop/basic
426 vect_unused_in_scope
= 0,
427 /* The def is in the inner loop, and the use is in the outer loop, and the
428 use is a reduction stmt. */
429 vect_used_in_outer_by_reduction
,
430 /* The def is in the inner loop, and the use is in the outer loop (and is
431 not part of reduction). */
434 /* defs that feed computations that end up (only) in a reduction. These
435 defs may be used by non-reduction stmts, but eventually, any
436 computations/values that are affected by these defs are used to compute
437 a reduction (i.e. don't get stored to memory, for example). We use this
438 to identify computations that we can change the order in which they are
440 vect_used_by_reduction
,
445 /* The type of vectorization that can be applied to the stmt: regular loop-based
446 vectorization; pure SLP - the stmt is a part of SLP instances and does not
447 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
448 a part of SLP instance and also must be loop-based vectorized, since it has
449 uses outside SLP sequences.
451 In the loop context the meanings of pure and hybrid SLP are slightly
452 different. By saying that pure SLP is applied to the loop, we mean that we
453 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
454 vectorized without doing any conceptual unrolling, cause we don't pack
455 together stmts from different iterations, only within a single iteration.
456 Loop hybrid SLP means that we exploit both intra-iteration and
457 inter-iteration parallelism (e.g., number of elements in the vector is 4
458 and the slp-group-size is 2, in which case we don't have enough parallelism
459 within an iteration, so we obtain the rest of the parallelism from subsequent
460 iterations by unrolling the loop by 2). */
468 typedef struct data_reference
*dr_p
;
470 DEF_VEC_ALLOC_P(dr_p
,heap
);
472 typedef struct _stmt_vec_info
{
474 enum stmt_vec_info_type type
;
476 /* Indicates whether this stmts is part of a computation whose result is
477 used outside the loop. */
480 /* Stmt is part of some pattern (computation idiom) */
483 /* For loads only, if there is a store with the same location, this field is
487 /* The stmt to which this info struct refers to. */
490 /* The loop_vec_info with respect to which STMT is vectorized. */
491 loop_vec_info loop_vinfo
;
493 /* The vector type to be used for the LHS of this statement. */
496 /* The vectorized version of the stmt. */
497 gimple vectorized_stmt
;
500 /** The following is relevant only for stmts that contain a non-scalar
501 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
502 at most one such data-ref. **/
504 /* Information about the data-ref (access function, etc),
505 relative to the inner-most containing loop. */
506 struct data_reference
*data_ref_info
;
508 /* Information about the data-ref relative to this loop
509 nest (the loop that is being considered for vectorization). */
510 tree dr_base_address
;
516 /* For loop PHI nodes, the evolution part of it. This makes sure
517 this information is still available in vect_update_ivs_after_vectorizer
518 where we may not be able to re-analyze the PHI nodes evolution as
519 peeling for the prologue loop can make it unanalyzable. The evolution
520 part is still correct though. */
521 tree loop_phi_evolution_part
;
523 /* Used for various bookkeeping purposes, generally holding a pointer to
524 some other stmt S that is in some way "related" to this stmt.
525 Current use of this field is:
526 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
527 true): S is the "pattern stmt" that represents (and replaces) the
528 sequence of stmts that constitutes the pattern. Similarly, the
529 related_stmt of the "pattern stmt" points back to this stmt (which is
530 the last stmt in the original sequence of stmts that constitutes the
534 /* Used to keep a sequence of def stmts of a pattern stmt if such exists. */
535 gimple_seq pattern_def_seq
;
537 /* List of datarefs that are known to have the same alignment as the dataref
539 VEC(dr_p
,heap
) *same_align_refs
;
541 /* Classify the def of this stmt. */
542 enum vect_def_type def_type
;
544 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
545 enum slp_vect_type slp_type
;
547 /* Interleaving and reduction chains info. */
548 /* First element in the group. */
549 gimple first_element
;
550 /* Pointer to the next element in the group. */
552 /* For data-refs, in case that two or more stmts share data-ref, this is the
553 pointer to the previously detected stmt with the same dr. */
555 /* The size of the group. */
557 /* For stores, number of stores from this group seen. We vectorize the last
559 unsigned int store_count
;
560 /* For loads only, the gap from the previous load. For consecutive loads, GAP
564 /* Not all stmts in the loop need to be vectorized. e.g, the increment
565 of the loop induction variable and computation of array indexes. relevant
566 indicates whether the stmt needs to be vectorized. */
567 enum vect_relevant relevant
;
569 /* Vectorization costs associated with statement. */
572 int outside_of_loop
; /* Statements generated outside loop. */
575 /* The bb_vec_info with respect to which STMT is vectorized. */
576 bb_vec_info bb_vinfo
;
578 /* Is this statement vectorizable or should it be skipped in (partial)
582 /* For loads only, true if this is a gather load. */
587 /* Access Functions. */
588 #define STMT_VINFO_TYPE(S) (S)->type
589 #define STMT_VINFO_STMT(S) (S)->stmt
590 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo
591 #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo
592 #define STMT_VINFO_RELEVANT(S) (S)->relevant
593 #define STMT_VINFO_LIVE_P(S) (S)->live
594 #define STMT_VINFO_VECTYPE(S) (S)->vectype
595 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
596 #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
597 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info
598 #define STMT_VINFO_GATHER_P(S) (S)->gather_p
599 #define STMT_VINFO_STRIDE_LOAD_P(S) (S)->stride_load_p
601 #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address
602 #define STMT_VINFO_DR_INIT(S) (S)->dr_init
603 #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset
604 #define STMT_VINFO_DR_STEP(S) (S)->dr_step
605 #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to
607 #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
608 #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
609 #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
610 #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
611 #define STMT_VINFO_DEF_TYPE(S) (S)->def_type
612 #define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element
613 #define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element
614 #define STMT_VINFO_GROUP_SIZE(S) (S)->size
615 #define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count
616 #define STMT_VINFO_GROUP_GAP(S) (S)->gap
617 #define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
618 #define STMT_VINFO_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
619 #define STMT_VINFO_GROUPED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info)
620 #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
622 #define GROUP_FIRST_ELEMENT(S) (S)->first_element
623 #define GROUP_NEXT_ELEMENT(S) (S)->next_element
624 #define GROUP_SIZE(S) (S)->size
625 #define GROUP_STORE_COUNT(S) (S)->store_count
626 #define GROUP_GAP(S) (S)->gap
627 #define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
628 #define GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
630 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
631 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
633 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
634 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
635 #define STMT_SLP_TYPE(S) (S)->slp_type
637 #define VECT_MAX_COST 1000
639 /* The maximum number of intermediate steps required in multi-step type
641 #define MAX_INTERM_CVT_STEPS 3
643 /* The maximum vectorization factor supported by any target (V32QI). */
644 #define MAX_VECTORIZATION_FACTOR 32
646 /* Avoid GTY(()) on stmt_vec_info. */
647 typedef void *vec_void_p
;
648 DEF_VEC_P (vec_void_p
);
649 DEF_VEC_ALLOC_P (vec_void_p
, heap
);
651 extern VEC(vec_void_p
,heap
) *stmt_vec_info_vec
;
653 void init_stmt_vec_info_vec (void);
654 void free_stmt_vec_info_vec (void);
656 /* Return a stmt_vec_info corresponding to STMT. */
658 static inline stmt_vec_info
659 vinfo_for_stmt (gimple stmt
)
661 unsigned int uid
= gimple_uid (stmt
);
665 return (stmt_vec_info
) VEC_index (vec_void_p
, stmt_vec_info_vec
, uid
- 1);
668 /* Set vectorizer information INFO for STMT. */
671 set_vinfo_for_stmt (gimple stmt
, stmt_vec_info info
)
673 unsigned int uid
= gimple_uid (stmt
);
676 gcc_checking_assert (info
);
677 uid
= VEC_length (vec_void_p
, stmt_vec_info_vec
) + 1;
678 gimple_set_uid (stmt
, uid
);
679 VEC_safe_push (vec_void_p
, heap
, stmt_vec_info_vec
, (vec_void_p
) info
);
682 VEC_replace (vec_void_p
, stmt_vec_info_vec
, uid
- 1, (vec_void_p
) info
);
685 /* Return the earlier statement between STMT1 and STMT2. */
688 get_earlier_stmt (gimple stmt1
, gimple stmt2
)
690 unsigned int uid1
, uid2
;
698 uid1
= gimple_uid (stmt1
);
699 uid2
= gimple_uid (stmt2
);
701 if (uid1
== 0 || uid2
== 0)
704 gcc_checking_assert (uid1
<= VEC_length (vec_void_p
, stmt_vec_info_vec
)
705 && uid2
<= VEC_length (vec_void_p
, stmt_vec_info_vec
));
713 /* Return the later statement between STMT1 and STMT2. */
716 get_later_stmt (gimple stmt1
, gimple stmt2
)
718 unsigned int uid1
, uid2
;
726 uid1
= gimple_uid (stmt1
);
727 uid2
= gimple_uid (stmt2
);
729 if (uid1
== 0 || uid2
== 0)
732 gcc_assert (uid1
<= VEC_length (vec_void_p
, stmt_vec_info_vec
));
733 gcc_assert (uid2
<= VEC_length (vec_void_p
, stmt_vec_info_vec
));
741 /* Return TRUE if a statement represented by STMT_INFO is a part of a
745 is_pattern_stmt_p (stmt_vec_info stmt_info
)
748 stmt_vec_info related_stmt_info
;
750 related_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
752 && (related_stmt_info
= vinfo_for_stmt (related_stmt
))
753 && STMT_VINFO_IN_PATTERN_P (related_stmt_info
))
759 /* Return true if BB is a loop header. */
762 is_loop_header_bb_p (basic_block bb
)
764 if (bb
== (bb
->loop_father
)->header
)
766 gcc_checking_assert (EDGE_COUNT (bb
->preds
) == 1);
770 /* Set outside loop vectorization cost. */
773 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info
, slp_tree slp_node
,
777 SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node
) = cost
;
779 STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info
) = cost
;
782 /* Return pow2 (X). */
789 for (i
= 0; i
< x
; i
++)
795 /* Get cost by calling cost target builtin. */
798 int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost
)
800 tree dummy_type
= NULL
;
803 return targetm
.vectorize
.builtin_vectorization_cost (type_of_cost
,
807 /* Alias targetm.vectorize.init_cost. */
810 init_cost (struct loop
*loop_info
)
812 return targetm
.vectorize
.init_cost (loop_info
);
815 /* Alias targetm.vectorize.add_stmt_cost. */
817 static inline unsigned
818 add_stmt_cost (void *data
, int count
, enum vect_cost_for_stmt kind
,
819 stmt_vec_info stmt_info
, int misalign
)
821 return targetm
.vectorize
.add_stmt_cost (data
, count
, kind
,
822 stmt_info
, misalign
);
825 /* Alias targetm.vectorize.finish_cost. */
827 static inline unsigned
828 finish_cost (void *data
)
830 return targetm
.vectorize
.finish_cost (data
);
833 /* Alias targetm.vectorize.destroy_cost_data. */
836 destroy_cost_data (void *data
)
838 targetm
.vectorize
.destroy_cost_data (data
);
842 /*-----------------------------------------------------------------*/
843 /* Info on data references alignment. */
844 /*-----------------------------------------------------------------*/
846 /* Reflects actual alignment of first access in the vectorized loop,
847 taking into account peeling/versioning if applied. */
848 #define DR_MISALIGNMENT(DR) ((int) (size_t) (DR)->aux)
849 #define SET_DR_MISALIGNMENT(DR, VAL) ((DR)->aux = (void *) (size_t) (VAL))
851 /* Return TRUE if the data access is aligned, and FALSE otherwise. */
854 aligned_access_p (struct data_reference
*data_ref_info
)
856 return (DR_MISALIGNMENT (data_ref_info
) == 0);
859 /* Return TRUE if the alignment of the data access is known, and FALSE
863 known_alignment_for_access_p (struct data_reference
*data_ref_info
)
865 return (DR_MISALIGNMENT (data_ref_info
) != -1);
868 /* vect_dump will be set to stderr or dump_file if exist. */
869 extern FILE *vect_dump
;
870 extern LOC vect_loop_location
;
872 /*-----------------------------------------------------------------*/
873 /* Function prototypes. */
874 /*-----------------------------------------------------------------*/
876 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
877 in tree-vect-loop-manip.c. */
878 extern void slpeel_make_loop_iterate_ntimes (struct loop
*, tree
);
879 extern bool slpeel_can_duplicate_loop_p (const struct loop
*, const_edge
);
880 extern void vect_loop_versioning (loop_vec_info
, unsigned int, bool);
881 extern void vect_do_peeling_for_loop_bound (loop_vec_info
, tree
*,
883 extern void vect_do_peeling_for_alignment (loop_vec_info
, unsigned int, bool);
884 extern LOC
find_loop_location (struct loop
*);
885 extern bool vect_can_advance_ivs_p (loop_vec_info
);
887 /* In tree-vect-stmts.c. */
888 extern unsigned int current_vector_size
;
889 extern tree
get_vectype_for_scalar_type (tree
);
890 extern tree
get_same_sized_vectype (tree
, tree
);
891 extern bool vect_is_simple_use (tree
, gimple
, loop_vec_info
,
892 bb_vec_info
, gimple
*,
893 tree
*, enum vect_def_type
*);
894 extern bool vect_is_simple_use_1 (tree
, gimple
, loop_vec_info
,
895 bb_vec_info
, gimple
*,
896 tree
*, enum vect_def_type
*, tree
*);
897 extern bool supportable_widening_operation (enum tree_code
, gimple
, tree
, tree
,
898 enum tree_code
*, enum tree_code
*,
899 int *, VEC (tree
, heap
) **);
900 extern bool supportable_narrowing_operation (enum tree_code
, tree
, tree
,
902 int *, VEC (tree
, heap
) **);
903 extern stmt_vec_info
new_stmt_vec_info (gimple stmt
, loop_vec_info
,
905 extern void free_stmt_vec_info (gimple stmt
);
906 extern tree
vectorizable_function (gimple
, tree
, tree
);
907 extern void vect_model_simple_cost (stmt_vec_info
, int, enum vect_def_type
*,
908 slp_tree
, stmt_vector_for_cost
*);
909 extern void vect_model_store_cost (stmt_vec_info
, int, bool,
910 enum vect_def_type
, slp_tree
,
911 stmt_vector_for_cost
*);
912 extern void vect_model_load_cost (stmt_vec_info
, int, bool, slp_tree
,
913 stmt_vector_for_cost
*);
914 extern unsigned record_stmt_cost (stmt_vector_for_cost
*, int,
915 enum vect_cost_for_stmt
, stmt_vec_info
, int);
916 extern void vect_finish_stmt_generation (gimple
, gimple
,
917 gimple_stmt_iterator
*);
918 extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info
);
919 extern tree
vect_get_vec_def_for_operand (tree
, gimple
, tree
*);
920 extern tree
vect_init_vector (gimple
, tree
, tree
,
921 gimple_stmt_iterator
*);
922 extern tree
vect_get_vec_def_for_stmt_copy (enum vect_def_type
, tree
);
923 extern bool vect_transform_stmt (gimple
, gimple_stmt_iterator
*,
924 bool *, slp_tree
, slp_instance
);
925 extern void vect_remove_stores (gimple
);
926 extern bool vect_analyze_stmt (gimple
, bool *, slp_tree
);
927 extern bool vectorizable_condition (gimple
, gimple_stmt_iterator
*, gimple
*,
928 tree
, int, slp_tree
);
929 extern void vect_get_load_cost (struct data_reference
*, int, bool,
930 unsigned int *, unsigned int *,
931 stmt_vector_for_cost
*);
932 extern void vect_get_store_cost (struct data_reference
*, int,
933 unsigned int *, stmt_vector_for_cost
*);
934 extern bool vect_supportable_shift (enum tree_code
, tree
);
935 extern void vect_get_vec_defs (tree
, tree
, gimple
, VEC (tree
, heap
) **,
936 VEC (tree
, heap
) **, slp_tree
, int);
937 extern tree
vect_gen_perm_mask (tree
, unsigned char *);
939 /* In tree-vect-data-refs.c. */
940 extern bool vect_can_force_dr_alignment_p (const_tree
, unsigned int);
941 extern enum dr_alignment_support vect_supportable_dr_alignment
942 (struct data_reference
*, bool);
943 extern tree
vect_get_smallest_scalar_type (gimple
, HOST_WIDE_INT
*,
945 extern bool vect_analyze_data_ref_dependences (loop_vec_info
, bb_vec_info
,
947 extern bool vect_enhance_data_refs_alignment (loop_vec_info
);
948 extern bool vect_analyze_data_refs_alignment (loop_vec_info
, bb_vec_info
);
949 extern bool vect_verify_datarefs_alignment (loop_vec_info
, bb_vec_info
);
950 extern bool vect_analyze_data_ref_accesses (loop_vec_info
, bb_vec_info
);
951 extern bool vect_prune_runtime_alias_test_list (loop_vec_info
);
952 extern tree
vect_check_gather (gimple
, loop_vec_info
, tree
*, tree
*,
954 extern bool vect_check_strided_load (gimple
, loop_vec_info
, tree
*, tree
*);
955 extern bool vect_analyze_data_refs (loop_vec_info
, bb_vec_info
, int *);
956 extern tree
vect_create_data_ref_ptr (gimple
, tree
, struct loop
*, tree
,
957 tree
*, gimple_stmt_iterator
*,
958 gimple
*, bool, bool *);
959 extern tree
bump_vector_ptr (tree
, gimple
, gimple_stmt_iterator
*, gimple
, tree
);
960 extern tree
vect_create_destination_var (tree
, tree
);
961 extern bool vect_grouped_store_supported (tree
, unsigned HOST_WIDE_INT
);
962 extern bool vect_store_lanes_supported (tree
, unsigned HOST_WIDE_INT
);
963 extern bool vect_grouped_load_supported (tree
, unsigned HOST_WIDE_INT
);
964 extern bool vect_load_lanes_supported (tree
, unsigned HOST_WIDE_INT
);
965 extern void vect_permute_store_chain (VEC(tree
,heap
) *,unsigned int, gimple
,
966 gimple_stmt_iterator
*, VEC(tree
,heap
) **);
967 extern tree
vect_setup_realignment (gimple
, gimple_stmt_iterator
*, tree
*,
968 enum dr_alignment_support
, tree
,
970 extern void vect_transform_grouped_load (gimple
, VEC(tree
,heap
) *, int,
971 gimple_stmt_iterator
*);
972 extern void vect_record_grouped_load_vectors (gimple
, VEC(tree
,heap
) *);
973 extern int vect_get_place_in_interleaving_chain (gimple
, gimple
);
974 extern tree
vect_get_new_vect_var (tree
, enum vect_var_kind
, const char *);
975 extern tree
vect_create_addr_base_for_vector_ref (gimple
, gimple_seq
*,
976 tree
, struct loop
*);
978 /* In tree-vect-loop.c. */
979 /* FORNOW: Used in tree-parloops.c. */
980 extern void destroy_loop_vec_info (loop_vec_info
, bool);
981 extern gimple
vect_force_simple_reduction (loop_vec_info
, gimple
, bool, bool *);
982 /* Drive for loop analysis stage. */
983 extern loop_vec_info
vect_analyze_loop (struct loop
*);
984 /* Drive for loop transformation stage. */
985 extern void vect_transform_loop (loop_vec_info
);
986 extern loop_vec_info
vect_analyze_loop_form (struct loop
*);
987 extern bool vectorizable_live_operation (gimple
, gimple_stmt_iterator
*,
989 extern bool vectorizable_reduction (gimple
, gimple_stmt_iterator
*, gimple
*,
991 extern bool vectorizable_induction (gimple
, gimple_stmt_iterator
*, gimple
*);
992 extern int vect_estimate_min_profitable_iters (loop_vec_info
);
993 extern tree
get_initial_def_for_reduction (gimple
, tree
, tree
*);
994 extern int vect_min_worthwhile_factor (enum tree_code
);
995 extern int vect_get_known_peeling_cost (loop_vec_info
, int, int *, int);
996 extern int vect_get_single_scalar_iteration_cost (loop_vec_info
);
998 /* In tree-vect-slp.c. */
999 extern void vect_free_slp_instance (slp_instance
);
1000 extern bool vect_transform_slp_perm_load (gimple
, VEC (tree
, heap
) *,
1001 gimple_stmt_iterator
*, int,
1002 slp_instance
, bool);
1003 extern bool vect_schedule_slp (loop_vec_info
, bb_vec_info
);
1004 extern void vect_update_slp_costs_according_to_vf (loop_vec_info
);
1005 extern bool vect_analyze_slp (loop_vec_info
, bb_vec_info
);
1006 extern bool vect_make_slp_decision (loop_vec_info
);
1007 extern void vect_detect_hybrid_slp (loop_vec_info
);
1008 extern void vect_get_slp_defs (VEC (tree
, heap
) *, slp_tree
,
1009 VEC (slp_void_p
, heap
) **, int);
1011 extern LOC
find_bb_location (basic_block
);
1012 extern bb_vec_info
vect_slp_analyze_bb (basic_block
);
1013 extern void vect_slp_transform_bb (basic_block
);
1015 /* In tree-vect-patterns.c. */
1016 /* Pattern recognition functions.
1017 Additional pattern recognition functions can (and will) be added
1019 typedef gimple (* vect_recog_func_ptr
) (VEC (gimple
, heap
) **, tree
*, tree
*);
1020 #define NUM_PATTERNS 10
1021 void vect_pattern_recog (loop_vec_info
, bb_vec_info
);
1023 /* In tree-vectorizer.c. */
1024 unsigned vectorize_loops (void);
1025 /* Vectorization debug information */
1026 extern bool vect_print_dump_info (enum vect_verbosity_levels
);
1028 #endif /* GCC_TREE_VECTORIZER_H */