2010-07-27 Paolo Carlini <paolo.carlini@oracle.com>
[official-gcc/alias-decl.git] / gcc / tree-vectorizer.h
blobed8ff58312aa6849e02dc021c4d2a4741e6d71c9
1 /* Vectorizer
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 #ifndef GCC_TREE_VECTORIZER_H
23 #define GCC_TREE_VECTORIZER_H
25 #include "tree-data-ref.h"
27 typedef source_location LOC;
28 #define UNKNOWN_LOC UNKNOWN_LOCATION
29 #define EXPR_LOC(e) EXPR_LOCATION(e)
30 #define LOC_FILE(l) LOCATION_FILE (l)
31 #define LOC_LINE(l) LOCATION_LINE (l)
33 /* Used for naming of new temporaries. */
34 enum vect_var_kind {
35 vect_simple_var,
36 vect_pointer_var,
37 vect_scalar_var
40 /* Defines type of operation. */
41 enum operation_type {
42 unary_op = 1,
43 binary_op,
44 ternary_op
47 /* Define type of available alignment support. */
48 enum dr_alignment_support {
49 dr_unaligned_unsupported,
50 dr_unaligned_supported,
51 dr_explicit_realign,
52 dr_explicit_realign_optimized,
53 dr_aligned
56 /* Define type of def-use cross-iteration cycle. */
57 enum vect_def_type {
58 vect_uninitialized_def = 0,
59 vect_constant_def = 1,
60 vect_external_def,
61 vect_internal_def,
62 vect_induction_def,
63 vect_reduction_def,
64 vect_double_reduction_def,
65 vect_nested_cycle,
66 vect_unknown_def_type
69 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
70 || ((D) == vect_double_reduction_def) \
71 || ((D) == vect_nested_cycle))
73 /* Define verbosity levels. */
74 enum verbosity_levels {
75 REPORT_NONE,
76 REPORT_VECTORIZED_LOCATIONS,
77 REPORT_UNVECTORIZED_LOCATIONS,
78 REPORT_COST,
79 REPORT_ALIGNMENT,
80 REPORT_DR_DETAILS,
81 REPORT_BAD_FORM_LOOPS,
82 REPORT_OUTER_LOOPS,
83 REPORT_SLP,
84 REPORT_DETAILS,
85 /* New verbosity levels should be added before this one. */
86 MAX_VERBOSITY_LEVEL
89 /************************************************************************
90 SLP
91 ************************************************************************/
93 /* A computation tree of an SLP instance. Each node corresponds to a group of
94 stmts to be packed in a SIMD stmt. */
95 typedef struct _slp_tree {
96 /* Only binary and unary operations are supported. LEFT child corresponds to
97 the first operand and RIGHT child to the second if the operation is
98 binary. */
99 struct _slp_tree *left;
100 struct _slp_tree *right;
101 /* A group of scalar stmts to be vectorized together. */
102 VEC (gimple, heap) *stmts;
103 /* Vectorized stmt/s. */
104 VEC (gimple, heap) *vec_stmts;
105 /* Number of vector stmts that are created to replace the group of scalar
106 stmts. It is calculated during the transformation phase as the number of
107 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
108 divided by vector size. */
109 unsigned int vec_stmts_size;
110 /* Vectorization costs associated with SLP node. */
111 struct
113 int outside_of_loop; /* Statements generated outside loop. */
114 int inside_of_loop; /* Statements generated inside loop. */
115 } cost;
116 } *slp_tree;
118 DEF_VEC_P(slp_tree);
119 DEF_VEC_ALLOC_P(slp_tree, heap);
121 /* SLP instance is a sequence of stmts in a loop that can be packed into
122 SIMD stmts. */
123 typedef struct _slp_instance {
124 /* The root of SLP tree. */
125 slp_tree root;
127 /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
128 unsigned int group_size;
130 /* The unrolling factor required to vectorized this SLP instance. */
131 unsigned int unrolling_factor;
133 /* Vectorization costs associated with SLP instance. */
134 struct
136 int outside_of_loop; /* Statements generated outside loop. */
137 int inside_of_loop; /* Statements generated inside loop. */
138 } cost;
140 /* Loads permutation relatively to the stores, NULL if there is no
141 permutation. */
142 VEC (int, heap) *load_permutation;
144 /* The group of nodes that contain loads of this SLP instance. */
145 VEC (slp_tree, heap) *loads;
147 /* The first scalar load of the instance. The created vector loads will be
148 inserted before this statement. */
149 gimple first_load;
150 } *slp_instance;
152 DEF_VEC_P(slp_instance);
153 DEF_VEC_ALLOC_P(slp_instance, heap);
155 /* Access Functions. */
156 #define SLP_INSTANCE_TREE(S) (S)->root
157 #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
158 #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
159 #define SLP_INSTANCE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
160 #define SLP_INSTANCE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
161 #define SLP_INSTANCE_LOAD_PERMUTATION(S) (S)->load_permutation
162 #define SLP_INSTANCE_LOADS(S) (S)->loads
163 #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load
165 #define SLP_TREE_LEFT(S) (S)->left
166 #define SLP_TREE_RIGHT(S) (S)->right
167 #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
168 #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
169 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
170 #define SLP_TREE_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
171 #define SLP_TREE_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
174 typedef struct _vect_peel_info
176 int npeel;
177 struct data_reference *dr;
178 unsigned int count;
179 } *vect_peel_info;
181 typedef struct _vect_peel_extended_info
183 struct _vect_peel_info peel_info;
184 unsigned int inside_cost;
185 unsigned int outside_cost;
186 } *vect_peel_extended_info;
188 /*-----------------------------------------------------------------*/
189 /* Info on vectorized loops. */
190 /*-----------------------------------------------------------------*/
191 typedef struct _loop_vec_info {
193 /* The loop to which this info struct refers to. */
194 struct loop *loop;
196 /* The loop basic blocks. */
197 basic_block *bbs;
199 /* Number of iterations. */
200 tree num_iters;
201 tree num_iters_unchanged;
203 /* Minimum number of iterations below which vectorization is expected to
204 not be profitable (as estimated by the cost model).
205 -1 indicates that vectorization will not be profitable.
206 FORNOW: This field is an int. Will be a tree in the future, to represent
207 values unknown at compile time. */
208 int min_profitable_iters;
210 /* Is the loop vectorizable? */
211 bool vectorizable;
213 /* Unrolling factor */
214 int vectorization_factor;
216 /* The loop location in the source. */
217 LOC loop_line_number;
219 /* Unknown DRs according to which loop was peeled. */
220 struct data_reference *unaligned_dr;
222 /* peeling_for_alignment indicates whether peeling for alignment will take
223 place, and what the peeling factor should be:
224 peeling_for_alignment = X means:
225 If X=0: Peeling for alignment will not be applied.
226 If X>0: Peel first X iterations.
227 If X=-1: Generate a runtime test to calculate the number of iterations
228 to be peeled, using the dataref recorded in the field
229 unaligned_dr. */
230 int peeling_for_alignment;
232 /* The mask used to check the alignment of pointers or arrays. */
233 int ptr_mask;
235 /* All data references in the loop. */
236 VEC (data_reference_p, heap) *datarefs;
238 /* All data dependences in the loop. */
239 VEC (ddr_p, heap) *ddrs;
241 /* Data Dependence Relations defining address ranges that are candidates
242 for a run-time aliasing check. */
243 VEC (ddr_p, heap) *may_alias_ddrs;
245 /* Statements in the loop that have data references that are candidates for a
246 runtime (loop versioning) misalignment check. */
247 VEC(gimple,heap) *may_misalign_stmts;
249 /* All interleaving chains of stores in the loop, represented by the first
250 stmt in the chain. */
251 VEC(gimple, heap) *strided_stores;
253 /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES
254 of the loop. */
255 VEC(slp_instance, heap) *slp_instances;
257 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
258 applied to the loop, i.e., no unrolling is needed, this is 1. */
259 unsigned slp_unrolling_factor;
261 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
262 VEC (gimple, heap) *reductions;
264 /* Hash table used to choose the best peeling option. */
265 htab_t peeling_htab;
267 } *loop_vec_info;
269 /* Access Functions. */
270 #define LOOP_VINFO_LOOP(L) (L)->loop
271 #define LOOP_VINFO_BBS(L) (L)->bbs
272 #define LOOP_VINFO_NITERS(L) (L)->num_iters
273 /* Since LOOP_VINFO_NITERS can change after prologue peeling
274 retain total unchanged scalar loop iterations for cost model. */
275 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
276 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
277 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
278 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
279 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
280 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs
281 #define LOOP_VINFO_DDRS(L) (L)->ddrs
282 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
283 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
284 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
285 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
286 #define LOOP_VINFO_LOC(L) (L)->loop_line_number
287 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
288 #define LOOP_VINFO_STRIDED_STORES(L) (L)->strided_stores
289 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
290 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
291 #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
292 #define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab
294 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
295 VEC_length (gimple, (L)->may_misalign_stmts) > 0
296 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
297 VEC_length (ddr_p, (L)->may_alias_ddrs) > 0
299 #define NITERS_KNOWN_P(n) \
300 (host_integerp ((n),0) \
301 && TREE_INT_CST_LOW ((n)) > 0)
303 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
304 NITERS_KNOWN_P((L)->num_iters)
306 static inline loop_vec_info
307 loop_vec_info_for_loop (struct loop *loop)
309 return (loop_vec_info) loop->aux;
312 static inline bool
313 nested_in_vect_loop_p (struct loop *loop, gimple stmt)
315 return (loop->inner
316 && (loop->inner == (gimple_bb (stmt))->loop_father));
319 typedef struct _bb_vec_info {
321 basic_block bb;
322 /* All interleaving chains of stores in the basic block, represented by the
323 first stmt in the chain. */
324 VEC(gimple, heap) *strided_stores;
326 /* All SLP instances in the basic block. This is a subset of the set of
327 STRIDED_STORES of the basic block. */
328 VEC(slp_instance, heap) *slp_instances;
330 /* All data references in the basic block. */
331 VEC (data_reference_p, heap) *datarefs;
333 /* All data dependences in the basic block. */
334 VEC (ddr_p, heap) *ddrs;
335 } *bb_vec_info;
337 #define BB_VINFO_BB(B) (B)->bb
338 #define BB_VINFO_STRIDED_STORES(B) (B)->strided_stores
339 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
340 #define BB_VINFO_DATAREFS(B) (B)->datarefs
341 #define BB_VINFO_DDRS(B) (B)->ddrs
343 static inline bb_vec_info
344 vec_info_for_bb (basic_block bb)
346 return (bb_vec_info) bb->aux;
349 /*-----------------------------------------------------------------*/
350 /* Info on vectorized defs. */
351 /*-----------------------------------------------------------------*/
352 enum stmt_vec_info_type {
353 undef_vec_info_type = 0,
354 load_vec_info_type,
355 store_vec_info_type,
356 op_vec_info_type,
357 call_vec_info_type,
358 assignment_vec_info_type,
359 condition_vec_info_type,
360 reduc_vec_info_type,
361 induc_vec_info_type,
362 type_promotion_vec_info_type,
363 type_demotion_vec_info_type,
364 type_conversion_vec_info_type,
365 loop_exit_ctrl_vec_info_type
368 /* Indicates whether/how a variable is used in the scope of loop/basic
369 block. */
370 enum vect_relevant {
371 vect_unused_in_scope = 0,
372 /* The def is in the inner loop, and the use is in the outer loop, and the
373 use is a reduction stmt. */
374 vect_used_in_outer_by_reduction,
375 /* The def is in the inner loop, and the use is in the outer loop (and is
376 not part of reduction). */
377 vect_used_in_outer,
379 /* defs that feed computations that end up (only) in a reduction. These
380 defs may be used by non-reduction stmts, but eventually, any
381 computations/values that are affected by these defs are used to compute
382 a reduction (i.e. don't get stored to memory, for example). We use this
383 to identify computations that we can change the order in which they are
384 computed. */
385 vect_used_by_reduction,
387 vect_used_in_scope
390 /* The type of vectorization that can be applied to the stmt: regular loop-based
391 vectorization; pure SLP - the stmt is a part of SLP instances and does not
392 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
393 a part of SLP instance and also must be loop-based vectorized, since it has
394 uses outside SLP sequences.
396 In the loop context the meanings of pure and hybrid SLP are slightly
397 different. By saying that pure SLP is applied to the loop, we mean that we
398 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
399 vectorized without doing any conceptual unrolling, cause we don't pack
400 together stmts from different iterations, only within a single iteration.
401 Loop hybrid SLP means that we exploit both intra-iteration and
402 inter-iteration parallelism (e.g., number of elements in the vector is 4
403 and the slp-group-size is 2, in which case we don't have enough parallelism
404 within an iteration, so we obtain the rest of the parallelism from subsequent
405 iterations by unrolling the loop by 2). */
406 enum slp_vect_type {
407 loop_vect = 0,
408 pure_slp,
409 hybrid
413 typedef struct data_reference *dr_p;
414 DEF_VEC_P(dr_p);
415 DEF_VEC_ALLOC_P(dr_p,heap);
417 typedef struct _stmt_vec_info {
419 enum stmt_vec_info_type type;
421 /* Indicates whether this stmts is part of a computation whose result is
422 used outside the loop. */
423 bool live;
425 /* Stmt is part of some pattern (computation idiom) */
426 bool in_pattern_p;
428 /* For loads only, if there is a store with the same location, this field is
429 TRUE. */
430 bool read_write_dep;
432 /* The stmt to which this info struct refers to. */
433 gimple stmt;
435 /* The loop_vec_info with respect to which STMT is vectorized. */
436 loop_vec_info loop_vinfo;
438 /* The vector type to be used for the LHS of this statement. */
439 tree vectype;
441 /* The vectorized version of the stmt. */
442 gimple vectorized_stmt;
445 /** The following is relevant only for stmts that contain a non-scalar
446 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
447 at most one such data-ref. **/
449 /* Information about the data-ref (access function, etc),
450 relative to the inner-most containing loop. */
451 struct data_reference *data_ref_info;
453 /* Information about the data-ref relative to this loop
454 nest (the loop that is being considered for vectorization). */
455 tree dr_base_address;
456 tree dr_init;
457 tree dr_offset;
458 tree dr_step;
459 tree dr_aligned_to;
461 /* Used for various bookkeeping purposes, generally holding a pointer to
462 some other stmt S that is in some way "related" to this stmt.
463 Current use of this field is:
464 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
465 true): S is the "pattern stmt" that represents (and replaces) the
466 sequence of stmts that constitutes the pattern. Similarly, the
467 related_stmt of the "pattern stmt" points back to this stmt (which is
468 the last stmt in the original sequence of stmts that constitutes the
469 pattern). */
470 gimple related_stmt;
472 /* List of datarefs that are known to have the same alignment as the dataref
473 of this stmt. */
474 VEC(dr_p,heap) *same_align_refs;
476 /* Classify the def of this stmt. */
477 enum vect_def_type def_type;
479 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
480 enum slp_vect_type slp_type;
482 /* Interleaving info. */
483 /* First data-ref in the interleaving group. */
484 gimple first_dr;
485 /* Pointer to the next data-ref in the group. */
486 gimple next_dr;
487 /* In case that two or more stmts share data-ref, this is the pointer to the
488 previously detected stmt with the same dr. */
489 gimple same_dr_stmt;
490 /* The size of the interleaving group. */
491 unsigned int size;
492 /* For stores, number of stores from this group seen. We vectorize the last
493 one. */
494 unsigned int store_count;
495 /* For loads only, the gap from the previous load. For consecutive loads, GAP
496 is 1. */
497 unsigned int gap;
499 /* Not all stmts in the loop need to be vectorized. e.g, the increment
500 of the loop induction variable and computation of array indexes. relevant
501 indicates whether the stmt needs to be vectorized. */
502 enum vect_relevant relevant;
504 /* Vectorization costs associated with statement. */
505 struct
507 int outside_of_loop; /* Statements generated outside loop. */
508 int inside_of_loop; /* Statements generated inside loop. */
509 } cost;
511 /* The bb_vec_info with respect to which STMT is vectorized. */
512 bb_vec_info bb_vinfo;
514 /* Is this statement vectorizable or should it be skipped in (partial)
515 vectorization. */
516 bool vectorizable;
517 } *stmt_vec_info;
519 /* Access Functions. */
520 #define STMT_VINFO_TYPE(S) (S)->type
521 #define STMT_VINFO_STMT(S) (S)->stmt
522 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo
523 #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo
524 #define STMT_VINFO_RELEVANT(S) (S)->relevant
525 #define STMT_VINFO_LIVE_P(S) (S)->live
526 #define STMT_VINFO_VECTYPE(S) (S)->vectype
527 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
528 #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
529 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info
531 #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address
532 #define STMT_VINFO_DR_INIT(S) (S)->dr_init
533 #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset
534 #define STMT_VINFO_DR_STEP(S) (S)->dr_step
535 #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to
537 #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
538 #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
539 #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
540 #define STMT_VINFO_DEF_TYPE(S) (S)->def_type
541 #define STMT_VINFO_DR_GROUP_FIRST_DR(S) (S)->first_dr
542 #define STMT_VINFO_DR_GROUP_NEXT_DR(S) (S)->next_dr
543 #define STMT_VINFO_DR_GROUP_SIZE(S) (S)->size
544 #define STMT_VINFO_DR_GROUP_STORE_COUNT(S) (S)->store_count
545 #define STMT_VINFO_DR_GROUP_GAP(S) (S)->gap
546 #define STMT_VINFO_DR_GROUP_SAME_DR_STMT(S)(S)->same_dr_stmt
547 #define STMT_VINFO_DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
548 #define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_dr != NULL)
550 #define DR_GROUP_FIRST_DR(S) (S)->first_dr
551 #define DR_GROUP_NEXT_DR(S) (S)->next_dr
552 #define DR_GROUP_SIZE(S) (S)->size
553 #define DR_GROUP_STORE_COUNT(S) (S)->store_count
554 #define DR_GROUP_GAP(S) (S)->gap
555 #define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
556 #define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep
558 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
559 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
560 #define STMT_VINFO_INSIDE_OF_LOOP_COST(S) (S)->cost.inside_of_loop
562 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
563 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
564 #define STMT_SLP_TYPE(S) (S)->slp_type
566 #define VECT_MAX_COST 1000
568 /* The maximum number of intermediate steps required in multi-step type
569 conversion. */
570 #define MAX_INTERM_CVT_STEPS 3
572 /* The maximum vectorization factor supported by any target (V32QI). */
573 #define MAX_VECTORIZATION_FACTOR 32
575 /* Avoid GTY(()) on stmt_vec_info. */
576 typedef void *vec_void_p;
577 DEF_VEC_P (vec_void_p);
578 DEF_VEC_ALLOC_P (vec_void_p, heap);
580 extern VEC(vec_void_p,heap) *stmt_vec_info_vec;
582 void init_stmt_vec_info_vec (void);
583 void free_stmt_vec_info_vec (void);
585 static inline stmt_vec_info
586 vinfo_for_stmt (gimple stmt)
588 unsigned int uid = gimple_uid (stmt);
589 if (uid == 0)
590 return NULL;
592 return (stmt_vec_info) VEC_index (vec_void_p, stmt_vec_info_vec, uid - 1);
595 static inline void
596 set_vinfo_for_stmt (gimple stmt, stmt_vec_info info)
598 unsigned int uid = gimple_uid (stmt);
599 if (uid == 0)
601 gcc_checking_assert (info);
602 uid = VEC_length (vec_void_p, stmt_vec_info_vec) + 1;
603 gimple_set_uid (stmt, uid);
604 VEC_safe_push (vec_void_p, heap, stmt_vec_info_vec, (vec_void_p) info);
606 else
607 VEC_replace (vec_void_p, stmt_vec_info_vec, uid - 1, (vec_void_p) info);
610 static inline gimple
611 get_earlier_stmt (gimple stmt1, gimple stmt2)
613 unsigned int uid1, uid2;
615 if (stmt1 == NULL)
616 return stmt2;
618 if (stmt2 == NULL)
619 return stmt1;
621 uid1 = gimple_uid (stmt1);
622 uid2 = gimple_uid (stmt2);
624 if (uid1 == 0 || uid2 == 0)
625 return NULL;
627 gcc_checking_assert (uid1 <= VEC_length (vec_void_p, stmt_vec_info_vec)
628 && uid2 <= VEC_length (vec_void_p, stmt_vec_info_vec));
630 if (uid1 < uid2)
631 return stmt1;
632 else
633 return stmt2;
636 static inline bool
637 is_pattern_stmt_p (stmt_vec_info stmt_info)
639 gimple related_stmt;
640 stmt_vec_info related_stmt_info;
642 related_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
643 if (related_stmt
644 && (related_stmt_info = vinfo_for_stmt (related_stmt))
645 && STMT_VINFO_IN_PATTERN_P (related_stmt_info))
646 return true;
648 return false;
651 static inline bool
652 is_loop_header_bb_p (basic_block bb)
654 if (bb == (bb->loop_father)->header)
655 return true;
656 gcc_checking_assert (EDGE_COUNT (bb->preds) == 1);
657 return false;
660 static inline void
661 stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
662 int cost)
664 if (slp_node)
665 SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost;
666 else
667 STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost;
670 static inline void
671 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
672 int cost)
674 if (slp_node)
675 SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost;
676 else
677 STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost;
680 static inline int
681 vect_pow2 (int x)
683 int i, res = 1;
685 for (i = 0; i < x; i++)
686 res *= 2;
688 return res;
691 /*-----------------------------------------------------------------*/
692 /* Info on data references alignment. */
693 /*-----------------------------------------------------------------*/
695 /* Reflects actual alignment of first access in the vectorized loop,
696 taking into account peeling/versioning if applied. */
697 #define DR_MISALIGNMENT(DR) ((int) (size_t) (DR)->aux)
698 #define SET_DR_MISALIGNMENT(DR, VAL) ((DR)->aux = (void *) (size_t) (VAL))
700 static inline bool
701 aligned_access_p (struct data_reference *data_ref_info)
703 return (DR_MISALIGNMENT (data_ref_info) == 0);
706 static inline bool
707 known_alignment_for_access_p (struct data_reference *data_ref_info)
709 return (DR_MISALIGNMENT (data_ref_info) != -1);
712 /* vect_dump will be set to stderr or dump_file if exist. */
713 extern FILE *vect_dump;
714 extern LOC vect_loop_location;
716 /*-----------------------------------------------------------------*/
717 /* Function prototypes. */
718 /*-----------------------------------------------------------------*/
720 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
721 in tree-vect-loop-manip.c. */
722 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
723 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
724 extern void vect_loop_versioning (loop_vec_info, bool, tree *, gimple_seq *);
725 extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
726 tree, gimple_seq);
727 extern void vect_do_peeling_for_alignment (loop_vec_info);
728 extern LOC find_loop_location (struct loop *);
729 extern bool vect_can_advance_ivs_p (loop_vec_info);
731 /* In tree-vect-stmts.c. */
732 extern tree get_vectype_for_scalar_type (tree);
733 extern tree get_same_sized_vectype (tree, tree);
734 extern bool vect_is_simple_use (tree, loop_vec_info, bb_vec_info, gimple *,
735 tree *, enum vect_def_type *);
736 extern bool vect_is_simple_use_1 (tree, loop_vec_info, bb_vec_info, gimple *,
737 tree *, enum vect_def_type *, tree *);
738 extern bool supportable_widening_operation (enum tree_code, gimple, tree, tree,
739 tree *, tree *, enum tree_code *,
740 enum tree_code *, int *,
741 VEC (tree, heap) **);
742 extern bool supportable_narrowing_operation (enum tree_code, tree, tree,
743 enum tree_code *,
744 int *, VEC (tree, heap) **);
745 extern stmt_vec_info new_stmt_vec_info (gimple stmt, loop_vec_info,
746 bb_vec_info);
747 extern void free_stmt_vec_info (gimple stmt);
748 extern tree vectorizable_function (gimple, tree, tree);
749 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
750 slp_tree);
751 extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type,
752 slp_tree);
753 extern void vect_model_load_cost (stmt_vec_info, int, slp_tree);
754 extern void vect_finish_stmt_generation (gimple, gimple,
755 gimple_stmt_iterator *);
756 extern bool vect_mark_stmts_to_be_vectorized (loop_vec_info);
757 extern int cost_for_stmt (gimple);
758 extern tree vect_get_vec_def_for_operand (tree, gimple, tree *);
759 extern tree vect_init_vector (gimple, tree, tree,
760 gimple_stmt_iterator *);
761 extern tree vect_get_vec_def_for_stmt_copy (enum vect_def_type, tree);
762 extern bool vect_transform_stmt (gimple, gimple_stmt_iterator *,
763 bool *, slp_tree, slp_instance);
764 extern void vect_remove_stores (gimple);
765 extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
766 extern bool vectorizable_condition (gimple, gimple_stmt_iterator *, gimple *,
767 tree, int);
768 extern void vect_get_load_cost (struct data_reference *, int, bool,
769 unsigned int *, unsigned int *);
770 extern void vect_get_store_cost (struct data_reference *, int, unsigned int *);
772 /* In tree-vect-data-refs.c. */
773 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
774 extern enum dr_alignment_support vect_supportable_dr_alignment
775 (struct data_reference *, bool);
776 extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *,
777 HOST_WIDE_INT *);
778 extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info,
779 int *);
780 extern bool vect_enhance_data_refs_alignment (loop_vec_info);
781 extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info);
782 extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info);
783 extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info);
784 extern bool vect_prune_runtime_alias_test_list (loop_vec_info);
785 extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *);
786 extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *,
787 gimple *, bool, bool *);
788 extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree);
789 extern tree vect_create_destination_var (tree, tree);
790 extern bool vect_strided_store_supported (tree);
791 extern bool vect_strided_load_supported (tree);
792 extern bool vect_permute_store_chain (VEC(tree,heap) *,unsigned int, gimple,
793 gimple_stmt_iterator *, VEC(tree,heap) **);
794 extern tree vect_setup_realignment (gimple, gimple_stmt_iterator *, tree *,
795 enum dr_alignment_support, tree,
796 struct loop **);
797 extern bool vect_permute_load_chain (VEC(tree,heap) *,unsigned int, gimple,
798 gimple_stmt_iterator *, VEC(tree,heap) **);
799 extern bool vect_transform_strided_load (gimple, VEC(tree,heap) *, int,
800 gimple_stmt_iterator *);
801 extern int vect_get_place_in_interleaving_chain (gimple, gimple);
802 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
803 extern tree vect_create_addr_base_for_vector_ref (gimple, gimple_seq *,
804 tree, struct loop *);
806 /* In tree-vect-loop.c. */
807 /* FORNOW: Used in tree-parloops.c. */
808 extern void destroy_loop_vec_info (loop_vec_info, bool);
809 extern gimple vect_force_simple_reduction (loop_vec_info, gimple, bool, bool *);
810 /* Drive for loop analysis stage. */
811 extern loop_vec_info vect_analyze_loop (struct loop *);
812 /* Drive for loop transformation stage. */
813 extern void vect_transform_loop (loop_vec_info);
814 extern loop_vec_info vect_analyze_loop_form (struct loop *);
815 extern bool vectorizable_live_operation (gimple, gimple_stmt_iterator *,
816 gimple *);
817 extern bool vectorizable_reduction (gimple, gimple_stmt_iterator *, gimple *,
818 slp_tree);
819 extern bool vectorizable_induction (gimple, gimple_stmt_iterator *, gimple *);
820 extern int vect_estimate_min_profitable_iters (loop_vec_info);
821 extern tree get_initial_def_for_reduction (gimple, tree, tree *);
822 extern int vect_min_worthwhile_factor (enum tree_code);
823 extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, int);
824 extern int vect_get_single_scalar_iteraion_cost (loop_vec_info);
826 /* In tree-vect-slp.c. */
827 extern void vect_free_slp_instance (slp_instance);
828 extern bool vect_transform_slp_perm_load (gimple, VEC (tree, heap) *,
829 gimple_stmt_iterator *, int,
830 slp_instance, bool);
831 extern bool vect_schedule_slp (loop_vec_info, bb_vec_info);
832 extern void vect_update_slp_costs_according_to_vf (loop_vec_info);
833 extern bool vect_analyze_slp (loop_vec_info, bb_vec_info);
834 extern void vect_make_slp_decision (loop_vec_info);
835 extern void vect_detect_hybrid_slp (loop_vec_info);
836 extern void vect_get_slp_defs (slp_tree, VEC (tree,heap) **,
837 VEC (tree,heap) **, int);
838 extern LOC find_bb_location (basic_block);
839 extern bb_vec_info vect_slp_analyze_bb (basic_block);
840 extern void vect_slp_transform_bb (basic_block);
842 /* In tree-vect-patterns.c. */
843 /* Pattern recognition functions.
844 Additional pattern recognition functions can (and will) be added
845 in the future. */
846 typedef gimple (* vect_recog_func_ptr) (gimple, tree *, tree *);
847 #define NUM_PATTERNS 4
848 void vect_pattern_recog (loop_vec_info);
850 /* In tree-vectorizer.c. */
851 unsigned vectorize_loops (void);
852 /* Vectorization debug information */
853 extern bool vect_print_dump_info (enum verbosity_levels);
855 #endif /* GCC_TREE_VECTORIZER_H */