2 Copyright (C) 2003-2021 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #ifndef GCC_TREE_VECTORIZER_H
22 #define GCC_TREE_VECTORIZER_H
24 typedef class _stmt_vec_info
*stmt_vec_info
;
26 #include "tree-data-ref.h"
27 #include "tree-hash-traits.h"
29 #include "internal-fn.h"
32 /* Used for naming of new temporaries. */
40 /* Defines type of operation. */
47 /* Define type of available alignment support. */
48 enum dr_alignment_support
{
49 dr_unaligned_unsupported
,
50 dr_unaligned_supported
,
52 dr_explicit_realign_optimized
,
56 /* Define type of def-use cross-iteration cycle. */
58 vect_uninitialized_def
= 0,
59 vect_constant_def
= 1,
64 vect_double_reduction_def
,
69 /* Define type of reduction. */
70 enum vect_reduction_type
{
73 INTEGER_INDUC_COND_REDUCTION
,
76 /* Retain a scalar phi and use a FOLD_EXTRACT_LAST within the loop
79 for (int i = 0; i < VF; ++i)
80 res = cond[i] ? val[i] : res; */
81 EXTRACT_LAST_REDUCTION
,
83 /* Use a folding reduction within the loop to implement:
85 for (int i = 0; i < VF; ++i)
88 (with no reassocation). */
92 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
93 || ((D) == vect_double_reduction_def) \
94 || ((D) == vect_nested_cycle))
96 /* Structure to encapsulate information about a group of like
97 instructions to be presented to the target cost model. */
98 struct stmt_info_for_cost
{
100 enum vect_cost_for_stmt kind
;
101 enum vect_cost_model_location where
;
102 stmt_vec_info stmt_info
;
107 typedef vec
<stmt_info_for_cost
> stmt_vector_for_cost
;
109 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
110 known alignment for that base. */
111 typedef hash_map
<tree_operand_hash
,
112 innermost_loop_behavior
*> vec_base_alignments
;
114 /************************************************************************
116 ************************************************************************/
117 typedef struct _slp_tree
*slp_tree
;
118 typedef vec
<std::pair
<unsigned, unsigned> > lane_permutation_t
;
119 typedef vec
<unsigned> load_permutation_t
;
121 /* A computation tree of an SLP instance. Each node corresponds to a group of
122 stmts to be packed in a SIMD stmt. */
127 /* Nodes that contain def-stmts of this node statements operands. */
128 vec
<slp_tree
> children
;
130 /* A group of scalar stmts to be vectorized together. */
131 vec
<stmt_vec_info
> stmts
;
132 /* A group of scalar operands to be vectorized together. */
134 /* The representative that should be used for analysis and
136 stmt_vec_info representative
;
138 /* Load permutation relative to the stores, NULL if there is no
140 load_permutation_t load_permutation
;
141 /* Lane permutation of the operands scalar lanes encoded as pairs
142 of { operand number, lane number }. The number of elements
143 denotes the number of output lanes. */
144 lane_permutation_t lane_permutation
;
147 /* Vectorized stmt/s. */
148 vec
<gimple
*> vec_stmts
;
150 /* Number of vector stmts that are created to replace the group of scalar
151 stmts. It is calculated during the transformation phase as the number of
152 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
153 divided by vector size. */
154 unsigned int vec_stmts_size
;
156 /* Reference count in the SLP graph. */
158 /* The maximum number of vector elements for the subtree rooted
160 poly_uint64 max_nunits
;
161 /* The DEF type of this node. */
162 enum vect_def_type def_type
;
163 /* The number of scalar lanes produced by this node. */
165 /* The operation of this node. */
170 /* Allocate from slp_tree_pool. */
171 static void *operator new (size_t);
173 /* Return memory to slp_tree_pool. */
174 static void operator delete (void *, size_t);
176 /* Linked list of nodes to release when we free the slp_tree_pool. */
181 /* The enum describes the type of operations that an SLP instance
184 enum slp_instance_kind
{
186 slp_inst_kind_reduc_group
,
187 slp_inst_kind_reduc_chain
,
191 /* SLP instance is a sequence of stmts in a loop that can be packed into
193 typedef class _slp_instance
{
195 /* The root of SLP tree. */
198 /* For vector constructors, the constructor stmt that the SLP tree is built
199 from, NULL otherwise. */
200 stmt_vec_info root_stmt
;
202 /* The unrolling factor required to vectorized this SLP instance. */
203 poly_uint64 unrolling_factor
;
205 /* The group of nodes that contain loads of this SLP instance. */
208 /* The SLP node containing the reduction PHIs. */
211 /* Vector cost of this entry to the SLP graph. */
212 stmt_vector_for_cost cost_vec
;
214 /* If this instance is the main entry of a subgraph the set of
215 entries into the same subgraph, including itself. */
216 vec
<_slp_instance
*> subgraph_entries
;
218 /* The type of operation the SLP instance is performing. */
219 slp_instance_kind kind
;
221 dump_user_location_t
location () const;
225 /* Access Functions. */
226 #define SLP_INSTANCE_TREE(S) (S)->root
227 #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
228 #define SLP_INSTANCE_LOADS(S) (S)->loads
229 #define SLP_INSTANCE_ROOT_STMT(S) (S)->root_stmt
230 #define SLP_INSTANCE_KIND(S) (S)->kind
232 #define SLP_TREE_CHILDREN(S) (S)->children
233 #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
234 #define SLP_TREE_SCALAR_OPS(S) (S)->ops
235 #define SLP_TREE_REF_COUNT(S) (S)->refcnt
236 #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
237 #define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
238 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
239 #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
240 #define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
241 #define SLP_TREE_DEF_TYPE(S) (S)->def_type
242 #define SLP_TREE_VECTYPE(S) (S)->vectype
243 #define SLP_TREE_REPRESENTATIVE(S) (S)->representative
244 #define SLP_TREE_LANES(S) (S)->lanes
245 #define SLP_TREE_CODE(S) (S)->code
247 /* Key for map that records association between
248 scalar conditions and corresponding loop mask, and
249 is populated by vect_record_loop_mask. */
251 struct scalar_cond_masked_key
253 scalar_cond_masked_key (tree t
, unsigned ncopies_
)
256 get_cond_ops_from_tree (t
);
259 void get_cond_ops_from_tree (tree
);
268 struct default_hash_traits
<scalar_cond_masked_key
>
270 typedef scalar_cond_masked_key compare_type
;
271 typedef scalar_cond_masked_key value_type
;
273 static inline hashval_t
278 inchash::add_expr (v
.op0
, h
, 0);
279 inchash::add_expr (v
.op1
, h
, 0);
280 h
.add_int (v
.ncopies
);
285 equal (value_type existing
, value_type candidate
)
287 return (existing
.ncopies
== candidate
.ncopies
288 && existing
.code
== candidate
.code
289 && operand_equal_p (existing
.op0
, candidate
.op0
, 0)
290 && operand_equal_p (existing
.op1
, candidate
.op1
, 0));
293 static const bool empty_zero_p
= true;
296 mark_empty (value_type
&v
)
302 is_empty (value_type v
)
304 return v
.ncopies
== 0;
307 static inline void mark_deleted (value_type
&) {}
309 static inline bool is_deleted (const value_type
&)
314 static inline void remove (value_type
&) {}
317 typedef hash_set
<scalar_cond_masked_key
> scalar_cond_masked_set_type
;
319 /* Describes two objects whose addresses must be unequal for the vectorized
321 typedef std::pair
<tree
, tree
> vec_object_pair
;
323 /* Records that vectorization is only possible if abs (EXPR) >= MIN_VALUE.
324 UNSIGNED_P is true if we can assume that abs (EXPR) == EXPR. */
325 class vec_lower_bound
{
327 vec_lower_bound () {}
328 vec_lower_bound (tree e
, bool u
, poly_uint64 m
)
329 : expr (e
), unsigned_p (u
), min_value (m
) {}
333 poly_uint64 min_value
;
336 /* Vectorizer state shared between different analyses like vector sizes
337 of the same CFG region. */
338 class vec_info_shared
{
343 void save_datarefs();
344 void check_datarefs();
346 /* All data references. Freed by free_data_refs, so not an auto_vec. */
347 vec
<data_reference_p
> datarefs
;
348 vec
<data_reference
> datarefs_copy
;
350 /* The loop nest in which the data dependences are computed. */
351 auto_vec
<loop_p
> loop_nest
;
353 /* All data dependences. Freed by free_dependence_relations, so not
358 /* Vectorizer state common between loop and basic-block vectorization. */
361 typedef hash_set
<int_hash
<machine_mode
, E_VOIDmode
, E_BLKmode
> > mode_set
;
362 enum vec_kind
{ bb
, loop
};
364 vec_info (vec_kind
, void *, vec_info_shared
*);
367 stmt_vec_info
add_stmt (gimple
*);
368 stmt_vec_info
add_pattern_stmt (gimple
*, stmt_vec_info
);
369 stmt_vec_info
lookup_stmt (gimple
*);
370 stmt_vec_info
lookup_def (tree
);
371 stmt_vec_info
lookup_single_use (tree
);
372 class dr_vec_info
*lookup_dr (data_reference
*);
373 void move_dr (stmt_vec_info
, stmt_vec_info
);
374 void remove_stmt (stmt_vec_info
);
375 void replace_stmt (gimple_stmt_iterator
*, stmt_vec_info
, gimple
*);
376 void insert_on_entry (stmt_vec_info
, gimple
*);
377 void insert_seq_on_entry (stmt_vec_info
, gimple_seq
);
379 /* The type of vectorization. */
382 /* Shared vectorizer state. */
383 vec_info_shared
*shared
;
385 /* The mapping of GIMPLE UID to stmt_vec_info. */
386 vec
<stmt_vec_info
> stmt_vec_infos
;
387 /* Whether the above mapping is complete. */
388 bool stmt_vec_info_ro
;
391 auto_vec
<slp_instance
> slp_instances
;
393 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
394 known alignment for that base. */
395 vec_base_alignments base_alignments
;
397 /* All interleaving chains of stores, represented by the first
398 stmt in the chain. */
399 auto_vec
<stmt_vec_info
> grouped_stores
;
401 /* Cost data used by the target cost model. */
402 void *target_cost_data
;
404 /* The set of vector modes used in the vectorized region. */
405 mode_set used_vector_modes
;
407 /* The argument we should pass to related_vector_mode when looking up
408 the vector mode for a scalar mode, or VOIDmode if we haven't yet
409 made any decisions about which vector modes to use. */
410 machine_mode vector_mode
;
413 stmt_vec_info
new_stmt_vec_info (gimple
*stmt
);
414 void set_vinfo_for_stmt (gimple
*, stmt_vec_info
, bool = true);
415 void free_stmt_vec_infos ();
416 void free_stmt_vec_info (stmt_vec_info
);
419 class _loop_vec_info
;
425 is_a_helper
<_loop_vec_info
*>::test (vec_info
*i
)
427 return i
->kind
== vec_info::loop
;
433 is_a_helper
<_bb_vec_info
*>::test (vec_info
*i
)
435 return i
->kind
== vec_info::bb
;
438 /* In general, we can divide the vector statements in a vectorized loop
439 into related groups ("rgroups") and say that for each rgroup there is
440 some nS such that the rgroup operates on nS values from one scalar
441 iteration followed by nS values from the next. That is, if VF is the
442 vectorization factor of the loop, the rgroup operates on a sequence:
444 (1,1) (1,2) ... (1,nS) (2,1) ... (2,nS) ... (VF,1) ... (VF,nS)
446 where (i,j) represents a scalar value with index j in a scalar
447 iteration with index i.
449 [ We use the term "rgroup" to emphasise that this grouping isn't
450 necessarily the same as the grouping of statements used elsewhere.
451 For example, if we implement a group of scalar loads using gather
452 loads, we'll use a separate gather load for each scalar load, and
453 thus each gather load will belong to its own rgroup. ]
455 In general this sequence will occupy nV vectors concatenated
456 together. If these vectors have nL lanes each, the total number
457 of scalar values N is given by:
459 N = nS * VF = nV * nL
461 None of nS, VF, nV and nL are required to be a power of 2. nS and nV
462 are compile-time constants but VF and nL can be variable (if the target
463 supports variable-length vectors).
465 In classical vectorization, each iteration of the vector loop would
466 handle exactly VF iterations of the original scalar loop. However,
467 in vector loops that are able to operate on partial vectors, a
468 particular iteration of the vector loop might handle fewer than VF
469 iterations of the scalar loop. The vector lanes that correspond to
470 iterations of the scalar loop are said to be "active" and the other
471 lanes are said to be "inactive".
473 In such vector loops, many rgroups need to be controlled to ensure
474 that they have no effect for the inactive lanes. Conceptually, each
475 such rgroup needs a sequence of booleans in the same order as above,
476 but with each (i,j) replaced by a boolean that indicates whether
477 iteration i is active. This sequence occupies nV vector controls
478 that again have nL lanes each. Thus the control sequence as a whole
479 consists of VF independent booleans that are each repeated nS times.
481 Taking mask-based approach as a partially-populated vectors example.
482 We make the simplifying assumption that if a sequence of nV masks is
483 suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
484 VIEW_CONVERTing it. This holds for all current targets that support
485 fully-masked loops. For example, suppose the scalar loop is:
489 for (int i = 0; i < n; ++i)
491 f[i * 2 + 0] += 1.0f;
492 f[i * 2 + 1] += 2.0f;
496 and suppose that vectors have 256 bits. The vectorized f accesses
497 will belong to one rgroup and the vectorized d access to another:
499 f rgroup: nS = 2, nV = 1, nL = 8
500 d rgroup: nS = 1, nV = 1, nL = 4
503 [ In this simple example the rgroups do correspond to the normal
504 SLP grouping scheme. ]
506 If only the first three lanes are active, the masks we need are:
508 f rgroup: 1 1 | 1 1 | 1 1 | 0 0
509 d rgroup: 1 | 1 | 1 | 0
511 Here we can use a mask calculated for f's rgroup for d's, but not
514 Thus for each value of nV, it is enough to provide nV masks, with the
515 mask being calculated based on the highest nL (or, equivalently, based
516 on the highest nS) required by any rgroup with that nV. We therefore
517 represent the entire collection of masks as a two-level table, with the
518 first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
519 the second being indexed by the mask index 0 <= i < nV. */
521 /* The controls (like masks or lengths) needed by rgroups with nV vectors,
522 according to the description above. */
523 struct rgroup_controls
{
524 /* The largest nS for all rgroups that use these controls. */
525 unsigned int max_nscalars_per_iter
;
527 /* For the largest nS recorded above, the loop controls divide each scalar
528 into FACTOR equal-sized pieces. This is useful if we need to split
529 element-based accesses into byte-based accesses. */
532 /* This is a vector type with MAX_NSCALARS_PER_ITER * VF / nV elements.
533 For mask-based controls, it is the type of the masks in CONTROLS.
534 For length-based controls, it can be any vector type that has the
535 specified number of elements; the type of the elements doesn't matter. */
538 /* A vector of nV controls, in iteration order. */
542 typedef auto_vec
<rgroup_controls
> vec_loop_masks
;
544 typedef auto_vec
<rgroup_controls
> vec_loop_lens
;
546 typedef auto_vec
<std::pair
<data_reference
*, tree
> > drs_init_vec
;
548 /*-----------------------------------------------------------------*/
549 /* Info on vectorized loops. */
550 /*-----------------------------------------------------------------*/
551 typedef class _loop_vec_info
: public vec_info
{
553 _loop_vec_info (class loop
*, vec_info_shared
*);
556 /* The loop to which this info struct refers to. */
559 /* The loop basic blocks. */
562 /* Number of latch executions. */
564 /* Number of iterations. */
566 /* Number of iterations of the original loop. */
567 tree num_iters_unchanged
;
568 /* Condition under which this loop is analyzed and versioned. */
569 tree num_iters_assumptions
;
571 /* Threshold of number of iterations below which vectorization will not be
572 performed. It is calculated from MIN_PROFITABLE_ITERS and
573 param_min_vect_loop_bound. */
576 /* When applying loop versioning, the vector form should only be used
577 if the number of scalar iterations is >= this value, on top of all
578 the other requirements. Ignored when loop versioning is not being
580 poly_uint64 versioning_threshold
;
582 /* Unrolling factor */
583 poly_uint64 vectorization_factor
;
585 /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR
586 if there is no particular limit. */
587 unsigned HOST_WIDE_INT max_vectorization_factor
;
589 /* The masks that a fully-masked loop should use to avoid operating
590 on inactive scalars. */
591 vec_loop_masks masks
;
593 /* The lengths that a loop with length should use to avoid operating
594 on inactive scalars. */
597 /* Set of scalar conditions that have loop mask applied. */
598 scalar_cond_masked_set_type scalar_cond_masked_set
;
600 /* If we are using a loop mask to align memory addresses, this variable
601 contains the number of vector elements that we should skip in the
602 first iteration of the vector loop (i.e. the number of leading
603 elements that should be false in the first mask). */
604 tree mask_skip_niters
;
606 /* The type that the loop control IV should be converted to before
607 testing which of the VF scalars are active and inactive.
608 Only meaningful if LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
609 tree rgroup_compare_type
;
611 /* For #pragma omp simd if (x) loops the x expression. If constant 0,
612 the loop should not be vectorized, if constant non-zero, simd_if_cond
613 shouldn't be set and loop vectorized normally, if SSA_NAME, the loop
614 should be versioned on that condition, using scalar loop if the condition
615 is false and vectorized loop otherwise. */
618 /* The type that the vector loop control IV should have when
619 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
622 /* Unknown DRs according to which loop was peeled. */
623 class dr_vec_info
*unaligned_dr
;
625 /* peeling_for_alignment indicates whether peeling for alignment will take
626 place, and what the peeling factor should be:
627 peeling_for_alignment = X means:
628 If X=0: Peeling for alignment will not be applied.
629 If X>0: Peel first X iterations.
630 If X=-1: Generate a runtime test to calculate the number of iterations
631 to be peeled, using the dataref recorded in the field
633 int peeling_for_alignment
;
635 /* The mask used to check the alignment of pointers or arrays. */
638 /* Data Dependence Relations defining address ranges that are candidates
639 for a run-time aliasing check. */
640 auto_vec
<ddr_p
> may_alias_ddrs
;
642 /* Data Dependence Relations defining address ranges together with segment
643 lengths from which the run-time aliasing check is built. */
644 auto_vec
<dr_with_seg_len_pair_t
> comp_alias_ddrs
;
646 /* Check that the addresses of each pair of objects is unequal. */
647 auto_vec
<vec_object_pair
> check_unequal_addrs
;
649 /* List of values that are required to be nonzero. This is used to check
650 whether things like "x[i * n] += 1;" are safe and eventually gets added
651 to the checks for lower bounds below. */
652 auto_vec
<tree
> check_nonzero
;
654 /* List of values that need to be checked for a minimum value. */
655 auto_vec
<vec_lower_bound
> lower_bounds
;
657 /* Statements in the loop that have data references that are candidates for a
658 runtime (loop versioning) misalignment check. */
659 auto_vec
<stmt_vec_info
> may_misalign_stmts
;
661 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
662 auto_vec
<stmt_vec_info
> reductions
;
664 /* All reduction chains in the loop, represented by the first
665 stmt in the chain. */
666 auto_vec
<stmt_vec_info
> reduction_chains
;
668 /* Cost vector for a single scalar iteration. */
669 auto_vec
<stmt_info_for_cost
> scalar_cost_vec
;
671 /* Map of IV base/step expressions to inserted name in the preheader. */
672 hash_map
<tree_operand_hash
, tree
> *ivexpr_map
;
674 /* Map of OpenMP "omp simd array" scan variables to corresponding
675 rhs of the store of the initializer. */
676 hash_map
<tree
, tree
> *scan_map
;
678 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
679 applied to the loop, i.e., no unrolling is needed, this is 1. */
680 poly_uint64 slp_unrolling_factor
;
682 /* Cost of a single scalar iteration. */
683 int single_scalar_iteration_cost
;
685 /* The cost of the vector prologue and epilogue, including peeled
686 iterations and set-up code. */
687 int vec_outside_cost
;
689 /* The cost of the vector loop body. */
692 /* The factor used to over weight those statements in an inner loop
693 relative to the loop being vectorized. */
694 unsigned int inner_loop_cost_factor
;
696 /* Is the loop vectorizable? */
699 /* Records whether we still have the option of vectorizing this loop
700 using partially-populated vectors; in other words, whether it is
701 still possible for one iteration of the vector loop to handle
702 fewer than VF scalars. */
703 bool can_use_partial_vectors_p
;
705 /* True if we've decided to use partially-populated vectors, so that
706 the vector loop can handle fewer than VF scalars. */
707 bool using_partial_vectors_p
;
709 /* True if we've decided to use partially-populated vectors for the
711 bool epil_using_partial_vectors_p
;
713 /* When we have grouped data accesses with gaps, we may introduce invalid
714 memory accesses. We peel the last iteration of the loop to prevent
716 bool peeling_for_gaps
;
718 /* When the number of iterations is not a multiple of the vector size
719 we need to peel off iterations at the end to form an epilogue loop. */
720 bool peeling_for_niter
;
722 /* True if there are no loop carried data dependencies in the loop.
723 If loop->safelen <= 1, then this is always true, either the loop
724 didn't have any loop carried data dependencies, or the loop is being
725 vectorized guarded with some runtime alias checks, or couldn't
726 be vectorized at all, but then this field shouldn't be used.
727 For loop->safelen >= 2, the user has asserted that there are no
728 backward dependencies, but there still could be loop carried forward
729 dependencies in such loops. This flag will be false if normal
730 vectorizer data dependency analysis would fail or require versioning
731 for alias, but because of loop->safelen >= 2 it has been vectorized
732 even without versioning for alias. E.g. in:
734 for (int i = 0; i < m; i++)
736 (or #pragma simd or #pragma ivdep) we can vectorize this and it will
737 DTRT even for k > 0 && k < m, but without safelen we would not
738 vectorize this, so this field would be false. */
739 bool no_data_dependencies
;
741 /* Mark loops having masked stores. */
744 /* Queued scaling factor for the scalar loop. */
745 profile_probability scalar_loop_scaling
;
747 /* If if-conversion versioned this loop before conversion, this is the
748 loop version without if-conversion. */
749 class loop
*scalar_loop
;
751 /* For loops being epilogues of already vectorized loops
752 this points to the original vectorized loop. Otherwise NULL. */
753 _loop_vec_info
*orig_loop_info
;
755 /* Used to store loop_vec_infos of epilogues of this loop during
757 vec
<_loop_vec_info
*> epilogue_vinfos
;
761 /* Access Functions. */
762 #define LOOP_VINFO_LOOP(L) (L)->loop
763 #define LOOP_VINFO_BBS(L) (L)->bbs
764 #define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
765 #define LOOP_VINFO_NITERS(L) (L)->num_iters
766 /* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after
767 prologue peeling retain total unchanged scalar loop iterations for
769 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
770 #define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
771 #define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
772 #define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
773 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
774 #define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
775 #define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
776 #define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
777 (L)->epil_using_partial_vectors_p
778 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
779 #define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
780 #define LOOP_VINFO_MASKS(L) (L)->masks
781 #define LOOP_VINFO_LENS(L) (L)->lens
782 #define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
783 #define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
784 #define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
785 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
786 #define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
787 #define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
788 #define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
789 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
790 #define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
791 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
792 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
793 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
794 #define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
795 #define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
796 #define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
797 #define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
798 #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
799 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
800 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
801 #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
802 #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
803 #define LOOP_VINFO_TARGET_COST_DATA(L) (L)->target_cost_data
804 #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
805 #define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
806 #define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
807 #define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
808 #define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
809 #define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
810 #define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
811 #define LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST(L) (L)->single_scalar_iteration_cost
812 #define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
813 #define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
814 #define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
816 #define LOOP_VINFO_FULLY_MASKED_P(L) \
817 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
818 && !LOOP_VINFO_MASKS (L).is_empty ())
820 #define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
821 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
822 && !LOOP_VINFO_LENS (L).is_empty ())
824 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
825 ((L)->may_misalign_stmts.length () > 0)
826 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
827 ((L)->comp_alias_ddrs.length () > 0 \
828 || (L)->check_unequal_addrs.length () > 0 \
829 || (L)->lower_bounds.length () > 0)
830 #define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
831 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
832 #define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
833 (LOOP_VINFO_SIMD_IF_COND (L))
834 #define LOOP_REQUIRES_VERSIONING(L) \
835 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
836 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
837 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
838 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
840 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
841 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
843 #define LOOP_VINFO_EPILOGUE_P(L) \
844 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
846 #define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
847 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
849 /* Wrapper for loop_vec_info, for tracking success/failure, where a non-NULL
850 value signifies success, and a NULL value signifies failure, supporting
851 propagating an opt_problem * describing the failure back up the call
853 typedef opt_pointer_wrapper
<loop_vec_info
> opt_loop_vec_info
;
855 static inline loop_vec_info
856 loop_vec_info_for_loop (class loop
*loop
)
858 return (loop_vec_info
) loop
->aux
;
863 slp_root (slp_instance_kind kind_
, vec
<stmt_vec_info
> stmts_
,
865 : kind(kind_
), stmts(stmts_
), root(root_
) {}
866 slp_instance_kind kind
;
867 vec
<stmt_vec_info
> stmts
;
871 typedef class _bb_vec_info
: public vec_info
874 _bb_vec_info (vec
<basic_block
> bbs
, vec_info_shared
*);
877 /* The region we are operating on. bbs[0] is the entry, excluding
878 its PHI nodes. In the future we might want to track an explicit
879 entry edge to cover bbs[0] PHI nodes and have a region entry
881 vec
<basic_block
> bbs
;
886 #define BB_VINFO_BB(B) (B)->bb
887 #define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
888 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
889 #define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
890 #define BB_VINFO_DDRS(B) (B)->shared->ddrs
892 static inline bb_vec_info
893 vec_info_for_bb (basic_block bb
)
895 return (bb_vec_info
) bb
->aux
;
898 /*-----------------------------------------------------------------*/
899 /* Info on vectorized defs. */
900 /*-----------------------------------------------------------------*/
901 enum stmt_vec_info_type
{
902 undef_vec_info_type
= 0,
908 call_simd_clone_vec_info_type
,
909 assignment_vec_info_type
,
910 condition_vec_info_type
,
911 comparison_vec_info_type
,
914 type_promotion_vec_info_type
,
915 type_demotion_vec_info_type
,
916 type_conversion_vec_info_type
,
920 loop_exit_ctrl_vec_info_type
923 /* Indicates whether/how a variable is used in the scope of loop/basic
926 vect_unused_in_scope
= 0,
928 /* The def is only used outside the loop. */
930 /* The def is in the inner loop, and the use is in the outer loop, and the
931 use is a reduction stmt. */
932 vect_used_in_outer_by_reduction
,
933 /* The def is in the inner loop, and the use is in the outer loop (and is
934 not part of reduction). */
937 /* defs that feed computations that end up (only) in a reduction. These
938 defs may be used by non-reduction stmts, but eventually, any
939 computations/values that are affected by these defs are used to compute
940 a reduction (i.e. don't get stored to memory, for example). We use this
941 to identify computations that we can change the order in which they are
943 vect_used_by_reduction
,
948 /* The type of vectorization that can be applied to the stmt: regular loop-based
949 vectorization; pure SLP - the stmt is a part of SLP instances and does not
950 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
951 a part of SLP instance and also must be loop-based vectorized, since it has
952 uses outside SLP sequences.
954 In the loop context the meanings of pure and hybrid SLP are slightly
955 different. By saying that pure SLP is applied to the loop, we mean that we
956 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
957 vectorized without doing any conceptual unrolling, cause we don't pack
958 together stmts from different iterations, only within a single iteration.
959 Loop hybrid SLP means that we exploit both intra-iteration and
960 inter-iteration parallelism (e.g., number of elements in the vector is 4
961 and the slp-group-size is 2, in which case we don't have enough parallelism
962 within an iteration, so we obtain the rest of the parallelism from subsequent
963 iterations by unrolling the loop by 2). */
970 /* Says whether a statement is a load, a store of a vectorized statement
971 result, or a store of an invariant value. */
972 enum vec_load_store_type
{
978 /* Describes how we're going to vectorize an individual load or store,
979 or a group of loads or stores. */
980 enum vect_memory_access_type
{
981 /* An access to an invariant address. This is used only for loads. */
984 /* A simple contiguous access. */
987 /* A contiguous access that goes down in memory rather than up,
988 with no additional permutation. This is used only for stores
990 VMAT_CONTIGUOUS_DOWN
,
992 /* A simple contiguous access in which the elements need to be permuted
993 after loading or before storing. Only used for loop vectorization;
994 SLP uses separate permutes. */
995 VMAT_CONTIGUOUS_PERMUTE
,
997 /* A simple contiguous access in which the elements need to be reversed
998 after loading or before storing. */
999 VMAT_CONTIGUOUS_REVERSE
,
1001 /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */
1002 VMAT_LOAD_STORE_LANES
,
1004 /* An access in which each scalar element is loaded or stored
1008 /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
1009 SLP accesses. Each unrolled iteration uses a contiguous load
1010 or store for the whole group, but the groups from separate iterations
1011 are combined in the same way as for VMAT_ELEMENTWISE. */
1014 /* The access uses gather loads or scatter stores. */
1020 /* The data reference itself. */
1022 /* The statement that contains the data reference. */
1024 /* The misalignment in bytes of the reference, or -1 if not known. */
1026 /* The byte alignment that we'd ideally like the reference to have,
1027 and the value that misalignment is measured against. */
1028 poly_uint64 target_alignment
;
1029 /* If true the alignment of base_decl needs to be increased. */
1030 bool base_misaligned
;
1033 /* Stores current vectorized loop's offset. To be added to the DR's
1034 offset to calculate current offset of data reference. */
1038 typedef struct data_reference
*dr_p
;
1040 class _stmt_vec_info
{
1043 enum stmt_vec_info_type type
;
1045 /* Indicates whether this stmts is part of a computation whose result is
1046 used outside the loop. */
1049 /* Stmt is part of some pattern (computation idiom) */
1052 /* True if the statement was created during pattern recognition as
1053 part of the replacement for RELATED_STMT. This implies that the
1054 statement isn't part of any basic block, although for convenience
1055 its gimple_bb is the same as for RELATED_STMT. */
1056 bool pattern_stmt_p
;
1058 /* Is this statement vectorizable or should it be skipped in (partial)
1062 /* The stmt to which this info struct refers to. */
1065 /* The vector type to be used for the LHS of this statement. */
1068 /* The vectorized stmts. */
1069 vec
<gimple
*> vec_stmts
;
1071 /* The following is relevant only for stmts that contain a non-scalar
1072 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
1073 at most one such data-ref. */
1077 /* Information about the data-ref relative to this loop
1078 nest (the loop that is being considered for vectorization). */
1079 innermost_loop_behavior dr_wrt_vec_loop
;
1081 /* For loop PHI nodes, the base and evolution part of it. This makes sure
1082 this information is still available in vect_update_ivs_after_vectorizer
1083 where we may not be able to re-analyze the PHI nodes evolution as
1084 peeling for the prologue loop can make it unanalyzable. The evolution
1085 part is still correct after peeling, but the base may have changed from
1086 the version here. */
1087 tree loop_phi_evolution_base_unchanged
;
1088 tree loop_phi_evolution_part
;
1090 /* Used for various bookkeeping purposes, generally holding a pointer to
1091 some other stmt S that is in some way "related" to this stmt.
1092 Current use of this field is:
1093 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
1094 true): S is the "pattern stmt" that represents (and replaces) the
1095 sequence of stmts that constitutes the pattern. Similarly, the
1096 related_stmt of the "pattern stmt" points back to this stmt (which is
1097 the last stmt in the original sequence of stmts that constitutes the
1099 stmt_vec_info related_stmt
;
1101 /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
1102 The sequence is attached to the original statement rather than the
1103 pattern statement. */
1104 gimple_seq pattern_def_seq
;
1106 /* Selected SIMD clone's function info. First vector element
1107 is SIMD clone's function decl, followed by a pair of trees (base + step)
1108 for linear arguments (pair of NULLs for other arguments). */
1109 vec
<tree
> simd_clone_info
;
1111 /* Classify the def of this stmt. */
1112 enum vect_def_type def_type
;
1114 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
1115 enum slp_vect_type slp_type
;
1117 /* Interleaving and reduction chains info. */
1118 /* First element in the group. */
1119 stmt_vec_info first_element
;
1120 /* Pointer to the next element in the group. */
1121 stmt_vec_info next_element
;
1122 /* The size of the group. */
1124 /* For stores, number of stores from this group seen. We vectorize the last
1126 unsigned int store_count
;
1127 /* For loads only, the gap from the previous load. For consecutive loads, GAP
1131 /* The minimum negative dependence distance this stmt participates in
1133 unsigned int min_neg_dist
;
1135 /* Not all stmts in the loop need to be vectorized. e.g, the increment
1136 of the loop induction variable and computation of array indexes. relevant
1137 indicates whether the stmt needs to be vectorized. */
1138 enum vect_relevant relevant
;
1140 /* For loads if this is a gather, for stores if this is a scatter. */
1141 bool gather_scatter_p
;
1143 /* True if this is an access with loop-invariant stride. */
1146 /* For both loads and stores. */
1147 unsigned simd_lane_access_p
: 3;
1149 /* Classifies how the load or store is going to be implemented
1150 for loop vectorization. */
1151 vect_memory_access_type memory_access_type
;
1153 /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
1154 tree induc_cond_initial_val
;
1156 /* If not NULL the value to be added to compute final reduction value. */
1157 tree reduc_epilogue_adjustment
;
1159 /* On a reduction PHI the reduction type as detected by
1160 vect_is_simple_reduction and vectorizable_reduction. */
1161 enum vect_reduction_type reduc_type
;
1163 /* The original reduction code, to be used in the epilogue. */
1164 enum tree_code reduc_code
;
1165 /* An internal function we should use in the epilogue. */
1166 internal_fn reduc_fn
;
1168 /* On a stmt participating in the reduction the index of the operand
1169 on the reduction SSA cycle. */
1172 /* On a reduction PHI the def returned by vect_force_simple_reduction.
1173 On the def returned by vect_force_simple_reduction the
1174 corresponding PHI. */
1175 stmt_vec_info reduc_def
;
1177 /* The vector input type relevant for reduction vectorization. */
1178 tree reduc_vectype_in
;
1180 /* The vector type for performing the actual reduction. */
1183 /* Whether we force a single cycle PHI during reduction vectorization. */
1184 bool force_single_cycle
;
1186 /* Whether on this stmt reduction meta is recorded. */
1189 /* If nonzero, the lhs of the statement could be truncated to this
1190 many bits without affecting any users of the result. */
1191 unsigned int min_output_precision
;
1193 /* If nonzero, all non-boolean input operands have the same precision,
1194 and they could each be truncated to this many bits without changing
1196 unsigned int min_input_precision
;
1198 /* If OPERATION_BITS is nonzero, the statement could be performed on
1199 an integer with the sign and number of bits given by OPERATION_SIGN
1200 and OPERATION_BITS without changing the result. */
1201 unsigned int operation_precision
;
1202 signop operation_sign
;
1204 /* If the statement produces a boolean result, this value describes
1205 how we should choose the associated vector type. The possible
1208 - an integer precision N if we should use the vector mask type
1209 associated with N-bit integers. This is only used if all relevant
1210 input booleans also want the vector mask type for N-bit integers,
1211 or if we can convert them into that form by pattern-matching.
1213 - ~0U if we considered choosing a vector mask type but decided
1214 to treat the boolean as a normal integer type instead.
1216 - 0 otherwise. This means either that the operation isn't one that
1217 could have a vector mask type (and so should have a normal vector
1218 type instead) or that we simply haven't made a choice either way. */
1219 unsigned int mask_precision
;
1221 /* True if this is only suitable for SLP vectorization. */
1222 bool slp_vect_only_p
;
1224 /* True if this is a pattern that can only be handled by SLP
1226 bool slp_vect_pattern_only_p
;
1229 /* Information about a gather/scatter call. */
1230 struct gather_scatter_info
{
1231 /* The internal function to use for the gather/scatter operation,
1232 or IFN_LAST if a built-in function should be used instead. */
1235 /* The FUNCTION_DECL for the built-in gather/scatter function,
1236 or null if an internal function should be used instead. */
1239 /* The loop-invariant base value. */
1242 /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
1245 /* Each offset element should be multiplied by this amount before
1246 being added to the base. */
1249 /* The definition type for the vectorized offset. */
1250 enum vect_def_type offset_dt
;
1252 /* The type of the vectorized offset. */
1253 tree offset_vectype
;
1255 /* The type of the scalar elements after loading or before storing. */
1258 /* The type of the scalar elements being loaded or stored. */
1262 /* Access Functions. */
1263 #define STMT_VINFO_TYPE(S) (S)->type
1264 #define STMT_VINFO_STMT(S) (S)->stmt
1265 #define STMT_VINFO_RELEVANT(S) (S)->relevant
1266 #define STMT_VINFO_LIVE_P(S) (S)->live
1267 #define STMT_VINFO_VECTYPE(S) (S)->vectype
1268 #define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1269 #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1270 #define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1271 #define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1272 #define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1273 #define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1274 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1275 #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1276 #define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1277 #define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1278 #define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1280 #define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1281 #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1282 #define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1283 #define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1284 #define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1285 #define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1286 #define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1287 (S)->dr_wrt_vec_loop.base_misalignment
1288 #define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1289 (S)->dr_wrt_vec_loop.offset_alignment
1290 #define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1291 (S)->dr_wrt_vec_loop.step_alignment
1293 #define STMT_VINFO_DR_INFO(S) \
1294 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1296 #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1297 #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1298 #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1299 #define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1300 #define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1301 #define STMT_VINFO_GROUPED_ACCESS(S) \
1302 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1303 #define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1304 #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1305 #define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1306 #define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1307 #define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1308 #define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1309 #define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1310 #define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1311 #define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1312 #define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1313 #define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1315 #define DR_GROUP_FIRST_ELEMENT(S) \
1316 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1317 #define DR_GROUP_NEXT_ELEMENT(S) \
1318 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1319 #define DR_GROUP_SIZE(S) \
1320 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1321 #define DR_GROUP_STORE_COUNT(S) \
1322 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1323 #define DR_GROUP_GAP(S) \
1324 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1326 #define REDUC_GROUP_FIRST_ELEMENT(S) \
1327 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1328 #define REDUC_GROUP_NEXT_ELEMENT(S) \
1329 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1330 #define REDUC_GROUP_SIZE(S) \
1331 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1333 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1335 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1336 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1337 #define STMT_SLP_TYPE(S) (S)->slp_type
1339 #define VECT_MAX_COST 1000
1341 /* The maximum number of intermediate steps required in multi-step type
1343 #define MAX_INTERM_CVT_STEPS 3
1345 #define MAX_VECTORIZATION_FACTOR INT_MAX
1347 /* Nonzero if TYPE represents a (scalar) boolean type or type
1348 in the middle-end compatible with it (unsigned precision 1 integral
1349 types). Used to determine which types should be vectorized as
1350 VECTOR_BOOLEAN_TYPE_P. */
1352 #define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1353 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1354 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1355 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1356 && TYPE_PRECISION (TYPE) == 1 \
1357 && TYPE_UNSIGNED (TYPE)))
1360 nested_in_vect_loop_p (class loop
*loop
, stmt_vec_info stmt_info
)
1363 && (loop
->inner
== (gimple_bb (stmt_info
->stmt
))->loop_father
));
1366 /* Return true if STMT_INFO should produce a vector mask type rather than
1367 a normal nonmask type. */
1370 vect_use_mask_type_p (stmt_vec_info stmt_info
)
1372 return stmt_info
->mask_precision
&& stmt_info
->mask_precision
!= ~0U;
1375 /* Return TRUE if a statement represented by STMT_INFO is a part of a
1379 is_pattern_stmt_p (stmt_vec_info stmt_info
)
1381 return stmt_info
->pattern_stmt_p
;
1384 /* If STMT_INFO is a pattern statement, return the statement that it
1385 replaces, otherwise return STMT_INFO itself. */
1387 inline stmt_vec_info
1388 vect_orig_stmt (stmt_vec_info stmt_info
)
1390 if (is_pattern_stmt_p (stmt_info
))
1391 return STMT_VINFO_RELATED_STMT (stmt_info
);
1395 /* Return the later statement between STMT1_INFO and STMT2_INFO. */
1397 static inline stmt_vec_info
1398 get_later_stmt (stmt_vec_info stmt1_info
, stmt_vec_info stmt2_info
)
1400 if (gimple_uid (vect_orig_stmt (stmt1_info
)->stmt
)
1401 > gimple_uid (vect_orig_stmt (stmt2_info
)->stmt
))
1407 /* If STMT_INFO has been replaced by a pattern statement, return the
1408 replacement statement, otherwise return STMT_INFO itself. */
1410 inline stmt_vec_info
1411 vect_stmt_to_vectorize (stmt_vec_info stmt_info
)
1413 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
1414 return STMT_VINFO_RELATED_STMT (stmt_info
);
1418 /* Return true if BB is a loop header. */
1421 is_loop_header_bb_p (basic_block bb
)
1423 if (bb
== (bb
->loop_father
)->header
)
1425 gcc_checking_assert (EDGE_COUNT (bb
->preds
) == 1);
1429 /* Return pow2 (X). */
1436 for (i
= 0; i
< x
; i
++)
1442 /* Alias targetm.vectorize.builtin_vectorization_cost. */
1445 builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost
,
1446 tree vectype
, int misalign
)
1448 return targetm
.vectorize
.builtin_vectorization_cost (type_of_cost
,
1452 /* Get cost by calling cost target builtin. */
1455 int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost
)
1457 return builtin_vectorization_cost (type_of_cost
, NULL
, 0);
1460 /* Alias targetm.vectorize.init_cost. */
1462 static inline void *
1463 init_cost (class loop
*loop_info
, bool costing_for_scalar
)
1465 return targetm
.vectorize
.init_cost (loop_info
, costing_for_scalar
);
1468 extern void dump_stmt_cost (FILE *, void *, int, enum vect_cost_for_stmt
,
1469 stmt_vec_info
, tree
, int, unsigned,
1470 enum vect_cost_model_location
);
1472 /* Alias targetm.vectorize.add_stmt_cost. */
1474 static inline unsigned
1475 add_stmt_cost (vec_info
*vinfo
, void *data
, int count
,
1476 enum vect_cost_for_stmt kind
,
1477 stmt_vec_info stmt_info
, tree vectype
, int misalign
,
1478 enum vect_cost_model_location where
)
1480 unsigned cost
= targetm
.vectorize
.add_stmt_cost (vinfo
, data
, count
, kind
,
1483 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1484 dump_stmt_cost (dump_file
, data
, count
, kind
, stmt_info
, vectype
, misalign
,
1489 /* Alias targetm.vectorize.add_stmt_cost. */
1491 static inline unsigned
1492 add_stmt_cost (vec_info
*vinfo
, void *data
, stmt_info_for_cost
*i
)
1494 return add_stmt_cost (vinfo
, data
, i
->count
, i
->kind
, i
->stmt_info
,
1495 i
->vectype
, i
->misalign
, i
->where
);
1498 /* Alias targetm.vectorize.finish_cost. */
1501 finish_cost (void *data
, unsigned *prologue_cost
,
1502 unsigned *body_cost
, unsigned *epilogue_cost
)
1504 targetm
.vectorize
.finish_cost (data
, prologue_cost
, body_cost
, epilogue_cost
);
1507 /* Alias targetm.vectorize.destroy_cost_data. */
1510 destroy_cost_data (void *data
)
1512 targetm
.vectorize
.destroy_cost_data (data
);
1516 add_stmt_costs (vec_info
*vinfo
, void *data
, stmt_vector_for_cost
*cost_vec
)
1518 stmt_info_for_cost
*cost
;
1520 FOR_EACH_VEC_ELT (*cost_vec
, i
, cost
)
1521 add_stmt_cost (vinfo
, data
, cost
->count
, cost
->kind
, cost
->stmt_info
,
1522 cost
->vectype
, cost
->misalign
, cost
->where
);
1525 /*-----------------------------------------------------------------*/
1526 /* Info on data references alignment. */
1527 /*-----------------------------------------------------------------*/
1528 #define DR_MISALIGNMENT_UNKNOWN (-1)
1529 #define DR_MISALIGNMENT_UNINITIALIZED (-2)
1532 set_dr_misalignment (dr_vec_info
*dr_info
, int val
)
1534 dr_info
->misalignment
= val
;
1538 dr_misalignment (dr_vec_info
*dr_info
)
1540 int misalign
= dr_info
->misalignment
;
1541 gcc_assert (misalign
!= DR_MISALIGNMENT_UNINITIALIZED
);
1545 /* Reflects actual alignment of first access in the vectorized loop,
1546 taking into account peeling/versioning if applied. */
1547 #define DR_MISALIGNMENT(DR) dr_misalignment (DR)
1548 #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
1550 /* Only defined once DR_MISALIGNMENT is defined. */
1551 #define DR_TARGET_ALIGNMENT(DR) ((DR)->target_alignment)
1553 /* Return true if data access DR_INFO is aligned to its target alignment
1554 (which may be less than a full vector). */
1557 aligned_access_p (dr_vec_info
*dr_info
)
1559 return (DR_MISALIGNMENT (dr_info
) == 0);
1562 /* Return TRUE if the alignment of the data access is known, and FALSE
1566 known_alignment_for_access_p (dr_vec_info
*dr_info
)
1568 return (DR_MISALIGNMENT (dr_info
) != DR_MISALIGNMENT_UNKNOWN
);
1571 /* Return the minimum alignment in bytes that the vectorized version
1572 of DR_INFO is guaranteed to have. */
1574 static inline unsigned int
1575 vect_known_alignment_in_bytes (dr_vec_info
*dr_info
)
1577 if (DR_MISALIGNMENT (dr_info
) == DR_MISALIGNMENT_UNKNOWN
)
1578 return TYPE_ALIGN_UNIT (TREE_TYPE (DR_REF (dr_info
->dr
)));
1579 if (DR_MISALIGNMENT (dr_info
) == 0)
1580 return known_alignment (DR_TARGET_ALIGNMENT (dr_info
));
1581 return DR_MISALIGNMENT (dr_info
) & -DR_MISALIGNMENT (dr_info
);
1584 /* Return the behavior of DR_INFO with respect to the vectorization context
1585 (which for outer loop vectorization might not be the behavior recorded
1586 in DR_INFO itself). */
1588 static inline innermost_loop_behavior
*
1589 vect_dr_behavior (vec_info
*vinfo
, dr_vec_info
*dr_info
)
1591 stmt_vec_info stmt_info
= dr_info
->stmt
;
1592 loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (vinfo
);
1593 if (loop_vinfo
== NULL
1594 || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), stmt_info
))
1595 return &DR_INNERMOST (dr_info
->dr
);
1597 return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info
);
1600 /* Return the offset calculated by adding the offset of this DR_INFO to the
1601 corresponding data_reference's offset. If CHECK_OUTER then use
1602 vect_dr_behavior to select the appropriate data_reference to use. */
1605 get_dr_vinfo_offset (vec_info
*vinfo
,
1606 dr_vec_info
*dr_info
, bool check_outer
= false)
1608 innermost_loop_behavior
*base
;
1610 base
= vect_dr_behavior (vinfo
, dr_info
);
1612 base
= &dr_info
->dr
->innermost
;
1614 tree offset
= base
->offset
;
1616 if (!dr_info
->offset
)
1619 offset
= fold_convert (sizetype
, offset
);
1620 return fold_build2 (PLUS_EXPR
, TREE_TYPE (dr_info
->offset
), offset
,
1625 /* Return true if the vect cost model is unlimited. */
1627 unlimited_cost_model (loop_p loop
)
1629 if (loop
!= NULL
&& loop
->force_vectorize
1630 && flag_simd_cost_model
!= VECT_COST_MODEL_DEFAULT
)
1631 return flag_simd_cost_model
== VECT_COST_MODEL_UNLIMITED
;
1632 return (flag_vect_cost_model
== VECT_COST_MODEL_UNLIMITED
);
1635 /* Return true if the loop described by LOOP_VINFO is fully-masked and
1636 if the first iteration should use a partial mask in order to achieve
1640 vect_use_loop_mask_for_alignment_p (loop_vec_info loop_vinfo
)
1642 return (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
1643 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
));
1646 /* Return the number of vectors of type VECTYPE that are needed to get
1647 NUNITS elements. NUNITS should be based on the vectorization factor,
1648 so it is always a known multiple of the number of elements in VECTYPE. */
1650 static inline unsigned int
1651 vect_get_num_vectors (poly_uint64 nunits
, tree vectype
)
1653 return exact_div (nunits
, TYPE_VECTOR_SUBPARTS (vectype
)).to_constant ();
1656 /* Return the number of copies needed for loop vectorization when
1657 a statement operates on vectors of type VECTYPE. This is the
1658 vectorization factor divided by the number of elements in
1659 VECTYPE and is always known at compile time. */
1661 static inline unsigned int
1662 vect_get_num_copies (loop_vec_info loop_vinfo
, tree vectype
)
1664 return vect_get_num_vectors (LOOP_VINFO_VECT_FACTOR (loop_vinfo
), vectype
);
1667 /* Update maximum unit count *MAX_NUNITS so that it accounts for
1668 NUNITS. *MAX_NUNITS can be 1 if we haven't yet recorded anything. */
1671 vect_update_max_nunits (poly_uint64
*max_nunits
, poly_uint64 nunits
)
1673 /* All unit counts have the form vec_info::vector_size * X for some
1674 rational X, so two unit sizes must have a common multiple.
1675 Everything is a multiple of the initial value of 1. */
1676 *max_nunits
= force_common_multiple (*max_nunits
, nunits
);
1679 /* Update maximum unit count *MAX_NUNITS so that it accounts for
1680 the number of units in vector type VECTYPE. *MAX_NUNITS can be 1
1681 if we haven't yet recorded any vector types. */
1684 vect_update_max_nunits (poly_uint64
*max_nunits
, tree vectype
)
1686 vect_update_max_nunits (max_nunits
, TYPE_VECTOR_SUBPARTS (vectype
));
1689 /* Return the vectorization factor that should be used for costing
1690 purposes while vectorizing the loop described by LOOP_VINFO.
1691 Pick a reasonable estimate if the vectorization factor isn't
1692 known at compile time. */
1694 static inline unsigned int
1695 vect_vf_for_cost (loop_vec_info loop_vinfo
)
1697 return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo
));
1700 /* Estimate the number of elements in VEC_TYPE for costing purposes.
1701 Pick a reasonable estimate if the exact number isn't known at
1704 static inline unsigned int
1705 vect_nunits_for_cost (tree vec_type
)
1707 return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type
));
1710 /* Return the maximum possible vectorization factor for LOOP_VINFO. */
1712 static inline unsigned HOST_WIDE_INT
1713 vect_max_vf (loop_vec_info loop_vinfo
)
1715 unsigned HOST_WIDE_INT vf
;
1716 if (LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant (&vf
))
1718 return MAX_VECTORIZATION_FACTOR
;
1721 /* Return the size of the value accessed by unvectorized data reference
1722 DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
1723 for the associated gimple statement, since that guarantees that DR_INFO
1724 accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
1725 here includes things like V1SI, which can be vectorized in the same way
1729 vect_get_scalar_dr_size (dr_vec_info
*dr_info
)
1731 return tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr_info
->dr
))));
1734 /* Return true if LOOP_VINFO requires a runtime check for whether the
1735 vector loop is profitable. */
1738 vect_apply_runtime_profitability_check_p (loop_vec_info loop_vinfo
)
1740 unsigned int th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
1741 return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1742 && th
>= vect_vf_for_cost (loop_vinfo
));
1745 /* Source location + hotness information. */
1746 extern dump_user_location_t vect_location
;
1748 /* A macro for calling:
1749 dump_begin_scope (MSG, vect_location);
1750 via an RAII object, thus printing "=== MSG ===\n" to the dumpfile etc,
1753 once the object goes out of scope, thus capturing the nesting of
1756 These scopes affect dump messages within them: dump messages at the
1757 top level implicitly default to MSG_PRIORITY_USER_FACING, whereas those
1758 in a nested scope implicitly default to MSG_PRIORITY_INTERNALS. */
1760 #define DUMP_VECT_SCOPE(MSG) \
1761 AUTO_DUMP_SCOPE (MSG, vect_location)
1763 /* A sentinel class for ensuring that the "vect_location" global gets
1764 reset at the end of a scope.
1766 The "vect_location" global is used during dumping and contains a
1767 location_t, which could contain references to a tree block via the
1768 ad-hoc data. This data is used for tracking inlining information,
1769 but it's not a GC root; it's simply assumed that such locations never
1770 get accessed if the blocks are optimized away.
1772 Hence we need to ensure that such locations are purged at the end
1773 of any operations using them (e.g. via this class). */
1775 class auto_purge_vect_location
1778 ~auto_purge_vect_location ();
1781 /*-----------------------------------------------------------------*/
1782 /* Function prototypes. */
1783 /*-----------------------------------------------------------------*/
1785 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
1786 in tree-vect-loop-manip.c. */
1787 extern void vect_set_loop_condition (class loop
*, loop_vec_info
,
1788 tree
, tree
, tree
, bool);
1789 extern bool slpeel_can_duplicate_loop_p (const class loop
*, const_edge
);
1790 class loop
*slpeel_tree_duplicate_loop_to_edge_cfg (class loop
*,
1791 class loop
*, edge
);
1792 class loop
*vect_loop_versioning (loop_vec_info
, gimple
*);
1793 extern class loop
*vect_do_peeling (loop_vec_info
, tree
, tree
,
1794 tree
*, tree
*, tree
*, int, bool, bool,
1796 extern void vect_prepare_for_masked_peels (loop_vec_info
);
1797 extern dump_user_location_t
find_loop_location (class loop
*);
1798 extern bool vect_can_advance_ivs_p (loop_vec_info
);
1799 extern void vect_update_inits_of_drs (loop_vec_info
, tree
, tree_code
);
1801 /* In tree-vect-stmts.c. */
1802 extern tree
get_related_vectype_for_scalar_type (machine_mode
, tree
,
1804 extern tree
get_vectype_for_scalar_type (vec_info
*, tree
, unsigned int = 0);
1805 extern tree
get_vectype_for_scalar_type (vec_info
*, tree
, slp_tree
);
1806 extern tree
get_mask_type_for_scalar_type (vec_info
*, tree
, unsigned int = 0);
1807 extern tree
get_same_sized_vectype (tree
, tree
);
1808 extern bool vect_chooses_same_modes_p (vec_info
*, machine_mode
);
1809 extern bool vect_get_loop_mask_type (loop_vec_info
);
1810 extern bool vect_is_simple_use (tree
, vec_info
*, enum vect_def_type
*,
1811 stmt_vec_info
* = NULL
, gimple
** = NULL
);
1812 extern bool vect_is_simple_use (tree
, vec_info
*, enum vect_def_type
*,
1813 tree
*, stmt_vec_info
* = NULL
,
1815 extern bool vect_is_simple_use (vec_info
*, stmt_vec_info
, slp_tree
,
1816 unsigned, tree
*, slp_tree
*,
1817 enum vect_def_type
*,
1818 tree
*, stmt_vec_info
* = NULL
);
1819 extern bool vect_maybe_update_slp_op_vectype (slp_tree
, tree
);
1820 extern bool supportable_widening_operation (vec_info
*,
1821 enum tree_code
, stmt_vec_info
,
1822 tree
, tree
, enum tree_code
*,
1823 enum tree_code
*, int *,
1825 extern bool supportable_narrowing_operation (enum tree_code
, tree
, tree
,
1826 enum tree_code
*, int *,
1829 extern unsigned record_stmt_cost (stmt_vector_for_cost
*, int,
1830 enum vect_cost_for_stmt
, stmt_vec_info
,
1831 tree
, int, enum vect_cost_model_location
);
1833 /* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
1835 static inline unsigned
1836 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
1837 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
1838 int misalign
, enum vect_cost_model_location where
)
1840 return record_stmt_cost (body_cost_vec
, count
, kind
, stmt_info
,
1841 STMT_VINFO_VECTYPE (stmt_info
), misalign
, where
);
1844 extern void vect_finish_replace_stmt (vec_info
*, stmt_vec_info
, gimple
*);
1845 extern void vect_finish_stmt_generation (vec_info
*, stmt_vec_info
, gimple
*,
1846 gimple_stmt_iterator
*);
1847 extern opt_result
vect_mark_stmts_to_be_vectorized (loop_vec_info
, bool *);
1848 extern tree
vect_get_store_rhs (stmt_vec_info
);
1849 void vect_get_vec_defs_for_operand (vec_info
*vinfo
, stmt_vec_info
, unsigned,
1850 tree op
, vec
<tree
> *, tree
= NULL
);
1851 void vect_get_vec_defs (vec_info
*, stmt_vec_info
, slp_tree
, unsigned,
1853 tree
= NULL
, vec
<tree
> * = NULL
,
1854 tree
= NULL
, vec
<tree
> * = NULL
,
1855 tree
= NULL
, vec
<tree
> * = NULL
);
1856 void vect_get_vec_defs (vec_info
*, stmt_vec_info
, slp_tree
, unsigned,
1857 tree
, vec
<tree
> *, tree
,
1858 tree
= NULL
, vec
<tree
> * = NULL
, tree
= NULL
,
1859 tree
= NULL
, vec
<tree
> * = NULL
, tree
= NULL
,
1860 tree
= NULL
, vec
<tree
> * = NULL
, tree
= NULL
);
1861 extern tree
vect_init_vector (vec_info
*, stmt_vec_info
, tree
, tree
,
1862 gimple_stmt_iterator
*);
1863 extern tree
vect_get_slp_vect_def (slp_tree
, unsigned);
1864 extern bool vect_transform_stmt (vec_info
*, stmt_vec_info
,
1865 gimple_stmt_iterator
*,
1866 slp_tree
, slp_instance
);
1867 extern void vect_remove_stores (vec_info
*, stmt_vec_info
);
1868 extern bool vect_nop_conversion_p (stmt_vec_info
);
1869 extern opt_result
vect_analyze_stmt (vec_info
*, stmt_vec_info
, bool *,
1871 slp_instance
, stmt_vector_for_cost
*);
1872 extern void vect_get_load_cost (vec_info
*, stmt_vec_info
, int, bool,
1873 unsigned int *, unsigned int *,
1874 stmt_vector_for_cost
*,
1875 stmt_vector_for_cost
*, bool);
1876 extern void vect_get_store_cost (vec_info
*, stmt_vec_info
, int,
1877 unsigned int *, stmt_vector_for_cost
*);
1878 extern bool vect_supportable_shift (vec_info
*, enum tree_code
, tree
);
1879 extern tree
vect_gen_perm_mask_any (tree
, const vec_perm_indices
&);
1880 extern tree
vect_gen_perm_mask_checked (tree
, const vec_perm_indices
&);
1881 extern void optimize_mask_stores (class loop
*);
1882 extern gcall
*vect_gen_while (tree
, tree
, tree
);
1883 extern tree
vect_gen_while_not (gimple_seq
*, tree
, tree
, tree
);
1884 extern opt_result
vect_get_vector_types_for_stmt (vec_info
*,
1885 stmt_vec_info
, tree
*,
1886 tree
*, unsigned int = 0);
1887 extern opt_tree
vect_get_mask_type_for_stmt (stmt_vec_info
, unsigned int = 0);
1889 /* In tree-vect-data-refs.c. */
1890 extern bool vect_can_force_dr_alignment_p (const_tree
, poly_uint64
);
1891 extern enum dr_alignment_support vect_supportable_dr_alignment
1892 (vec_info
*, dr_vec_info
*, bool);
1893 extern tree
vect_get_smallest_scalar_type (stmt_vec_info
, HOST_WIDE_INT
*,
1895 extern opt_result
vect_analyze_data_ref_dependences (loop_vec_info
, unsigned int *);
1896 extern bool vect_slp_analyze_instance_dependence (vec_info
*, slp_instance
);
1897 extern opt_result
vect_enhance_data_refs_alignment (loop_vec_info
);
1898 extern opt_result
vect_analyze_data_refs_alignment (loop_vec_info
);
1899 extern bool vect_slp_analyze_instance_alignment (vec_info
*, slp_instance
);
1900 extern opt_result
vect_analyze_data_ref_accesses (vec_info
*, vec
<int> *);
1901 extern opt_result
vect_prune_runtime_alias_test_list (loop_vec_info
);
1902 extern bool vect_gather_scatter_fn_p (vec_info
*, bool, bool, tree
, tree
,
1903 tree
, int, internal_fn
*, tree
*);
1904 extern bool vect_check_gather_scatter (stmt_vec_info
, loop_vec_info
,
1905 gather_scatter_info
*);
1906 extern opt_result
vect_find_stmt_data_reference (loop_p
, gimple
*,
1907 vec
<data_reference_p
> *,
1909 extern opt_result
vect_analyze_data_refs (vec_info
*, poly_uint64
*, bool *);
1910 extern void vect_record_base_alignments (vec_info
*);
1911 extern tree
vect_create_data_ref_ptr (vec_info
*,
1912 stmt_vec_info
, tree
, class loop
*, tree
,
1913 tree
*, gimple_stmt_iterator
*,
1915 tree
= NULL_TREE
, tree
= NULL_TREE
);
1916 extern tree
bump_vector_ptr (vec_info
*, tree
, gimple
*, gimple_stmt_iterator
*,
1917 stmt_vec_info
, tree
);
1918 extern void vect_copy_ref_info (tree
, tree
);
1919 extern tree
vect_create_destination_var (tree
, tree
);
1920 extern bool vect_grouped_store_supported (tree
, unsigned HOST_WIDE_INT
);
1921 extern bool vect_store_lanes_supported (tree
, unsigned HOST_WIDE_INT
, bool);
1922 extern bool vect_grouped_load_supported (tree
, bool, unsigned HOST_WIDE_INT
);
1923 extern bool vect_load_lanes_supported (tree
, unsigned HOST_WIDE_INT
, bool);
1924 extern void vect_permute_store_chain (vec_info
*,
1925 vec
<tree
> ,unsigned int, stmt_vec_info
,
1926 gimple_stmt_iterator
*, vec
<tree
> *);
1927 extern tree
vect_setup_realignment (vec_info
*,
1928 stmt_vec_info
, gimple_stmt_iterator
*,
1929 tree
*, enum dr_alignment_support
, tree
,
1931 extern void vect_transform_grouped_load (vec_info
*, stmt_vec_info
, vec
<tree
>,
1932 int, gimple_stmt_iterator
*);
1933 extern void vect_record_grouped_load_vectors (vec_info
*,
1934 stmt_vec_info
, vec
<tree
>);
1935 extern tree
vect_get_new_vect_var (tree
, enum vect_var_kind
, const char *);
1936 extern tree
vect_get_new_ssa_name (tree
, enum vect_var_kind
,
1937 const char * = NULL
);
1938 extern tree
vect_create_addr_base_for_vector_ref (vec_info
*,
1939 stmt_vec_info
, gimple_seq
*,
1940 tree
, tree
= NULL_TREE
);
1942 /* In tree-vect-loop.c. */
1943 extern widest_int
vect_iv_limit_for_partial_vectors (loop_vec_info loop_vinfo
);
1944 bool vect_rgroup_iv_might_wrap_p (loop_vec_info
, rgroup_controls
*);
1945 /* Used in tree-vect-loop-manip.c */
1946 extern opt_result
vect_determine_partial_vectors_and_peeling (loop_vec_info
,
1948 /* Used in gimple-loop-interchange.c and tree-parloops.c. */
1949 extern bool check_reduction_path (dump_user_location_t
, loop_p
, gphi
*, tree
,
1951 extern bool needs_fold_left_reduction_p (tree
, tree_code
);
1952 /* Drive for loop analysis stage. */
1953 extern opt_loop_vec_info
vect_analyze_loop (class loop
*, vec_info_shared
*);
1954 extern tree
vect_build_loop_niters (loop_vec_info
, bool * = NULL
);
1955 extern void vect_gen_vector_loop_niters (loop_vec_info
, tree
, tree
*,
1957 extern tree
vect_halve_mask_nunits (tree
, machine_mode
);
1958 extern tree
vect_double_mask_nunits (tree
, machine_mode
);
1959 extern void vect_record_loop_mask (loop_vec_info
, vec_loop_masks
*,
1960 unsigned int, tree
, tree
);
1961 extern tree
vect_get_loop_mask (gimple_stmt_iterator
*, vec_loop_masks
*,
1962 unsigned int, tree
, unsigned int);
1963 extern void vect_record_loop_len (loop_vec_info
, vec_loop_lens
*, unsigned int,
1964 tree
, unsigned int);
1965 extern tree
vect_get_loop_len (loop_vec_info
, vec_loop_lens
*, unsigned int,
1967 extern gimple_seq
vect_gen_len (tree
, tree
, tree
, tree
);
1968 extern stmt_vec_info
info_for_reduction (vec_info
*, stmt_vec_info
);
1970 /* Drive for loop transformation stage. */
1971 extern class loop
*vect_transform_loop (loop_vec_info
, gimple
*);
1972 extern opt_loop_vec_info
vect_analyze_loop_form (class loop
*,
1974 extern bool vectorizable_live_operation (vec_info
*,
1975 stmt_vec_info
, gimple_stmt_iterator
*,
1976 slp_tree
, slp_instance
, int,
1977 bool, stmt_vector_for_cost
*);
1978 extern bool vectorizable_reduction (loop_vec_info
, stmt_vec_info
,
1979 slp_tree
, slp_instance
,
1980 stmt_vector_for_cost
*);
1981 extern bool vectorizable_induction (loop_vec_info
, stmt_vec_info
,
1982 gimple
**, slp_tree
,
1983 stmt_vector_for_cost
*);
1984 extern bool vect_transform_reduction (loop_vec_info
, stmt_vec_info
,
1985 gimple_stmt_iterator
*,
1986 gimple
**, slp_tree
);
1987 extern bool vect_transform_cycle_phi (loop_vec_info
, stmt_vec_info
,
1989 slp_tree
, slp_instance
);
1990 extern bool vectorizable_lc_phi (loop_vec_info
, stmt_vec_info
,
1991 gimple
**, slp_tree
);
1992 extern bool vectorizable_phi (vec_info
*, stmt_vec_info
, gimple
**, slp_tree
,
1993 stmt_vector_for_cost
*);
1994 extern bool vect_worthwhile_without_simd_p (vec_info
*, tree_code
);
1995 extern int vect_get_known_peeling_cost (loop_vec_info
, int, int *,
1996 stmt_vector_for_cost
*,
1997 stmt_vector_for_cost
*,
1998 stmt_vector_for_cost
*);
1999 extern tree
cse_and_gimplify_to_preheader (loop_vec_info
, tree
);
2001 /* In tree-vect-slp.c. */
2002 extern void vect_slp_init (void);
2003 extern void vect_slp_fini (void);
2004 extern void vect_free_slp_instance (slp_instance
);
2005 extern bool vect_transform_slp_perm_load (vec_info
*, slp_tree
, vec
<tree
>,
2006 gimple_stmt_iterator
*, poly_uint64
,
2008 unsigned * = nullptr);
2009 extern bool vect_slp_analyze_operations (vec_info
*);
2010 extern void vect_schedule_slp (vec_info
*, vec
<slp_instance
>);
2011 extern opt_result
vect_analyze_slp (vec_info
*, unsigned);
2012 extern bool vect_make_slp_decision (loop_vec_info
);
2013 extern void vect_detect_hybrid_slp (loop_vec_info
);
2014 extern void vect_optimize_slp (vec_info
*);
2015 extern void vect_gather_slp_loads (vec_info
*);
2016 extern void vect_get_slp_defs (slp_tree
, vec
<tree
> *);
2017 extern void vect_get_slp_defs (vec_info
*, slp_tree
, vec
<vec
<tree
> > *,
2019 extern bool vect_slp_bb (basic_block
);
2020 extern bool vect_slp_function (function
*);
2021 extern stmt_vec_info
vect_find_last_scalar_stmt_in_slp (slp_tree
);
2022 extern stmt_vec_info
vect_find_first_scalar_stmt_in_slp (slp_tree
);
2023 extern bool is_simple_and_all_uses_invariant (stmt_vec_info
, loop_vec_info
);
2024 extern bool can_duplicate_and_interleave_p (vec_info
*, unsigned int, tree
,
2025 unsigned int * = NULL
,
2026 tree
* = NULL
, tree
* = NULL
);
2027 extern void duplicate_and_interleave (vec_info
*, gimple_seq
*, tree
,
2028 vec
<tree
>, unsigned int, vec
<tree
> &);
2029 extern int vect_get_place_in_interleaving_chain (stmt_vec_info
, stmt_vec_info
);
2030 extern bool vect_update_shared_vectype (stmt_vec_info
, tree
);
2031 extern slp_tree
vect_create_new_slp_node (unsigned, tree_code
);
2032 extern void vect_free_slp_tree (slp_tree
);
2034 /* In tree-vect-patterns.c. */
2036 vect_mark_pattern_stmts (vec_info
*, stmt_vec_info
, gimple
*, tree
);
2038 /* Pattern recognition functions.
2039 Additional pattern recognition functions can (and will) be added
2041 void vect_pattern_recog (vec_info
*);
2043 /* In tree-vectorizer.c. */
2044 unsigned vectorize_loops (void);
2045 void vect_free_loop_info_assumptions (class loop
*);
2046 gimple
*vect_loop_vectorized_call (class loop
*, gcond
**cond
= NULL
);
2047 bool vect_stmt_dominates_stmt_p (gimple
*, gimple
*);
2049 /* SLP Pattern matcher types, tree-vect-slp-patterns.c. */
2051 /* Forward declaration of possible two operands operation that can be matched
2052 by the complex numbers pattern matchers. */
2053 enum _complex_operation
: unsigned;
2055 /* All possible load permute values that could result from the partial data-flow
2057 typedef enum _complex_perm_kinds
{
2063 /* Can be combined with any other PERM values. */
2065 } complex_perm_kinds_t
;
2067 /* Cache from nodes to the load permutation they represent. */
2068 typedef hash_map
<slp_tree
, complex_perm_kinds_t
>
2069 slp_tree_to_load_perm_map_t
;
2071 /* Vector pattern matcher base class. All SLP pattern matchers must inherit
2077 /* The number of arguments that the IFN requires. */
2078 unsigned m_num_args
;
2080 /* The internal function that will be used when a pattern is created. */
2083 /* The current node being inspected. */
2086 /* The list of operands to be the children for the node produced when the
2087 internal function is created. */
2088 vec
<slp_tree
> m_ops
;
2090 /* Default constructor where NODE is the root of the tree to inspect. */
2091 vect_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
2094 this->m_node
= node
;
2095 this->m_ops
.create (0);
2096 this->m_ops
.safe_splice (*m_ops
);
2101 /* Create a new instance of the pattern matcher class of the given type. */
2102 static vect_pattern
* recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
2104 /* Build the pattern from the data collected so far. */
2105 virtual void build (vec_info
*) = 0;
2107 /* Default destructor. */
2108 virtual ~vect_pattern ()
2110 this->m_ops
.release ();
2114 /* Function pointer to create a new pattern matcher from a generic type. */
2115 typedef vect_pattern
* (*vect_pattern_decl_t
) (slp_tree_to_load_perm_map_t
*,
2118 /* List of supported pattern matchers. */
2119 extern vect_pattern_decl_t slp_patterns
[];
2121 /* Number of supported pattern matchers. */
2122 extern size_t num__slp_patterns
;
2124 #endif /* GCC_TREE_VECTORIZER_H */