1 /* Scalar Replacement of Aggregates (SRA) converts some structure
2 references into scalar references, exposing them to the scalar
4 Copyright (C) 2008-2020 Free Software Foundation, Inc.
5 Contributed by Martin Jambor <mjambor@suse.cz>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
23 /* This file implements Scalar Reduction of Aggregates (SRA). SRA is run
24 twice, once in the early stages of compilation (early SRA) and once in the
25 late stages (late SRA). The aim of both is to turn references to scalar
26 parts of aggregates into uses of independent scalar variables.
28 The two passes are nearly identical, the only difference is that early SRA
29 does not scalarize unions which are used as the result in a GIMPLE_RETURN
30 statement because together with inlining this can lead to weird type
33 Both passes operate in four stages:
35 1. The declarations that have properties which make them candidates for
36 scalarization are identified in function find_var_candidates(). The
37 candidates are stored in candidate_bitmap.
39 2. The function body is scanned. In the process, declarations which are
40 used in a manner that prevent their scalarization are removed from the
41 candidate bitmap. More importantly, for every access into an aggregate,
42 an access structure (struct access) is created by create_access() and
43 stored in a vector associated with the aggregate. Among other
44 information, the aggregate declaration, the offset and size of the access
45 and its type are stored in the structure.
47 On a related note, assign_link structures are created for every assign
48 statement between candidate aggregates and attached to the related
51 3. The vectors of accesses are analyzed. They are first sorted according to
52 their offset and size and then scanned for partially overlapping accesses
53 (i.e. those which overlap but one is not entirely within another). Such
54 an access disqualifies the whole aggregate from being scalarized.
56 If there is no such inhibiting overlap, a representative access structure
57 is chosen for every unique combination of offset and size. Afterwards,
58 the pass builds a set of trees from these structures, in which children
59 of an access are within their parent (in terms of offset and size).
61 Then accesses are propagated whenever possible (i.e. in cases when it
62 does not create a partially overlapping access) across assign_links from
63 the right hand side to the left hand side.
65 Then the set of trees for each declaration is traversed again and those
66 accesses which should be replaced by a scalar are identified.
68 4. The function is traversed again, and for every reference into an
69 aggregate that has some component which is about to be scalarized,
70 statements are amended and new statements are created as necessary.
71 Finally, if a parameter got scalarized, the scalar replacements are
72 initialized with values from respective parameter aggregates. */
76 #include "coretypes.h"
83 #include "alloc-pool.h"
84 #include "tree-pass.h"
87 #include "gimple-pretty-print.h"
89 #include "fold-const.h"
91 #include "stor-layout.h"
93 #include "gimple-iterator.h"
94 #include "gimplify-me.h"
95 #include "gimple-walk.h"
100 #include "builtins.h"
101 #include "tree-sra.h"
104 /* Enumeration of all aggregate reductions we can do. */
105 enum sra_mode
{ SRA_MODE_EARLY_IPA
, /* early call regularization */
106 SRA_MODE_EARLY_INTRA
, /* early intraprocedural SRA */
107 SRA_MODE_INTRA
}; /* late intraprocedural SRA */
109 /* Global variable describing which aggregate reduction we are performing at
111 static enum sra_mode sra_mode
;
115 /* ACCESS represents each access to an aggregate variable (as a whole or a
116 part). It can also represent a group of accesses that refer to exactly the
117 same fragment of an aggregate (i.e. those that have exactly the same offset
118 and size). Such representatives for a single aggregate, once determined,
119 are linked in a linked list and have the group fields set.
121 Moreover, when doing intraprocedural SRA, a tree is built from those
122 representatives (by the means of first_child and next_sibling pointers), in
123 which all items in a subtree are "within" the root, i.e. their offset is
124 greater or equal to offset of the root and offset+size is smaller or equal
125 to offset+size of the root. Children of an access are sorted by offset.
127 Note that accesses to parts of vector and complex number types always
128 represented by an access to the whole complex number or a vector. It is a
129 duty of the modifying functions to replace them appropriately. */
133 /* Values returned by `get_ref_base_and_extent' for each component reference
134 If EXPR isn't a component reference just set `BASE = EXPR', `OFFSET = 0',
135 `SIZE = TREE_SIZE (TREE_TYPE (expr))'. */
136 HOST_WIDE_INT offset
;
140 /* Expression. It is context dependent so do not use it to create new
141 expressions to access the original aggregate. See PR 42154 for a
147 /* The statement this access belongs to. */
150 /* Next group representative for this aggregate. */
151 struct access
*next_grp
;
153 /* Pointer to the group representative. Pointer to itself if the struct is
154 the representative. */
155 struct access
*group_representative
;
157 /* After access tree has been constructed, this points to the parent of the
158 current access, if there is one. NULL for roots. */
159 struct access
*parent
;
161 /* If this access has any children (in terms of the definition above), this
162 points to the first one. */
163 struct access
*first_child
;
165 /* In intraprocedural SRA, pointer to the next sibling in the access tree as
167 struct access
*next_sibling
;
169 /* Pointers to the first and last element in the linked list of assign
171 struct assign_link
*first_link
, *last_link
;
173 /* Pointer to the next access in the work queue. */
174 struct access
*next_queued
;
176 /* Replacement variable for this access "region." Never to be accessed
177 directly, always only by the means of get_access_replacement() and only
178 when grp_to_be_replaced flag is set. */
179 tree replacement_decl
;
181 /* Is this access made in reverse storage order? */
182 unsigned reverse
: 1;
184 /* Is this particular access write access? */
187 /* Is this access currently in the work queue? */
188 unsigned grp_queued
: 1;
190 /* Does this group contain a write access? This flag is propagated down the
192 unsigned grp_write
: 1;
194 /* Does this group contain a read access? This flag is propagated down the
196 unsigned grp_read
: 1;
198 /* Does this group contain a read access that comes from an assignment
199 statement? This flag is propagated down the access tree. */
200 unsigned grp_assignment_read
: 1;
202 /* Does this group contain a write access that comes from an assignment
203 statement? This flag is propagated down the access tree. */
204 unsigned grp_assignment_write
: 1;
206 /* Does this group contain a read access through a scalar type? This flag is
207 not propagated in the access tree in any direction. */
208 unsigned grp_scalar_read
: 1;
210 /* Does this group contain a write access through a scalar type? This flag
211 is not propagated in the access tree in any direction. */
212 unsigned grp_scalar_write
: 1;
214 /* Is this access an artificial one created to scalarize some record
216 unsigned grp_total_scalarization
: 1;
218 /* Other passes of the analysis use this bit to make function
219 analyze_access_subtree create scalar replacements for this group if
221 unsigned grp_hint
: 1;
223 /* Is the subtree rooted in this access fully covered by scalar
225 unsigned grp_covered
: 1;
227 /* If set to true, this access and all below it in an access tree must not be
229 unsigned grp_unscalarizable_region
: 1;
231 /* Whether data have been written to parts of the aggregate covered by this
232 access which is not to be scalarized. This flag is propagated up in the
234 unsigned grp_unscalarized_data
: 1;
236 /* Set if all accesses in the group consist of the same chain of
237 COMPONENT_REFs and ARRAY_REFs. */
238 unsigned grp_same_access_path
: 1;
240 /* Does this access and/or group contain a write access through a
242 unsigned grp_partial_lhs
: 1;
244 /* Set when a scalar replacement should be created for this variable. */
245 unsigned grp_to_be_replaced
: 1;
247 /* Set when we want a replacement for the sole purpose of having it in
248 generated debug statements. */
249 unsigned grp_to_be_debug_replaced
: 1;
251 /* Should TREE_NO_WARNING of a replacement be set? */
252 unsigned grp_no_warning
: 1;
255 typedef struct access
*access_p
;
258 /* Alloc pool for allocating access structures. */
259 static object_allocator
<struct access
> access_pool ("SRA accesses");
261 /* A structure linking lhs and rhs accesses from an aggregate assignment. They
262 are used to propagate subaccesses from rhs to lhs as long as they don't
263 conflict with what is already there. */
266 struct access
*lacc
, *racc
;
267 struct assign_link
*next
;
270 /* Alloc pool for allocating assign link structures. */
271 static object_allocator
<assign_link
> assign_link_pool ("SRA links");
273 /* Base (tree) -> Vector (vec<access_p> *) map. */
274 static hash_map
<tree
, auto_vec
<access_p
> > *base_access_vec
;
276 /* Candidate hash table helpers. */
278 struct uid_decl_hasher
: nofree_ptr_hash
<tree_node
>
280 static inline hashval_t
hash (const tree_node
*);
281 static inline bool equal (const tree_node
*, const tree_node
*);
284 /* Hash a tree in a uid_decl_map. */
287 uid_decl_hasher::hash (const tree_node
*item
)
289 return item
->decl_minimal
.uid
;
292 /* Return true if the DECL_UID in both trees are equal. */
295 uid_decl_hasher::equal (const tree_node
*a
, const tree_node
*b
)
297 return (a
->decl_minimal
.uid
== b
->decl_minimal
.uid
);
300 /* Set of candidates. */
301 static bitmap candidate_bitmap
;
302 static hash_table
<uid_decl_hasher
> *candidates
;
304 /* For a candidate UID return the candidates decl. */
307 candidate (unsigned uid
)
310 t
.decl_minimal
.uid
= uid
;
311 return candidates
->find_with_hash (&t
, static_cast <hashval_t
> (uid
));
314 /* Bitmap of candidates which we should try to entirely scalarize away and
315 those which cannot be (because they are and need be used as a whole). */
316 static bitmap should_scalarize_away_bitmap
, cannot_scalarize_away_bitmap
;
318 /* Bitmap of candidates in the constant pool, which cannot be scalarized
319 because this would produce non-constant expressions (e.g. Ada). */
320 static bitmap disqualified_constants
;
322 /* Obstack for creation of fancy names. */
323 static struct obstack name_obstack
;
325 /* Head of a linked list of accesses that need to have its subaccesses
326 propagated to their assignment counterparts. */
327 static struct access
*work_queue_head
;
329 /* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
330 representative fields are dumped, otherwise those which only describe the
331 individual access are. */
335 /* Number of processed aggregates is readily available in
336 analyze_all_variable_accesses and so is not stored here. */
338 /* Number of created scalar replacements. */
341 /* Number of times sra_modify_expr or sra_modify_assign themselves changed an
345 /* Number of statements created by generate_subtree_copies. */
348 /* Number of statements created by load_assign_lhs_subreplacements. */
351 /* Number of times sra_modify_assign has deleted a statement. */
354 /* Number of times sra_modify_assign has to deal with subaccesses of LHS and
355 RHS reparately due to type conversions or nonexistent matching
357 int separate_lhs_rhs_handling
;
359 /* Number of parameters that were removed because they were unused. */
360 int deleted_unused_parameters
;
362 /* Number of scalars passed as parameters by reference that have been
363 converted to be passed by value. */
364 int scalar_by_ref_to_by_val
;
366 /* Number of aggregate parameters that were replaced by one or more of their
368 int aggregate_params_reduced
;
370 /* Numbber of components created when splitting aggregate parameters. */
371 int param_reductions_created
;
375 dump_access (FILE *f
, struct access
*access
, bool grp
)
377 fprintf (f
, "access { ");
378 fprintf (f
, "base = (%d)'", DECL_UID (access
->base
));
379 print_generic_expr (f
, access
->base
);
380 fprintf (f
, "', offset = " HOST_WIDE_INT_PRINT_DEC
, access
->offset
);
381 fprintf (f
, ", size = " HOST_WIDE_INT_PRINT_DEC
, access
->size
);
382 fprintf (f
, ", expr = ");
383 print_generic_expr (f
, access
->expr
);
384 fprintf (f
, ", type = ");
385 print_generic_expr (f
, access
->type
);
386 fprintf (f
, ", reverse = %d", access
->reverse
);
388 fprintf (f
, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
389 "grp_assignment_write = %d, grp_scalar_read = %d, "
390 "grp_scalar_write = %d, grp_total_scalarization = %d, "
391 "grp_hint = %d, grp_covered = %d, "
392 "grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
393 "grp_same_access_path = %d, grp_partial_lhs = %d, "
394 "grp_to_be_replaced = %d, grp_to_be_debug_replaced = %d}\n",
395 access
->grp_read
, access
->grp_write
, access
->grp_assignment_read
,
396 access
->grp_assignment_write
, access
->grp_scalar_read
,
397 access
->grp_scalar_write
, access
->grp_total_scalarization
,
398 access
->grp_hint
, access
->grp_covered
,
399 access
->grp_unscalarizable_region
, access
->grp_unscalarized_data
,
400 access
->grp_same_access_path
, access
->grp_partial_lhs
,
401 access
->grp_to_be_replaced
, access
->grp_to_be_debug_replaced
);
403 fprintf (f
, ", write = %d, grp_total_scalarization = %d, "
404 "grp_partial_lhs = %d}\n",
405 access
->write
, access
->grp_total_scalarization
,
406 access
->grp_partial_lhs
);
409 /* Dump a subtree rooted in ACCESS to file F, indent by LEVEL. */
412 dump_access_tree_1 (FILE *f
, struct access
*access
, int level
)
418 for (i
= 0; i
< level
; i
++)
421 dump_access (f
, access
, true);
423 if (access
->first_child
)
424 dump_access_tree_1 (f
, access
->first_child
, level
+ 1);
426 access
= access
->next_sibling
;
431 /* Dump all access trees for a variable, given the pointer to the first root in
435 dump_access_tree (FILE *f
, struct access
*access
)
437 for (; access
; access
= access
->next_grp
)
438 dump_access_tree_1 (f
, access
, 0);
441 /* Return true iff ACC is non-NULL and has subaccesses. */
444 access_has_children_p (struct access
*acc
)
446 return acc
&& acc
->first_child
;
449 /* Return true iff ACC is (partly) covered by at least one replacement. */
452 access_has_replacements_p (struct access
*acc
)
454 struct access
*child
;
455 if (acc
->grp_to_be_replaced
)
457 for (child
= acc
->first_child
; child
; child
= child
->next_sibling
)
458 if (access_has_replacements_p (child
))
463 /* Return a vector of pointers to accesses for the variable given in BASE or
464 NULL if there is none. */
466 static vec
<access_p
> *
467 get_base_access_vector (tree base
)
469 return base_access_vec
->get (base
);
472 /* Find an access with required OFFSET and SIZE in a subtree of accesses rooted
473 in ACCESS. Return NULL if it cannot be found. */
475 static struct access
*
476 find_access_in_subtree (struct access
*access
, HOST_WIDE_INT offset
,
479 while (access
&& (access
->offset
!= offset
|| access
->size
!= size
))
481 struct access
*child
= access
->first_child
;
483 while (child
&& (child
->offset
+ child
->size
<= offset
))
484 child
= child
->next_sibling
;
491 /* Return the first group representative for DECL or NULL if none exists. */
493 static struct access
*
494 get_first_repr_for_decl (tree base
)
496 vec
<access_p
> *access_vec
;
498 access_vec
= get_base_access_vector (base
);
502 return (*access_vec
)[0];
505 /* Find an access representative for the variable BASE and given OFFSET and
506 SIZE. Requires that access trees have already been built. Return NULL if
507 it cannot be found. */
509 static struct access
*
510 get_var_base_offset_size_access (tree base
, HOST_WIDE_INT offset
,
513 struct access
*access
;
515 access
= get_first_repr_for_decl (base
);
516 while (access
&& (access
->offset
+ access
->size
<= offset
))
517 access
= access
->next_grp
;
521 return find_access_in_subtree (access
, offset
, size
);
524 /* Add LINK to the linked list of assign links of RACC. */
526 add_link_to_rhs (struct access
*racc
, struct assign_link
*link
)
528 gcc_assert (link
->racc
== racc
);
530 if (!racc
->first_link
)
532 gcc_assert (!racc
->last_link
);
533 racc
->first_link
= link
;
536 racc
->last_link
->next
= link
;
538 racc
->last_link
= link
;
542 /* Move all link structures in their linked list in OLD_RACC to the linked list
545 relink_to_new_repr (struct access
*new_racc
, struct access
*old_racc
)
547 if (!old_racc
->first_link
)
549 gcc_assert (!old_racc
->last_link
);
553 if (new_racc
->first_link
)
555 gcc_assert (!new_racc
->last_link
->next
);
556 gcc_assert (!old_racc
->last_link
|| !old_racc
->last_link
->next
);
558 new_racc
->last_link
->next
= old_racc
->first_link
;
559 new_racc
->last_link
= old_racc
->last_link
;
563 gcc_assert (!new_racc
->last_link
);
565 new_racc
->first_link
= old_racc
->first_link
;
566 new_racc
->last_link
= old_racc
->last_link
;
568 old_racc
->first_link
= old_racc
->last_link
= NULL
;
571 /* Add ACCESS to the work queue (which is actually a stack). */
574 add_access_to_work_queue (struct access
*access
)
576 if (access
->first_link
&& !access
->grp_queued
)
578 gcc_assert (!access
->next_queued
);
579 access
->next_queued
= work_queue_head
;
580 access
->grp_queued
= 1;
581 work_queue_head
= access
;
585 /* Pop an access from the work queue, and return it, assuming there is one. */
587 static struct access
*
588 pop_access_from_work_queue (void)
590 struct access
*access
= work_queue_head
;
592 work_queue_head
= access
->next_queued
;
593 access
->next_queued
= NULL
;
594 access
->grp_queued
= 0;
599 /* Allocate necessary structures. */
602 sra_initialize (void)
604 candidate_bitmap
= BITMAP_ALLOC (NULL
);
605 candidates
= new hash_table
<uid_decl_hasher
>
606 (vec_safe_length (cfun
->local_decls
) / 2);
607 should_scalarize_away_bitmap
= BITMAP_ALLOC (NULL
);
608 cannot_scalarize_away_bitmap
= BITMAP_ALLOC (NULL
);
609 disqualified_constants
= BITMAP_ALLOC (NULL
);
610 gcc_obstack_init (&name_obstack
);
611 base_access_vec
= new hash_map
<tree
, auto_vec
<access_p
> >;
612 memset (&sra_stats
, 0, sizeof (sra_stats
));
615 /* Deallocate all general structures. */
618 sra_deinitialize (void)
620 BITMAP_FREE (candidate_bitmap
);
623 BITMAP_FREE (should_scalarize_away_bitmap
);
624 BITMAP_FREE (cannot_scalarize_away_bitmap
);
625 BITMAP_FREE (disqualified_constants
);
626 access_pool
.release ();
627 assign_link_pool
.release ();
628 obstack_free (&name_obstack
, NULL
);
630 delete base_access_vec
;
633 /* Return true if DECL is a VAR_DECL in the constant pool, false otherwise. */
635 static bool constant_decl_p (tree decl
)
637 return VAR_P (decl
) && DECL_IN_CONSTANT_POOL (decl
);
640 /* Remove DECL from candidates for SRA and write REASON to the dump file if
644 disqualify_candidate (tree decl
, const char *reason
)
646 if (bitmap_clear_bit (candidate_bitmap
, DECL_UID (decl
)))
647 candidates
->remove_elt_with_hash (decl
, DECL_UID (decl
));
648 if (constant_decl_p (decl
))
649 bitmap_set_bit (disqualified_constants
, DECL_UID (decl
));
651 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
653 fprintf (dump_file
, "! Disqualifying ");
654 print_generic_expr (dump_file
, decl
);
655 fprintf (dump_file
, " - %s\n", reason
);
659 /* Return true iff the type contains a field or an element which does not allow
660 scalarization. Use VISITED_TYPES to avoid re-checking already checked
664 type_internals_preclude_sra_p_1 (tree type
, const char **msg
,
665 hash_set
<tree
> *visited_types
)
670 if (visited_types
->contains (type
))
672 visited_types
->add (type
);
674 switch (TREE_CODE (type
))
678 case QUAL_UNION_TYPE
:
679 for (fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
680 if (TREE_CODE (fld
) == FIELD_DECL
)
682 if (TREE_CODE (fld
) == FUNCTION_DECL
)
684 tree ft
= TREE_TYPE (fld
);
686 if (TREE_THIS_VOLATILE (fld
))
688 *msg
= "volatile structure field";
691 if (!DECL_FIELD_OFFSET (fld
))
693 *msg
= "no structure field offset";
696 if (!DECL_SIZE (fld
))
698 *msg
= "zero structure field size";
701 if (!tree_fits_uhwi_p (DECL_FIELD_OFFSET (fld
)))
703 *msg
= "structure field offset not fixed";
706 if (!tree_fits_uhwi_p (DECL_SIZE (fld
)))
708 *msg
= "structure field size not fixed";
711 if (!tree_fits_shwi_p (bit_position (fld
)))
713 *msg
= "structure field size too big";
716 if (AGGREGATE_TYPE_P (ft
)
717 && int_bit_position (fld
) % BITS_PER_UNIT
!= 0)
719 *msg
= "structure field is bit field";
723 if (AGGREGATE_TYPE_P (ft
)
724 && type_internals_preclude_sra_p_1 (ft
, msg
, visited_types
))
731 et
= TREE_TYPE (type
);
733 if (TYPE_VOLATILE (et
))
735 *msg
= "element type is volatile";
739 if (AGGREGATE_TYPE_P (et
)
740 && type_internals_preclude_sra_p_1 (et
, msg
, visited_types
))
750 /* Return true iff the type contains a field or an element which does not allow
754 type_internals_preclude_sra_p (tree type
, const char **msg
)
756 hash_set
<tree
> visited_types
;
757 return type_internals_preclude_sra_p_1 (type
, msg
, &visited_types
);
761 /* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
762 the three fields. Also add it to the vector of accesses corresponding to
763 the base. Finally, return the new access. */
765 static struct access
*
766 create_access_1 (tree base
, HOST_WIDE_INT offset
, HOST_WIDE_INT size
)
768 struct access
*access
= access_pool
.allocate ();
770 memset (access
, 0, sizeof (struct access
));
772 access
->offset
= offset
;
775 base_access_vec
->get_or_insert (base
).safe_push (access
);
780 static bool maybe_add_sra_candidate (tree
);
782 /* Create and insert access for EXPR. Return created access, or NULL if it is
783 not possible. Also scan for uses of constant pool as we go along and add
786 static struct access
*
787 create_access (tree expr
, gimple
*stmt
, bool write
)
789 struct access
*access
;
790 poly_int64 poffset
, psize
, pmax_size
;
792 bool reverse
, unscalarizable_region
= false;
794 base
= get_ref_base_and_extent (expr
, &poffset
, &psize
, &pmax_size
,
797 /* For constant-pool entries, check we can substitute the constant value. */
798 if (constant_decl_p (base
))
800 gcc_assert (!bitmap_bit_p (disqualified_constants
, DECL_UID (base
)));
802 && !is_gimple_reg_type (TREE_TYPE (expr
))
803 && dump_file
&& (dump_flags
& TDF_DETAILS
))
805 /* This occurs in Ada with accesses to ARRAY_RANGE_REFs,
806 and elements of multidimensional arrays (which are
807 multi-element arrays in their own right). */
808 fprintf (dump_file
, "Allowing non-reg-type load of part"
809 " of constant-pool entry: ");
810 print_generic_expr (dump_file
, expr
);
812 maybe_add_sra_candidate (base
);
815 if (!DECL_P (base
) || !bitmap_bit_p (candidate_bitmap
, DECL_UID (base
)))
818 HOST_WIDE_INT offset
, size
, max_size
;
819 if (!poffset
.is_constant (&offset
)
820 || !psize
.is_constant (&size
)
821 || !pmax_size
.is_constant (&max_size
))
823 disqualify_candidate (base
, "Encountered a polynomial-sized access.");
827 if (size
!= max_size
)
830 unscalarizable_region
= true;
834 disqualify_candidate (base
, "Encountered an unconstrained access.");
838 access
= create_access_1 (base
, offset
, size
);
840 access
->type
= TREE_TYPE (expr
);
841 access
->write
= write
;
842 access
->grp_unscalarizable_region
= unscalarizable_region
;
844 access
->reverse
= reverse
;
850 /* Return true iff TYPE is scalarizable - i.e. a RECORD_TYPE or fixed-length
851 ARRAY_TYPE with fields that are either of gimple register types (excluding
852 bit-fields) or (recursively) scalarizable types. CONST_DECL must be true if
853 we are considering a decl from constant pool. If it is false, char arrays
857 scalarizable_type_p (tree type
, bool const_decl
)
859 gcc_assert (!is_gimple_reg_type (type
));
860 if (type_contains_placeholder_p (type
))
863 switch (TREE_CODE (type
))
866 for (tree fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
867 if (TREE_CODE (fld
) == FIELD_DECL
)
869 tree ft
= TREE_TYPE (fld
);
871 if (DECL_BIT_FIELD (fld
))
874 if (!is_gimple_reg_type (ft
)
875 && !scalarizable_type_p (ft
, const_decl
))
883 HOST_WIDE_INT min_elem_size
;
887 min_elem_size
= BITS_PER_UNIT
;
889 if (TYPE_DOMAIN (type
) == NULL_TREE
890 || !tree_fits_shwi_p (TYPE_SIZE (type
))
891 || !tree_fits_shwi_p (TYPE_SIZE (TREE_TYPE (type
)))
892 || (tree_to_shwi (TYPE_SIZE (TREE_TYPE (type
))) <= min_elem_size
)
893 || !tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type
))))
895 if (tree_to_shwi (TYPE_SIZE (type
)) == 0
896 && TYPE_MAX_VALUE (TYPE_DOMAIN (type
)) == NULL_TREE
)
897 /* Zero-element array, should not prevent scalarization. */
899 else if ((tree_to_shwi (TYPE_SIZE (type
)) <= 0)
900 || !tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type
))))
901 /* Variable-length array, do not allow scalarization. */
904 tree elem
= TREE_TYPE (type
);
905 if (!is_gimple_reg_type (elem
)
906 && !scalarizable_type_p (elem
, const_decl
))
915 static void scalarize_elem (tree
, HOST_WIDE_INT
, HOST_WIDE_INT
, bool, tree
, tree
);
917 /* Create total_scalarization accesses for all scalar fields of a member
918 of type DECL_TYPE conforming to scalarizable_type_p. BASE
919 must be the top-most VAR_DECL representing the variable; within that,
920 OFFSET locates the member and REF must be the memory reference expression for
924 completely_scalarize (tree base
, tree decl_type
, HOST_WIDE_INT offset
, tree ref
)
926 switch (TREE_CODE (decl_type
))
929 for (tree fld
= TYPE_FIELDS (decl_type
); fld
; fld
= DECL_CHAIN (fld
))
930 if (TREE_CODE (fld
) == FIELD_DECL
)
932 HOST_WIDE_INT pos
= offset
+ int_bit_position (fld
);
933 tree ft
= TREE_TYPE (fld
);
934 tree nref
= build3 (COMPONENT_REF
, ft
, ref
, fld
, NULL_TREE
);
936 scalarize_elem (base
, pos
, tree_to_uhwi (DECL_SIZE (fld
)),
937 TYPE_REVERSE_STORAGE_ORDER (decl_type
),
943 tree elemtype
= TREE_TYPE (decl_type
);
944 tree elem_size
= TYPE_SIZE (elemtype
);
945 gcc_assert (elem_size
&& tree_fits_shwi_p (elem_size
));
946 HOST_WIDE_INT el_size
= tree_to_shwi (elem_size
);
947 gcc_assert (el_size
> 0);
949 tree minidx
= TYPE_MIN_VALUE (TYPE_DOMAIN (decl_type
));
950 gcc_assert (TREE_CODE (minidx
) == INTEGER_CST
);
951 tree maxidx
= TYPE_MAX_VALUE (TYPE_DOMAIN (decl_type
));
952 /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
955 gcc_assert (TREE_CODE (maxidx
) == INTEGER_CST
);
956 tree domain
= TYPE_DOMAIN (decl_type
);
957 /* MINIDX and MAXIDX are inclusive, and must be interpreted in
958 DOMAIN (e.g. signed int, whereas min/max may be size_int). */
959 offset_int idx
= wi::to_offset (minidx
);
960 offset_int max
= wi::to_offset (maxidx
);
961 if (!TYPE_UNSIGNED (domain
))
963 idx
= wi::sext (idx
, TYPE_PRECISION (domain
));
964 max
= wi::sext (max
, TYPE_PRECISION (domain
));
966 for (int el_off
= offset
; idx
<= max
; ++idx
)
968 tree nref
= build4 (ARRAY_REF
, elemtype
,
970 wide_int_to_tree (domain
, idx
),
971 NULL_TREE
, NULL_TREE
);
972 scalarize_elem (base
, el_off
, el_size
,
973 TYPE_REVERSE_STORAGE_ORDER (decl_type
),
985 /* Create total_scalarization accesses for a member of type TYPE, which must
986 satisfy either is_gimple_reg_type or scalarizable_type_p. BASE must be the
987 top-most VAR_DECL representing the variable; within that, POS and SIZE locate
988 the member, REVERSE gives its torage order. and REF must be the reference
989 expression for it. */
992 scalarize_elem (tree base
, HOST_WIDE_INT pos
, HOST_WIDE_INT size
, bool reverse
,
995 if (is_gimple_reg_type (type
))
997 struct access
*access
= create_access_1 (base
, pos
, size
);
1000 access
->grp_total_scalarization
= 1;
1001 access
->reverse
= reverse
;
1002 /* Accesses for intraprocedural SRA can have their stmt NULL. */
1005 completely_scalarize (base
, type
, pos
, ref
);
1008 /* Create a total_scalarization access for VAR as a whole. VAR must be of a
1009 RECORD_TYPE or ARRAY_TYPE conforming to scalarizable_type_p. */
1012 create_total_scalarization_access (tree var
)
1014 HOST_WIDE_INT size
= tree_to_uhwi (DECL_SIZE (var
));
1015 struct access
*access
;
1017 access
= create_access_1 (var
, 0, size
);
1019 access
->type
= TREE_TYPE (var
);
1020 access
->grp_total_scalarization
= 1;
1023 /* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
1026 contains_view_convert_expr_p (const_tree ref
)
1028 while (handled_component_p (ref
))
1030 if (TREE_CODE (ref
) == VIEW_CONVERT_EXPR
)
1032 ref
= TREE_OPERAND (ref
, 0);
1038 /* Return true if REF contains a VIEW_CONVERT_EXPR or a COMPONENT_REF with a
1039 bit-field field declaration. If TYPE_CHANGING_P is non-NULL, set the bool
1040 it points to will be set if REF contains any of the above or a MEM_REF
1041 expression that effectively performs type conversion. */
1044 contains_vce_or_bfcref_p (const_tree ref
, bool *type_changing_p
= NULL
)
1046 while (handled_component_p (ref
))
1048 if (TREE_CODE (ref
) == VIEW_CONVERT_EXPR
1049 || (TREE_CODE (ref
) == COMPONENT_REF
1050 && DECL_BIT_FIELD (TREE_OPERAND (ref
, 1))))
1052 if (type_changing_p
)
1053 *type_changing_p
= true;
1056 ref
= TREE_OPERAND (ref
, 0);
1059 if (!type_changing_p
1060 || TREE_CODE (ref
) != MEM_REF
1061 || TREE_CODE (TREE_OPERAND (ref
, 0)) != ADDR_EXPR
)
1064 tree mem
= TREE_OPERAND (TREE_OPERAND (ref
, 0), 0);
1065 if (TYPE_MAIN_VARIANT (TREE_TYPE (ref
))
1066 != TYPE_MAIN_VARIANT (TREE_TYPE (mem
)))
1067 *type_changing_p
= true;
1072 /* Search the given tree for a declaration by skipping handled components and
1073 exclude it from the candidates. */
1076 disqualify_base_of_expr (tree t
, const char *reason
)
1078 t
= get_base_address (t
);
1079 if (t
&& DECL_P (t
))
1080 disqualify_candidate (t
, reason
);
1083 /* Scan expression EXPR and create access structures for all accesses to
1084 candidates for scalarization. Return the created access or NULL if none is
1087 static struct access
*
1088 build_access_from_expr_1 (tree expr
, gimple
*stmt
, bool write
)
1090 struct access
*ret
= NULL
;
1093 if (TREE_CODE (expr
) == BIT_FIELD_REF
1094 || TREE_CODE (expr
) == IMAGPART_EXPR
1095 || TREE_CODE (expr
) == REALPART_EXPR
)
1097 expr
= TREE_OPERAND (expr
, 0);
1101 partial_ref
= false;
1103 if (storage_order_barrier_p (expr
))
1105 disqualify_base_of_expr (expr
, "storage order barrier.");
1109 /* We need to dive through V_C_Es in order to get the size of its parameter
1110 and not the result type. Ada produces such statements. We are also
1111 capable of handling the topmost V_C_E but not any of those buried in other
1112 handled components. */
1113 if (TREE_CODE (expr
) == VIEW_CONVERT_EXPR
)
1114 expr
= TREE_OPERAND (expr
, 0);
1116 if (contains_view_convert_expr_p (expr
))
1118 disqualify_base_of_expr (expr
, "V_C_E under a different handled "
1122 if (TREE_THIS_VOLATILE (expr
))
1124 disqualify_base_of_expr (expr
, "part of a volatile reference.");
1128 switch (TREE_CODE (expr
))
1131 if (TREE_CODE (TREE_OPERAND (expr
, 0)) != ADDR_EXPR
)
1139 case ARRAY_RANGE_REF
:
1140 ret
= create_access (expr
, stmt
, write
);
1147 if (write
&& partial_ref
&& ret
)
1148 ret
->grp_partial_lhs
= 1;
1153 /* Scan expression EXPR and create access structures for all accesses to
1154 candidates for scalarization. Return true if any access has been inserted.
1155 STMT must be the statement from which the expression is taken, WRITE must be
1156 true if the expression is a store and false otherwise. */
1159 build_access_from_expr (tree expr
, gimple
*stmt
, bool write
)
1161 struct access
*access
;
1163 access
= build_access_from_expr_1 (expr
, stmt
, write
);
1166 /* This means the aggregate is accesses as a whole in a way other than an
1167 assign statement and thus cannot be removed even if we had a scalar
1168 replacement for everything. */
1169 if (cannot_scalarize_away_bitmap
)
1170 bitmap_set_bit (cannot_scalarize_away_bitmap
, DECL_UID (access
->base
));
1176 /* Return the single non-EH successor edge of BB or NULL if there is none or
1180 single_non_eh_succ (basic_block bb
)
1185 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1186 if (!(e
->flags
& EDGE_EH
))
1196 /* Disqualify LHS and RHS for scalarization if STMT has to terminate its BB and
1197 there is no alternative spot where to put statements SRA might need to
1198 generate after it. The spot we are looking for is an edge leading to a
1199 single non-EH successor, if it exists and is indeed single. RHS may be
1200 NULL, in that case ignore it. */
1203 disqualify_if_bad_bb_terminating_stmt (gimple
*stmt
, tree lhs
, tree rhs
)
1205 if (stmt_ends_bb_p (stmt
))
1207 if (single_non_eh_succ (gimple_bb (stmt
)))
1210 disqualify_base_of_expr (lhs
, "LHS of a throwing stmt.");
1212 disqualify_base_of_expr (rhs
, "RHS of a throwing stmt.");
1218 /* Return true if the nature of BASE is such that it contains data even if
1219 there is no write to it in the function. */
1222 comes_initialized_p (tree base
)
1224 return TREE_CODE (base
) == PARM_DECL
|| constant_decl_p (base
);
1227 /* Scan expressions occurring in STMT, create access structures for all accesses
1228 to candidates for scalarization and remove those candidates which occur in
1229 statements or expressions that prevent them from being split apart. Return
1230 true if any access has been inserted. */
1233 build_accesses_from_assign (gimple
*stmt
)
1236 struct access
*lacc
, *racc
;
1238 if (!gimple_assign_single_p (stmt
)
1239 /* Scope clobbers don't influence scalarization. */
1240 || gimple_clobber_p (stmt
))
1243 lhs
= gimple_assign_lhs (stmt
);
1244 rhs
= gimple_assign_rhs1 (stmt
);
1246 if (disqualify_if_bad_bb_terminating_stmt (stmt
, lhs
, rhs
))
1249 racc
= build_access_from_expr_1 (rhs
, stmt
, false);
1250 lacc
= build_access_from_expr_1 (lhs
, stmt
, true);
1254 lacc
->grp_assignment_write
= 1;
1255 if (storage_order_barrier_p (rhs
))
1256 lacc
->grp_unscalarizable_region
= 1;
1258 if (should_scalarize_away_bitmap
&& !is_gimple_reg_type (lacc
->type
))
1260 bool type_changing_p
= false;
1261 contains_vce_or_bfcref_p (lhs
, &type_changing_p
);
1262 if (type_changing_p
)
1263 bitmap_set_bit (cannot_scalarize_away_bitmap
,
1264 DECL_UID (lacc
->base
));
1270 racc
->grp_assignment_read
= 1;
1271 if (should_scalarize_away_bitmap
&& !is_gimple_reg_type (racc
->type
))
1273 bool type_changing_p
= false;
1274 contains_vce_or_bfcref_p (rhs
, &type_changing_p
);
1276 if (type_changing_p
|| gimple_has_volatile_ops (stmt
))
1277 bitmap_set_bit (cannot_scalarize_away_bitmap
,
1278 DECL_UID (racc
->base
));
1280 bitmap_set_bit (should_scalarize_away_bitmap
,
1281 DECL_UID (racc
->base
));
1283 if (storage_order_barrier_p (lhs
))
1284 racc
->grp_unscalarizable_region
= 1;
1288 && (sra_mode
== SRA_MODE_EARLY_INTRA
|| sra_mode
== SRA_MODE_INTRA
)
1289 && !lacc
->grp_unscalarizable_region
1290 && !racc
->grp_unscalarizable_region
1291 && AGGREGATE_TYPE_P (TREE_TYPE (lhs
))
1292 && lacc
->size
== racc
->size
1293 && useless_type_conversion_p (lacc
->type
, racc
->type
))
1295 struct assign_link
*link
;
1297 link
= assign_link_pool
.allocate ();
1298 memset (link
, 0, sizeof (struct assign_link
));
1302 add_link_to_rhs (racc
, link
);
1303 add_access_to_work_queue (racc
);
1305 /* Let's delay marking the areas as written until propagation of accesses
1306 across link, unless the nature of rhs tells us that its data comes
1308 if (!comes_initialized_p (racc
->base
))
1309 lacc
->write
= false;
1312 return lacc
|| racc
;
1315 /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1316 GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1319 asm_visit_addr (gimple
*, tree op
, tree
, void *)
1321 op
= get_base_address (op
);
1324 disqualify_candidate (op
, "Non-scalarizable GIMPLE_ASM operand.");
1329 /* Scan function and look for interesting expressions and create access
1330 structures for them. Return true iff any access is created. */
1333 scan_function (void)
1338 FOR_EACH_BB_FN (bb
, cfun
)
1340 gimple_stmt_iterator gsi
;
1341 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1343 gimple
*stmt
= gsi_stmt (gsi
);
1347 switch (gimple_code (stmt
))
1350 t
= gimple_return_retval (as_a
<greturn
*> (stmt
));
1352 ret
|= build_access_from_expr (t
, stmt
, false);
1356 ret
|= build_accesses_from_assign (stmt
);
1360 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
1361 ret
|= build_access_from_expr (gimple_call_arg (stmt
, i
),
1364 t
= gimple_call_lhs (stmt
);
1365 if (t
&& !disqualify_if_bad_bb_terminating_stmt (stmt
, t
, NULL
))
1366 ret
|= build_access_from_expr (t
, stmt
, true);
1371 gasm
*asm_stmt
= as_a
<gasm
*> (stmt
);
1372 walk_stmt_load_store_addr_ops (asm_stmt
, NULL
, NULL
, NULL
,
1374 for (i
= 0; i
< gimple_asm_ninputs (asm_stmt
); i
++)
1376 t
= TREE_VALUE (gimple_asm_input_op (asm_stmt
, i
));
1377 ret
|= build_access_from_expr (t
, asm_stmt
, false);
1379 for (i
= 0; i
< gimple_asm_noutputs (asm_stmt
); i
++)
1381 t
= TREE_VALUE (gimple_asm_output_op (asm_stmt
, i
));
1382 ret
|= build_access_from_expr (t
, asm_stmt
, true);
1396 /* Helper of QSORT function. There are pointers to accesses in the array. An
1397 access is considered smaller than another if it has smaller offset or if the
1398 offsets are the same but is size is bigger. */
1401 compare_access_positions (const void *a
, const void *b
)
1403 const access_p
*fp1
= (const access_p
*) a
;
1404 const access_p
*fp2
= (const access_p
*) b
;
1405 const access_p f1
= *fp1
;
1406 const access_p f2
= *fp2
;
1408 if (f1
->offset
!= f2
->offset
)
1409 return f1
->offset
< f2
->offset
? -1 : 1;
1411 if (f1
->size
== f2
->size
)
1413 if (f1
->type
== f2
->type
)
1415 /* Put any non-aggregate type before any aggregate type. */
1416 else if (!is_gimple_reg_type (f1
->type
)
1417 && is_gimple_reg_type (f2
->type
))
1419 else if (is_gimple_reg_type (f1
->type
)
1420 && !is_gimple_reg_type (f2
->type
))
1422 /* Put any complex or vector type before any other scalar type. */
1423 else if (TREE_CODE (f1
->type
) != COMPLEX_TYPE
1424 && TREE_CODE (f1
->type
) != VECTOR_TYPE
1425 && (TREE_CODE (f2
->type
) == COMPLEX_TYPE
1426 || TREE_CODE (f2
->type
) == VECTOR_TYPE
))
1428 else if ((TREE_CODE (f1
->type
) == COMPLEX_TYPE
1429 || TREE_CODE (f1
->type
) == VECTOR_TYPE
)
1430 && TREE_CODE (f2
->type
) != COMPLEX_TYPE
1431 && TREE_CODE (f2
->type
) != VECTOR_TYPE
)
1433 /* Put any integral type before any non-integral type. When splicing, we
1434 make sure that those with insufficient precision and occupying the
1435 same space are not scalarized. */
1436 else if (INTEGRAL_TYPE_P (f1
->type
)
1437 && !INTEGRAL_TYPE_P (f2
->type
))
1439 else if (!INTEGRAL_TYPE_P (f1
->type
)
1440 && INTEGRAL_TYPE_P (f2
->type
))
1442 /* Put the integral type with the bigger precision first. */
1443 else if (INTEGRAL_TYPE_P (f1
->type
)
1444 && INTEGRAL_TYPE_P (f2
->type
)
1445 && (TYPE_PRECISION (f2
->type
) != TYPE_PRECISION (f1
->type
)))
1446 return TYPE_PRECISION (f2
->type
) - TYPE_PRECISION (f1
->type
);
1447 /* Stabilize the sort. */
1448 return TYPE_UID (f1
->type
) - TYPE_UID (f2
->type
);
1451 /* We want the bigger accesses first, thus the opposite operator in the next
1453 return f1
->size
> f2
->size
? -1 : 1;
1457 /* Append a name of the declaration to the name obstack. A helper function for
1461 make_fancy_decl_name (tree decl
)
1465 tree name
= DECL_NAME (decl
);
1467 obstack_grow (&name_obstack
, IDENTIFIER_POINTER (name
),
1468 IDENTIFIER_LENGTH (name
));
1471 sprintf (buffer
, "D%u", DECL_UID (decl
));
1472 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1476 /* Helper for make_fancy_name. */
1479 make_fancy_name_1 (tree expr
)
1486 make_fancy_decl_name (expr
);
1490 switch (TREE_CODE (expr
))
1493 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1494 obstack_1grow (&name_obstack
, '$');
1495 make_fancy_decl_name (TREE_OPERAND (expr
, 1));
1499 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1500 obstack_1grow (&name_obstack
, '$');
1501 /* Arrays with only one element may not have a constant as their
1503 index
= TREE_OPERAND (expr
, 1);
1504 if (TREE_CODE (index
) != INTEGER_CST
)
1506 sprintf (buffer
, HOST_WIDE_INT_PRINT_DEC
, TREE_INT_CST_LOW (index
));
1507 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1511 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1515 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1516 if (!integer_zerop (TREE_OPERAND (expr
, 1)))
1518 obstack_1grow (&name_obstack
, '$');
1519 sprintf (buffer
, HOST_WIDE_INT_PRINT_DEC
,
1520 TREE_INT_CST_LOW (TREE_OPERAND (expr
, 1)));
1521 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1528 gcc_unreachable (); /* we treat these as scalars. */
1535 /* Create a human readable name for replacement variable of ACCESS. */
1538 make_fancy_name (tree expr
)
1540 make_fancy_name_1 (expr
);
1541 obstack_1grow (&name_obstack
, '\0');
1542 return XOBFINISH (&name_obstack
, char *);
1545 /* Construct a MEM_REF that would reference a part of aggregate BASE of type
1546 EXP_TYPE at the given OFFSET and with storage order REVERSE. If BASE is
1547 something for which get_addr_base_and_unit_offset returns NULL, gsi must
1548 be non-NULL and is used to insert new statements either before or below
1549 the current one as specified by INSERT_AFTER. This function is not capable
1550 of handling bitfields. */
1553 build_ref_for_offset (location_t loc
, tree base
, poly_int64 offset
,
1554 bool reverse
, tree exp_type
, gimple_stmt_iterator
*gsi
,
1557 tree prev_base
= base
;
1560 poly_int64 base_offset
;
1561 unsigned HOST_WIDE_INT misalign
;
1564 /* Preserve address-space information. */
1565 addr_space_t as
= TYPE_ADDR_SPACE (TREE_TYPE (base
));
1566 if (as
!= TYPE_ADDR_SPACE (exp_type
))
1567 exp_type
= build_qualified_type (exp_type
,
1568 TYPE_QUALS (exp_type
)
1569 | ENCODE_QUAL_ADDR_SPACE (as
));
1571 poly_int64 byte_offset
= exact_div (offset
, BITS_PER_UNIT
);
1572 get_object_alignment_1 (base
, &align
, &misalign
);
1573 base
= get_addr_base_and_unit_offset (base
, &base_offset
);
1575 /* get_addr_base_and_unit_offset returns NULL for references with a variable
1576 offset such as array[var_index]. */
1582 gcc_checking_assert (gsi
);
1583 tmp
= make_ssa_name (build_pointer_type (TREE_TYPE (prev_base
)));
1584 addr
= build_fold_addr_expr (unshare_expr (prev_base
));
1585 STRIP_USELESS_TYPE_CONVERSION (addr
);
1586 stmt
= gimple_build_assign (tmp
, addr
);
1587 gimple_set_location (stmt
, loc
);
1589 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
1591 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
1593 off
= build_int_cst (reference_alias_ptr_type (prev_base
), byte_offset
);
1596 else if (TREE_CODE (base
) == MEM_REF
)
1598 off
= build_int_cst (TREE_TYPE (TREE_OPERAND (base
, 1)),
1599 base_offset
+ byte_offset
);
1600 off
= int_const_binop (PLUS_EXPR
, TREE_OPERAND (base
, 1), off
);
1601 base
= unshare_expr (TREE_OPERAND (base
, 0));
1605 off
= build_int_cst (reference_alias_ptr_type (prev_base
),
1606 base_offset
+ byte_offset
);
1607 base
= build_fold_addr_expr (unshare_expr (base
));
1610 unsigned int align_bound
= known_alignment (misalign
+ offset
);
1611 if (align_bound
!= 0)
1612 align
= MIN (align
, align_bound
);
1613 if (align
!= TYPE_ALIGN (exp_type
))
1614 exp_type
= build_aligned_type (exp_type
, align
);
1616 mem_ref
= fold_build2_loc (loc
, MEM_REF
, exp_type
, base
, off
);
1617 REF_REVERSE_STORAGE_ORDER (mem_ref
) = reverse
;
1618 if (TREE_THIS_VOLATILE (prev_base
))
1619 TREE_THIS_VOLATILE (mem_ref
) = 1;
1620 if (TREE_SIDE_EFFECTS (prev_base
))
1621 TREE_SIDE_EFFECTS (mem_ref
) = 1;
1625 /* Construct and return a memory reference that is equal to a portion of
1626 MODEL->expr but is based on BASE. If this cannot be done, return NULL. */
1629 build_reconstructed_reference (location_t
, tree base
, struct access
*model
)
1631 tree expr
= model
->expr
, prev_expr
= NULL
;
1632 while (!types_compatible_p (TREE_TYPE (expr
), TREE_TYPE (base
)))
1634 if (!handled_component_p (expr
))
1637 expr
= TREE_OPERAND (expr
, 0);
1640 /* Guard against broken VIEW_CONVERT_EXPRs... */
1644 TREE_OPERAND (prev_expr
, 0) = base
;
1645 tree ref
= unshare_expr (model
->expr
);
1646 TREE_OPERAND (prev_expr
, 0) = expr
;
1650 /* Construct a memory reference to a part of an aggregate BASE at the given
1651 OFFSET and of the same type as MODEL. In case this is a reference to a
1652 bit-field, the function will replicate the last component_ref of model's
1653 expr to access it. GSI and INSERT_AFTER have the same meaning as in
1654 build_ref_for_offset. */
1657 build_ref_for_model (location_t loc
, tree base
, HOST_WIDE_INT offset
,
1658 struct access
*model
, gimple_stmt_iterator
*gsi
,
1661 if (TREE_CODE (model
->expr
) == COMPONENT_REF
1662 && DECL_BIT_FIELD (TREE_OPERAND (model
->expr
, 1)))
1664 /* This access represents a bit-field. */
1665 tree t
, exp_type
, fld
= TREE_OPERAND (model
->expr
, 1);
1667 offset
-= int_bit_position (fld
);
1668 exp_type
= TREE_TYPE (TREE_OPERAND (model
->expr
, 0));
1669 t
= build_ref_for_offset (loc
, base
, offset
, model
->reverse
, exp_type
,
1671 /* The flag will be set on the record type. */
1672 REF_REVERSE_STORAGE_ORDER (t
) = 0;
1673 return fold_build3_loc (loc
, COMPONENT_REF
, TREE_TYPE (fld
), t
, fld
,
1679 if (model
->grp_same_access_path
1680 && !TREE_THIS_VOLATILE (base
)
1681 && (TYPE_ADDR_SPACE (TREE_TYPE (base
))
1682 == TYPE_ADDR_SPACE (TREE_TYPE (model
->expr
)))
1683 && offset
<= model
->offset
1684 /* build_reconstructed_reference can still fail if we have already
1685 massaged BASE because of another type incompatibility. */
1686 && (res
= build_reconstructed_reference (loc
, base
, model
)))
1689 return build_ref_for_offset (loc
, base
, offset
, model
->reverse
,
1690 model
->type
, gsi
, insert_after
);
1694 /* Attempt to build a memory reference that we could but into a gimple
1695 debug_bind statement. Similar to build_ref_for_model but punts if it has to
1696 create statements and return s NULL instead. This function also ignores
1697 alignment issues and so its results should never end up in non-debug
1701 build_debug_ref_for_model (location_t loc
, tree base
, HOST_WIDE_INT offset
,
1702 struct access
*model
)
1704 poly_int64 base_offset
;
1707 if (TREE_CODE (model
->expr
) == COMPONENT_REF
1708 && DECL_BIT_FIELD (TREE_OPERAND (model
->expr
, 1)))
1711 base
= get_addr_base_and_unit_offset (base
, &base_offset
);
1714 if (TREE_CODE (base
) == MEM_REF
)
1716 off
= build_int_cst (TREE_TYPE (TREE_OPERAND (base
, 1)),
1717 base_offset
+ offset
/ BITS_PER_UNIT
);
1718 off
= int_const_binop (PLUS_EXPR
, TREE_OPERAND (base
, 1), off
);
1719 base
= unshare_expr (TREE_OPERAND (base
, 0));
1723 off
= build_int_cst (reference_alias_ptr_type (base
),
1724 base_offset
+ offset
/ BITS_PER_UNIT
);
1725 base
= build_fold_addr_expr (unshare_expr (base
));
1728 return fold_build2_loc (loc
, MEM_REF
, model
->type
, base
, off
);
1731 /* Construct a memory reference consisting of component_refs and array_refs to
1732 a part of an aggregate *RES (which is of type TYPE). The requested part
1733 should have type EXP_TYPE at be the given OFFSET. This function might not
1734 succeed, it returns true when it does and only then *RES points to something
1735 meaningful. This function should be used only to build expressions that we
1736 might need to present to user (e.g. in warnings). In all other situations,
1737 build_ref_for_model or build_ref_for_offset should be used instead. */
1740 build_user_friendly_ref_for_offset (tree
*res
, tree type
, HOST_WIDE_INT offset
,
1746 tree tr_size
, index
, minidx
;
1747 HOST_WIDE_INT el_size
;
1749 if (offset
== 0 && exp_type
1750 && types_compatible_p (exp_type
, type
))
1753 switch (TREE_CODE (type
))
1756 case QUAL_UNION_TYPE
:
1758 for (fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
1760 HOST_WIDE_INT pos
, size
;
1761 tree tr_pos
, expr
, *expr_ptr
;
1763 if (TREE_CODE (fld
) != FIELD_DECL
)
1766 tr_pos
= bit_position (fld
);
1767 if (!tr_pos
|| !tree_fits_uhwi_p (tr_pos
))
1769 pos
= tree_to_uhwi (tr_pos
);
1770 gcc_assert (TREE_CODE (type
) == RECORD_TYPE
|| pos
== 0);
1771 tr_size
= DECL_SIZE (fld
);
1772 if (!tr_size
|| !tree_fits_uhwi_p (tr_size
))
1774 size
= tree_to_uhwi (tr_size
);
1780 else if (pos
> offset
|| (pos
+ size
) <= offset
)
1783 expr
= build3 (COMPONENT_REF
, TREE_TYPE (fld
), *res
, fld
,
1786 if (build_user_friendly_ref_for_offset (expr_ptr
, TREE_TYPE (fld
),
1787 offset
- pos
, exp_type
))
1796 tr_size
= TYPE_SIZE (TREE_TYPE (type
));
1797 if (!tr_size
|| !tree_fits_uhwi_p (tr_size
))
1799 el_size
= tree_to_uhwi (tr_size
);
1801 minidx
= TYPE_MIN_VALUE (TYPE_DOMAIN (type
));
1802 if (TREE_CODE (minidx
) != INTEGER_CST
|| el_size
== 0)
1804 index
= build_int_cst (TYPE_DOMAIN (type
), offset
/ el_size
);
1805 if (!integer_zerop (minidx
))
1806 index
= int_const_binop (PLUS_EXPR
, index
, minidx
);
1807 *res
= build4 (ARRAY_REF
, TREE_TYPE (type
), *res
, index
,
1808 NULL_TREE
, NULL_TREE
);
1809 offset
= offset
% el_size
;
1810 type
= TREE_TYPE (type
);
1825 /* Print message to dump file why a variable was rejected. */
1828 reject (tree var
, const char *msg
)
1830 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1832 fprintf (dump_file
, "Rejected (%d): %s: ", DECL_UID (var
), msg
);
1833 print_generic_expr (dump_file
, var
);
1834 fprintf (dump_file
, "\n");
1838 /* Return true if VAR is a candidate for SRA. */
1841 maybe_add_sra_candidate (tree var
)
1843 tree type
= TREE_TYPE (var
);
1847 if (!AGGREGATE_TYPE_P (type
))
1849 reject (var
, "not aggregate");
1852 /* Allow constant-pool entries that "need to live in memory". */
1853 if (needs_to_live_in_memory (var
) && !constant_decl_p (var
))
1855 reject (var
, "needs to live in memory");
1858 if (TREE_THIS_VOLATILE (var
))
1860 reject (var
, "is volatile");
1863 if (!COMPLETE_TYPE_P (type
))
1865 reject (var
, "has incomplete type");
1868 if (!tree_fits_uhwi_p (TYPE_SIZE (type
)))
1870 reject (var
, "type size not fixed");
1873 if (tree_to_uhwi (TYPE_SIZE (type
)) == 0)
1875 reject (var
, "type size is zero");
1878 if (type_internals_preclude_sra_p (type
, &msg
))
1883 if (/* Fix for PR 41089. tree-stdarg.c needs to have va_lists intact but
1884 we also want to schedule it rather late. Thus we ignore it in
1886 (sra_mode
== SRA_MODE_EARLY_INTRA
1887 && is_va_list_type (type
)))
1889 reject (var
, "is va_list");
1893 bitmap_set_bit (candidate_bitmap
, DECL_UID (var
));
1894 slot
= candidates
->find_slot_with_hash (var
, DECL_UID (var
), INSERT
);
1897 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1899 fprintf (dump_file
, "Candidate (%d): ", DECL_UID (var
));
1900 print_generic_expr (dump_file
, var
);
1901 fprintf (dump_file
, "\n");
1907 /* The very first phase of intraprocedural SRA. It marks in candidate_bitmap
1908 those with type which is suitable for scalarization. */
1911 find_var_candidates (void)
1917 for (parm
= DECL_ARGUMENTS (current_function_decl
);
1919 parm
= DECL_CHAIN (parm
))
1920 ret
|= maybe_add_sra_candidate (parm
);
1922 FOR_EACH_LOCAL_DECL (cfun
, i
, var
)
1927 ret
|= maybe_add_sra_candidate (var
);
1933 /* Return true if EXP is a reference chain of COMPONENT_REFs and AREAY_REFs
1934 ending either with a DECL or a MEM_REF with zero offset. */
1937 path_comparable_for_same_access (tree expr
)
1939 while (handled_component_p (expr
))
1941 if (TREE_CODE (expr
) == ARRAY_REF
)
1943 /* SSA name indices can occur here too when the array is of sie one.
1944 But we cannot just re-use array_refs with SSA names elsewhere in
1945 the function, so disallow non-constant indices. TODO: Remove this
1946 limitation after teaching build_reconstructed_reference to replace
1947 the index with the index type lower bound. */
1948 if (TREE_CODE (TREE_OPERAND (expr
, 1)) != INTEGER_CST
)
1951 expr
= TREE_OPERAND (expr
, 0);
1954 if (TREE_CODE (expr
) == MEM_REF
)
1956 if (!zerop (TREE_OPERAND (expr
, 1)))
1960 gcc_assert (DECL_P (expr
));
1965 /* Assuming that EXP1 consists of only COMPONENT_REFs and ARRAY_REFs, return
1966 true if the chain of these handled components are exactly the same as EXP2
1967 and the expression under them is the same DECL or an equivalent MEM_REF.
1968 The reference picked by compare_access_positions must go to EXP1. */
1971 same_access_path_p (tree exp1
, tree exp2
)
1973 if (TREE_CODE (exp1
) != TREE_CODE (exp2
))
1975 /* Special case single-field structures loaded sometimes as the field
1976 and sometimes as the structure. If the field is of a scalar type,
1977 compare_access_positions will put it into exp1.
1979 TODO: The gimple register type condition can be removed if teach
1980 compare_access_positions to put inner types first. */
1981 if (is_gimple_reg_type (TREE_TYPE (exp1
))
1982 && TREE_CODE (exp1
) == COMPONENT_REF
1983 && (TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (exp1
, 0)))
1984 == TYPE_MAIN_VARIANT (TREE_TYPE (exp2
))))
1985 exp1
= TREE_OPERAND (exp1
, 0);
1990 if (!operand_equal_p (exp1
, exp2
, OEP_ADDRESS_OF
))
1996 /* Sort all accesses for the given variable, check for partial overlaps and
1997 return NULL if there are any. If there are none, pick a representative for
1998 each combination of offset and size and create a linked list out of them.
1999 Return the pointer to the first representative and make sure it is the first
2000 one in the vector of accesses. */
2002 static struct access
*
2003 sort_and_splice_var_accesses (tree var
)
2005 int i
, j
, access_count
;
2006 struct access
*res
, **prev_acc_ptr
= &res
;
2007 vec
<access_p
> *access_vec
;
2009 HOST_WIDE_INT low
= -1, high
= 0;
2011 access_vec
= get_base_access_vector (var
);
2014 access_count
= access_vec
->length ();
2016 /* Sort by <OFFSET, SIZE>. */
2017 access_vec
->qsort (compare_access_positions
);
2020 while (i
< access_count
)
2022 struct access
*access
= (*access_vec
)[i
];
2023 bool grp_write
= access
->write
;
2024 bool grp_read
= !access
->write
;
2025 bool grp_scalar_write
= access
->write
2026 && is_gimple_reg_type (access
->type
);
2027 bool grp_scalar_read
= !access
->write
2028 && is_gimple_reg_type (access
->type
);
2029 bool grp_assignment_read
= access
->grp_assignment_read
;
2030 bool grp_assignment_write
= access
->grp_assignment_write
;
2031 bool multiple_scalar_reads
= false;
2032 bool total_scalarization
= access
->grp_total_scalarization
;
2033 bool grp_partial_lhs
= access
->grp_partial_lhs
;
2034 bool first_scalar
= is_gimple_reg_type (access
->type
);
2035 bool unscalarizable_region
= access
->grp_unscalarizable_region
;
2036 bool grp_same_access_path
= true;
2037 bool bf_non_full_precision
2038 = (INTEGRAL_TYPE_P (access
->type
)
2039 && TYPE_PRECISION (access
->type
) != access
->size
2040 && TREE_CODE (access
->expr
) == COMPONENT_REF
2041 && DECL_BIT_FIELD (TREE_OPERAND (access
->expr
, 1)));
2043 if (first
|| access
->offset
>= high
)
2046 low
= access
->offset
;
2047 high
= access
->offset
+ access
->size
;
2049 else if (access
->offset
> low
&& access
->offset
+ access
->size
> high
)
2052 gcc_assert (access
->offset
>= low
2053 && access
->offset
+ access
->size
<= high
);
2055 grp_same_access_path
= path_comparable_for_same_access (access
->expr
);
2058 while (j
< access_count
)
2060 struct access
*ac2
= (*access_vec
)[j
];
2061 if (ac2
->offset
!= access
->offset
|| ac2
->size
!= access
->size
)
2066 grp_scalar_write
= (grp_scalar_write
2067 || is_gimple_reg_type (ac2
->type
));
2072 if (is_gimple_reg_type (ac2
->type
))
2074 if (grp_scalar_read
)
2075 multiple_scalar_reads
= true;
2077 grp_scalar_read
= true;
2080 grp_assignment_read
|= ac2
->grp_assignment_read
;
2081 grp_assignment_write
|= ac2
->grp_assignment_write
;
2082 grp_partial_lhs
|= ac2
->grp_partial_lhs
;
2083 unscalarizable_region
|= ac2
->grp_unscalarizable_region
;
2084 total_scalarization
|= ac2
->grp_total_scalarization
;
2085 relink_to_new_repr (access
, ac2
);
2087 /* If there are both aggregate-type and scalar-type accesses with
2088 this combination of size and offset, the comparison function
2089 should have put the scalars first. */
2090 gcc_assert (first_scalar
|| !is_gimple_reg_type (ac2
->type
));
2091 /* It also prefers integral types to non-integral. However, when the
2092 precision of the selected type does not span the entire area and
2093 should also be used for a non-integer (i.e. float), we must not
2094 let that happen. Normally analyze_access_subtree expands the type
2095 to cover the entire area but for bit-fields it doesn't. */
2096 if (bf_non_full_precision
&& !INTEGRAL_TYPE_P (ac2
->type
))
2098 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2100 fprintf (dump_file
, "Cannot scalarize the following access "
2101 "because insufficient precision integer type was "
2103 dump_access (dump_file
, access
, false);
2105 unscalarizable_region
= true;
2108 if (grp_same_access_path
2109 && !same_access_path_p (access
->expr
, ac2
->expr
))
2110 grp_same_access_path
= false;
2112 ac2
->group_representative
= access
;
2118 access
->group_representative
= access
;
2119 access
->grp_write
= grp_write
;
2120 access
->grp_read
= grp_read
;
2121 access
->grp_scalar_read
= grp_scalar_read
;
2122 access
->grp_scalar_write
= grp_scalar_write
;
2123 access
->grp_assignment_read
= grp_assignment_read
;
2124 access
->grp_assignment_write
= grp_assignment_write
;
2125 access
->grp_hint
= total_scalarization
2126 || (multiple_scalar_reads
&& !constant_decl_p (var
));
2127 access
->grp_total_scalarization
= total_scalarization
;
2128 access
->grp_partial_lhs
= grp_partial_lhs
;
2129 access
->grp_unscalarizable_region
= unscalarizable_region
;
2130 access
->grp_same_access_path
= grp_same_access_path
;
2132 *prev_acc_ptr
= access
;
2133 prev_acc_ptr
= &access
->next_grp
;
2136 gcc_assert (res
== (*access_vec
)[0]);
2140 /* Create a variable for the given ACCESS which determines the type, name and a
2141 few other properties. Return the variable declaration and store it also to
2142 ACCESS->replacement. REG_TREE is used when creating a declaration to base a
2143 default-definition SSA name on on in order to facilitate an uninitialized
2144 warning. It is used instead of the actual ACCESS type if that is not of a
2145 gimple register type. */
2148 create_access_replacement (struct access
*access
, tree reg_type
= NULL_TREE
)
2152 tree type
= access
->type
;
2153 if (reg_type
&& !is_gimple_reg_type (type
))
2156 if (access
->grp_to_be_debug_replaced
)
2158 repl
= create_tmp_var_raw (access
->type
);
2159 DECL_CONTEXT (repl
) = current_function_decl
;
2162 /* Drop any special alignment on the type if it's not on the main
2163 variant. This avoids issues with weirdo ABIs like AAPCS. */
2164 repl
= create_tmp_var (build_qualified_type (TYPE_MAIN_VARIANT (type
),
2165 TYPE_QUALS (type
)), "SR");
2166 if (TREE_CODE (type
) == COMPLEX_TYPE
2167 || TREE_CODE (type
) == VECTOR_TYPE
)
2169 if (!access
->grp_partial_lhs
)
2170 DECL_GIMPLE_REG_P (repl
) = 1;
2172 else if (access
->grp_partial_lhs
2173 && is_gimple_reg_type (type
))
2174 TREE_ADDRESSABLE (repl
) = 1;
2176 DECL_SOURCE_LOCATION (repl
) = DECL_SOURCE_LOCATION (access
->base
);
2177 DECL_ARTIFICIAL (repl
) = 1;
2178 DECL_IGNORED_P (repl
) = DECL_IGNORED_P (access
->base
);
2180 if (DECL_NAME (access
->base
)
2181 && !DECL_IGNORED_P (access
->base
)
2182 && !DECL_ARTIFICIAL (access
->base
))
2184 char *pretty_name
= make_fancy_name (access
->expr
);
2185 tree debug_expr
= unshare_expr_without_location (access
->expr
), d
;
2188 DECL_NAME (repl
) = get_identifier (pretty_name
);
2189 DECL_NAMELESS (repl
) = 1;
2190 obstack_free (&name_obstack
, pretty_name
);
2192 /* Get rid of any SSA_NAMEs embedded in debug_expr,
2193 as DECL_DEBUG_EXPR isn't considered when looking for still
2194 used SSA_NAMEs and thus they could be freed. All debug info
2195 generation cares is whether something is constant or variable
2196 and that get_ref_base_and_extent works properly on the
2197 expression. It cannot handle accesses at a non-constant offset
2198 though, so just give up in those cases. */
2199 for (d
= debug_expr
;
2200 !fail
&& (handled_component_p (d
) || TREE_CODE (d
) == MEM_REF
);
2201 d
= TREE_OPERAND (d
, 0))
2202 switch (TREE_CODE (d
))
2205 case ARRAY_RANGE_REF
:
2206 if (TREE_OPERAND (d
, 1)
2207 && TREE_CODE (TREE_OPERAND (d
, 1)) != INTEGER_CST
)
2209 if (TREE_OPERAND (d
, 3)
2210 && TREE_CODE (TREE_OPERAND (d
, 3)) != INTEGER_CST
)
2214 if (TREE_OPERAND (d
, 2)
2215 && TREE_CODE (TREE_OPERAND (d
, 2)) != INTEGER_CST
)
2219 if (TREE_CODE (TREE_OPERAND (d
, 0)) != ADDR_EXPR
)
2222 d
= TREE_OPERAND (d
, 0);
2229 SET_DECL_DEBUG_EXPR (repl
, debug_expr
);
2230 DECL_HAS_DEBUG_EXPR_P (repl
) = 1;
2232 if (access
->grp_no_warning
)
2233 TREE_NO_WARNING (repl
) = 1;
2235 TREE_NO_WARNING (repl
) = TREE_NO_WARNING (access
->base
);
2238 TREE_NO_WARNING (repl
) = 1;
2242 if (access
->grp_to_be_debug_replaced
)
2244 fprintf (dump_file
, "Created a debug-only replacement for ");
2245 print_generic_expr (dump_file
, access
->base
);
2246 fprintf (dump_file
, " offset: %u, size: %u\n",
2247 (unsigned) access
->offset
, (unsigned) access
->size
);
2251 fprintf (dump_file
, "Created a replacement for ");
2252 print_generic_expr (dump_file
, access
->base
);
2253 fprintf (dump_file
, " offset: %u, size: %u: ",
2254 (unsigned) access
->offset
, (unsigned) access
->size
);
2255 print_generic_expr (dump_file
, repl
);
2256 fprintf (dump_file
, "\n");
2259 sra_stats
.replacements
++;
2264 /* Return ACCESS scalar replacement, which must exist. */
2267 get_access_replacement (struct access
*access
)
2269 gcc_checking_assert (access
->replacement_decl
);
2270 return access
->replacement_decl
;
2274 /* Build a subtree of accesses rooted in *ACCESS, and move the pointer in the
2275 linked list along the way. Stop when *ACCESS is NULL or the access pointed
2276 to it is not "within" the root. Return false iff some accesses partially
2280 build_access_subtree (struct access
**access
)
2282 struct access
*root
= *access
, *last_child
= NULL
;
2283 HOST_WIDE_INT limit
= root
->offset
+ root
->size
;
2285 *access
= (*access
)->next_grp
;
2286 while (*access
&& (*access
)->offset
+ (*access
)->size
<= limit
)
2289 root
->first_child
= *access
;
2291 last_child
->next_sibling
= *access
;
2292 last_child
= *access
;
2293 (*access
)->parent
= root
;
2294 (*access
)->grp_write
|= root
->grp_write
;
2296 if (!build_access_subtree (access
))
2300 if (*access
&& (*access
)->offset
< limit
)
2306 /* Build a tree of access representatives, ACCESS is the pointer to the first
2307 one, others are linked in a list by the next_grp field. Return false iff
2308 some accesses partially overlap. */
2311 build_access_trees (struct access
*access
)
2315 struct access
*root
= access
;
2317 if (!build_access_subtree (&access
))
2319 root
->next_grp
= access
;
2324 /* Return true if expr contains some ARRAY_REFs into a variable bounded
2328 expr_with_var_bounded_array_refs_p (tree expr
)
2330 while (handled_component_p (expr
))
2332 if (TREE_CODE (expr
) == ARRAY_REF
2333 && !tree_fits_shwi_p (array_ref_low_bound (expr
)))
2335 expr
= TREE_OPERAND (expr
, 0);
2340 /* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
2341 both seeming beneficial and when ALLOW_REPLACEMENTS allows it. Also set all
2342 sorts of access flags appropriately along the way, notably always set
2343 grp_read and grp_assign_read according to MARK_READ and grp_write when
2346 Creating a replacement for a scalar access is considered beneficial if its
2347 grp_hint is set (this means we are either attempting total scalarization or
2348 there is more than one direct read access) or according to the following
2351 Access written to through a scalar type (once or more times)
2353 | Written to in an assignment statement
2355 | | Access read as scalar _once_
2357 | | | Read in an assignment statement
2359 | | | | Scalarize Comment
2360 -----------------------------------------------------------------------------
2361 0 0 0 0 No access for the scalar
2362 0 0 0 1 No access for the scalar
2363 0 0 1 0 No Single read - won't help
2364 0 0 1 1 No The same case
2365 0 1 0 0 No access for the scalar
2366 0 1 0 1 No access for the scalar
2367 0 1 1 0 Yes s = *g; return s.i;
2368 0 1 1 1 Yes The same case as above
2369 1 0 0 0 No Won't help
2370 1 0 0 1 Yes s.i = 1; *g = s;
2371 1 0 1 0 Yes s.i = 5; g = s.i;
2372 1 0 1 1 Yes The same case as above
2373 1 1 0 0 No Won't help.
2374 1 1 0 1 Yes s.i = 1; *g = s;
2375 1 1 1 0 Yes s = *g; return s.i;
2376 1 1 1 1 Yes Any of the above yeses */
2379 analyze_access_subtree (struct access
*root
, struct access
*parent
,
2380 bool allow_replacements
)
2382 struct access
*child
;
2383 HOST_WIDE_INT limit
= root
->offset
+ root
->size
;
2384 HOST_WIDE_INT covered_to
= root
->offset
;
2385 bool scalar
= is_gimple_reg_type (root
->type
);
2386 bool hole
= false, sth_created
= false;
2390 if (parent
->grp_read
)
2392 if (parent
->grp_assignment_read
)
2393 root
->grp_assignment_read
= 1;
2394 if (parent
->grp_write
)
2395 root
->grp_write
= 1;
2396 if (parent
->grp_assignment_write
)
2397 root
->grp_assignment_write
= 1;
2398 if (parent
->grp_total_scalarization
)
2399 root
->grp_total_scalarization
= 1;
2400 if (!parent
->grp_same_access_path
)
2401 root
->grp_same_access_path
= 0;
2404 if (root
->grp_unscalarizable_region
)
2405 allow_replacements
= false;
2407 if (allow_replacements
&& expr_with_var_bounded_array_refs_p (root
->expr
))
2408 allow_replacements
= false;
2410 for (child
= root
->first_child
; child
; child
= child
->next_sibling
)
2412 hole
|= covered_to
< child
->offset
;
2413 sth_created
|= analyze_access_subtree (child
, root
,
2414 allow_replacements
&& !scalar
);
2416 root
->grp_unscalarized_data
|= child
->grp_unscalarized_data
;
2417 root
->grp_total_scalarization
&= child
->grp_total_scalarization
;
2418 if (child
->grp_covered
)
2419 covered_to
+= child
->size
;
2424 if (allow_replacements
&& scalar
&& !root
->first_child
2426 || ((root
->grp_scalar_read
|| root
->grp_assignment_read
)
2427 && (root
->grp_scalar_write
|| root
->grp_assignment_write
))))
2429 /* Always create access replacements that cover the whole access.
2430 For integral types this means the precision has to match.
2431 Avoid assumptions based on the integral type kind, too. */
2432 if (INTEGRAL_TYPE_P (root
->type
)
2433 && (TREE_CODE (root
->type
) != INTEGER_TYPE
2434 || TYPE_PRECISION (root
->type
) != root
->size
)
2435 /* But leave bitfield accesses alone. */
2436 && (TREE_CODE (root
->expr
) != COMPONENT_REF
2437 || !DECL_BIT_FIELD (TREE_OPERAND (root
->expr
, 1))))
2439 tree rt
= root
->type
;
2440 gcc_assert ((root
->offset
% BITS_PER_UNIT
) == 0
2441 && (root
->size
% BITS_PER_UNIT
) == 0);
2442 root
->type
= build_nonstandard_integer_type (root
->size
,
2443 TYPE_UNSIGNED (rt
));
2444 root
->expr
= build_ref_for_offset (UNKNOWN_LOCATION
, root
->base
,
2445 root
->offset
, root
->reverse
,
2446 root
->type
, NULL
, false);
2448 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2450 fprintf (dump_file
, "Changing the type of a replacement for ");
2451 print_generic_expr (dump_file
, root
->base
);
2452 fprintf (dump_file
, " offset: %u, size: %u ",
2453 (unsigned) root
->offset
, (unsigned) root
->size
);
2454 fprintf (dump_file
, " to an integer.\n");
2458 root
->grp_to_be_replaced
= 1;
2459 root
->replacement_decl
= create_access_replacement (root
);
2465 if (allow_replacements
2466 && scalar
&& !root
->first_child
2467 && (root
->grp_scalar_write
|| root
->grp_assignment_write
)
2468 && !bitmap_bit_p (cannot_scalarize_away_bitmap
,
2469 DECL_UID (root
->base
)))
2471 gcc_checking_assert (!root
->grp_scalar_read
2472 && !root
->grp_assignment_read
);
2474 if (MAY_HAVE_DEBUG_BIND_STMTS
)
2476 root
->grp_to_be_debug_replaced
= 1;
2477 root
->replacement_decl
= create_access_replacement (root
);
2481 if (covered_to
< limit
)
2483 if (scalar
|| !allow_replacements
)
2484 root
->grp_total_scalarization
= 0;
2487 if (!hole
|| root
->grp_total_scalarization
)
2488 root
->grp_covered
= 1;
2489 else if (root
->grp_write
|| comes_initialized_p (root
->base
))
2490 root
->grp_unscalarized_data
= 1; /* not covered and written to */
2494 /* Analyze all access trees linked by next_grp by the means of
2495 analyze_access_subtree. */
2497 analyze_access_trees (struct access
*access
)
2503 if (analyze_access_subtree (access
, NULL
, true))
2505 access
= access
->next_grp
;
2511 /* Return true iff a potential new child of LACC at offset OFFSET and with size
2512 SIZE would conflict with an already existing one. If exactly such a child
2513 already exists in LACC, store a pointer to it in EXACT_MATCH. */
2516 child_would_conflict_in_lacc (struct access
*lacc
, HOST_WIDE_INT norm_offset
,
2517 HOST_WIDE_INT size
, struct access
**exact_match
)
2519 struct access
*child
;
2521 for (child
= lacc
->first_child
; child
; child
= child
->next_sibling
)
2523 if (child
->offset
== norm_offset
&& child
->size
== size
)
2525 *exact_match
= child
;
2529 if (child
->offset
< norm_offset
+ size
2530 && child
->offset
+ child
->size
> norm_offset
)
2537 /* Create a new child access of PARENT, with all properties just like MODEL
2538 except for its offset and with its grp_write false and grp_read true.
2539 Return the new access or NULL if it cannot be created. Note that this
2540 access is created long after all splicing and sorting, it's not located in
2541 any access vector and is automatically a representative of its group. Set
2542 the gpr_write flag of the new accesss if SET_GRP_WRITE is true. */
2544 static struct access
*
2545 create_artificial_child_access (struct access
*parent
, struct access
*model
,
2546 HOST_WIDE_INT new_offset
,
2549 struct access
**child
;
2550 tree expr
= parent
->base
;
2552 gcc_assert (!model
->grp_unscalarizable_region
);
2554 struct access
*access
= access_pool
.allocate ();
2555 memset (access
, 0, sizeof (struct access
));
2556 if (!build_user_friendly_ref_for_offset (&expr
, TREE_TYPE (expr
), new_offset
,
2559 access
->grp_no_warning
= true;
2560 expr
= build_ref_for_model (EXPR_LOCATION (parent
->base
), parent
->base
,
2561 new_offset
, model
, NULL
, false);
2564 access
->base
= parent
->base
;
2565 access
->expr
= expr
;
2566 access
->offset
= new_offset
;
2567 access
->size
= model
->size
;
2568 access
->type
= model
->type
;
2569 access
->grp_write
= set_grp_write
;
2570 access
->grp_read
= false;
2571 access
->reverse
= model
->reverse
;
2573 child
= &parent
->first_child
;
2574 while (*child
&& (*child
)->offset
< new_offset
)
2575 child
= &(*child
)->next_sibling
;
2577 access
->next_sibling
= *child
;
2584 /* Beginning with ACCESS, traverse its whole access subtree and mark all
2585 sub-trees as written to. If any of them has not been marked so previously
2586 and has assignment links leading from it, re-enqueue it. */
2589 subtree_mark_written_and_enqueue (struct access
*access
)
2591 if (access
->grp_write
)
2593 access
->grp_write
= true;
2594 add_access_to_work_queue (access
);
2596 struct access
*child
;
2597 for (child
= access
->first_child
; child
; child
= child
->next_sibling
)
2598 subtree_mark_written_and_enqueue (child
);
2601 /* Propagate subaccesses and grp_write flags of RACC across an assignment link
2602 to LACC. Enqueue sub-accesses as necessary so that the write flag is
2603 propagated transitively. Return true if anything changed. Additionally, if
2604 RACC is a scalar access but LACC is not, change the type of the latter, if
2608 propagate_subaccesses_across_link (struct access
*lacc
, struct access
*racc
)
2610 struct access
*rchild
;
2611 HOST_WIDE_INT norm_delta
= lacc
->offset
- racc
->offset
;
2614 /* IF the LHS is still not marked as being written to, we only need to do so
2615 if the RHS at this level actually was. */
2616 if (!lacc
->grp_write
)
2618 gcc_checking_assert (!comes_initialized_p (racc
->base
));
2619 if (racc
->grp_write
)
2621 subtree_mark_written_and_enqueue (lacc
);
2626 if (is_gimple_reg_type (lacc
->type
)
2627 || lacc
->grp_unscalarizable_region
2628 || racc
->grp_unscalarizable_region
)
2630 if (!lacc
->grp_write
)
2633 subtree_mark_written_and_enqueue (lacc
);
2638 if (is_gimple_reg_type (racc
->type
))
2640 if (!lacc
->grp_write
)
2643 subtree_mark_written_and_enqueue (lacc
);
2645 if (!lacc
->first_child
&& !racc
->first_child
)
2647 tree t
= lacc
->base
;
2649 lacc
->type
= racc
->type
;
2650 if (build_user_friendly_ref_for_offset (&t
, TREE_TYPE (t
),
2651 lacc
->offset
, racc
->type
))
2654 lacc
->grp_same_access_path
= true;
2658 lacc
->expr
= build_ref_for_model (EXPR_LOCATION (lacc
->base
),
2659 lacc
->base
, lacc
->offset
,
2661 lacc
->grp_no_warning
= true;
2662 lacc
->grp_same_access_path
= false;
2668 for (rchild
= racc
->first_child
; rchild
; rchild
= rchild
->next_sibling
)
2670 struct access
*new_acc
= NULL
;
2671 HOST_WIDE_INT norm_offset
= rchild
->offset
+ norm_delta
;
2673 if (child_would_conflict_in_lacc (lacc
, norm_offset
, rchild
->size
,
2678 if (!new_acc
->grp_write
&& rchild
->grp_write
)
2680 gcc_assert (!lacc
->grp_write
);
2681 subtree_mark_written_and_enqueue (new_acc
);
2685 rchild
->grp_hint
= 1;
2686 new_acc
->grp_hint
|= new_acc
->grp_read
;
2687 if (rchild
->first_child
2688 && propagate_subaccesses_across_link (new_acc
, rchild
))
2691 add_access_to_work_queue (new_acc
);
2696 if (!lacc
->grp_write
)
2699 subtree_mark_written_and_enqueue (lacc
);
2705 if (rchild
->grp_unscalarizable_region
)
2707 if (rchild
->grp_write
&& !lacc
->grp_write
)
2710 subtree_mark_written_and_enqueue (lacc
);
2715 rchild
->grp_hint
= 1;
2716 new_acc
= create_artificial_child_access (lacc
, rchild
, norm_offset
,
2718 || rchild
->grp_write
);
2719 gcc_checking_assert (new_acc
);
2720 if (racc
->first_child
)
2721 propagate_subaccesses_across_link (new_acc
, rchild
);
2723 add_access_to_work_queue (lacc
);
2730 /* Propagate all subaccesses across assignment links. */
2733 propagate_all_subaccesses (void)
2735 while (work_queue_head
)
2737 struct access
*racc
= pop_access_from_work_queue ();
2738 struct assign_link
*link
;
2740 if (racc
->group_representative
)
2741 racc
= racc
->group_representative
;
2742 gcc_assert (racc
->first_link
);
2744 for (link
= racc
->first_link
; link
; link
= link
->next
)
2746 struct access
*lacc
= link
->lacc
;
2748 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (lacc
->base
)))
2750 lacc
= lacc
->group_representative
;
2752 bool reque_parents
= false;
2753 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (racc
->base
)))
2755 if (!lacc
->grp_write
)
2757 subtree_mark_written_and_enqueue (lacc
);
2758 reque_parents
= true;
2761 else if (propagate_subaccesses_across_link (lacc
, racc
))
2762 reque_parents
= true;
2767 add_access_to_work_queue (lacc
);
2768 lacc
= lacc
->parent
;
2775 /* Go through all accesses collected throughout the (intraprocedural) analysis
2776 stage, exclude overlapping ones, identify representatives and build trees
2777 out of them, making decisions about scalarization on the way. Return true
2778 iff there are any to-be-scalarized variables after this stage. */
2781 analyze_all_variable_accesses (void)
2784 bitmap tmp
= BITMAP_ALLOC (NULL
);
2787 bool optimize_speed_p
= !optimize_function_for_size_p (cfun
);
2789 /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
2790 fall back to a target default. */
2791 unsigned HOST_WIDE_INT max_scalarization_size
2792 = get_move_ratio (optimize_speed_p
) * UNITS_PER_WORD
;
2794 if (optimize_speed_p
)
2796 if (global_options_set
.x_param_sra_max_scalarization_size_speed
)
2797 max_scalarization_size
= param_sra_max_scalarization_size_speed
;
2801 if (global_options_set
.x_param_sra_max_scalarization_size_size
)
2802 max_scalarization_size
= param_sra_max_scalarization_size_size
;
2805 max_scalarization_size
*= BITS_PER_UNIT
;
2807 EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap
, 0, i
, bi
)
2808 if (bitmap_bit_p (should_scalarize_away_bitmap
, i
)
2809 && !bitmap_bit_p (cannot_scalarize_away_bitmap
, i
))
2811 tree var
= candidate (i
);
2813 if (VAR_P (var
) && scalarizable_type_p (TREE_TYPE (var
),
2814 constant_decl_p (var
)))
2816 if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var
)))
2817 <= max_scalarization_size
)
2819 create_total_scalarization_access (var
);
2820 completely_scalarize (var
, TREE_TYPE (var
), 0, var
);
2821 statistics_counter_event (cfun
,
2822 "Totally-scalarized aggregates", 1);
2823 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2825 fprintf (dump_file
, "Will attempt to totally scalarize ");
2826 print_generic_expr (dump_file
, var
);
2827 fprintf (dump_file
, " (UID: %u): \n", DECL_UID (var
));
2830 else if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2832 fprintf (dump_file
, "Too big to totally scalarize: ");
2833 print_generic_expr (dump_file
, var
);
2834 fprintf (dump_file
, " (UID: %u)\n", DECL_UID (var
));
2839 bitmap_copy (tmp
, candidate_bitmap
);
2840 EXECUTE_IF_SET_IN_BITMAP (tmp
, 0, i
, bi
)
2842 tree var
= candidate (i
);
2843 struct access
*access
;
2845 access
= sort_and_splice_var_accesses (var
);
2846 if (!access
|| !build_access_trees (access
))
2847 disqualify_candidate (var
,
2848 "No or inhibitingly overlapping accesses.");
2851 propagate_all_subaccesses ();
2853 bitmap_copy (tmp
, candidate_bitmap
);
2854 EXECUTE_IF_SET_IN_BITMAP (tmp
, 0, i
, bi
)
2856 tree var
= candidate (i
);
2857 struct access
*access
= get_first_repr_for_decl (var
);
2859 if (analyze_access_trees (access
))
2862 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2864 fprintf (dump_file
, "\nAccess trees for ");
2865 print_generic_expr (dump_file
, var
);
2866 fprintf (dump_file
, " (UID: %u): \n", DECL_UID (var
));
2867 dump_access_tree (dump_file
, access
);
2868 fprintf (dump_file
, "\n");
2872 disqualify_candidate (var
, "No scalar replacements to be created.");
2879 statistics_counter_event (cfun
, "Scalarized aggregates", res
);
2886 /* Generate statements copying scalar replacements of accesses within a subtree
2887 into or out of AGG. ACCESS, all its children, siblings and their children
2888 are to be processed. AGG is an aggregate type expression (can be a
2889 declaration but does not have to be, it can for example also be a mem_ref or
2890 a series of handled components). TOP_OFFSET is the offset of the processed
2891 subtree which has to be subtracted from offsets of individual accesses to
2892 get corresponding offsets for AGG. If CHUNK_SIZE is non-null, copy only
2893 replacements in the interval <start_offset, start_offset + chunk_size>,
2894 otherwise copy all. GSI is a statement iterator used to place the new
2895 statements. WRITE should be true when the statements should write from AGG
2896 to the replacement and false if vice versa. if INSERT_AFTER is true, new
2897 statements will be added after the current statement in GSI, they will be
2898 added before the statement otherwise. */
2901 generate_subtree_copies (struct access
*access
, tree agg
,
2902 HOST_WIDE_INT top_offset
,
2903 HOST_WIDE_INT start_offset
, HOST_WIDE_INT chunk_size
,
2904 gimple_stmt_iterator
*gsi
, bool write
,
2905 bool insert_after
, location_t loc
)
2907 /* Never write anything into constant pool decls. See PR70602. */
2908 if (!write
&& constant_decl_p (agg
))
2912 if (chunk_size
&& access
->offset
>= start_offset
+ chunk_size
)
2915 if (access
->grp_to_be_replaced
2917 || access
->offset
+ access
->size
> start_offset
))
2919 tree expr
, repl
= get_access_replacement (access
);
2922 expr
= build_ref_for_model (loc
, agg
, access
->offset
- top_offset
,
2923 access
, gsi
, insert_after
);
2927 if (access
->grp_partial_lhs
)
2928 expr
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
2930 insert_after
? GSI_NEW_STMT
2932 stmt
= gimple_build_assign (repl
, expr
);
2936 TREE_NO_WARNING (repl
) = 1;
2937 if (access
->grp_partial_lhs
)
2938 repl
= force_gimple_operand_gsi (gsi
, repl
, true, NULL_TREE
,
2940 insert_after
? GSI_NEW_STMT
2942 stmt
= gimple_build_assign (expr
, repl
);
2944 gimple_set_location (stmt
, loc
);
2947 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
2949 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
2951 sra_stats
.subtree_copies
++;
2954 && access
->grp_to_be_debug_replaced
2956 || access
->offset
+ access
->size
> start_offset
))
2959 tree drhs
= build_debug_ref_for_model (loc
, agg
,
2960 access
->offset
- top_offset
,
2962 ds
= gimple_build_debug_bind (get_access_replacement (access
),
2963 drhs
, gsi_stmt (*gsi
));
2965 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
2967 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
2970 if (access
->first_child
)
2971 generate_subtree_copies (access
->first_child
, agg
, top_offset
,
2972 start_offset
, chunk_size
, gsi
,
2973 write
, insert_after
, loc
);
2975 access
= access
->next_sibling
;
2980 /* Assign zero to all scalar replacements in an access subtree. ACCESS is the
2981 root of the subtree to be processed. GSI is the statement iterator used
2982 for inserting statements which are added after the current statement if
2983 INSERT_AFTER is true or before it otherwise. */
2986 init_subtree_with_zero (struct access
*access
, gimple_stmt_iterator
*gsi
,
2987 bool insert_after
, location_t loc
)
2990 struct access
*child
;
2992 if (access
->grp_to_be_replaced
)
2996 stmt
= gimple_build_assign (get_access_replacement (access
),
2997 build_zero_cst (access
->type
));
2999 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
3001 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
3003 gimple_set_location (stmt
, loc
);
3005 else if (access
->grp_to_be_debug_replaced
)
3008 = gimple_build_debug_bind (get_access_replacement (access
),
3009 build_zero_cst (access
->type
),
3012 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
3014 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
3017 for (child
= access
->first_child
; child
; child
= child
->next_sibling
)
3018 init_subtree_with_zero (child
, gsi
, insert_after
, loc
);
3021 /* Clobber all scalar replacements in an access subtree. ACCESS is the
3022 root of the subtree to be processed. GSI is the statement iterator used
3023 for inserting statements which are added after the current statement if
3024 INSERT_AFTER is true or before it otherwise. */
3027 clobber_subtree (struct access
*access
, gimple_stmt_iterator
*gsi
,
3028 bool insert_after
, location_t loc
)
3031 struct access
*child
;
3033 if (access
->grp_to_be_replaced
)
3035 tree rep
= get_access_replacement (access
);
3036 tree clobber
= build_clobber (access
->type
);
3037 gimple
*stmt
= gimple_build_assign (rep
, clobber
);
3040 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
3042 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
3044 gimple_set_location (stmt
, loc
);
3047 for (child
= access
->first_child
; child
; child
= child
->next_sibling
)
3048 clobber_subtree (child
, gsi
, insert_after
, loc
);
3051 /* Search for an access representative for the given expression EXPR and
3052 return it or NULL if it cannot be found. */
3054 static struct access
*
3055 get_access_for_expr (tree expr
)
3057 poly_int64 poffset
, psize
, pmax_size
;
3058 HOST_WIDE_INT offset
, max_size
;
3062 /* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
3063 a different size than the size of its argument and we need the latter
3065 if (TREE_CODE (expr
) == VIEW_CONVERT_EXPR
)
3066 expr
= TREE_OPERAND (expr
, 0);
3068 base
= get_ref_base_and_extent (expr
, &poffset
, &psize
, &pmax_size
,
3070 if (!known_size_p (pmax_size
)
3071 || !pmax_size
.is_constant (&max_size
)
3072 || !poffset
.is_constant (&offset
)
3076 if (tree basesize
= DECL_SIZE (base
))
3078 poly_int64 sz
= tree_to_poly_int64 (basesize
);
3079 if (offset
< 0 || known_le (sz
, offset
))
3083 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (base
)))
3086 return get_var_base_offset_size_access (base
, offset
, max_size
);
3089 /* Replace the expression EXPR with a scalar replacement if there is one and
3090 generate other statements to do type conversion or subtree copying if
3091 necessary. GSI is used to place newly created statements, WRITE is true if
3092 the expression is being written to (it is on a LHS of a statement or output
3093 in an assembly statement). */
3096 sra_modify_expr (tree
*expr
, gimple_stmt_iterator
*gsi
, bool write
)
3099 struct access
*access
;
3100 tree type
, bfr
, orig_expr
;
3102 if (TREE_CODE (*expr
) == BIT_FIELD_REF
)
3105 expr
= &TREE_OPERAND (*expr
, 0);
3110 if (TREE_CODE (*expr
) == REALPART_EXPR
|| TREE_CODE (*expr
) == IMAGPART_EXPR
)
3111 expr
= &TREE_OPERAND (*expr
, 0);
3112 access
= get_access_for_expr (*expr
);
3115 type
= TREE_TYPE (*expr
);
3118 loc
= gimple_location (gsi_stmt (*gsi
));
3119 gimple_stmt_iterator alt_gsi
= gsi_none ();
3120 if (write
&& stmt_ends_bb_p (gsi_stmt (*gsi
)))
3122 alt_gsi
= gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi
)));
3126 if (access
->grp_to_be_replaced
)
3128 tree repl
= get_access_replacement (access
);
3129 /* If we replace a non-register typed access simply use the original
3130 access expression to extract the scalar component afterwards.
3131 This happens if scalarizing a function return value or parameter
3132 like in gcc.c-torture/execute/20041124-1.c, 20050316-1.c and
3133 gcc.c-torture/compile/20011217-1.c.
3135 We also want to use this when accessing a complex or vector which can
3136 be accessed as a different type too, potentially creating a need for
3137 type conversion (see PR42196) and when scalarized unions are involved
3138 in assembler statements (see PR42398). */
3139 if (!useless_type_conversion_p (type
, access
->type
))
3143 ref
= build_ref_for_model (loc
, orig_expr
, 0, access
, gsi
, false);
3149 if (access
->grp_partial_lhs
)
3150 ref
= force_gimple_operand_gsi (gsi
, ref
, true, NULL_TREE
,
3151 false, GSI_NEW_STMT
);
3152 stmt
= gimple_build_assign (repl
, ref
);
3153 gimple_set_location (stmt
, loc
);
3154 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
3160 if (access
->grp_partial_lhs
)
3161 repl
= force_gimple_operand_gsi (gsi
, repl
, true, NULL_TREE
,
3162 true, GSI_SAME_STMT
);
3163 stmt
= gimple_build_assign (ref
, repl
);
3164 gimple_set_location (stmt
, loc
);
3165 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
3172 else if (write
&& access
->grp_to_be_debug_replaced
)
3174 gdebug
*ds
= gimple_build_debug_bind (get_access_replacement (access
),
3177 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
3180 if (access
->first_child
)
3182 HOST_WIDE_INT start_offset
, chunk_size
;
3184 && tree_fits_uhwi_p (TREE_OPERAND (bfr
, 1))
3185 && tree_fits_uhwi_p (TREE_OPERAND (bfr
, 2)))
3187 chunk_size
= tree_to_uhwi (TREE_OPERAND (bfr
, 1));
3188 start_offset
= access
->offset
3189 + tree_to_uhwi (TREE_OPERAND (bfr
, 2));
3192 start_offset
= chunk_size
= 0;
3194 generate_subtree_copies (access
->first_child
, orig_expr
, access
->offset
,
3195 start_offset
, chunk_size
, gsi
, write
, write
,
3201 /* Where scalar replacements of the RHS have been written to when a replacement
3202 of a LHS of an assigments cannot be direclty loaded from a replacement of
3204 enum unscalarized_data_handling
{ SRA_UDH_NONE
, /* Nothing done so far. */
3205 SRA_UDH_RIGHT
, /* Data flushed to the RHS. */
3206 SRA_UDH_LEFT
}; /* Data flushed to the LHS. */
3208 struct subreplacement_assignment_data
3210 /* Offset of the access representing the lhs of the assignment. */
3211 HOST_WIDE_INT left_offset
;
3213 /* LHS and RHS of the original assignment. */
3214 tree assignment_lhs
, assignment_rhs
;
3216 /* Access representing the rhs of the whole assignment. */
3217 struct access
*top_racc
;
3219 /* Stmt iterator used for statement insertions after the original assignment.
3220 It points to the main GSI used to traverse a BB during function body
3222 gimple_stmt_iterator
*new_gsi
;
3224 /* Stmt iterator used for statement insertions before the original
3225 assignment. Keeps on pointing to the original statement. */
3226 gimple_stmt_iterator old_gsi
;
3228 /* Location of the assignment. */
3231 /* Keeps the information whether we have needed to refresh replacements of
3232 the LHS and from which side of the assignments this takes place. */
3233 enum unscalarized_data_handling refreshed
;
3236 /* Store all replacements in the access tree rooted in TOP_RACC either to their
3237 base aggregate if there are unscalarized data or directly to LHS of the
3238 statement that is pointed to by GSI otherwise. */
3241 handle_unscalarized_data_in_subtree (struct subreplacement_assignment_data
*sad
)
3244 if (sad
->top_racc
->grp_unscalarized_data
)
3246 src
= sad
->assignment_rhs
;
3247 sad
->refreshed
= SRA_UDH_RIGHT
;
3251 src
= sad
->assignment_lhs
;
3252 sad
->refreshed
= SRA_UDH_LEFT
;
3254 generate_subtree_copies (sad
->top_racc
->first_child
, src
,
3255 sad
->top_racc
->offset
, 0, 0,
3256 &sad
->old_gsi
, false, false, sad
->loc
);
3259 /* Try to generate statements to load all sub-replacements in an access subtree
3260 formed by children of LACC from scalar replacements in the SAD->top_racc
3261 subtree. If that is not possible, refresh the SAD->top_racc base aggregate
3262 and load the accesses from it. */
3265 load_assign_lhs_subreplacements (struct access
*lacc
,
3266 struct subreplacement_assignment_data
*sad
)
3268 for (lacc
= lacc
->first_child
; lacc
; lacc
= lacc
->next_sibling
)
3270 HOST_WIDE_INT offset
;
3271 offset
= lacc
->offset
- sad
->left_offset
+ sad
->top_racc
->offset
;
3273 if (lacc
->grp_to_be_replaced
)
3275 struct access
*racc
;
3279 racc
= find_access_in_subtree (sad
->top_racc
, offset
, lacc
->size
);
3280 if (racc
&& racc
->grp_to_be_replaced
)
3282 rhs
= get_access_replacement (racc
);
3283 if (!useless_type_conversion_p (lacc
->type
, racc
->type
))
3284 rhs
= fold_build1_loc (sad
->loc
, VIEW_CONVERT_EXPR
,
3287 if (racc
->grp_partial_lhs
&& lacc
->grp_partial_lhs
)
3288 rhs
= force_gimple_operand_gsi (&sad
->old_gsi
, rhs
, true,
3289 NULL_TREE
, true, GSI_SAME_STMT
);
3293 /* No suitable access on the right hand side, need to load from
3294 the aggregate. See if we have to update it first... */
3295 if (sad
->refreshed
== SRA_UDH_NONE
)
3296 handle_unscalarized_data_in_subtree (sad
);
3298 if (sad
->refreshed
== SRA_UDH_LEFT
)
3299 rhs
= build_ref_for_model (sad
->loc
, sad
->assignment_lhs
,
3300 lacc
->offset
- sad
->left_offset
,
3301 lacc
, sad
->new_gsi
, true);
3303 rhs
= build_ref_for_model (sad
->loc
, sad
->assignment_rhs
,
3304 lacc
->offset
- sad
->left_offset
,
3305 lacc
, sad
->new_gsi
, true);
3306 if (lacc
->grp_partial_lhs
)
3307 rhs
= force_gimple_operand_gsi (sad
->new_gsi
,
3308 rhs
, true, NULL_TREE
,
3309 false, GSI_NEW_STMT
);
3312 stmt
= gimple_build_assign (get_access_replacement (lacc
), rhs
);
3313 gsi_insert_after (sad
->new_gsi
, stmt
, GSI_NEW_STMT
);
3314 gimple_set_location (stmt
, sad
->loc
);
3316 sra_stats
.subreplacements
++;
3320 if (sad
->refreshed
== SRA_UDH_NONE
3321 && lacc
->grp_read
&& !lacc
->grp_covered
)
3322 handle_unscalarized_data_in_subtree (sad
);
3324 if (lacc
&& lacc
->grp_to_be_debug_replaced
)
3328 struct access
*racc
= find_access_in_subtree (sad
->top_racc
,
3332 if (racc
&& racc
->grp_to_be_replaced
)
3334 if (racc
->grp_write
|| constant_decl_p (racc
->base
))
3335 drhs
= get_access_replacement (racc
);
3339 else if (sad
->refreshed
== SRA_UDH_LEFT
)
3340 drhs
= build_debug_ref_for_model (sad
->loc
, lacc
->base
,
3341 lacc
->offset
, lacc
);
3342 else if (sad
->refreshed
== SRA_UDH_RIGHT
)
3343 drhs
= build_debug_ref_for_model (sad
->loc
, sad
->top_racc
->base
,
3348 && !useless_type_conversion_p (lacc
->type
, TREE_TYPE (drhs
)))
3349 drhs
= fold_build1_loc (sad
->loc
, VIEW_CONVERT_EXPR
,
3351 ds
= gimple_build_debug_bind (get_access_replacement (lacc
),
3352 drhs
, gsi_stmt (sad
->old_gsi
));
3353 gsi_insert_after (sad
->new_gsi
, ds
, GSI_NEW_STMT
);
3357 if (lacc
->first_child
)
3358 load_assign_lhs_subreplacements (lacc
, sad
);
3362 /* Result code for SRA assignment modification. */
3363 enum assignment_mod_result
{ SRA_AM_NONE
, /* nothing done for the stmt */
3364 SRA_AM_MODIFIED
, /* stmt changed but not
3366 SRA_AM_REMOVED
}; /* stmt eliminated */
3368 /* Modify assignments with a CONSTRUCTOR on their RHS. STMT contains a pointer
3369 to the assignment and GSI is the statement iterator pointing at it. Returns
3370 the same values as sra_modify_assign. */
3372 static enum assignment_mod_result
3373 sra_modify_constructor_assign (gimple
*stmt
, gimple_stmt_iterator
*gsi
)
3375 tree lhs
= gimple_assign_lhs (stmt
);
3376 struct access
*acc
= get_access_for_expr (lhs
);
3379 location_t loc
= gimple_location (stmt
);
3381 if (gimple_clobber_p (stmt
))
3383 /* Clobber the replacement variable. */
3384 clobber_subtree (acc
, gsi
, !acc
->grp_covered
, loc
);
3385 /* Remove clobbers of fully scalarized variables, they are dead. */
3386 if (acc
->grp_covered
)
3388 unlink_stmt_vdef (stmt
);
3389 gsi_remove (gsi
, true);
3390 release_defs (stmt
);
3391 return SRA_AM_REMOVED
;
3394 return SRA_AM_MODIFIED
;
3397 if (CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt
)) > 0)
3399 /* I have never seen this code path trigger but if it can happen the
3400 following should handle it gracefully. */
3401 if (access_has_children_p (acc
))
3402 generate_subtree_copies (acc
->first_child
, lhs
, acc
->offset
, 0, 0, gsi
,
3404 return SRA_AM_MODIFIED
;
3407 if (acc
->grp_covered
)
3409 init_subtree_with_zero (acc
, gsi
, false, loc
);
3410 unlink_stmt_vdef (stmt
);
3411 gsi_remove (gsi
, true);
3412 release_defs (stmt
);
3413 return SRA_AM_REMOVED
;
3417 init_subtree_with_zero (acc
, gsi
, true, loc
);
3418 return SRA_AM_MODIFIED
;
3422 /* Create and return a new suitable default definition SSA_NAME for RACC which
3423 is an access describing an uninitialized part of an aggregate that is being
3424 loaded. REG_TREE is used instead of the actual RACC type if that is not of
3425 a gimple register type. */
3428 get_repl_default_def_ssa_name (struct access
*racc
, tree reg_type
)
3430 gcc_checking_assert (!racc
->grp_to_be_replaced
3431 && !racc
->grp_to_be_debug_replaced
);
3432 if (!racc
->replacement_decl
)
3433 racc
->replacement_decl
= create_access_replacement (racc
, reg_type
);
3434 return get_or_create_ssa_default_def (cfun
, racc
->replacement_decl
);
3437 /* Examine both sides of the assignment statement pointed to by STMT, replace
3438 them with a scalare replacement if there is one and generate copying of
3439 replacements if scalarized aggregates have been used in the assignment. GSI
3440 is used to hold generated statements for type conversions and subtree
3443 static enum assignment_mod_result
3444 sra_modify_assign (gimple
*stmt
, gimple_stmt_iterator
*gsi
)
3446 struct access
*lacc
, *racc
;
3448 bool modify_this_stmt
= false;
3449 bool force_gimple_rhs
= false;
3451 gimple_stmt_iterator orig_gsi
= *gsi
;
3453 if (!gimple_assign_single_p (stmt
))
3455 lhs
= gimple_assign_lhs (stmt
);
3456 rhs
= gimple_assign_rhs1 (stmt
);
3458 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
3459 return sra_modify_constructor_assign (stmt
, gsi
);
3461 if (TREE_CODE (rhs
) == REALPART_EXPR
|| TREE_CODE (lhs
) == REALPART_EXPR
3462 || TREE_CODE (rhs
) == IMAGPART_EXPR
|| TREE_CODE (lhs
) == IMAGPART_EXPR
3463 || TREE_CODE (rhs
) == BIT_FIELD_REF
|| TREE_CODE (lhs
) == BIT_FIELD_REF
)
3465 modify_this_stmt
= sra_modify_expr (gimple_assign_rhs1_ptr (stmt
),
3467 modify_this_stmt
|= sra_modify_expr (gimple_assign_lhs_ptr (stmt
),
3469 return modify_this_stmt
? SRA_AM_MODIFIED
: SRA_AM_NONE
;
3472 lacc
= get_access_for_expr (lhs
);
3473 racc
= get_access_for_expr (rhs
);
3476 /* Avoid modifying initializations of constant-pool replacements. */
3477 if (racc
&& (racc
->replacement_decl
== lhs
))
3480 loc
= gimple_location (stmt
);
3481 if (lacc
&& lacc
->grp_to_be_replaced
)
3483 lhs
= get_access_replacement (lacc
);
3484 gimple_assign_set_lhs (stmt
, lhs
);
3485 modify_this_stmt
= true;
3486 if (lacc
->grp_partial_lhs
)
3487 force_gimple_rhs
= true;
3491 if (racc
&& racc
->grp_to_be_replaced
)
3493 rhs
= get_access_replacement (racc
);
3494 modify_this_stmt
= true;
3495 if (racc
->grp_partial_lhs
)
3496 force_gimple_rhs
= true;
3500 && !racc
->grp_unscalarized_data
3501 && !racc
->grp_unscalarizable_region
3502 && TREE_CODE (lhs
) == SSA_NAME
3503 && !access_has_replacements_p (racc
))
3505 rhs
= get_repl_default_def_ssa_name (racc
, TREE_TYPE (lhs
));
3506 modify_this_stmt
= true;
3510 if (modify_this_stmt
)
3512 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3514 /* If we can avoid creating a VIEW_CONVERT_EXPR do so.
3515 ??? This should move to fold_stmt which we simply should
3516 call after building a VIEW_CONVERT_EXPR here. */
3517 if (AGGREGATE_TYPE_P (TREE_TYPE (lhs
))
3518 && !contains_bitfld_component_ref_p (lhs
))
3520 lhs
= build_ref_for_model (loc
, lhs
, 0, racc
, gsi
, false);
3521 gimple_assign_set_lhs (stmt
, lhs
);
3524 && AGGREGATE_TYPE_P (TREE_TYPE (rhs
))
3525 && !contains_vce_or_bfcref_p (rhs
))
3526 rhs
= build_ref_for_model (loc
, rhs
, 0, lacc
, gsi
, false);
3528 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3530 rhs
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
3532 if (is_gimple_reg_type (TREE_TYPE (lhs
))
3533 && TREE_CODE (lhs
) != SSA_NAME
)
3534 force_gimple_rhs
= true;
3539 if (lacc
&& lacc
->grp_to_be_debug_replaced
)
3541 tree dlhs
= get_access_replacement (lacc
);
3542 tree drhs
= unshare_expr (rhs
);
3543 if (!useless_type_conversion_p (TREE_TYPE (dlhs
), TREE_TYPE (drhs
)))
3545 if (AGGREGATE_TYPE_P (TREE_TYPE (drhs
))
3546 && !contains_vce_or_bfcref_p (drhs
))
3547 drhs
= build_debug_ref_for_model (loc
, drhs
, 0, lacc
);
3549 && !useless_type_conversion_p (TREE_TYPE (dlhs
),
3551 drhs
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
,
3552 TREE_TYPE (dlhs
), drhs
);
3554 gdebug
*ds
= gimple_build_debug_bind (dlhs
, drhs
, stmt
);
3555 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
3558 /* From this point on, the function deals with assignments in between
3559 aggregates when at least one has scalar reductions of some of its
3560 components. There are three possible scenarios: Both the LHS and RHS have
3561 to-be-scalarized components, 2) only the RHS has or 3) only the LHS has.
3563 In the first case, we would like to load the LHS components from RHS
3564 components whenever possible. If that is not possible, we would like to
3565 read it directly from the RHS (after updating it by storing in it its own
3566 components). If there are some necessary unscalarized data in the LHS,
3567 those will be loaded by the original assignment too. If neither of these
3568 cases happen, the original statement can be removed. Most of this is done
3569 by load_assign_lhs_subreplacements.
3571 In the second case, we would like to store all RHS scalarized components
3572 directly into LHS and if they cover the aggregate completely, remove the
3573 statement too. In the third case, we want the LHS components to be loaded
3574 directly from the RHS (DSE will remove the original statement if it
3577 This is a bit complex but manageable when types match and when unions do
3578 not cause confusion in a way that we cannot really load a component of LHS
3579 from the RHS or vice versa (the access representing this level can have
3580 subaccesses that are accessible only through a different union field at a
3581 higher level - different from the one used in the examined expression).
3584 Therefore, I specially handle a fourth case, happening when there is a
3585 specific type cast or it is impossible to locate a scalarized subaccess on
3586 the other side of the expression. If that happens, I simply "refresh" the
3587 RHS by storing in it is scalarized components leave the original statement
3588 there to do the copying and then load the scalar replacements of the LHS.
3589 This is what the first branch does. */
3591 if (modify_this_stmt
3592 || gimple_has_volatile_ops (stmt
)
3593 || contains_vce_or_bfcref_p (rhs
)
3594 || contains_vce_or_bfcref_p (lhs
)
3595 || stmt_ends_bb_p (stmt
))
3597 /* No need to copy into a constant-pool, it comes pre-initialized. */
3598 if (access_has_children_p (racc
) && !constant_decl_p (racc
->base
))
3599 generate_subtree_copies (racc
->first_child
, rhs
, racc
->offset
, 0, 0,
3600 gsi
, false, false, loc
);
3601 if (access_has_children_p (lacc
))
3603 gimple_stmt_iterator alt_gsi
= gsi_none ();
3604 if (stmt_ends_bb_p (stmt
))
3606 alt_gsi
= gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi
)));
3609 generate_subtree_copies (lacc
->first_child
, lhs
, lacc
->offset
, 0, 0,
3610 gsi
, true, true, loc
);
3612 sra_stats
.separate_lhs_rhs_handling
++;
3614 /* This gimplification must be done after generate_subtree_copies,
3615 lest we insert the subtree copies in the middle of the gimplified
3617 if (force_gimple_rhs
)
3618 rhs
= force_gimple_operand_gsi (&orig_gsi
, rhs
, true, NULL_TREE
,
3619 true, GSI_SAME_STMT
);
3620 if (gimple_assign_rhs1 (stmt
) != rhs
)
3622 modify_this_stmt
= true;
3623 gimple_assign_set_rhs_from_tree (&orig_gsi
, rhs
);
3624 gcc_assert (stmt
== gsi_stmt (orig_gsi
));
3627 return modify_this_stmt
? SRA_AM_MODIFIED
: SRA_AM_NONE
;
3631 if (access_has_children_p (lacc
)
3632 && access_has_children_p (racc
)
3633 /* When an access represents an unscalarizable region, it usually
3634 represents accesses with variable offset and thus must not be used
3635 to generate new memory accesses. */
3636 && !lacc
->grp_unscalarizable_region
3637 && !racc
->grp_unscalarizable_region
)
3639 struct subreplacement_assignment_data sad
;
3641 sad
.left_offset
= lacc
->offset
;
3642 sad
.assignment_lhs
= lhs
;
3643 sad
.assignment_rhs
= rhs
;
3644 sad
.top_racc
= racc
;
3647 sad
.loc
= gimple_location (stmt
);
3648 sad
.refreshed
= SRA_UDH_NONE
;
3650 if (lacc
->grp_read
&& !lacc
->grp_covered
)
3651 handle_unscalarized_data_in_subtree (&sad
);
3653 load_assign_lhs_subreplacements (lacc
, &sad
);
3654 if (sad
.refreshed
!= SRA_UDH_RIGHT
)
3657 unlink_stmt_vdef (stmt
);
3658 gsi_remove (&sad
.old_gsi
, true);
3659 release_defs (stmt
);
3660 sra_stats
.deleted
++;
3661 return SRA_AM_REMOVED
;
3666 if (access_has_children_p (racc
)
3667 && !racc
->grp_unscalarized_data
3668 && TREE_CODE (lhs
) != SSA_NAME
)
3672 fprintf (dump_file
, "Removing load: ");
3673 print_gimple_stmt (dump_file
, stmt
, 0);
3675 generate_subtree_copies (racc
->first_child
, lhs
,
3676 racc
->offset
, 0, 0, gsi
,
3678 gcc_assert (stmt
== gsi_stmt (*gsi
));
3679 unlink_stmt_vdef (stmt
);
3680 gsi_remove (gsi
, true);
3681 release_defs (stmt
);
3682 sra_stats
.deleted
++;
3683 return SRA_AM_REMOVED
;
3685 /* Restore the aggregate RHS from its components so the
3686 prevailing aggregate copy does the right thing. */
3687 if (access_has_children_p (racc
))
3688 generate_subtree_copies (racc
->first_child
, rhs
, racc
->offset
, 0, 0,
3689 gsi
, false, false, loc
);
3690 /* Re-load the components of the aggregate copy destination.
3691 But use the RHS aggregate to load from to expose more
3692 optimization opportunities. */
3693 if (access_has_children_p (lacc
))
3694 generate_subtree_copies (lacc
->first_child
, rhs
, lacc
->offset
,
3695 0, 0, gsi
, true, true, loc
);
3702 /* Set any scalar replacements of values in the constant pool to the initial
3703 value of the constant. (Constant-pool decls like *.LC0 have effectively
3704 been initialized before the program starts, we must do the same for their
3705 replacements.) Thus, we output statements like 'SR.1 = *.LC0[0];' into
3706 the function's entry block. */
3709 initialize_constant_pool_replacements (void)
3711 gimple_seq seq
= NULL
;
3712 gimple_stmt_iterator gsi
= gsi_start (seq
);
3716 EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap
, 0, i
, bi
)
3718 tree var
= candidate (i
);
3719 if (!constant_decl_p (var
))
3721 vec
<access_p
> *access_vec
= get_base_access_vector (var
);
3724 for (unsigned i
= 0; i
< access_vec
->length (); i
++)
3726 struct access
*access
= (*access_vec
)[i
];
3727 if (!access
->replacement_decl
)
3730 = gimple_build_assign (get_access_replacement (access
),
3731 unshare_expr (access
->expr
));
3732 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3734 fprintf (dump_file
, "Generating constant initializer: ");
3735 print_gimple_stmt (dump_file
, stmt
, 0);
3736 fprintf (dump_file
, "\n");
3738 gsi_insert_after (&gsi
, stmt
, GSI_NEW_STMT
);
3743 seq
= gsi_seq (gsi
);
3745 gsi_insert_seq_on_edge_immediate (
3746 single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
3749 /* Traverse the function body and all modifications as decided in
3750 analyze_all_variable_accesses. Return true iff the CFG has been
3754 sra_modify_function_body (void)
3756 bool cfg_changed
= false;
3759 initialize_constant_pool_replacements ();
3761 FOR_EACH_BB_FN (bb
, cfun
)
3763 gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
3764 while (!gsi_end_p (gsi
))
3766 gimple
*stmt
= gsi_stmt (gsi
);
3767 enum assignment_mod_result assign_result
;
3768 bool modified
= false, deleted
= false;
3772 switch (gimple_code (stmt
))
3775 t
= gimple_return_retval_ptr (as_a
<greturn
*> (stmt
));
3776 if (*t
!= NULL_TREE
)
3777 modified
|= sra_modify_expr (t
, &gsi
, false);
3781 assign_result
= sra_modify_assign (stmt
, &gsi
);
3782 modified
|= assign_result
== SRA_AM_MODIFIED
;
3783 deleted
= assign_result
== SRA_AM_REMOVED
;
3787 /* Operands must be processed before the lhs. */
3788 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
3790 t
= gimple_call_arg_ptr (stmt
, i
);
3791 modified
|= sra_modify_expr (t
, &gsi
, false);
3794 if (gimple_call_lhs (stmt
))
3796 t
= gimple_call_lhs_ptr (stmt
);
3797 modified
|= sra_modify_expr (t
, &gsi
, true);
3803 gasm
*asm_stmt
= as_a
<gasm
*> (stmt
);
3804 for (i
= 0; i
< gimple_asm_ninputs (asm_stmt
); i
++)
3806 t
= &TREE_VALUE (gimple_asm_input_op (asm_stmt
, i
));
3807 modified
|= sra_modify_expr (t
, &gsi
, false);
3809 for (i
= 0; i
< gimple_asm_noutputs (asm_stmt
); i
++)
3811 t
= &TREE_VALUE (gimple_asm_output_op (asm_stmt
, i
));
3812 modified
|= sra_modify_expr (t
, &gsi
, true);
3824 if (maybe_clean_eh_stmt (stmt
)
3825 && gimple_purge_dead_eh_edges (gimple_bb (stmt
)))
3833 gsi_commit_edge_inserts ();
3837 /* Generate statements initializing scalar replacements of parts of function
3841 initialize_parameter_reductions (void)
3843 gimple_stmt_iterator gsi
;
3844 gimple_seq seq
= NULL
;
3847 gsi
= gsi_start (seq
);
3848 for (parm
= DECL_ARGUMENTS (current_function_decl
);
3850 parm
= DECL_CHAIN (parm
))
3852 vec
<access_p
> *access_vec
;
3853 struct access
*access
;
3855 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (parm
)))
3857 access_vec
= get_base_access_vector (parm
);
3861 for (access
= (*access_vec
)[0];
3863 access
= access
->next_grp
)
3864 generate_subtree_copies (access
, parm
, 0, 0, 0, &gsi
, true, true,
3865 EXPR_LOCATION (parm
));
3868 seq
= gsi_seq (gsi
);
3870 gsi_insert_seq_on_edge_immediate (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
3873 /* The "main" function of intraprocedural SRA passes. Runs the analysis and if
3874 it reveals there are components of some aggregates to be scalarized, it runs
3875 the required transformations. */
3877 perform_intra_sra (void)
3882 if (!find_var_candidates ())
3885 if (!scan_function ())
3888 if (!analyze_all_variable_accesses ())
3891 if (sra_modify_function_body ())
3892 ret
= TODO_update_ssa
| TODO_cleanup_cfg
;
3894 ret
= TODO_update_ssa
;
3895 initialize_parameter_reductions ();
3897 statistics_counter_event (cfun
, "Scalar replacements created",
3898 sra_stats
.replacements
);
3899 statistics_counter_event (cfun
, "Modified expressions", sra_stats
.exprs
);
3900 statistics_counter_event (cfun
, "Subtree copy stmts",
3901 sra_stats
.subtree_copies
);
3902 statistics_counter_event (cfun
, "Subreplacement stmts",
3903 sra_stats
.subreplacements
);
3904 statistics_counter_event (cfun
, "Deleted stmts", sra_stats
.deleted
);
3905 statistics_counter_event (cfun
, "Separate LHS and RHS handling",
3906 sra_stats
.separate_lhs_rhs_handling
);
3909 sra_deinitialize ();
3913 /* Perform early intraprocedural SRA. */
3915 early_intra_sra (void)
3917 sra_mode
= SRA_MODE_EARLY_INTRA
;
3918 return perform_intra_sra ();
3921 /* Perform "late" intraprocedural SRA. */
3923 late_intra_sra (void)
3925 sra_mode
= SRA_MODE_INTRA
;
3926 return perform_intra_sra ();
3931 gate_intra_sra (void)
3933 return flag_tree_sra
!= 0 && dbg_cnt (tree_sra
);
3939 const pass_data pass_data_sra_early
=
3941 GIMPLE_PASS
, /* type */
3943 OPTGROUP_NONE
, /* optinfo_flags */
3944 TV_TREE_SRA
, /* tv_id */
3945 ( PROP_cfg
| PROP_ssa
), /* properties_required */
3946 0, /* properties_provided */
3947 0, /* properties_destroyed */
3948 0, /* todo_flags_start */
3949 TODO_update_ssa
, /* todo_flags_finish */
3952 class pass_sra_early
: public gimple_opt_pass
3955 pass_sra_early (gcc::context
*ctxt
)
3956 : gimple_opt_pass (pass_data_sra_early
, ctxt
)
3959 /* opt_pass methods: */
3960 virtual bool gate (function
*) { return gate_intra_sra (); }
3961 virtual unsigned int execute (function
*) { return early_intra_sra (); }
3963 }; // class pass_sra_early
3968 make_pass_sra_early (gcc::context
*ctxt
)
3970 return new pass_sra_early (ctxt
);
3975 const pass_data pass_data_sra
=
3977 GIMPLE_PASS
, /* type */
3979 OPTGROUP_NONE
, /* optinfo_flags */
3980 TV_TREE_SRA
, /* tv_id */
3981 ( PROP_cfg
| PROP_ssa
), /* properties_required */
3982 0, /* properties_provided */
3983 0, /* properties_destroyed */
3984 TODO_update_address_taken
, /* todo_flags_start */
3985 TODO_update_ssa
, /* todo_flags_finish */
3988 class pass_sra
: public gimple_opt_pass
3991 pass_sra (gcc::context
*ctxt
)
3992 : gimple_opt_pass (pass_data_sra
, ctxt
)
3995 /* opt_pass methods: */
3996 virtual bool gate (function
*) { return gate_intra_sra (); }
3997 virtual unsigned int execute (function
*) { return late_intra_sra (); }
3999 }; // class pass_sra
4004 make_pass_sra (gcc::context
*ctxt
)
4006 return new pass_sra (ctxt
);