1 /* Scalar Replacement of Aggregates (SRA) converts some structure
2 references into scalar references, exposing them to the scalar
4 Copyright (C) 2008-2015 Free Software Foundation, Inc.
5 Contributed by Martin Jambor <mjambor@suse.cz>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
23 /* This file implements Scalar Reduction of Aggregates (SRA). SRA is run
24 twice, once in the early stages of compilation (early SRA) and once in the
25 late stages (late SRA). The aim of both is to turn references to scalar
26 parts of aggregates into uses of independent scalar variables.
28 The two passes are nearly identical, the only difference is that early SRA
29 does not scalarize unions which are used as the result in a GIMPLE_RETURN
30 statement because together with inlining this can lead to weird type
33 Both passes operate in four stages:
35 1. The declarations that have properties which make them candidates for
36 scalarization are identified in function find_var_candidates(). The
37 candidates are stored in candidate_bitmap.
39 2. The function body is scanned. In the process, declarations which are
40 used in a manner that prevent their scalarization are removed from the
41 candidate bitmap. More importantly, for every access into an aggregate,
42 an access structure (struct access) is created by create_access() and
43 stored in a vector associated with the aggregate. Among other
44 information, the aggregate declaration, the offset and size of the access
45 and its type are stored in the structure.
47 On a related note, assign_link structures are created for every assign
48 statement between candidate aggregates and attached to the related
51 3. The vectors of accesses are analyzed. They are first sorted according to
52 their offset and size and then scanned for partially overlapping accesses
53 (i.e. those which overlap but one is not entirely within another). Such
54 an access disqualifies the whole aggregate from being scalarized.
56 If there is no such inhibiting overlap, a representative access structure
57 is chosen for every unique combination of offset and size. Afterwards,
58 the pass builds a set of trees from these structures, in which children
59 of an access are within their parent (in terms of offset and size).
61 Then accesses are propagated whenever possible (i.e. in cases when it
62 does not create a partially overlapping access) across assign_links from
63 the right hand side to the left hand side.
65 Then the set of trees for each declaration is traversed again and those
66 accesses which should be replaced by a scalar are identified.
68 4. The function is traversed again, and for every reference into an
69 aggregate that has some component which is about to be scalarized,
70 statements are amended and new statements are created as necessary.
71 Finally, if a parameter got scalarized, the scalar replacements are
72 initialized with values from respective parameter aggregates. */
76 #include "coretypes.h"
83 #include "alloc-pool.h"
84 #include "tree-pass.h"
87 #include "gimple-pretty-print.h"
89 #include "fold-const.h"
91 #include "stor-layout.h"
93 #include "gimple-iterator.h"
94 #include "gimplify-me.h"
95 #include "gimple-walk.h"
99 #include "symbol-summary.h"
100 #include "ipa-prop.h"
103 #include "tree-inline.h"
104 #include "ipa-inline.h"
105 #include "ipa-utils.h"
106 #include "builtins.h"
108 /* Enumeration of all aggregate reductions we can do. */
109 enum sra_mode
{ SRA_MODE_EARLY_IPA
, /* early call regularization */
110 SRA_MODE_EARLY_INTRA
, /* early intraprocedural SRA */
111 SRA_MODE_INTRA
}; /* late intraprocedural SRA */
113 /* Global variable describing which aggregate reduction we are performing at
115 static enum sra_mode sra_mode
;
119 /* ACCESS represents each access to an aggregate variable (as a whole or a
120 part). It can also represent a group of accesses that refer to exactly the
121 same fragment of an aggregate (i.e. those that have exactly the same offset
122 and size). Such representatives for a single aggregate, once determined,
123 are linked in a linked list and have the group fields set.
125 Moreover, when doing intraprocedural SRA, a tree is built from those
126 representatives (by the means of first_child and next_sibling pointers), in
127 which all items in a subtree are "within" the root, i.e. their offset is
128 greater or equal to offset of the root and offset+size is smaller or equal
129 to offset+size of the root. Children of an access are sorted by offset.
131 Note that accesses to parts of vector and complex number types always
132 represented by an access to the whole complex number or a vector. It is a
133 duty of the modifying functions to replace them appropriately. */
137 /* Values returned by `get_ref_base_and_extent' for each component reference
138 If EXPR isn't a component reference just set `BASE = EXPR', `OFFSET = 0',
139 `SIZE = TREE_SIZE (TREE_TYPE (expr))'. */
140 HOST_WIDE_INT offset
;
144 /* Expression. It is context dependent so do not use it to create new
145 expressions to access the original aggregate. See PR 42154 for a
151 /* The statement this access belongs to. */
154 /* Next group representative for this aggregate. */
155 struct access
*next_grp
;
157 /* Pointer to the group representative. Pointer to itself if the struct is
158 the representative. */
159 struct access
*group_representative
;
161 /* If this access has any children (in terms of the definition above), this
162 points to the first one. */
163 struct access
*first_child
;
165 /* In intraprocedural SRA, pointer to the next sibling in the access tree as
166 described above. In IPA-SRA this is a pointer to the next access
167 belonging to the same group (having the same representative). */
168 struct access
*next_sibling
;
170 /* Pointers to the first and last element in the linked list of assign
172 struct assign_link
*first_link
, *last_link
;
174 /* Pointer to the next access in the work queue. */
175 struct access
*next_queued
;
177 /* Replacement variable for this access "region." Never to be accessed
178 directly, always only by the means of get_access_replacement() and only
179 when grp_to_be_replaced flag is set. */
180 tree replacement_decl
;
182 /* Is this particular access write access? */
185 /* Is this access an access to a non-addressable field? */
186 unsigned non_addressable
: 1;
188 /* Is this access currently in the work queue? */
189 unsigned grp_queued
: 1;
191 /* Does this group contain a write access? This flag is propagated down the
193 unsigned grp_write
: 1;
195 /* Does this group contain a read access? This flag is propagated down the
197 unsigned grp_read
: 1;
199 /* Does this group contain a read access that comes from an assignment
200 statement? This flag is propagated down the access tree. */
201 unsigned grp_assignment_read
: 1;
203 /* Does this group contain a write access that comes from an assignment
204 statement? This flag is propagated down the access tree. */
205 unsigned grp_assignment_write
: 1;
207 /* Does this group contain a read access through a scalar type? This flag is
208 not propagated in the access tree in any direction. */
209 unsigned grp_scalar_read
: 1;
211 /* Does this group contain a write access through a scalar type? This flag
212 is not propagated in the access tree in any direction. */
213 unsigned grp_scalar_write
: 1;
215 /* Is this access an artificial one created to scalarize some record
217 unsigned grp_total_scalarization
: 1;
219 /* Other passes of the analysis use this bit to make function
220 analyze_access_subtree create scalar replacements for this group if
222 unsigned grp_hint
: 1;
224 /* Is the subtree rooted in this access fully covered by scalar
226 unsigned grp_covered
: 1;
228 /* If set to true, this access and all below it in an access tree must not be
230 unsigned grp_unscalarizable_region
: 1;
232 /* Whether data have been written to parts of the aggregate covered by this
233 access which is not to be scalarized. This flag is propagated up in the
235 unsigned grp_unscalarized_data
: 1;
237 /* Does this access and/or group contain a write access through a
239 unsigned grp_partial_lhs
: 1;
241 /* Set when a scalar replacement should be created for this variable. */
242 unsigned grp_to_be_replaced
: 1;
244 /* Set when we want a replacement for the sole purpose of having it in
245 generated debug statements. */
246 unsigned grp_to_be_debug_replaced
: 1;
248 /* Should TREE_NO_WARNING of a replacement be set? */
249 unsigned grp_no_warning
: 1;
251 /* Is it possible that the group refers to data which might be (directly or
252 otherwise) modified? */
253 unsigned grp_maybe_modified
: 1;
255 /* Set when this is a representative of a pointer to scalar (i.e. by
256 reference) parameter which we consider for turning into a plain scalar
257 (i.e. a by value parameter). */
258 unsigned grp_scalar_ptr
: 1;
260 /* Set when we discover that this pointer is not safe to dereference in the
262 unsigned grp_not_necessarilly_dereferenced
: 1;
265 typedef struct access
*access_p
;
268 /* Alloc pool for allocating access structures. */
269 static object_allocator
<struct access
> access_pool ("SRA accesses");
271 /* A structure linking lhs and rhs accesses from an aggregate assignment. They
272 are used to propagate subaccesses from rhs to lhs as long as they don't
273 conflict with what is already there. */
276 struct access
*lacc
, *racc
;
277 struct assign_link
*next
;
280 /* Alloc pool for allocating assign link structures. */
281 static object_allocator
<assign_link
> assign_link_pool ("SRA links");
283 /* Base (tree) -> Vector (vec<access_p> *) map. */
284 static hash_map
<tree
, auto_vec
<access_p
> > *base_access_vec
;
286 /* Candidate hash table helpers. */
288 struct uid_decl_hasher
: nofree_ptr_hash
<tree_node
>
290 static inline hashval_t
hash (const tree_node
*);
291 static inline bool equal (const tree_node
*, const tree_node
*);
294 /* Hash a tree in a uid_decl_map. */
297 uid_decl_hasher::hash (const tree_node
*item
)
299 return item
->decl_minimal
.uid
;
302 /* Return true if the DECL_UID in both trees are equal. */
305 uid_decl_hasher::equal (const tree_node
*a
, const tree_node
*b
)
307 return (a
->decl_minimal
.uid
== b
->decl_minimal
.uid
);
310 /* Set of candidates. */
311 static bitmap candidate_bitmap
;
312 static hash_table
<uid_decl_hasher
> *candidates
;
314 /* For a candidate UID return the candidates decl. */
317 candidate (unsigned uid
)
320 t
.decl_minimal
.uid
= uid
;
321 return candidates
->find_with_hash (&t
, static_cast <hashval_t
> (uid
));
324 /* Bitmap of candidates which we should try to entirely scalarize away and
325 those which cannot be (because they are and need be used as a whole). */
326 static bitmap should_scalarize_away_bitmap
, cannot_scalarize_away_bitmap
;
328 /* Obstack for creation of fancy names. */
329 static struct obstack name_obstack
;
331 /* Head of a linked list of accesses that need to have its subaccesses
332 propagated to their assignment counterparts. */
333 static struct access
*work_queue_head
;
335 /* Number of parameters of the analyzed function when doing early ipa SRA. */
336 static int func_param_count
;
338 /* scan_function sets the following to true if it encounters a call to
339 __builtin_apply_args. */
340 static bool encountered_apply_args
;
342 /* Set by scan_function when it finds a recursive call. */
343 static bool encountered_recursive_call
;
345 /* Set by scan_function when it finds a recursive call with less actual
346 arguments than formal parameters.. */
347 static bool encountered_unchangable_recursive_call
;
349 /* This is a table in which for each basic block and parameter there is a
350 distance (offset + size) in that parameter which is dereferenced and
351 accessed in that BB. */
352 static HOST_WIDE_INT
*bb_dereferences
;
353 /* Bitmap of BBs that can cause the function to "stop" progressing by
354 returning, throwing externally, looping infinitely or calling a function
355 which might abort etc.. */
356 static bitmap final_bbs
;
358 /* Representative of no accesses at all. */
359 static struct access no_accesses_representant
;
361 /* Predicate to test the special value. */
364 no_accesses_p (struct access
*access
)
366 return access
== &no_accesses_representant
;
369 /* Dump contents of ACCESS to file F in a human friendly way. If GRP is true,
370 representative fields are dumped, otherwise those which only describe the
371 individual access are. */
375 /* Number of processed aggregates is readily available in
376 analyze_all_variable_accesses and so is not stored here. */
378 /* Number of created scalar replacements. */
381 /* Number of times sra_modify_expr or sra_modify_assign themselves changed an
385 /* Number of statements created by generate_subtree_copies. */
388 /* Number of statements created by load_assign_lhs_subreplacements. */
391 /* Number of times sra_modify_assign has deleted a statement. */
394 /* Number of times sra_modify_assign has to deal with subaccesses of LHS and
395 RHS reparately due to type conversions or nonexistent matching
397 int separate_lhs_rhs_handling
;
399 /* Number of parameters that were removed because they were unused. */
400 int deleted_unused_parameters
;
402 /* Number of scalars passed as parameters by reference that have been
403 converted to be passed by value. */
404 int scalar_by_ref_to_by_val
;
406 /* Number of aggregate parameters that were replaced by one or more of their
408 int aggregate_params_reduced
;
410 /* Numbber of components created when splitting aggregate parameters. */
411 int param_reductions_created
;
415 dump_access (FILE *f
, struct access
*access
, bool grp
)
417 fprintf (f
, "access { ");
418 fprintf (f
, "base = (%d)'", DECL_UID (access
->base
));
419 print_generic_expr (f
, access
->base
, 0);
420 fprintf (f
, "', offset = " HOST_WIDE_INT_PRINT_DEC
, access
->offset
);
421 fprintf (f
, ", size = " HOST_WIDE_INT_PRINT_DEC
, access
->size
);
422 fprintf (f
, ", expr = ");
423 print_generic_expr (f
, access
->expr
, 0);
424 fprintf (f
, ", type = ");
425 print_generic_expr (f
, access
->type
, 0);
427 fprintf (f
, ", grp_read = %d, grp_write = %d, grp_assignment_read = %d, "
428 "grp_assignment_write = %d, grp_scalar_read = %d, "
429 "grp_scalar_write = %d, grp_total_scalarization = %d, "
430 "grp_hint = %d, grp_covered = %d, "
431 "grp_unscalarizable_region = %d, grp_unscalarized_data = %d, "
432 "grp_partial_lhs = %d, grp_to_be_replaced = %d, "
433 "grp_to_be_debug_replaced = %d, grp_maybe_modified = %d, "
434 "grp_not_necessarilly_dereferenced = %d\n",
435 access
->grp_read
, access
->grp_write
, access
->grp_assignment_read
,
436 access
->grp_assignment_write
, access
->grp_scalar_read
,
437 access
->grp_scalar_write
, access
->grp_total_scalarization
,
438 access
->grp_hint
, access
->grp_covered
,
439 access
->grp_unscalarizable_region
, access
->grp_unscalarized_data
,
440 access
->grp_partial_lhs
, access
->grp_to_be_replaced
,
441 access
->grp_to_be_debug_replaced
, access
->grp_maybe_modified
,
442 access
->grp_not_necessarilly_dereferenced
);
444 fprintf (f
, ", write = %d, grp_total_scalarization = %d, "
445 "grp_partial_lhs = %d\n",
446 access
->write
, access
->grp_total_scalarization
,
447 access
->grp_partial_lhs
);
450 /* Dump a subtree rooted in ACCESS to file F, indent by LEVEL. */
453 dump_access_tree_1 (FILE *f
, struct access
*access
, int level
)
459 for (i
= 0; i
< level
; i
++)
460 fputs ("* ", dump_file
);
462 dump_access (f
, access
, true);
464 if (access
->first_child
)
465 dump_access_tree_1 (f
, access
->first_child
, level
+ 1);
467 access
= access
->next_sibling
;
472 /* Dump all access trees for a variable, given the pointer to the first root in
476 dump_access_tree (FILE *f
, struct access
*access
)
478 for (; access
; access
= access
->next_grp
)
479 dump_access_tree_1 (f
, access
, 0);
482 /* Return true iff ACC is non-NULL and has subaccesses. */
485 access_has_children_p (struct access
*acc
)
487 return acc
&& acc
->first_child
;
490 /* Return true iff ACC is (partly) covered by at least one replacement. */
493 access_has_replacements_p (struct access
*acc
)
495 struct access
*child
;
496 if (acc
->grp_to_be_replaced
)
498 for (child
= acc
->first_child
; child
; child
= child
->next_sibling
)
499 if (access_has_replacements_p (child
))
504 /* Return a vector of pointers to accesses for the variable given in BASE or
505 NULL if there is none. */
507 static vec
<access_p
> *
508 get_base_access_vector (tree base
)
510 return base_access_vec
->get (base
);
513 /* Find an access with required OFFSET and SIZE in a subtree of accesses rooted
514 in ACCESS. Return NULL if it cannot be found. */
516 static struct access
*
517 find_access_in_subtree (struct access
*access
, HOST_WIDE_INT offset
,
520 while (access
&& (access
->offset
!= offset
|| access
->size
!= size
))
522 struct access
*child
= access
->first_child
;
524 while (child
&& (child
->offset
+ child
->size
<= offset
))
525 child
= child
->next_sibling
;
532 /* Return the first group representative for DECL or NULL if none exists. */
534 static struct access
*
535 get_first_repr_for_decl (tree base
)
537 vec
<access_p
> *access_vec
;
539 access_vec
= get_base_access_vector (base
);
543 return (*access_vec
)[0];
546 /* Find an access representative for the variable BASE and given OFFSET and
547 SIZE. Requires that access trees have already been built. Return NULL if
548 it cannot be found. */
550 static struct access
*
551 get_var_base_offset_size_access (tree base
, HOST_WIDE_INT offset
,
554 struct access
*access
;
556 access
= get_first_repr_for_decl (base
);
557 while (access
&& (access
->offset
+ access
->size
<= offset
))
558 access
= access
->next_grp
;
562 return find_access_in_subtree (access
, offset
, size
);
565 /* Add LINK to the linked list of assign links of RACC. */
567 add_link_to_rhs (struct access
*racc
, struct assign_link
*link
)
569 gcc_assert (link
->racc
== racc
);
571 if (!racc
->first_link
)
573 gcc_assert (!racc
->last_link
);
574 racc
->first_link
= link
;
577 racc
->last_link
->next
= link
;
579 racc
->last_link
= link
;
583 /* Move all link structures in their linked list in OLD_RACC to the linked list
586 relink_to_new_repr (struct access
*new_racc
, struct access
*old_racc
)
588 if (!old_racc
->first_link
)
590 gcc_assert (!old_racc
->last_link
);
594 if (new_racc
->first_link
)
596 gcc_assert (!new_racc
->last_link
->next
);
597 gcc_assert (!old_racc
->last_link
|| !old_racc
->last_link
->next
);
599 new_racc
->last_link
->next
= old_racc
->first_link
;
600 new_racc
->last_link
= old_racc
->last_link
;
604 gcc_assert (!new_racc
->last_link
);
606 new_racc
->first_link
= old_racc
->first_link
;
607 new_racc
->last_link
= old_racc
->last_link
;
609 old_racc
->first_link
= old_racc
->last_link
= NULL
;
612 /* Add ACCESS to the work queue (which is actually a stack). */
615 add_access_to_work_queue (struct access
*access
)
617 if (!access
->grp_queued
)
619 gcc_assert (!access
->next_queued
);
620 access
->next_queued
= work_queue_head
;
621 access
->grp_queued
= 1;
622 work_queue_head
= access
;
626 /* Pop an access from the work queue, and return it, assuming there is one. */
628 static struct access
*
629 pop_access_from_work_queue (void)
631 struct access
*access
= work_queue_head
;
633 work_queue_head
= access
->next_queued
;
634 access
->next_queued
= NULL
;
635 access
->grp_queued
= 0;
640 /* Allocate necessary structures. */
643 sra_initialize (void)
645 candidate_bitmap
= BITMAP_ALLOC (NULL
);
646 candidates
= new hash_table
<uid_decl_hasher
>
647 (vec_safe_length (cfun
->local_decls
) / 2);
648 should_scalarize_away_bitmap
= BITMAP_ALLOC (NULL
);
649 cannot_scalarize_away_bitmap
= BITMAP_ALLOC (NULL
);
650 gcc_obstack_init (&name_obstack
);
651 base_access_vec
= new hash_map
<tree
, auto_vec
<access_p
> >;
652 memset (&sra_stats
, 0, sizeof (sra_stats
));
653 encountered_apply_args
= false;
654 encountered_recursive_call
= false;
655 encountered_unchangable_recursive_call
= false;
658 /* Deallocate all general structures. */
661 sra_deinitialize (void)
663 BITMAP_FREE (candidate_bitmap
);
666 BITMAP_FREE (should_scalarize_away_bitmap
);
667 BITMAP_FREE (cannot_scalarize_away_bitmap
);
668 access_pool
.release ();
669 assign_link_pool
.release ();
670 obstack_free (&name_obstack
, NULL
);
672 delete base_access_vec
;
675 /* Remove DECL from candidates for SRA and write REASON to the dump file if
678 disqualify_candidate (tree decl
, const char *reason
)
680 if (bitmap_clear_bit (candidate_bitmap
, DECL_UID (decl
)))
681 candidates
->remove_elt_with_hash (decl
, DECL_UID (decl
));
683 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
685 fprintf (dump_file
, "! Disqualifying ");
686 print_generic_expr (dump_file
, decl
, 0);
687 fprintf (dump_file
, " - %s\n", reason
);
691 /* Return true iff the type contains a field or an element which does not allow
695 type_internals_preclude_sra_p (tree type
, const char **msg
)
700 switch (TREE_CODE (type
))
704 case QUAL_UNION_TYPE
:
705 for (fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
706 if (TREE_CODE (fld
) == FIELD_DECL
)
708 tree ft
= TREE_TYPE (fld
);
710 if (TREE_THIS_VOLATILE (fld
))
712 *msg
= "volatile structure field";
715 if (!DECL_FIELD_OFFSET (fld
))
717 *msg
= "no structure field offset";
720 if (!DECL_SIZE (fld
))
722 *msg
= "zero structure field size";
725 if (!tree_fits_uhwi_p (DECL_FIELD_OFFSET (fld
)))
727 *msg
= "structure field offset not fixed";
730 if (!tree_fits_uhwi_p (DECL_SIZE (fld
)))
732 *msg
= "structure field size not fixed";
735 if (!tree_fits_shwi_p (bit_position (fld
)))
737 *msg
= "structure field size too big";
740 if (AGGREGATE_TYPE_P (ft
)
741 && int_bit_position (fld
) % BITS_PER_UNIT
!= 0)
743 *msg
= "structure field is bit field";
747 if (AGGREGATE_TYPE_P (ft
) && type_internals_preclude_sra_p (ft
, msg
))
754 et
= TREE_TYPE (type
);
756 if (TYPE_VOLATILE (et
))
758 *msg
= "element type is volatile";
762 if (AGGREGATE_TYPE_P (et
) && type_internals_preclude_sra_p (et
, msg
))
772 /* If T is an SSA_NAME, return NULL if it is not a default def or return its
773 base variable if it is. Return T if it is not an SSA_NAME. */
776 get_ssa_base_param (tree t
)
778 if (TREE_CODE (t
) == SSA_NAME
)
780 if (SSA_NAME_IS_DEFAULT_DEF (t
))
781 return SSA_NAME_VAR (t
);
788 /* Mark a dereference of BASE of distance DIST in a basic block tht STMT
789 belongs to, unless the BB has already been marked as a potentially
793 mark_parm_dereference (tree base
, HOST_WIDE_INT dist
, gimple
*stmt
)
795 basic_block bb
= gimple_bb (stmt
);
796 int idx
, parm_index
= 0;
799 if (bitmap_bit_p (final_bbs
, bb
->index
))
802 for (parm
= DECL_ARGUMENTS (current_function_decl
);
803 parm
&& parm
!= base
;
804 parm
= DECL_CHAIN (parm
))
807 gcc_assert (parm_index
< func_param_count
);
809 idx
= bb
->index
* func_param_count
+ parm_index
;
810 if (bb_dereferences
[idx
] < dist
)
811 bb_dereferences
[idx
] = dist
;
814 /* Allocate an access structure for BASE, OFFSET and SIZE, clear it, fill in
815 the three fields. Also add it to the vector of accesses corresponding to
816 the base. Finally, return the new access. */
818 static struct access
*
819 create_access_1 (tree base
, HOST_WIDE_INT offset
, HOST_WIDE_INT size
)
821 struct access
*access
= access_pool
.allocate ();
823 memset (access
, 0, sizeof (struct access
));
825 access
->offset
= offset
;
828 base_access_vec
->get_or_insert (base
).safe_push (access
);
833 /* Create and insert access for EXPR. Return created access, or NULL if it is
836 static struct access
*
837 create_access (tree expr
, gimple
*stmt
, bool write
)
839 struct access
*access
;
840 HOST_WIDE_INT offset
, size
, max_size
;
842 bool ptr
, unscalarizable_region
= false;
844 base
= get_ref_base_and_extent (expr
, &offset
, &size
, &max_size
);
846 if (sra_mode
== SRA_MODE_EARLY_IPA
847 && TREE_CODE (base
) == MEM_REF
)
849 base
= get_ssa_base_param (TREE_OPERAND (base
, 0));
857 if (!DECL_P (base
) || !bitmap_bit_p (candidate_bitmap
, DECL_UID (base
)))
860 if (sra_mode
== SRA_MODE_EARLY_IPA
)
862 if (size
< 0 || size
!= max_size
)
864 disqualify_candidate (base
, "Encountered a variable sized access.");
867 if (TREE_CODE (expr
) == COMPONENT_REF
868 && DECL_BIT_FIELD (TREE_OPERAND (expr
, 1)))
870 disqualify_candidate (base
, "Encountered a bit-field access.");
873 gcc_checking_assert ((offset
% BITS_PER_UNIT
) == 0);
876 mark_parm_dereference (base
, offset
+ size
, stmt
);
880 if (size
!= max_size
)
883 unscalarizable_region
= true;
887 disqualify_candidate (base
, "Encountered an unconstrained access.");
892 access
= create_access_1 (base
, offset
, size
);
894 access
->type
= TREE_TYPE (expr
);
895 access
->write
= write
;
896 access
->grp_unscalarizable_region
= unscalarizable_region
;
899 if (TREE_CODE (expr
) == COMPONENT_REF
900 && DECL_NONADDRESSABLE_P (TREE_OPERAND (expr
, 1)))
901 access
->non_addressable
= 1;
907 /* Return true iff TYPE is scalarizable - i.e. a RECORD_TYPE or fixed-length
908 ARRAY_TYPE with fields that are either of gimple register types (excluding
909 bit-fields) or (recursively) scalarizable types. */
912 scalarizable_type_p (tree type
)
914 gcc_assert (!is_gimple_reg_type (type
));
916 switch (TREE_CODE (type
))
919 for (tree fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
920 if (TREE_CODE (fld
) == FIELD_DECL
)
922 tree ft
= TREE_TYPE (fld
);
924 if (DECL_BIT_FIELD (fld
))
927 if (!is_gimple_reg_type (ft
)
928 && !scalarizable_type_p (ft
))
936 if (TYPE_DOMAIN (type
) == NULL_TREE
937 || !tree_fits_shwi_p (TYPE_SIZE (type
))
938 || !tree_fits_shwi_p (TYPE_SIZE (TREE_TYPE (type
)))
939 || (tree_to_shwi (TYPE_SIZE (TREE_TYPE (type
))) <= 0)
940 || !tree_fits_shwi_p (TYPE_MIN_VALUE (TYPE_DOMAIN (type
))))
942 if (tree_to_shwi (TYPE_SIZE (type
)) == 0
943 && TYPE_MAX_VALUE (TYPE_DOMAIN (type
)) == NULL_TREE
)
944 /* Zero-element array, should not prevent scalarization. */
946 else if ((tree_to_shwi (TYPE_SIZE (type
)) <= 0)
947 || !tree_fits_shwi_p (TYPE_MAX_VALUE (TYPE_DOMAIN (type
))))
948 /* Variable-length array, do not allow scalarization. */
951 tree elem
= TREE_TYPE (type
);
952 if (!is_gimple_reg_type (elem
)
953 && !scalarizable_type_p (elem
))
962 static void scalarize_elem (tree
, HOST_WIDE_INT
, HOST_WIDE_INT
, tree
, tree
);
964 /* Create total_scalarization accesses for all scalar fields of a member
965 of type DECL_TYPE conforming to scalarizable_type_p. BASE
966 must be the top-most VAR_DECL representing the variable; within that,
967 OFFSET locates the member and REF must be the memory reference expression for
971 completely_scalarize (tree base
, tree decl_type
, HOST_WIDE_INT offset
, tree ref
)
973 switch (TREE_CODE (decl_type
))
976 for (tree fld
= TYPE_FIELDS (decl_type
); fld
; fld
= DECL_CHAIN (fld
))
977 if (TREE_CODE (fld
) == FIELD_DECL
)
979 HOST_WIDE_INT pos
= offset
+ int_bit_position (fld
);
980 tree ft
= TREE_TYPE (fld
);
981 tree nref
= build3 (COMPONENT_REF
, ft
, ref
, fld
, NULL_TREE
);
983 scalarize_elem (base
, pos
, tree_to_uhwi (DECL_SIZE (fld
)), nref
,
989 tree elemtype
= TREE_TYPE (decl_type
);
990 tree elem_size
= TYPE_SIZE (elemtype
);
991 gcc_assert (elem_size
&& tree_fits_shwi_p (elem_size
));
992 HOST_WIDE_INT el_size
= tree_to_shwi (elem_size
);
993 gcc_assert (el_size
> 0);
995 tree minidx
= TYPE_MIN_VALUE (TYPE_DOMAIN (decl_type
));
996 gcc_assert (TREE_CODE (minidx
) == INTEGER_CST
);
997 tree maxidx
= TYPE_MAX_VALUE (TYPE_DOMAIN (decl_type
));
998 /* Skip (some) zero-length arrays; others have MAXIDX == MINIDX - 1. */
1001 gcc_assert (TREE_CODE (maxidx
) == INTEGER_CST
);
1002 tree domain
= TYPE_DOMAIN (decl_type
);
1003 /* MINIDX and MAXIDX are inclusive, and must be interpreted in
1004 DOMAIN (e.g. signed int, whereas min/max may be size_int). */
1005 offset_int idx
= wi::to_offset (minidx
);
1006 offset_int max
= wi::to_offset (maxidx
);
1007 if (!TYPE_UNSIGNED (domain
))
1009 idx
= wi::sext (idx
, TYPE_PRECISION (domain
));
1010 max
= wi::sext (max
, TYPE_PRECISION (domain
));
1012 for (int el_off
= offset
; wi::les_p (idx
, max
); ++idx
)
1014 tree nref
= build4 (ARRAY_REF
, elemtype
,
1016 wide_int_to_tree (domain
, idx
),
1017 NULL_TREE
, NULL_TREE
);
1018 scalarize_elem (base
, el_off
, el_size
, nref
, elemtype
);
1029 /* Create total_scalarization accesses for a member of type TYPE, which must
1030 satisfy either is_gimple_reg_type or scalarizable_type_p. BASE must be the
1031 top-most VAR_DECL representing the variable; within that, POS and SIZE locate
1032 the member and REF must be the reference expression for it. */
1035 scalarize_elem (tree base
, HOST_WIDE_INT pos
, HOST_WIDE_INT size
,
1036 tree ref
, tree type
)
1038 if (is_gimple_reg_type (type
))
1040 struct access
*access
= create_access_1 (base
, pos
, size
);
1042 access
->type
= type
;
1043 access
->grp_total_scalarization
= 1;
1044 /* Accesses for intraprocedural SRA can have their stmt NULL. */
1047 completely_scalarize (base
, type
, pos
, ref
);
1050 /* Create a total_scalarization access for VAR as a whole. VAR must be of a
1051 RECORD_TYPE or ARRAY_TYPE conforming to scalarizable_type_p. */
1054 create_total_scalarization_access (tree var
)
1056 HOST_WIDE_INT size
= tree_to_uhwi (DECL_SIZE (var
));
1057 struct access
*access
;
1059 access
= create_access_1 (var
, 0, size
);
1061 access
->type
= TREE_TYPE (var
);
1062 access
->grp_total_scalarization
= 1;
1065 /* Return true if REF has an VIEW_CONVERT_EXPR somewhere in it. */
1068 contains_view_convert_expr_p (const_tree ref
)
1070 while (handled_component_p (ref
))
1072 if (TREE_CODE (ref
) == VIEW_CONVERT_EXPR
)
1074 ref
= TREE_OPERAND (ref
, 0);
1080 /* Search the given tree for a declaration by skipping handled components and
1081 exclude it from the candidates. */
1084 disqualify_base_of_expr (tree t
, const char *reason
)
1086 t
= get_base_address (t
);
1087 if (sra_mode
== SRA_MODE_EARLY_IPA
1088 && TREE_CODE (t
) == MEM_REF
)
1089 t
= get_ssa_base_param (TREE_OPERAND (t
, 0));
1091 if (t
&& DECL_P (t
))
1092 disqualify_candidate (t
, reason
);
1095 /* Scan expression EXPR and create access structures for all accesses to
1096 candidates for scalarization. Return the created access or NULL if none is
1099 static struct access
*
1100 build_access_from_expr_1 (tree expr
, gimple
*stmt
, bool write
)
1102 struct access
*ret
= NULL
;
1105 if (TREE_CODE (expr
) == BIT_FIELD_REF
1106 || TREE_CODE (expr
) == IMAGPART_EXPR
1107 || TREE_CODE (expr
) == REALPART_EXPR
)
1109 expr
= TREE_OPERAND (expr
, 0);
1113 partial_ref
= false;
1115 /* We need to dive through V_C_Es in order to get the size of its parameter
1116 and not the result type. Ada produces such statements. We are also
1117 capable of handling the topmost V_C_E but not any of those buried in other
1118 handled components. */
1119 if (TREE_CODE (expr
) == VIEW_CONVERT_EXPR
)
1120 expr
= TREE_OPERAND (expr
, 0);
1122 if (contains_view_convert_expr_p (expr
))
1124 disqualify_base_of_expr (expr
, "V_C_E under a different handled "
1128 if (TREE_THIS_VOLATILE (expr
))
1130 disqualify_base_of_expr (expr
, "part of a volatile reference.");
1134 switch (TREE_CODE (expr
))
1137 if (TREE_CODE (TREE_OPERAND (expr
, 0)) != ADDR_EXPR
1138 && sra_mode
!= SRA_MODE_EARLY_IPA
)
1146 case ARRAY_RANGE_REF
:
1147 ret
= create_access (expr
, stmt
, write
);
1154 if (write
&& partial_ref
&& ret
)
1155 ret
->grp_partial_lhs
= 1;
1160 /* Scan expression EXPR and create access structures for all accesses to
1161 candidates for scalarization. Return true if any access has been inserted.
1162 STMT must be the statement from which the expression is taken, WRITE must be
1163 true if the expression is a store and false otherwise. */
1166 build_access_from_expr (tree expr
, gimple
*stmt
, bool write
)
1168 struct access
*access
;
1170 access
= build_access_from_expr_1 (expr
, stmt
, write
);
1173 /* This means the aggregate is accesses as a whole in a way other than an
1174 assign statement and thus cannot be removed even if we had a scalar
1175 replacement for everything. */
1176 if (cannot_scalarize_away_bitmap
)
1177 bitmap_set_bit (cannot_scalarize_away_bitmap
, DECL_UID (access
->base
));
1183 /* Return the single non-EH successor edge of BB or NULL if there is none or
1187 single_non_eh_succ (basic_block bb
)
1192 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1193 if (!(e
->flags
& EDGE_EH
))
1203 /* Disqualify LHS and RHS for scalarization if STMT has to terminate its BB and
1204 there is no alternative spot where to put statements SRA might need to
1205 generate after it. The spot we are looking for is an edge leading to a
1206 single non-EH successor, if it exists and is indeed single. RHS may be
1207 NULL, in that case ignore it. */
1210 disqualify_if_bad_bb_terminating_stmt (gimple
*stmt
, tree lhs
, tree rhs
)
1212 if ((sra_mode
== SRA_MODE_EARLY_INTRA
|| sra_mode
== SRA_MODE_INTRA
)
1213 && stmt_ends_bb_p (stmt
))
1215 if (single_non_eh_succ (gimple_bb (stmt
)))
1218 disqualify_base_of_expr (lhs
, "LHS of a throwing stmt.");
1220 disqualify_base_of_expr (rhs
, "RHS of a throwing stmt.");
1226 /* Scan expressions occurring in STMT, create access structures for all accesses
1227 to candidates for scalarization and remove those candidates which occur in
1228 statements or expressions that prevent them from being split apart. Return
1229 true if any access has been inserted. */
1232 build_accesses_from_assign (gimple
*stmt
)
1235 struct access
*lacc
, *racc
;
1237 if (!gimple_assign_single_p (stmt
)
1238 /* Scope clobbers don't influence scalarization. */
1239 || gimple_clobber_p (stmt
))
1242 lhs
= gimple_assign_lhs (stmt
);
1243 rhs
= gimple_assign_rhs1 (stmt
);
1245 if (disqualify_if_bad_bb_terminating_stmt (stmt
, lhs
, rhs
))
1248 racc
= build_access_from_expr_1 (rhs
, stmt
, false);
1249 lacc
= build_access_from_expr_1 (lhs
, stmt
, true);
1252 lacc
->grp_assignment_write
= 1;
1256 racc
->grp_assignment_read
= 1;
1257 if (should_scalarize_away_bitmap
&& !gimple_has_volatile_ops (stmt
)
1258 && !is_gimple_reg_type (racc
->type
))
1259 bitmap_set_bit (should_scalarize_away_bitmap
, DECL_UID (racc
->base
));
1263 && (sra_mode
== SRA_MODE_EARLY_INTRA
|| sra_mode
== SRA_MODE_INTRA
)
1264 && !lacc
->grp_unscalarizable_region
1265 && !racc
->grp_unscalarizable_region
1266 && AGGREGATE_TYPE_P (TREE_TYPE (lhs
))
1267 && lacc
->size
== racc
->size
1268 && useless_type_conversion_p (lacc
->type
, racc
->type
))
1270 struct assign_link
*link
;
1272 link
= assign_link_pool
.allocate ();
1273 memset (link
, 0, sizeof (struct assign_link
));
1278 add_link_to_rhs (racc
, link
);
1281 return lacc
|| racc
;
1284 /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
1285 GIMPLE_ASM operands with memory constrains which cannot be scalarized. */
1288 asm_visit_addr (gimple
*, tree op
, tree
, void *)
1290 op
= get_base_address (op
);
1293 disqualify_candidate (op
, "Non-scalarizable GIMPLE_ASM operand.");
1298 /* Return true iff callsite CALL has at least as many actual arguments as there
1299 are formal parameters of the function currently processed by IPA-SRA and
1300 that their types match. */
1303 callsite_arguments_match_p (gimple
*call
)
1305 if (gimple_call_num_args (call
) < (unsigned) func_param_count
)
1310 for (parm
= DECL_ARGUMENTS (current_function_decl
), i
= 0;
1312 parm
= DECL_CHAIN (parm
), i
++)
1314 tree arg
= gimple_call_arg (call
, i
);
1315 if (!useless_type_conversion_p (TREE_TYPE (parm
), TREE_TYPE (arg
)))
1321 /* Scan function and look for interesting expressions and create access
1322 structures for them. Return true iff any access is created. */
1325 scan_function (void)
1330 FOR_EACH_BB_FN (bb
, cfun
)
1332 gimple_stmt_iterator gsi
;
1333 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1335 gimple
*stmt
= gsi_stmt (gsi
);
1339 if (final_bbs
&& stmt_can_throw_external (stmt
))
1340 bitmap_set_bit (final_bbs
, bb
->index
);
1341 switch (gimple_code (stmt
))
1344 t
= gimple_return_retval (as_a
<greturn
*> (stmt
));
1346 ret
|= build_access_from_expr (t
, stmt
, false);
1348 bitmap_set_bit (final_bbs
, bb
->index
);
1352 ret
|= build_accesses_from_assign (stmt
);
1356 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
1357 ret
|= build_access_from_expr (gimple_call_arg (stmt
, i
),
1360 if (sra_mode
== SRA_MODE_EARLY_IPA
)
1362 tree dest
= gimple_call_fndecl (stmt
);
1363 int flags
= gimple_call_flags (stmt
);
1367 if (DECL_BUILT_IN_CLASS (dest
) == BUILT_IN_NORMAL
1368 && DECL_FUNCTION_CODE (dest
) == BUILT_IN_APPLY_ARGS
)
1369 encountered_apply_args
= true;
1370 if (recursive_call_p (current_function_decl
, dest
))
1372 encountered_recursive_call
= true;
1373 if (!callsite_arguments_match_p (stmt
))
1374 encountered_unchangable_recursive_call
= true;
1379 && (flags
& (ECF_CONST
| ECF_PURE
)) == 0)
1380 bitmap_set_bit (final_bbs
, bb
->index
);
1383 t
= gimple_call_lhs (stmt
);
1384 if (t
&& !disqualify_if_bad_bb_terminating_stmt (stmt
, t
, NULL
))
1385 ret
|= build_access_from_expr (t
, stmt
, true);
1390 gasm
*asm_stmt
= as_a
<gasm
*> (stmt
);
1391 walk_stmt_load_store_addr_ops (asm_stmt
, NULL
, NULL
, NULL
,
1394 bitmap_set_bit (final_bbs
, bb
->index
);
1396 for (i
= 0; i
< gimple_asm_ninputs (asm_stmt
); i
++)
1398 t
= TREE_VALUE (gimple_asm_input_op (asm_stmt
, i
));
1399 ret
|= build_access_from_expr (t
, asm_stmt
, false);
1401 for (i
= 0; i
< gimple_asm_noutputs (asm_stmt
); i
++)
1403 t
= TREE_VALUE (gimple_asm_output_op (asm_stmt
, i
));
1404 ret
|= build_access_from_expr (t
, asm_stmt
, true);
1418 /* Helper of QSORT function. There are pointers to accesses in the array. An
1419 access is considered smaller than another if it has smaller offset or if the
1420 offsets are the same but is size is bigger. */
1423 compare_access_positions (const void *a
, const void *b
)
1425 const access_p
*fp1
= (const access_p
*) a
;
1426 const access_p
*fp2
= (const access_p
*) b
;
1427 const access_p f1
= *fp1
;
1428 const access_p f2
= *fp2
;
1430 if (f1
->offset
!= f2
->offset
)
1431 return f1
->offset
< f2
->offset
? -1 : 1;
1433 if (f1
->size
== f2
->size
)
1435 if (f1
->type
== f2
->type
)
1437 /* Put any non-aggregate type before any aggregate type. */
1438 else if (!is_gimple_reg_type (f1
->type
)
1439 && is_gimple_reg_type (f2
->type
))
1441 else if (is_gimple_reg_type (f1
->type
)
1442 && !is_gimple_reg_type (f2
->type
))
1444 /* Put any complex or vector type before any other scalar type. */
1445 else if (TREE_CODE (f1
->type
) != COMPLEX_TYPE
1446 && TREE_CODE (f1
->type
) != VECTOR_TYPE
1447 && (TREE_CODE (f2
->type
) == COMPLEX_TYPE
1448 || TREE_CODE (f2
->type
) == VECTOR_TYPE
))
1450 else if ((TREE_CODE (f1
->type
) == COMPLEX_TYPE
1451 || TREE_CODE (f1
->type
) == VECTOR_TYPE
)
1452 && TREE_CODE (f2
->type
) != COMPLEX_TYPE
1453 && TREE_CODE (f2
->type
) != VECTOR_TYPE
)
1455 /* Put the integral type with the bigger precision first. */
1456 else if (INTEGRAL_TYPE_P (f1
->type
)
1457 && INTEGRAL_TYPE_P (f2
->type
))
1458 return TYPE_PRECISION (f2
->type
) - TYPE_PRECISION (f1
->type
);
1459 /* Put any integral type with non-full precision last. */
1460 else if (INTEGRAL_TYPE_P (f1
->type
)
1461 && (TREE_INT_CST_LOW (TYPE_SIZE (f1
->type
))
1462 != TYPE_PRECISION (f1
->type
)))
1464 else if (INTEGRAL_TYPE_P (f2
->type
)
1465 && (TREE_INT_CST_LOW (TYPE_SIZE (f2
->type
))
1466 != TYPE_PRECISION (f2
->type
)))
1468 /* Stabilize the sort. */
1469 return TYPE_UID (f1
->type
) - TYPE_UID (f2
->type
);
1472 /* We want the bigger accesses first, thus the opposite operator in the next
1474 return f1
->size
> f2
->size
? -1 : 1;
1478 /* Append a name of the declaration to the name obstack. A helper function for
1482 make_fancy_decl_name (tree decl
)
1486 tree name
= DECL_NAME (decl
);
1488 obstack_grow (&name_obstack
, IDENTIFIER_POINTER (name
),
1489 IDENTIFIER_LENGTH (name
));
1492 sprintf (buffer
, "D%u", DECL_UID (decl
));
1493 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1497 /* Helper for make_fancy_name. */
1500 make_fancy_name_1 (tree expr
)
1507 make_fancy_decl_name (expr
);
1511 switch (TREE_CODE (expr
))
1514 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1515 obstack_1grow (&name_obstack
, '$');
1516 make_fancy_decl_name (TREE_OPERAND (expr
, 1));
1520 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1521 obstack_1grow (&name_obstack
, '$');
1522 /* Arrays with only one element may not have a constant as their
1524 index
= TREE_OPERAND (expr
, 1);
1525 if (TREE_CODE (index
) != INTEGER_CST
)
1527 sprintf (buffer
, HOST_WIDE_INT_PRINT_DEC
, TREE_INT_CST_LOW (index
));
1528 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1532 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1536 make_fancy_name_1 (TREE_OPERAND (expr
, 0));
1537 if (!integer_zerop (TREE_OPERAND (expr
, 1)))
1539 obstack_1grow (&name_obstack
, '$');
1540 sprintf (buffer
, HOST_WIDE_INT_PRINT_DEC
,
1541 TREE_INT_CST_LOW (TREE_OPERAND (expr
, 1)));
1542 obstack_grow (&name_obstack
, buffer
, strlen (buffer
));
1549 gcc_unreachable (); /* we treat these as scalars. */
1556 /* Create a human readable name for replacement variable of ACCESS. */
1559 make_fancy_name (tree expr
)
1561 make_fancy_name_1 (expr
);
1562 obstack_1grow (&name_obstack
, '\0');
1563 return XOBFINISH (&name_obstack
, char *);
1566 /* Construct a MEM_REF that would reference a part of aggregate BASE of type
1567 EXP_TYPE at the given OFFSET. If BASE is something for which
1568 get_addr_base_and_unit_offset returns NULL, gsi must be non-NULL and is used
1569 to insert new statements either before or below the current one as specified
1570 by INSERT_AFTER. This function is not capable of handling bitfields.
1572 BASE must be either a declaration or a memory reference that has correct
1573 alignment ifformation embeded in it (e.g. a pre-existing one in SRA). */
1576 build_ref_for_offset (location_t loc
, tree base
, HOST_WIDE_INT offset
,
1577 tree exp_type
, gimple_stmt_iterator
*gsi
,
1580 tree prev_base
= base
;
1583 HOST_WIDE_INT base_offset
;
1584 unsigned HOST_WIDE_INT misalign
;
1587 gcc_checking_assert (offset
% BITS_PER_UNIT
== 0);
1588 get_object_alignment_1 (base
, &align
, &misalign
);
1589 base
= get_addr_base_and_unit_offset (base
, &base_offset
);
1591 /* get_addr_base_and_unit_offset returns NULL for references with a variable
1592 offset such as array[var_index]. */
1598 gcc_checking_assert (gsi
);
1599 tmp
= make_ssa_name (build_pointer_type (TREE_TYPE (prev_base
)));
1600 addr
= build_fold_addr_expr (unshare_expr (prev_base
));
1601 STRIP_USELESS_TYPE_CONVERSION (addr
);
1602 stmt
= gimple_build_assign (tmp
, addr
);
1603 gimple_set_location (stmt
, loc
);
1605 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
1607 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
1609 off
= build_int_cst (reference_alias_ptr_type (prev_base
),
1610 offset
/ BITS_PER_UNIT
);
1613 else if (TREE_CODE (base
) == MEM_REF
)
1615 off
= build_int_cst (TREE_TYPE (TREE_OPERAND (base
, 1)),
1616 base_offset
+ offset
/ BITS_PER_UNIT
);
1617 off
= int_const_binop (PLUS_EXPR
, TREE_OPERAND (base
, 1), off
);
1618 base
= unshare_expr (TREE_OPERAND (base
, 0));
1622 off
= build_int_cst (reference_alias_ptr_type (base
),
1623 base_offset
+ offset
/ BITS_PER_UNIT
);
1624 base
= build_fold_addr_expr (unshare_expr (base
));
1627 misalign
= (misalign
+ offset
) & (align
- 1);
1629 align
= (misalign
& -misalign
);
1630 if (align
!= TYPE_ALIGN (exp_type
))
1631 exp_type
= build_aligned_type (exp_type
, align
);
1633 mem_ref
= fold_build2_loc (loc
, MEM_REF
, exp_type
, base
, off
);
1634 if (TREE_THIS_VOLATILE (prev_base
))
1635 TREE_THIS_VOLATILE (mem_ref
) = 1;
1636 if (TREE_SIDE_EFFECTS (prev_base
))
1637 TREE_SIDE_EFFECTS (mem_ref
) = 1;
1641 /* Construct a memory reference to a part of an aggregate BASE at the given
1642 OFFSET and of the same type as MODEL. In case this is a reference to a
1643 bit-field, the function will replicate the last component_ref of model's
1644 expr to access it. GSI and INSERT_AFTER have the same meaning as in
1645 build_ref_for_offset. */
1648 build_ref_for_model (location_t loc
, tree base
, HOST_WIDE_INT offset
,
1649 struct access
*model
, gimple_stmt_iterator
*gsi
,
1652 if (TREE_CODE (model
->expr
) == COMPONENT_REF
1653 && DECL_BIT_FIELD (TREE_OPERAND (model
->expr
, 1)))
1655 /* This access represents a bit-field. */
1656 tree t
, exp_type
, fld
= TREE_OPERAND (model
->expr
, 1);
1658 offset
-= int_bit_position (fld
);
1659 exp_type
= TREE_TYPE (TREE_OPERAND (model
->expr
, 0));
1660 t
= build_ref_for_offset (loc
, base
, offset
, exp_type
, gsi
, insert_after
);
1661 return fold_build3_loc (loc
, COMPONENT_REF
, TREE_TYPE (fld
), t
, fld
,
1665 return build_ref_for_offset (loc
, base
, offset
, model
->type
,
1669 /* Attempt to build a memory reference that we could but into a gimple
1670 debug_bind statement. Similar to build_ref_for_model but punts if it has to
1671 create statements and return s NULL instead. This function also ignores
1672 alignment issues and so its results should never end up in non-debug
1676 build_debug_ref_for_model (location_t loc
, tree base
, HOST_WIDE_INT offset
,
1677 struct access
*model
)
1679 HOST_WIDE_INT base_offset
;
1682 if (TREE_CODE (model
->expr
) == COMPONENT_REF
1683 && DECL_BIT_FIELD (TREE_OPERAND (model
->expr
, 1)))
1686 base
= get_addr_base_and_unit_offset (base
, &base_offset
);
1689 if (TREE_CODE (base
) == MEM_REF
)
1691 off
= build_int_cst (TREE_TYPE (TREE_OPERAND (base
, 1)),
1692 base_offset
+ offset
/ BITS_PER_UNIT
);
1693 off
= int_const_binop (PLUS_EXPR
, TREE_OPERAND (base
, 1), off
);
1694 base
= unshare_expr (TREE_OPERAND (base
, 0));
1698 off
= build_int_cst (reference_alias_ptr_type (base
),
1699 base_offset
+ offset
/ BITS_PER_UNIT
);
1700 base
= build_fold_addr_expr (unshare_expr (base
));
1703 return fold_build2_loc (loc
, MEM_REF
, model
->type
, base
, off
);
1706 /* Construct a memory reference consisting of component_refs and array_refs to
1707 a part of an aggregate *RES (which is of type TYPE). The requested part
1708 should have type EXP_TYPE at be the given OFFSET. This function might not
1709 succeed, it returns true when it does and only then *RES points to something
1710 meaningful. This function should be used only to build expressions that we
1711 might need to present to user (e.g. in warnings). In all other situations,
1712 build_ref_for_model or build_ref_for_offset should be used instead. */
1715 build_user_friendly_ref_for_offset (tree
*res
, tree type
, HOST_WIDE_INT offset
,
1721 tree tr_size
, index
, minidx
;
1722 HOST_WIDE_INT el_size
;
1724 if (offset
== 0 && exp_type
1725 && types_compatible_p (exp_type
, type
))
1728 switch (TREE_CODE (type
))
1731 case QUAL_UNION_TYPE
:
1733 for (fld
= TYPE_FIELDS (type
); fld
; fld
= DECL_CHAIN (fld
))
1735 HOST_WIDE_INT pos
, size
;
1736 tree tr_pos
, expr
, *expr_ptr
;
1738 if (TREE_CODE (fld
) != FIELD_DECL
)
1741 tr_pos
= bit_position (fld
);
1742 if (!tr_pos
|| !tree_fits_uhwi_p (tr_pos
))
1744 pos
= tree_to_uhwi (tr_pos
);
1745 gcc_assert (TREE_CODE (type
) == RECORD_TYPE
|| pos
== 0);
1746 tr_size
= DECL_SIZE (fld
);
1747 if (!tr_size
|| !tree_fits_uhwi_p (tr_size
))
1749 size
= tree_to_uhwi (tr_size
);
1755 else if (pos
> offset
|| (pos
+ size
) <= offset
)
1758 expr
= build3 (COMPONENT_REF
, TREE_TYPE (fld
), *res
, fld
,
1761 if (build_user_friendly_ref_for_offset (expr_ptr
, TREE_TYPE (fld
),
1762 offset
- pos
, exp_type
))
1771 tr_size
= TYPE_SIZE (TREE_TYPE (type
));
1772 if (!tr_size
|| !tree_fits_uhwi_p (tr_size
))
1774 el_size
= tree_to_uhwi (tr_size
);
1776 minidx
= TYPE_MIN_VALUE (TYPE_DOMAIN (type
));
1777 if (TREE_CODE (minidx
) != INTEGER_CST
|| el_size
== 0)
1779 index
= build_int_cst (TYPE_DOMAIN (type
), offset
/ el_size
);
1780 if (!integer_zerop (minidx
))
1781 index
= int_const_binop (PLUS_EXPR
, index
, minidx
);
1782 *res
= build4 (ARRAY_REF
, TREE_TYPE (type
), *res
, index
,
1783 NULL_TREE
, NULL_TREE
);
1784 offset
= offset
% el_size
;
1785 type
= TREE_TYPE (type
);
1800 /* Return true iff TYPE is stdarg va_list type. */
1803 is_va_list_type (tree type
)
1805 return TYPE_MAIN_VARIANT (type
) == TYPE_MAIN_VARIANT (va_list_type_node
);
1808 /* Print message to dump file why a variable was rejected. */
1811 reject (tree var
, const char *msg
)
1813 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1815 fprintf (dump_file
, "Rejected (%d): %s: ", DECL_UID (var
), msg
);
1816 print_generic_expr (dump_file
, var
, 0);
1817 fprintf (dump_file
, "\n");
1821 /* Return true if VAR is a candidate for SRA. */
1824 maybe_add_sra_candidate (tree var
)
1826 tree type
= TREE_TYPE (var
);
1830 if (!AGGREGATE_TYPE_P (type
))
1832 reject (var
, "not aggregate");
1835 if (needs_to_live_in_memory (var
))
1837 reject (var
, "needs to live in memory");
1840 if (TREE_THIS_VOLATILE (var
))
1842 reject (var
, "is volatile");
1845 if (!COMPLETE_TYPE_P (type
))
1847 reject (var
, "has incomplete type");
1850 if (!tree_fits_uhwi_p (TYPE_SIZE (type
)))
1852 reject (var
, "type size not fixed");
1855 if (tree_to_uhwi (TYPE_SIZE (type
)) == 0)
1857 reject (var
, "type size is zero");
1860 if (type_internals_preclude_sra_p (type
, &msg
))
1865 if (/* Fix for PR 41089. tree-stdarg.c needs to have va_lists intact but
1866 we also want to schedule it rather late. Thus we ignore it in
1868 (sra_mode
== SRA_MODE_EARLY_INTRA
1869 && is_va_list_type (type
)))
1871 reject (var
, "is va_list");
1875 bitmap_set_bit (candidate_bitmap
, DECL_UID (var
));
1876 slot
= candidates
->find_slot_with_hash (var
, DECL_UID (var
), INSERT
);
1879 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1881 fprintf (dump_file
, "Candidate (%d): ", DECL_UID (var
));
1882 print_generic_expr (dump_file
, var
, 0);
1883 fprintf (dump_file
, "\n");
1889 /* The very first phase of intraprocedural SRA. It marks in candidate_bitmap
1890 those with type which is suitable for scalarization. */
1893 find_var_candidates (void)
1899 for (parm
= DECL_ARGUMENTS (current_function_decl
);
1901 parm
= DECL_CHAIN (parm
))
1902 ret
|= maybe_add_sra_candidate (parm
);
1904 FOR_EACH_LOCAL_DECL (cfun
, i
, var
)
1906 if (TREE_CODE (var
) != VAR_DECL
)
1909 ret
|= maybe_add_sra_candidate (var
);
1915 /* Sort all accesses for the given variable, check for partial overlaps and
1916 return NULL if there are any. If there are none, pick a representative for
1917 each combination of offset and size and create a linked list out of them.
1918 Return the pointer to the first representative and make sure it is the first
1919 one in the vector of accesses. */
1921 static struct access
*
1922 sort_and_splice_var_accesses (tree var
)
1924 int i
, j
, access_count
;
1925 struct access
*res
, **prev_acc_ptr
= &res
;
1926 vec
<access_p
> *access_vec
;
1928 HOST_WIDE_INT low
= -1, high
= 0;
1930 access_vec
= get_base_access_vector (var
);
1933 access_count
= access_vec
->length ();
1935 /* Sort by <OFFSET, SIZE>. */
1936 access_vec
->qsort (compare_access_positions
);
1939 while (i
< access_count
)
1941 struct access
*access
= (*access_vec
)[i
];
1942 bool grp_write
= access
->write
;
1943 bool grp_read
= !access
->write
;
1944 bool grp_scalar_write
= access
->write
1945 && is_gimple_reg_type (access
->type
);
1946 bool grp_scalar_read
= !access
->write
1947 && is_gimple_reg_type (access
->type
);
1948 bool grp_assignment_read
= access
->grp_assignment_read
;
1949 bool grp_assignment_write
= access
->grp_assignment_write
;
1950 bool multiple_scalar_reads
= false;
1951 bool total_scalarization
= access
->grp_total_scalarization
;
1952 bool grp_partial_lhs
= access
->grp_partial_lhs
;
1953 bool first_scalar
= is_gimple_reg_type (access
->type
);
1954 bool unscalarizable_region
= access
->grp_unscalarizable_region
;
1956 if (first
|| access
->offset
>= high
)
1959 low
= access
->offset
;
1960 high
= access
->offset
+ access
->size
;
1962 else if (access
->offset
> low
&& access
->offset
+ access
->size
> high
)
1965 gcc_assert (access
->offset
>= low
1966 && access
->offset
+ access
->size
<= high
);
1969 while (j
< access_count
)
1971 struct access
*ac2
= (*access_vec
)[j
];
1972 if (ac2
->offset
!= access
->offset
|| ac2
->size
!= access
->size
)
1977 grp_scalar_write
= (grp_scalar_write
1978 || is_gimple_reg_type (ac2
->type
));
1983 if (is_gimple_reg_type (ac2
->type
))
1985 if (grp_scalar_read
)
1986 multiple_scalar_reads
= true;
1988 grp_scalar_read
= true;
1991 grp_assignment_read
|= ac2
->grp_assignment_read
;
1992 grp_assignment_write
|= ac2
->grp_assignment_write
;
1993 grp_partial_lhs
|= ac2
->grp_partial_lhs
;
1994 unscalarizable_region
|= ac2
->grp_unscalarizable_region
;
1995 total_scalarization
|= ac2
->grp_total_scalarization
;
1996 relink_to_new_repr (access
, ac2
);
1998 /* If there are both aggregate-type and scalar-type accesses with
1999 this combination of size and offset, the comparison function
2000 should have put the scalars first. */
2001 gcc_assert (first_scalar
|| !is_gimple_reg_type (ac2
->type
));
2002 ac2
->group_representative
= access
;
2008 access
->group_representative
= access
;
2009 access
->grp_write
= grp_write
;
2010 access
->grp_read
= grp_read
;
2011 access
->grp_scalar_read
= grp_scalar_read
;
2012 access
->grp_scalar_write
= grp_scalar_write
;
2013 access
->grp_assignment_read
= grp_assignment_read
;
2014 access
->grp_assignment_write
= grp_assignment_write
;
2015 access
->grp_hint
= multiple_scalar_reads
|| total_scalarization
;
2016 access
->grp_total_scalarization
= total_scalarization
;
2017 access
->grp_partial_lhs
= grp_partial_lhs
;
2018 access
->grp_unscalarizable_region
= unscalarizable_region
;
2019 if (access
->first_link
)
2020 add_access_to_work_queue (access
);
2022 *prev_acc_ptr
= access
;
2023 prev_acc_ptr
= &access
->next_grp
;
2026 gcc_assert (res
== (*access_vec
)[0]);
2030 /* Create a variable for the given ACCESS which determines the type, name and a
2031 few other properties. Return the variable declaration and store it also to
2032 ACCESS->replacement. */
2035 create_access_replacement (struct access
*access
)
2039 if (access
->grp_to_be_debug_replaced
)
2041 repl
= create_tmp_var_raw (access
->type
);
2042 DECL_CONTEXT (repl
) = current_function_decl
;
2045 /* Drop any special alignment on the type if it's not on the main
2046 variant. This avoids issues with weirdo ABIs like AAPCS. */
2047 repl
= create_tmp_var (build_qualified_type
2048 (TYPE_MAIN_VARIANT (access
->type
),
2049 TYPE_QUALS (access
->type
)), "SR");
2050 if (TREE_CODE (access
->type
) == COMPLEX_TYPE
2051 || TREE_CODE (access
->type
) == VECTOR_TYPE
)
2053 if (!access
->grp_partial_lhs
)
2054 DECL_GIMPLE_REG_P (repl
) = 1;
2056 else if (access
->grp_partial_lhs
2057 && is_gimple_reg_type (access
->type
))
2058 TREE_ADDRESSABLE (repl
) = 1;
2060 DECL_SOURCE_LOCATION (repl
) = DECL_SOURCE_LOCATION (access
->base
);
2061 DECL_ARTIFICIAL (repl
) = 1;
2062 DECL_IGNORED_P (repl
) = DECL_IGNORED_P (access
->base
);
2064 if (DECL_NAME (access
->base
)
2065 && !DECL_IGNORED_P (access
->base
)
2066 && !DECL_ARTIFICIAL (access
->base
))
2068 char *pretty_name
= make_fancy_name (access
->expr
);
2069 tree debug_expr
= unshare_expr_without_location (access
->expr
), d
;
2072 DECL_NAME (repl
) = get_identifier (pretty_name
);
2073 obstack_free (&name_obstack
, pretty_name
);
2075 /* Get rid of any SSA_NAMEs embedded in debug_expr,
2076 as DECL_DEBUG_EXPR isn't considered when looking for still
2077 used SSA_NAMEs and thus they could be freed. All debug info
2078 generation cares is whether something is constant or variable
2079 and that get_ref_base_and_extent works properly on the
2080 expression. It cannot handle accesses at a non-constant offset
2081 though, so just give up in those cases. */
2082 for (d
= debug_expr
;
2083 !fail
&& (handled_component_p (d
) || TREE_CODE (d
) == MEM_REF
);
2084 d
= TREE_OPERAND (d
, 0))
2085 switch (TREE_CODE (d
))
2088 case ARRAY_RANGE_REF
:
2089 if (TREE_OPERAND (d
, 1)
2090 && TREE_CODE (TREE_OPERAND (d
, 1)) != INTEGER_CST
)
2092 if (TREE_OPERAND (d
, 3)
2093 && TREE_CODE (TREE_OPERAND (d
, 3)) != INTEGER_CST
)
2097 if (TREE_OPERAND (d
, 2)
2098 && TREE_CODE (TREE_OPERAND (d
, 2)) != INTEGER_CST
)
2102 if (TREE_CODE (TREE_OPERAND (d
, 0)) != ADDR_EXPR
)
2105 d
= TREE_OPERAND (d
, 0);
2112 SET_DECL_DEBUG_EXPR (repl
, debug_expr
);
2113 DECL_HAS_DEBUG_EXPR_P (repl
) = 1;
2115 if (access
->grp_no_warning
)
2116 TREE_NO_WARNING (repl
) = 1;
2118 TREE_NO_WARNING (repl
) = TREE_NO_WARNING (access
->base
);
2121 TREE_NO_WARNING (repl
) = 1;
2125 if (access
->grp_to_be_debug_replaced
)
2127 fprintf (dump_file
, "Created a debug-only replacement for ");
2128 print_generic_expr (dump_file
, access
->base
, 0);
2129 fprintf (dump_file
, " offset: %u, size: %u\n",
2130 (unsigned) access
->offset
, (unsigned) access
->size
);
2134 fprintf (dump_file
, "Created a replacement for ");
2135 print_generic_expr (dump_file
, access
->base
, 0);
2136 fprintf (dump_file
, " offset: %u, size: %u: ",
2137 (unsigned) access
->offset
, (unsigned) access
->size
);
2138 print_generic_expr (dump_file
, repl
, 0);
2139 fprintf (dump_file
, "\n");
2142 sra_stats
.replacements
++;
2147 /* Return ACCESS scalar replacement, which must exist. */
2150 get_access_replacement (struct access
*access
)
2152 gcc_checking_assert (access
->replacement_decl
);
2153 return access
->replacement_decl
;
2157 /* Build a subtree of accesses rooted in *ACCESS, and move the pointer in the
2158 linked list along the way. Stop when *ACCESS is NULL or the access pointed
2159 to it is not "within" the root. Return false iff some accesses partially
2163 build_access_subtree (struct access
**access
)
2165 struct access
*root
= *access
, *last_child
= NULL
;
2166 HOST_WIDE_INT limit
= root
->offset
+ root
->size
;
2168 *access
= (*access
)->next_grp
;
2169 while (*access
&& (*access
)->offset
+ (*access
)->size
<= limit
)
2172 root
->first_child
= *access
;
2174 last_child
->next_sibling
= *access
;
2175 last_child
= *access
;
2177 if (!build_access_subtree (access
))
2181 if (*access
&& (*access
)->offset
< limit
)
2187 /* Build a tree of access representatives, ACCESS is the pointer to the first
2188 one, others are linked in a list by the next_grp field. Return false iff
2189 some accesses partially overlap. */
2192 build_access_trees (struct access
*access
)
2196 struct access
*root
= access
;
2198 if (!build_access_subtree (&access
))
2200 root
->next_grp
= access
;
2205 /* Return true if expr contains some ARRAY_REFs into a variable bounded
2209 expr_with_var_bounded_array_refs_p (tree expr
)
2211 while (handled_component_p (expr
))
2213 if (TREE_CODE (expr
) == ARRAY_REF
2214 && !tree_fits_shwi_p (array_ref_low_bound (expr
)))
2216 expr
= TREE_OPERAND (expr
, 0);
2221 /* Analyze the subtree of accesses rooted in ROOT, scheduling replacements when
2222 both seeming beneficial and when ALLOW_REPLACEMENTS allows it. Also set all
2223 sorts of access flags appropriately along the way, notably always set
2224 grp_read and grp_assign_read according to MARK_READ and grp_write when
2227 Creating a replacement for a scalar access is considered beneficial if its
2228 grp_hint is set (this means we are either attempting total scalarization or
2229 there is more than one direct read access) or according to the following
2232 Access written to through a scalar type (once or more times)
2234 | Written to in an assignment statement
2236 | | Access read as scalar _once_
2238 | | | Read in an assignment statement
2240 | | | | Scalarize Comment
2241 -----------------------------------------------------------------------------
2242 0 0 0 0 No access for the scalar
2243 0 0 0 1 No access for the scalar
2244 0 0 1 0 No Single read - won't help
2245 0 0 1 1 No The same case
2246 0 1 0 0 No access for the scalar
2247 0 1 0 1 No access for the scalar
2248 0 1 1 0 Yes s = *g; return s.i;
2249 0 1 1 1 Yes The same case as above
2250 1 0 0 0 No Won't help
2251 1 0 0 1 Yes s.i = 1; *g = s;
2252 1 0 1 0 Yes s.i = 5; g = s.i;
2253 1 0 1 1 Yes The same case as above
2254 1 1 0 0 No Won't help.
2255 1 1 0 1 Yes s.i = 1; *g = s;
2256 1 1 1 0 Yes s = *g; return s.i;
2257 1 1 1 1 Yes Any of the above yeses */
2260 analyze_access_subtree (struct access
*root
, struct access
*parent
,
2261 bool allow_replacements
)
2263 struct access
*child
;
2264 HOST_WIDE_INT limit
= root
->offset
+ root
->size
;
2265 HOST_WIDE_INT covered_to
= root
->offset
;
2266 bool scalar
= is_gimple_reg_type (root
->type
);
2267 bool hole
= false, sth_created
= false;
2271 if (parent
->grp_read
)
2273 if (parent
->grp_assignment_read
)
2274 root
->grp_assignment_read
= 1;
2275 if (parent
->grp_write
)
2276 root
->grp_write
= 1;
2277 if (parent
->grp_assignment_write
)
2278 root
->grp_assignment_write
= 1;
2279 if (parent
->grp_total_scalarization
)
2280 root
->grp_total_scalarization
= 1;
2283 if (root
->grp_unscalarizable_region
)
2284 allow_replacements
= false;
2286 if (allow_replacements
&& expr_with_var_bounded_array_refs_p (root
->expr
))
2287 allow_replacements
= false;
2289 for (child
= root
->first_child
; child
; child
= child
->next_sibling
)
2291 hole
|= covered_to
< child
->offset
;
2292 sth_created
|= analyze_access_subtree (child
, root
,
2293 allow_replacements
&& !scalar
);
2295 root
->grp_unscalarized_data
|= child
->grp_unscalarized_data
;
2296 root
->grp_total_scalarization
&= child
->grp_total_scalarization
;
2297 if (child
->grp_covered
)
2298 covered_to
+= child
->size
;
2303 if (allow_replacements
&& scalar
&& !root
->first_child
2305 || ((root
->grp_scalar_read
|| root
->grp_assignment_read
)
2306 && (root
->grp_scalar_write
|| root
->grp_assignment_write
))))
2308 /* Always create access replacements that cover the whole access.
2309 For integral types this means the precision has to match.
2310 Avoid assumptions based on the integral type kind, too. */
2311 if (INTEGRAL_TYPE_P (root
->type
)
2312 && (TREE_CODE (root
->type
) != INTEGER_TYPE
2313 || TYPE_PRECISION (root
->type
) != root
->size
)
2314 /* But leave bitfield accesses alone. */
2315 && (TREE_CODE (root
->expr
) != COMPONENT_REF
2316 || !DECL_BIT_FIELD (TREE_OPERAND (root
->expr
, 1))))
2318 tree rt
= root
->type
;
2319 gcc_assert ((root
->offset
% BITS_PER_UNIT
) == 0
2320 && (root
->size
% BITS_PER_UNIT
) == 0);
2321 root
->type
= build_nonstandard_integer_type (root
->size
,
2322 TYPE_UNSIGNED (rt
));
2323 root
->expr
= build_ref_for_offset (UNKNOWN_LOCATION
,
2324 root
->base
, root
->offset
,
2325 root
->type
, NULL
, false);
2327 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2329 fprintf (dump_file
, "Changing the type of a replacement for ");
2330 print_generic_expr (dump_file
, root
->base
, 0);
2331 fprintf (dump_file
, " offset: %u, size: %u ",
2332 (unsigned) root
->offset
, (unsigned) root
->size
);
2333 fprintf (dump_file
, " to an integer.\n");
2337 root
->grp_to_be_replaced
= 1;
2338 root
->replacement_decl
= create_access_replacement (root
);
2344 if (allow_replacements
2345 && scalar
&& !root
->first_child
2346 && (root
->grp_scalar_write
|| root
->grp_assignment_write
)
2347 && !bitmap_bit_p (cannot_scalarize_away_bitmap
,
2348 DECL_UID (root
->base
)))
2350 gcc_checking_assert (!root
->grp_scalar_read
2351 && !root
->grp_assignment_read
);
2353 if (MAY_HAVE_DEBUG_STMTS
)
2355 root
->grp_to_be_debug_replaced
= 1;
2356 root
->replacement_decl
= create_access_replacement (root
);
2360 if (covered_to
< limit
)
2363 root
->grp_total_scalarization
= 0;
2366 if (!hole
|| root
->grp_total_scalarization
)
2367 root
->grp_covered
= 1;
2368 else if (root
->grp_write
|| TREE_CODE (root
->base
) == PARM_DECL
)
2369 root
->grp_unscalarized_data
= 1; /* not covered and written to */
2373 /* Analyze all access trees linked by next_grp by the means of
2374 analyze_access_subtree. */
2376 analyze_access_trees (struct access
*access
)
2382 if (analyze_access_subtree (access
, NULL
, true))
2384 access
= access
->next_grp
;
2390 /* Return true iff a potential new child of LACC at offset OFFSET and with size
2391 SIZE would conflict with an already existing one. If exactly such a child
2392 already exists in LACC, store a pointer to it in EXACT_MATCH. */
2395 child_would_conflict_in_lacc (struct access
*lacc
, HOST_WIDE_INT norm_offset
,
2396 HOST_WIDE_INT size
, struct access
**exact_match
)
2398 struct access
*child
;
2400 for (child
= lacc
->first_child
; child
; child
= child
->next_sibling
)
2402 if (child
->offset
== norm_offset
&& child
->size
== size
)
2404 *exact_match
= child
;
2408 if (child
->offset
< norm_offset
+ size
2409 && child
->offset
+ child
->size
> norm_offset
)
2416 /* Create a new child access of PARENT, with all properties just like MODEL
2417 except for its offset and with its grp_write false and grp_read true.
2418 Return the new access or NULL if it cannot be created. Note that this access
2419 is created long after all splicing and sorting, it's not located in any
2420 access vector and is automatically a representative of its group. */
2422 static struct access
*
2423 create_artificial_child_access (struct access
*parent
, struct access
*model
,
2424 HOST_WIDE_INT new_offset
)
2426 struct access
**child
;
2427 tree expr
= parent
->base
;
2429 gcc_assert (!model
->grp_unscalarizable_region
);
2431 struct access
*access
= access_pool
.allocate ();
2432 memset (access
, 0, sizeof (struct access
));
2433 if (!build_user_friendly_ref_for_offset (&expr
, TREE_TYPE (expr
), new_offset
,
2436 access
->grp_no_warning
= true;
2437 expr
= build_ref_for_model (EXPR_LOCATION (parent
->base
), parent
->base
,
2438 new_offset
, model
, NULL
, false);
2441 access
->base
= parent
->base
;
2442 access
->expr
= expr
;
2443 access
->offset
= new_offset
;
2444 access
->size
= model
->size
;
2445 access
->type
= model
->type
;
2446 access
->grp_write
= true;
2447 access
->grp_read
= false;
2449 child
= &parent
->first_child
;
2450 while (*child
&& (*child
)->offset
< new_offset
)
2451 child
= &(*child
)->next_sibling
;
2453 access
->next_sibling
= *child
;
2460 /* Propagate all subaccesses of RACC across an assignment link to LACC. Return
2461 true if any new subaccess was created. Additionally, if RACC is a scalar
2462 access but LACC is not, change the type of the latter, if possible. */
2465 propagate_subaccesses_across_link (struct access
*lacc
, struct access
*racc
)
2467 struct access
*rchild
;
2468 HOST_WIDE_INT norm_delta
= lacc
->offset
- racc
->offset
;
2471 if (is_gimple_reg_type (lacc
->type
)
2472 || lacc
->grp_unscalarizable_region
2473 || racc
->grp_unscalarizable_region
)
2476 if (is_gimple_reg_type (racc
->type
))
2478 if (!lacc
->first_child
&& !racc
->first_child
)
2480 tree t
= lacc
->base
;
2482 lacc
->type
= racc
->type
;
2483 if (build_user_friendly_ref_for_offset (&t
, TREE_TYPE (t
),
2484 lacc
->offset
, racc
->type
))
2488 lacc
->expr
= build_ref_for_model (EXPR_LOCATION (lacc
->base
),
2489 lacc
->base
, lacc
->offset
,
2491 lacc
->grp_no_warning
= true;
2497 for (rchild
= racc
->first_child
; rchild
; rchild
= rchild
->next_sibling
)
2499 struct access
*new_acc
= NULL
;
2500 HOST_WIDE_INT norm_offset
= rchild
->offset
+ norm_delta
;
2502 if (rchild
->grp_unscalarizable_region
)
2505 if (child_would_conflict_in_lacc (lacc
, norm_offset
, rchild
->size
,
2510 rchild
->grp_hint
= 1;
2511 new_acc
->grp_hint
|= new_acc
->grp_read
;
2512 if (rchild
->first_child
)
2513 ret
|= propagate_subaccesses_across_link (new_acc
, rchild
);
2518 rchild
->grp_hint
= 1;
2519 new_acc
= create_artificial_child_access (lacc
, rchild
, norm_offset
);
2523 if (racc
->first_child
)
2524 propagate_subaccesses_across_link (new_acc
, rchild
);
2531 /* Propagate all subaccesses across assignment links. */
2534 propagate_all_subaccesses (void)
2536 while (work_queue_head
)
2538 struct access
*racc
= pop_access_from_work_queue ();
2539 struct assign_link
*link
;
2541 gcc_assert (racc
->first_link
);
2543 for (link
= racc
->first_link
; link
; link
= link
->next
)
2545 struct access
*lacc
= link
->lacc
;
2547 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (lacc
->base
)))
2549 lacc
= lacc
->group_representative
;
2550 if (propagate_subaccesses_across_link (lacc
, racc
)
2551 && lacc
->first_link
)
2552 add_access_to_work_queue (lacc
);
2557 /* Go through all accesses collected throughout the (intraprocedural) analysis
2558 stage, exclude overlapping ones, identify representatives and build trees
2559 out of them, making decisions about scalarization on the way. Return true
2560 iff there are any to-be-scalarized variables after this stage. */
2563 analyze_all_variable_accesses (void)
2566 bitmap tmp
= BITMAP_ALLOC (NULL
);
2569 bool optimize_speed_p
= !optimize_function_for_size_p (cfun
);
2571 enum compiler_param param
= optimize_speed_p
2572 ? PARAM_SRA_MAX_SCALARIZATION_SIZE_SPEED
2573 : PARAM_SRA_MAX_SCALARIZATION_SIZE_SIZE
;
2575 /* If the user didn't set PARAM_SRA_MAX_SCALARIZATION_SIZE_<...>,
2576 fall back to a target default. */
2577 unsigned HOST_WIDE_INT max_scalarization_size
2578 = global_options_set
.x_param_values
[param
]
2579 ? PARAM_VALUE (param
)
2580 : get_move_ratio (optimize_speed_p
) * UNITS_PER_WORD
;
2582 max_scalarization_size
*= BITS_PER_UNIT
;
2584 EXECUTE_IF_SET_IN_BITMAP (candidate_bitmap
, 0, i
, bi
)
2585 if (bitmap_bit_p (should_scalarize_away_bitmap
, i
)
2586 && !bitmap_bit_p (cannot_scalarize_away_bitmap
, i
))
2588 tree var
= candidate (i
);
2590 if (TREE_CODE (var
) == VAR_DECL
2591 && scalarizable_type_p (TREE_TYPE (var
)))
2593 if (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var
)))
2594 <= max_scalarization_size
)
2596 create_total_scalarization_access (var
);
2597 completely_scalarize (var
, TREE_TYPE (var
), 0, var
);
2598 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2600 fprintf (dump_file
, "Will attempt to totally scalarize ");
2601 print_generic_expr (dump_file
, var
, 0);
2602 fprintf (dump_file
, " (UID: %u): \n", DECL_UID (var
));
2605 else if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2607 fprintf (dump_file
, "Too big to totally scalarize: ");
2608 print_generic_expr (dump_file
, var
, 0);
2609 fprintf (dump_file
, " (UID: %u)\n", DECL_UID (var
));
2614 bitmap_copy (tmp
, candidate_bitmap
);
2615 EXECUTE_IF_SET_IN_BITMAP (tmp
, 0, i
, bi
)
2617 tree var
= candidate (i
);
2618 struct access
*access
;
2620 access
= sort_and_splice_var_accesses (var
);
2621 if (!access
|| !build_access_trees (access
))
2622 disqualify_candidate (var
,
2623 "No or inhibitingly overlapping accesses.");
2626 propagate_all_subaccesses ();
2628 bitmap_copy (tmp
, candidate_bitmap
);
2629 EXECUTE_IF_SET_IN_BITMAP (tmp
, 0, i
, bi
)
2631 tree var
= candidate (i
);
2632 struct access
*access
= get_first_repr_for_decl (var
);
2634 if (analyze_access_trees (access
))
2637 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2639 fprintf (dump_file
, "\nAccess trees for ");
2640 print_generic_expr (dump_file
, var
, 0);
2641 fprintf (dump_file
, " (UID: %u): \n", DECL_UID (var
));
2642 dump_access_tree (dump_file
, access
);
2643 fprintf (dump_file
, "\n");
2647 disqualify_candidate (var
, "No scalar replacements to be created.");
2654 statistics_counter_event (cfun
, "Scalarized aggregates", res
);
2661 /* Generate statements copying scalar replacements of accesses within a subtree
2662 into or out of AGG. ACCESS, all its children, siblings and their children
2663 are to be processed. AGG is an aggregate type expression (can be a
2664 declaration but does not have to be, it can for example also be a mem_ref or
2665 a series of handled components). TOP_OFFSET is the offset of the processed
2666 subtree which has to be subtracted from offsets of individual accesses to
2667 get corresponding offsets for AGG. If CHUNK_SIZE is non-null, copy only
2668 replacements in the interval <start_offset, start_offset + chunk_size>,
2669 otherwise copy all. GSI is a statement iterator used to place the new
2670 statements. WRITE should be true when the statements should write from AGG
2671 to the replacement and false if vice versa. if INSERT_AFTER is true, new
2672 statements will be added after the current statement in GSI, they will be
2673 added before the statement otherwise. */
2676 generate_subtree_copies (struct access
*access
, tree agg
,
2677 HOST_WIDE_INT top_offset
,
2678 HOST_WIDE_INT start_offset
, HOST_WIDE_INT chunk_size
,
2679 gimple_stmt_iterator
*gsi
, bool write
,
2680 bool insert_after
, location_t loc
)
2684 if (chunk_size
&& access
->offset
>= start_offset
+ chunk_size
)
2687 if (access
->grp_to_be_replaced
2689 || access
->offset
+ access
->size
> start_offset
))
2691 tree expr
, repl
= get_access_replacement (access
);
2694 expr
= build_ref_for_model (loc
, agg
, access
->offset
- top_offset
,
2695 access
, gsi
, insert_after
);
2699 if (access
->grp_partial_lhs
)
2700 expr
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
2702 insert_after
? GSI_NEW_STMT
2704 stmt
= gimple_build_assign (repl
, expr
);
2708 TREE_NO_WARNING (repl
) = 1;
2709 if (access
->grp_partial_lhs
)
2710 repl
= force_gimple_operand_gsi (gsi
, repl
, true, NULL_TREE
,
2712 insert_after
? GSI_NEW_STMT
2714 stmt
= gimple_build_assign (expr
, repl
);
2716 gimple_set_location (stmt
, loc
);
2719 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
2721 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
2723 sra_stats
.subtree_copies
++;
2726 && access
->grp_to_be_debug_replaced
2728 || access
->offset
+ access
->size
> start_offset
))
2731 tree drhs
= build_debug_ref_for_model (loc
, agg
,
2732 access
->offset
- top_offset
,
2734 ds
= gimple_build_debug_bind (get_access_replacement (access
),
2735 drhs
, gsi_stmt (*gsi
));
2737 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
2739 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
2742 if (access
->first_child
)
2743 generate_subtree_copies (access
->first_child
, agg
, top_offset
,
2744 start_offset
, chunk_size
, gsi
,
2745 write
, insert_after
, loc
);
2747 access
= access
->next_sibling
;
2752 /* Assign zero to all scalar replacements in an access subtree. ACCESS is the
2753 the root of the subtree to be processed. GSI is the statement iterator used
2754 for inserting statements which are added after the current statement if
2755 INSERT_AFTER is true or before it otherwise. */
2758 init_subtree_with_zero (struct access
*access
, gimple_stmt_iterator
*gsi
,
2759 bool insert_after
, location_t loc
)
2762 struct access
*child
;
2764 if (access
->grp_to_be_replaced
)
2768 stmt
= gimple_build_assign (get_access_replacement (access
),
2769 build_zero_cst (access
->type
));
2771 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
2773 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
2775 gimple_set_location (stmt
, loc
);
2777 else if (access
->grp_to_be_debug_replaced
)
2780 = gimple_build_debug_bind (get_access_replacement (access
),
2781 build_zero_cst (access
->type
),
2784 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
2786 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
2789 for (child
= access
->first_child
; child
; child
= child
->next_sibling
)
2790 init_subtree_with_zero (child
, gsi
, insert_after
, loc
);
2793 /* Clobber all scalar replacements in an access subtree. ACCESS is the the
2794 root of the subtree to be processed. GSI is the statement iterator used
2795 for inserting statements which are added after the current statement if
2796 INSERT_AFTER is true or before it otherwise. */
2799 clobber_subtree (struct access
*access
, gimple_stmt_iterator
*gsi
,
2800 bool insert_after
, location_t loc
)
2803 struct access
*child
;
2805 if (access
->grp_to_be_replaced
)
2807 tree rep
= get_access_replacement (access
);
2808 tree clobber
= build_constructor (access
->type
, NULL
);
2809 TREE_THIS_VOLATILE (clobber
) = 1;
2810 gimple
*stmt
= gimple_build_assign (rep
, clobber
);
2813 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
2815 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
2817 gimple_set_location (stmt
, loc
);
2820 for (child
= access
->first_child
; child
; child
= child
->next_sibling
)
2821 clobber_subtree (child
, gsi
, insert_after
, loc
);
2824 /* Search for an access representative for the given expression EXPR and
2825 return it or NULL if it cannot be found. */
2827 static struct access
*
2828 get_access_for_expr (tree expr
)
2830 HOST_WIDE_INT offset
, size
, max_size
;
2833 /* FIXME: This should not be necessary but Ada produces V_C_Es with a type of
2834 a different size than the size of its argument and we need the latter
2836 if (TREE_CODE (expr
) == VIEW_CONVERT_EXPR
)
2837 expr
= TREE_OPERAND (expr
, 0);
2839 base
= get_ref_base_and_extent (expr
, &offset
, &size
, &max_size
);
2840 if (max_size
== -1 || !DECL_P (base
))
2843 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (base
)))
2846 return get_var_base_offset_size_access (base
, offset
, max_size
);
2849 /* Replace the expression EXPR with a scalar replacement if there is one and
2850 generate other statements to do type conversion or subtree copying if
2851 necessary. GSI is used to place newly created statements, WRITE is true if
2852 the expression is being written to (it is on a LHS of a statement or output
2853 in an assembly statement). */
2856 sra_modify_expr (tree
*expr
, gimple_stmt_iterator
*gsi
, bool write
)
2859 struct access
*access
;
2860 tree type
, bfr
, orig_expr
;
2862 if (TREE_CODE (*expr
) == BIT_FIELD_REF
)
2865 expr
= &TREE_OPERAND (*expr
, 0);
2870 if (TREE_CODE (*expr
) == REALPART_EXPR
|| TREE_CODE (*expr
) == IMAGPART_EXPR
)
2871 expr
= &TREE_OPERAND (*expr
, 0);
2872 access
= get_access_for_expr (*expr
);
2875 type
= TREE_TYPE (*expr
);
2878 loc
= gimple_location (gsi_stmt (*gsi
));
2879 gimple_stmt_iterator alt_gsi
= gsi_none ();
2880 if (write
&& stmt_ends_bb_p (gsi_stmt (*gsi
)))
2882 alt_gsi
= gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi
)));
2886 if (access
->grp_to_be_replaced
)
2888 tree repl
= get_access_replacement (access
);
2889 /* If we replace a non-register typed access simply use the original
2890 access expression to extract the scalar component afterwards.
2891 This happens if scalarizing a function return value or parameter
2892 like in gcc.c-torture/execute/20041124-1.c, 20050316-1.c and
2893 gcc.c-torture/compile/20011217-1.c.
2895 We also want to use this when accessing a complex or vector which can
2896 be accessed as a different type too, potentially creating a need for
2897 type conversion (see PR42196) and when scalarized unions are involved
2898 in assembler statements (see PR42398). */
2899 if (!useless_type_conversion_p (type
, access
->type
))
2903 ref
= build_ref_for_model (loc
, orig_expr
, 0, access
, gsi
, false);
2909 if (access
->grp_partial_lhs
)
2910 ref
= force_gimple_operand_gsi (gsi
, ref
, true, NULL_TREE
,
2911 false, GSI_NEW_STMT
);
2912 stmt
= gimple_build_assign (repl
, ref
);
2913 gimple_set_location (stmt
, loc
);
2914 gsi_insert_after (gsi
, stmt
, GSI_NEW_STMT
);
2920 if (access
->grp_partial_lhs
)
2921 repl
= force_gimple_operand_gsi (gsi
, repl
, true, NULL_TREE
,
2922 true, GSI_SAME_STMT
);
2923 stmt
= gimple_build_assign (ref
, repl
);
2924 gimple_set_location (stmt
, loc
);
2925 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
2932 else if (write
&& access
->grp_to_be_debug_replaced
)
2934 gdebug
*ds
= gimple_build_debug_bind (get_access_replacement (access
),
2937 gsi_insert_after (gsi
, ds
, GSI_NEW_STMT
);
2940 if (access
->first_child
)
2942 HOST_WIDE_INT start_offset
, chunk_size
;
2944 && tree_fits_uhwi_p (TREE_OPERAND (bfr
, 1))
2945 && tree_fits_uhwi_p (TREE_OPERAND (bfr
, 2)))
2947 chunk_size
= tree_to_uhwi (TREE_OPERAND (bfr
, 1));
2948 start_offset
= access
->offset
2949 + tree_to_uhwi (TREE_OPERAND (bfr
, 2));
2952 start_offset
= chunk_size
= 0;
2954 generate_subtree_copies (access
->first_child
, orig_expr
, access
->offset
,
2955 start_offset
, chunk_size
, gsi
, write
, write
,
2961 /* Where scalar replacements of the RHS have been written to when a replacement
2962 of a LHS of an assigments cannot be direclty loaded from a replacement of
2964 enum unscalarized_data_handling
{ SRA_UDH_NONE
, /* Nothing done so far. */
2965 SRA_UDH_RIGHT
, /* Data flushed to the RHS. */
2966 SRA_UDH_LEFT
}; /* Data flushed to the LHS. */
2968 struct subreplacement_assignment_data
2970 /* Offset of the access representing the lhs of the assignment. */
2971 HOST_WIDE_INT left_offset
;
2973 /* LHS and RHS of the original assignment. */
2974 tree assignment_lhs
, assignment_rhs
;
2976 /* Access representing the rhs of the whole assignment. */
2977 struct access
*top_racc
;
2979 /* Stmt iterator used for statement insertions after the original assignment.
2980 It points to the main GSI used to traverse a BB during function body
2982 gimple_stmt_iterator
*new_gsi
;
2984 /* Stmt iterator used for statement insertions before the original
2985 assignment. Keeps on pointing to the original statement. */
2986 gimple_stmt_iterator old_gsi
;
2988 /* Location of the assignment. */
2991 /* Keeps the information whether we have needed to refresh replacements of
2992 the LHS and from which side of the assignments this takes place. */
2993 enum unscalarized_data_handling refreshed
;
2996 /* Store all replacements in the access tree rooted in TOP_RACC either to their
2997 base aggregate if there are unscalarized data or directly to LHS of the
2998 statement that is pointed to by GSI otherwise. */
3001 handle_unscalarized_data_in_subtree (struct subreplacement_assignment_data
*sad
)
3004 if (sad
->top_racc
->grp_unscalarized_data
)
3006 src
= sad
->assignment_rhs
;
3007 sad
->refreshed
= SRA_UDH_RIGHT
;
3011 src
= sad
->assignment_lhs
;
3012 sad
->refreshed
= SRA_UDH_LEFT
;
3014 generate_subtree_copies (sad
->top_racc
->first_child
, src
,
3015 sad
->top_racc
->offset
, 0, 0,
3016 &sad
->old_gsi
, false, false, sad
->loc
);
3019 /* Try to generate statements to load all sub-replacements in an access subtree
3020 formed by children of LACC from scalar replacements in the SAD->top_racc
3021 subtree. If that is not possible, refresh the SAD->top_racc base aggregate
3022 and load the accesses from it. */
3025 load_assign_lhs_subreplacements (struct access
*lacc
,
3026 struct subreplacement_assignment_data
*sad
)
3028 for (lacc
= lacc
->first_child
; lacc
; lacc
= lacc
->next_sibling
)
3030 HOST_WIDE_INT offset
;
3031 offset
= lacc
->offset
- sad
->left_offset
+ sad
->top_racc
->offset
;
3033 if (lacc
->grp_to_be_replaced
)
3035 struct access
*racc
;
3039 racc
= find_access_in_subtree (sad
->top_racc
, offset
, lacc
->size
);
3040 if (racc
&& racc
->grp_to_be_replaced
)
3042 rhs
= get_access_replacement (racc
);
3043 if (!useless_type_conversion_p (lacc
->type
, racc
->type
))
3044 rhs
= fold_build1_loc (sad
->loc
, VIEW_CONVERT_EXPR
,
3047 if (racc
->grp_partial_lhs
&& lacc
->grp_partial_lhs
)
3048 rhs
= force_gimple_operand_gsi (&sad
->old_gsi
, rhs
, true,
3049 NULL_TREE
, true, GSI_SAME_STMT
);
3053 /* No suitable access on the right hand side, need to load from
3054 the aggregate. See if we have to update it first... */
3055 if (sad
->refreshed
== SRA_UDH_NONE
)
3056 handle_unscalarized_data_in_subtree (sad
);
3058 if (sad
->refreshed
== SRA_UDH_LEFT
)
3059 rhs
= build_ref_for_model (sad
->loc
, sad
->assignment_lhs
,
3060 lacc
->offset
- sad
->left_offset
,
3061 lacc
, sad
->new_gsi
, true);
3063 rhs
= build_ref_for_model (sad
->loc
, sad
->assignment_rhs
,
3064 lacc
->offset
- sad
->left_offset
,
3065 lacc
, sad
->new_gsi
, true);
3066 if (lacc
->grp_partial_lhs
)
3067 rhs
= force_gimple_operand_gsi (sad
->new_gsi
,
3068 rhs
, true, NULL_TREE
,
3069 false, GSI_NEW_STMT
);
3072 stmt
= gimple_build_assign (get_access_replacement (lacc
), rhs
);
3073 gsi_insert_after (sad
->new_gsi
, stmt
, GSI_NEW_STMT
);
3074 gimple_set_location (stmt
, sad
->loc
);
3076 sra_stats
.subreplacements
++;
3080 if (sad
->refreshed
== SRA_UDH_NONE
3081 && lacc
->grp_read
&& !lacc
->grp_covered
)
3082 handle_unscalarized_data_in_subtree (sad
);
3084 if (lacc
&& lacc
->grp_to_be_debug_replaced
)
3088 struct access
*racc
= find_access_in_subtree (sad
->top_racc
,
3092 if (racc
&& racc
->grp_to_be_replaced
)
3094 if (racc
->grp_write
)
3095 drhs
= get_access_replacement (racc
);
3099 else if (sad
->refreshed
== SRA_UDH_LEFT
)
3100 drhs
= build_debug_ref_for_model (sad
->loc
, lacc
->base
,
3101 lacc
->offset
, lacc
);
3102 else if (sad
->refreshed
== SRA_UDH_RIGHT
)
3103 drhs
= build_debug_ref_for_model (sad
->loc
, sad
->top_racc
->base
,
3108 && !useless_type_conversion_p (lacc
->type
, TREE_TYPE (drhs
)))
3109 drhs
= fold_build1_loc (sad
->loc
, VIEW_CONVERT_EXPR
,
3111 ds
= gimple_build_debug_bind (get_access_replacement (lacc
),
3112 drhs
, gsi_stmt (sad
->old_gsi
));
3113 gsi_insert_after (sad
->new_gsi
, ds
, GSI_NEW_STMT
);
3117 if (lacc
->first_child
)
3118 load_assign_lhs_subreplacements (lacc
, sad
);
3122 /* Result code for SRA assignment modification. */
3123 enum assignment_mod_result
{ SRA_AM_NONE
, /* nothing done for the stmt */
3124 SRA_AM_MODIFIED
, /* stmt changed but not
3126 SRA_AM_REMOVED
}; /* stmt eliminated */
3128 /* Modify assignments with a CONSTRUCTOR on their RHS. STMT contains a pointer
3129 to the assignment and GSI is the statement iterator pointing at it. Returns
3130 the same values as sra_modify_assign. */
3132 static enum assignment_mod_result
3133 sra_modify_constructor_assign (gimple
*stmt
, gimple_stmt_iterator
*gsi
)
3135 tree lhs
= gimple_assign_lhs (stmt
);
3136 struct access
*acc
= get_access_for_expr (lhs
);
3139 location_t loc
= gimple_location (stmt
);
3141 if (gimple_clobber_p (stmt
))
3143 /* Clobber the replacement variable. */
3144 clobber_subtree (acc
, gsi
, !acc
->grp_covered
, loc
);
3145 /* Remove clobbers of fully scalarized variables, they are dead. */
3146 if (acc
->grp_covered
)
3148 unlink_stmt_vdef (stmt
);
3149 gsi_remove (gsi
, true);
3150 release_defs (stmt
);
3151 return SRA_AM_REMOVED
;
3154 return SRA_AM_MODIFIED
;
3157 if (vec_safe_length (CONSTRUCTOR_ELTS (gimple_assign_rhs1 (stmt
))) > 0)
3159 /* I have never seen this code path trigger but if it can happen the
3160 following should handle it gracefully. */
3161 if (access_has_children_p (acc
))
3162 generate_subtree_copies (acc
->first_child
, lhs
, acc
->offset
, 0, 0, gsi
,
3164 return SRA_AM_MODIFIED
;
3167 if (acc
->grp_covered
)
3169 init_subtree_with_zero (acc
, gsi
, false, loc
);
3170 unlink_stmt_vdef (stmt
);
3171 gsi_remove (gsi
, true);
3172 release_defs (stmt
);
3173 return SRA_AM_REMOVED
;
3177 init_subtree_with_zero (acc
, gsi
, true, loc
);
3178 return SRA_AM_MODIFIED
;
3182 /* Create and return a new suitable default definition SSA_NAME for RACC which
3183 is an access describing an uninitialized part of an aggregate that is being
3187 get_repl_default_def_ssa_name (struct access
*racc
)
3189 gcc_checking_assert (!racc
->grp_to_be_replaced
3190 && !racc
->grp_to_be_debug_replaced
);
3191 if (!racc
->replacement_decl
)
3192 racc
->replacement_decl
= create_access_replacement (racc
);
3193 return get_or_create_ssa_default_def (cfun
, racc
->replacement_decl
);
3196 /* Return true if REF has an VIEW_CONVERT_EXPR or a COMPONENT_REF with a
3197 bit-field field declaration somewhere in it. */
3200 contains_vce_or_bfcref_p (const_tree ref
)
3202 while (handled_component_p (ref
))
3204 if (TREE_CODE (ref
) == VIEW_CONVERT_EXPR
3205 || (TREE_CODE (ref
) == COMPONENT_REF
3206 && DECL_BIT_FIELD (TREE_OPERAND (ref
, 1))))
3208 ref
= TREE_OPERAND (ref
, 0);
3214 /* Examine both sides of the assignment statement pointed to by STMT, replace
3215 them with a scalare replacement if there is one and generate copying of
3216 replacements if scalarized aggregates have been used in the assignment. GSI
3217 is used to hold generated statements for type conversions and subtree
3220 static enum assignment_mod_result
3221 sra_modify_assign (gimple
*stmt
, gimple_stmt_iterator
*gsi
)
3223 struct access
*lacc
, *racc
;
3225 bool modify_this_stmt
= false;
3226 bool force_gimple_rhs
= false;
3228 gimple_stmt_iterator orig_gsi
= *gsi
;
3230 if (!gimple_assign_single_p (stmt
))
3232 lhs
= gimple_assign_lhs (stmt
);
3233 rhs
= gimple_assign_rhs1 (stmt
);
3235 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
3236 return sra_modify_constructor_assign (stmt
, gsi
);
3238 if (TREE_CODE (rhs
) == REALPART_EXPR
|| TREE_CODE (lhs
) == REALPART_EXPR
3239 || TREE_CODE (rhs
) == IMAGPART_EXPR
|| TREE_CODE (lhs
) == IMAGPART_EXPR
3240 || TREE_CODE (rhs
) == BIT_FIELD_REF
|| TREE_CODE (lhs
) == BIT_FIELD_REF
)
3242 modify_this_stmt
= sra_modify_expr (gimple_assign_rhs1_ptr (stmt
),
3244 modify_this_stmt
|= sra_modify_expr (gimple_assign_lhs_ptr (stmt
),
3246 return modify_this_stmt
? SRA_AM_MODIFIED
: SRA_AM_NONE
;
3249 lacc
= get_access_for_expr (lhs
);
3250 racc
= get_access_for_expr (rhs
);
3254 loc
= gimple_location (stmt
);
3255 if (lacc
&& lacc
->grp_to_be_replaced
)
3257 lhs
= get_access_replacement (lacc
);
3258 gimple_assign_set_lhs (stmt
, lhs
);
3259 modify_this_stmt
= true;
3260 if (lacc
->grp_partial_lhs
)
3261 force_gimple_rhs
= true;
3265 if (racc
&& racc
->grp_to_be_replaced
)
3267 rhs
= get_access_replacement (racc
);
3268 modify_this_stmt
= true;
3269 if (racc
->grp_partial_lhs
)
3270 force_gimple_rhs
= true;
3274 && !racc
->grp_unscalarized_data
3275 && TREE_CODE (lhs
) == SSA_NAME
3276 && !access_has_replacements_p (racc
))
3278 rhs
= get_repl_default_def_ssa_name (racc
);
3279 modify_this_stmt
= true;
3283 if (modify_this_stmt
)
3285 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3287 /* If we can avoid creating a VIEW_CONVERT_EXPR do so.
3288 ??? This should move to fold_stmt which we simply should
3289 call after building a VIEW_CONVERT_EXPR here. */
3290 if (AGGREGATE_TYPE_P (TREE_TYPE (lhs
))
3291 && !contains_bitfld_component_ref_p (lhs
))
3293 lhs
= build_ref_for_model (loc
, lhs
, 0, racc
, gsi
, false);
3294 gimple_assign_set_lhs (stmt
, lhs
);
3296 else if (AGGREGATE_TYPE_P (TREE_TYPE (rhs
))
3297 && !contains_vce_or_bfcref_p (rhs
))
3298 rhs
= build_ref_for_model (loc
, rhs
, 0, lacc
, gsi
, false);
3300 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (rhs
)))
3302 rhs
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
3304 if (is_gimple_reg_type (TREE_TYPE (lhs
))
3305 && TREE_CODE (lhs
) != SSA_NAME
)
3306 force_gimple_rhs
= true;
3311 if (lacc
&& lacc
->grp_to_be_debug_replaced
)
3313 tree dlhs
= get_access_replacement (lacc
);
3314 tree drhs
= unshare_expr (rhs
);
3315 if (!useless_type_conversion_p (TREE_TYPE (dlhs
), TREE_TYPE (drhs
)))
3317 if (AGGREGATE_TYPE_P (TREE_TYPE (drhs
))
3318 && !contains_vce_or_bfcref_p (drhs
))
3319 drhs
= build_debug_ref_for_model (loc
, drhs
, 0, lacc
);
3321 && !useless_type_conversion_p (TREE_TYPE (dlhs
),
3323 drhs
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
,
3324 TREE_TYPE (dlhs
), drhs
);
3326 gdebug
*ds
= gimple_build_debug_bind (dlhs
, drhs
, stmt
);
3327 gsi_insert_before (gsi
, ds
, GSI_SAME_STMT
);
3330 /* From this point on, the function deals with assignments in between
3331 aggregates when at least one has scalar reductions of some of its
3332 components. There are three possible scenarios: Both the LHS and RHS have
3333 to-be-scalarized components, 2) only the RHS has or 3) only the LHS has.
3335 In the first case, we would like to load the LHS components from RHS
3336 components whenever possible. If that is not possible, we would like to
3337 read it directly from the RHS (after updating it by storing in it its own
3338 components). If there are some necessary unscalarized data in the LHS,
3339 those will be loaded by the original assignment too. If neither of these
3340 cases happen, the original statement can be removed. Most of this is done
3341 by load_assign_lhs_subreplacements.
3343 In the second case, we would like to store all RHS scalarized components
3344 directly into LHS and if they cover the aggregate completely, remove the
3345 statement too. In the third case, we want the LHS components to be loaded
3346 directly from the RHS (DSE will remove the original statement if it
3349 This is a bit complex but manageable when types match and when unions do
3350 not cause confusion in a way that we cannot really load a component of LHS
3351 from the RHS or vice versa (the access representing this level can have
3352 subaccesses that are accessible only through a different union field at a
3353 higher level - different from the one used in the examined expression).
3356 Therefore, I specially handle a fourth case, happening when there is a
3357 specific type cast or it is impossible to locate a scalarized subaccess on
3358 the other side of the expression. If that happens, I simply "refresh" the
3359 RHS by storing in it is scalarized components leave the original statement
3360 there to do the copying and then load the scalar replacements of the LHS.
3361 This is what the first branch does. */
3363 if (modify_this_stmt
3364 || gimple_has_volatile_ops (stmt
)
3365 || contains_vce_or_bfcref_p (rhs
)
3366 || contains_vce_or_bfcref_p (lhs
)
3367 || stmt_ends_bb_p (stmt
))
3369 if (access_has_children_p (racc
))
3370 generate_subtree_copies (racc
->first_child
, rhs
, racc
->offset
, 0, 0,
3371 gsi
, false, false, loc
);
3372 if (access_has_children_p (lacc
))
3374 gimple_stmt_iterator alt_gsi
= gsi_none ();
3375 if (stmt_ends_bb_p (stmt
))
3377 alt_gsi
= gsi_start_edge (single_non_eh_succ (gsi_bb (*gsi
)));
3380 generate_subtree_copies (lacc
->first_child
, lhs
, lacc
->offset
, 0, 0,
3381 gsi
, true, true, loc
);
3383 sra_stats
.separate_lhs_rhs_handling
++;
3385 /* This gimplification must be done after generate_subtree_copies,
3386 lest we insert the subtree copies in the middle of the gimplified
3388 if (force_gimple_rhs
)
3389 rhs
= force_gimple_operand_gsi (&orig_gsi
, rhs
, true, NULL_TREE
,
3390 true, GSI_SAME_STMT
);
3391 if (gimple_assign_rhs1 (stmt
) != rhs
)
3393 modify_this_stmt
= true;
3394 gimple_assign_set_rhs_from_tree (&orig_gsi
, rhs
);
3395 gcc_assert (stmt
== gsi_stmt (orig_gsi
));
3398 return modify_this_stmt
? SRA_AM_MODIFIED
: SRA_AM_NONE
;
3402 if (access_has_children_p (lacc
)
3403 && access_has_children_p (racc
)
3404 /* When an access represents an unscalarizable region, it usually
3405 represents accesses with variable offset and thus must not be used
3406 to generate new memory accesses. */
3407 && !lacc
->grp_unscalarizable_region
3408 && !racc
->grp_unscalarizable_region
)
3410 struct subreplacement_assignment_data sad
;
3412 sad
.left_offset
= lacc
->offset
;
3413 sad
.assignment_lhs
= lhs
;
3414 sad
.assignment_rhs
= rhs
;
3415 sad
.top_racc
= racc
;
3418 sad
.loc
= gimple_location (stmt
);
3419 sad
.refreshed
= SRA_UDH_NONE
;
3421 if (lacc
->grp_read
&& !lacc
->grp_covered
)
3422 handle_unscalarized_data_in_subtree (&sad
);
3424 load_assign_lhs_subreplacements (lacc
, &sad
);
3425 if (sad
.refreshed
!= SRA_UDH_RIGHT
)
3428 unlink_stmt_vdef (stmt
);
3429 gsi_remove (&sad
.old_gsi
, true);
3430 release_defs (stmt
);
3431 sra_stats
.deleted
++;
3432 return SRA_AM_REMOVED
;
3437 if (access_has_children_p (racc
)
3438 && !racc
->grp_unscalarized_data
)
3442 fprintf (dump_file
, "Removing load: ");
3443 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3445 generate_subtree_copies (racc
->first_child
, lhs
,
3446 racc
->offset
, 0, 0, gsi
,
3448 gcc_assert (stmt
== gsi_stmt (*gsi
));
3449 unlink_stmt_vdef (stmt
);
3450 gsi_remove (gsi
, true);
3451 release_defs (stmt
);
3452 sra_stats
.deleted
++;
3453 return SRA_AM_REMOVED
;
3455 /* Restore the aggregate RHS from its components so the
3456 prevailing aggregate copy does the right thing. */
3457 if (access_has_children_p (racc
))
3458 generate_subtree_copies (racc
->first_child
, rhs
, racc
->offset
, 0, 0,
3459 gsi
, false, false, loc
);
3460 /* Re-load the components of the aggregate copy destination.
3461 But use the RHS aggregate to load from to expose more
3462 optimization opportunities. */
3463 if (access_has_children_p (lacc
))
3464 generate_subtree_copies (lacc
->first_child
, rhs
, lacc
->offset
,
3465 0, 0, gsi
, true, true, loc
);
3472 /* Traverse the function body and all modifications as decided in
3473 analyze_all_variable_accesses. Return true iff the CFG has been
3477 sra_modify_function_body (void)
3479 bool cfg_changed
= false;
3482 FOR_EACH_BB_FN (bb
, cfun
)
3484 gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
3485 while (!gsi_end_p (gsi
))
3487 gimple
*stmt
= gsi_stmt (gsi
);
3488 enum assignment_mod_result assign_result
;
3489 bool modified
= false, deleted
= false;
3493 switch (gimple_code (stmt
))
3496 t
= gimple_return_retval_ptr (as_a
<greturn
*> (stmt
));
3497 if (*t
!= NULL_TREE
)
3498 modified
|= sra_modify_expr (t
, &gsi
, false);
3502 assign_result
= sra_modify_assign (stmt
, &gsi
);
3503 modified
|= assign_result
== SRA_AM_MODIFIED
;
3504 deleted
= assign_result
== SRA_AM_REMOVED
;
3508 /* Operands must be processed before the lhs. */
3509 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
3511 t
= gimple_call_arg_ptr (stmt
, i
);
3512 modified
|= sra_modify_expr (t
, &gsi
, false);
3515 if (gimple_call_lhs (stmt
))
3517 t
= gimple_call_lhs_ptr (stmt
);
3518 modified
|= sra_modify_expr (t
, &gsi
, true);
3524 gasm
*asm_stmt
= as_a
<gasm
*> (stmt
);
3525 for (i
= 0; i
< gimple_asm_ninputs (asm_stmt
); i
++)
3527 t
= &TREE_VALUE (gimple_asm_input_op (asm_stmt
, i
));
3528 modified
|= sra_modify_expr (t
, &gsi
, false);
3530 for (i
= 0; i
< gimple_asm_noutputs (asm_stmt
); i
++)
3532 t
= &TREE_VALUE (gimple_asm_output_op (asm_stmt
, i
));
3533 modified
|= sra_modify_expr (t
, &gsi
, true);
3545 if (maybe_clean_eh_stmt (stmt
)
3546 && gimple_purge_dead_eh_edges (gimple_bb (stmt
)))
3554 gsi_commit_edge_inserts ();
3558 /* Generate statements initializing scalar replacements of parts of function
3562 initialize_parameter_reductions (void)
3564 gimple_stmt_iterator gsi
;
3565 gimple_seq seq
= NULL
;
3568 gsi
= gsi_start (seq
);
3569 for (parm
= DECL_ARGUMENTS (current_function_decl
);
3571 parm
= DECL_CHAIN (parm
))
3573 vec
<access_p
> *access_vec
;
3574 struct access
*access
;
3576 if (!bitmap_bit_p (candidate_bitmap
, DECL_UID (parm
)))
3578 access_vec
= get_base_access_vector (parm
);
3582 for (access
= (*access_vec
)[0];
3584 access
= access
->next_grp
)
3585 generate_subtree_copies (access
, parm
, 0, 0, 0, &gsi
, true, true,
3586 EXPR_LOCATION (parm
));
3589 seq
= gsi_seq (gsi
);
3591 gsi_insert_seq_on_edge_immediate (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
3594 /* The "main" function of intraprocedural SRA passes. Runs the analysis and if
3595 it reveals there are components of some aggregates to be scalarized, it runs
3596 the required transformations. */
3598 perform_intra_sra (void)
3603 if (!find_var_candidates ())
3606 if (!scan_function ())
3609 if (!analyze_all_variable_accesses ())
3612 if (sra_modify_function_body ())
3613 ret
= TODO_update_ssa
| TODO_cleanup_cfg
;
3615 ret
= TODO_update_ssa
;
3616 initialize_parameter_reductions ();
3618 statistics_counter_event (cfun
, "Scalar replacements created",
3619 sra_stats
.replacements
);
3620 statistics_counter_event (cfun
, "Modified expressions", sra_stats
.exprs
);
3621 statistics_counter_event (cfun
, "Subtree copy stmts",
3622 sra_stats
.subtree_copies
);
3623 statistics_counter_event (cfun
, "Subreplacement stmts",
3624 sra_stats
.subreplacements
);
3625 statistics_counter_event (cfun
, "Deleted stmts", sra_stats
.deleted
);
3626 statistics_counter_event (cfun
, "Separate LHS and RHS handling",
3627 sra_stats
.separate_lhs_rhs_handling
);
3630 sra_deinitialize ();
3634 /* Perform early intraprocedural SRA. */
3636 early_intra_sra (void)
3638 sra_mode
= SRA_MODE_EARLY_INTRA
;
3639 return perform_intra_sra ();
3642 /* Perform "late" intraprocedural SRA. */
3644 late_intra_sra (void)
3646 sra_mode
= SRA_MODE_INTRA
;
3647 return perform_intra_sra ();
3652 gate_intra_sra (void)
3654 return flag_tree_sra
!= 0 && dbg_cnt (tree_sra
);
3660 const pass_data pass_data_sra_early
=
3662 GIMPLE_PASS
, /* type */
3664 OPTGROUP_NONE
, /* optinfo_flags */
3665 TV_TREE_SRA
, /* tv_id */
3666 ( PROP_cfg
| PROP_ssa
), /* properties_required */
3667 0, /* properties_provided */
3668 0, /* properties_destroyed */
3669 0, /* todo_flags_start */
3670 TODO_update_ssa
, /* todo_flags_finish */
3673 class pass_sra_early
: public gimple_opt_pass
3676 pass_sra_early (gcc::context
*ctxt
)
3677 : gimple_opt_pass (pass_data_sra_early
, ctxt
)
3680 /* opt_pass methods: */
3681 virtual bool gate (function
*) { return gate_intra_sra (); }
3682 virtual unsigned int execute (function
*) { return early_intra_sra (); }
3684 }; // class pass_sra_early
3689 make_pass_sra_early (gcc::context
*ctxt
)
3691 return new pass_sra_early (ctxt
);
3696 const pass_data pass_data_sra
=
3698 GIMPLE_PASS
, /* type */
3700 OPTGROUP_NONE
, /* optinfo_flags */
3701 TV_TREE_SRA
, /* tv_id */
3702 ( PROP_cfg
| PROP_ssa
), /* properties_required */
3703 0, /* properties_provided */
3704 0, /* properties_destroyed */
3705 TODO_update_address_taken
, /* todo_flags_start */
3706 TODO_update_ssa
, /* todo_flags_finish */
3709 class pass_sra
: public gimple_opt_pass
3712 pass_sra (gcc::context
*ctxt
)
3713 : gimple_opt_pass (pass_data_sra
, ctxt
)
3716 /* opt_pass methods: */
3717 virtual bool gate (function
*) { return gate_intra_sra (); }
3718 virtual unsigned int execute (function
*) { return late_intra_sra (); }
3720 }; // class pass_sra
3725 make_pass_sra (gcc::context
*ctxt
)
3727 return new pass_sra (ctxt
);
3731 /* Return true iff PARM (which must be a parm_decl) is an unused scalar
3735 is_unused_scalar_param (tree parm
)
3738 return (is_gimple_reg (parm
)
3739 && (!(name
= ssa_default_def (cfun
, parm
))
3740 || has_zero_uses (name
)));
3743 /* Scan immediate uses of a default definition SSA name of a parameter PARM and
3744 examine whether there are any direct or otherwise infeasible ones. If so,
3745 return true, otherwise return false. PARM must be a gimple register with a
3746 non-NULL default definition. */
3749 ptr_parm_has_direct_uses (tree parm
)
3751 imm_use_iterator ui
;
3753 tree name
= ssa_default_def (cfun
, parm
);
3756 FOR_EACH_IMM_USE_STMT (stmt
, ui
, name
)
3759 use_operand_p use_p
;
3761 if (is_gimple_debug (stmt
))
3764 /* Valid uses include dereferences on the lhs and the rhs. */
3765 if (gimple_has_lhs (stmt
))
3767 tree lhs
= gimple_get_lhs (stmt
);
3768 while (handled_component_p (lhs
))
3769 lhs
= TREE_OPERAND (lhs
, 0);
3770 if (TREE_CODE (lhs
) == MEM_REF
3771 && TREE_OPERAND (lhs
, 0) == name
3772 && integer_zerop (TREE_OPERAND (lhs
, 1))
3773 && types_compatible_p (TREE_TYPE (lhs
),
3774 TREE_TYPE (TREE_TYPE (name
)))
3775 && !TREE_THIS_VOLATILE (lhs
))
3778 if (gimple_assign_single_p (stmt
))
3780 tree rhs
= gimple_assign_rhs1 (stmt
);
3781 while (handled_component_p (rhs
))
3782 rhs
= TREE_OPERAND (rhs
, 0);
3783 if (TREE_CODE (rhs
) == MEM_REF
3784 && TREE_OPERAND (rhs
, 0) == name
3785 && integer_zerop (TREE_OPERAND (rhs
, 1))
3786 && types_compatible_p (TREE_TYPE (rhs
),
3787 TREE_TYPE (TREE_TYPE (name
)))
3788 && !TREE_THIS_VOLATILE (rhs
))
3791 else if (is_gimple_call (stmt
))
3794 for (i
= 0; i
< gimple_call_num_args (stmt
); ++i
)
3796 tree arg
= gimple_call_arg (stmt
, i
);
3797 while (handled_component_p (arg
))
3798 arg
= TREE_OPERAND (arg
, 0);
3799 if (TREE_CODE (arg
) == MEM_REF
3800 && TREE_OPERAND (arg
, 0) == name
3801 && integer_zerop (TREE_OPERAND (arg
, 1))
3802 && types_compatible_p (TREE_TYPE (arg
),
3803 TREE_TYPE (TREE_TYPE (name
)))
3804 && !TREE_THIS_VOLATILE (arg
))
3809 /* If the number of valid uses does not match the number of
3810 uses in this stmt there is an unhandled use. */
3811 FOR_EACH_IMM_USE_ON_STMT (use_p
, ui
)
3818 BREAK_FROM_IMM_USE_STMT (ui
);
3824 /* Identify candidates for reduction for IPA-SRA based on their type and mark
3825 them in candidate_bitmap. Note that these do not necessarily include
3826 parameter which are unused and thus can be removed. Return true iff any
3827 such candidate has been found. */
3830 find_param_candidates (void)
3837 for (parm
= DECL_ARGUMENTS (current_function_decl
);
3839 parm
= DECL_CHAIN (parm
))
3841 tree type
= TREE_TYPE (parm
);
3846 if (TREE_THIS_VOLATILE (parm
)
3847 || TREE_ADDRESSABLE (parm
)
3848 || (!is_gimple_reg_type (type
) && is_va_list_type (type
)))
3851 if (is_unused_scalar_param (parm
))
3857 if (POINTER_TYPE_P (type
))
3859 type
= TREE_TYPE (type
);
3861 if (TREE_CODE (type
) == FUNCTION_TYPE
3862 || TYPE_VOLATILE (type
)
3863 || (TREE_CODE (type
) == ARRAY_TYPE
3864 && TYPE_NONALIASED_COMPONENT (type
))
3865 || !is_gimple_reg (parm
)
3866 || is_va_list_type (type
)
3867 || ptr_parm_has_direct_uses (parm
))
3870 else if (!AGGREGATE_TYPE_P (type
))
3873 if (!COMPLETE_TYPE_P (type
)
3874 || !tree_fits_uhwi_p (TYPE_SIZE (type
))
3875 || tree_to_uhwi (TYPE_SIZE (type
)) == 0
3876 || (AGGREGATE_TYPE_P (type
)
3877 && type_internals_preclude_sra_p (type
, &msg
)))
3880 bitmap_set_bit (candidate_bitmap
, DECL_UID (parm
));
3881 slot
= candidates
->find_slot_with_hash (parm
, DECL_UID (parm
), INSERT
);
3885 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3887 fprintf (dump_file
, "Candidate (%d): ", DECL_UID (parm
));
3888 print_generic_expr (dump_file
, parm
, 0);
3889 fprintf (dump_file
, "\n");
3893 func_param_count
= count
;
3897 /* Callback of walk_aliased_vdefs, marks the access passed as DATA as
3901 mark_maybe_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
3904 struct access
*repr
= (struct access
*) data
;
3906 repr
->grp_maybe_modified
= 1;
3910 /* Analyze what representatives (in linked lists accessible from
3911 REPRESENTATIVES) can be modified by side effects of statements in the
3912 current function. */
3915 analyze_modified_params (vec
<access_p
> representatives
)
3919 for (i
= 0; i
< func_param_count
; i
++)
3921 struct access
*repr
;
3923 for (repr
= representatives
[i
];
3925 repr
= repr
->next_grp
)
3927 struct access
*access
;
3931 if (no_accesses_p (repr
))
3933 if (!POINTER_TYPE_P (TREE_TYPE (repr
->base
))
3934 || repr
->grp_maybe_modified
)
3937 ao_ref_init (&ar
, repr
->expr
);
3938 visited
= BITMAP_ALLOC (NULL
);
3939 for (access
= repr
; access
; access
= access
->next_sibling
)
3941 /* All accesses are read ones, otherwise grp_maybe_modified would
3942 be trivially set. */
3943 walk_aliased_vdefs (&ar
, gimple_vuse (access
->stmt
),
3944 mark_maybe_modified
, repr
, &visited
);
3945 if (repr
->grp_maybe_modified
)
3948 BITMAP_FREE (visited
);
3953 /* Propagate distances in bb_dereferences in the opposite direction than the
3954 control flow edges, in each step storing the maximum of the current value
3955 and the minimum of all successors. These steps are repeated until the table
3956 stabilizes. Note that BBs which might terminate the functions (according to
3957 final_bbs bitmap) never updated in this way. */
3960 propagate_dereference_distances (void)
3964 auto_vec
<basic_block
> queue (last_basic_block_for_fn (cfun
));
3965 queue
.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
3966 FOR_EACH_BB_FN (bb
, cfun
)
3968 queue
.quick_push (bb
);
3972 while (!queue
.is_empty ())
3976 bool change
= false;
3982 if (bitmap_bit_p (final_bbs
, bb
->index
))
3985 for (i
= 0; i
< func_param_count
; i
++)
3987 int idx
= bb
->index
* func_param_count
+ i
;
3989 HOST_WIDE_INT inh
= 0;
3991 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3993 int succ_idx
= e
->dest
->index
* func_param_count
+ i
;
3995 if (e
->src
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
4001 inh
= bb_dereferences
[succ_idx
];
4003 else if (bb_dereferences
[succ_idx
] < inh
)
4004 inh
= bb_dereferences
[succ_idx
];
4007 if (!first
&& bb_dereferences
[idx
] < inh
)
4009 bb_dereferences
[idx
] = inh
;
4014 if (change
&& !bitmap_bit_p (final_bbs
, bb
->index
))
4015 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
4020 e
->src
->aux
= e
->src
;
4021 queue
.quick_push (e
->src
);
4026 /* Dump a dereferences TABLE with heading STR to file F. */
4029 dump_dereferences_table (FILE *f
, const char *str
, HOST_WIDE_INT
*table
)
4033 fprintf (dump_file
, "%s", str
);
4034 FOR_BB_BETWEEN (bb
, ENTRY_BLOCK_PTR_FOR_FN (cfun
),
4035 EXIT_BLOCK_PTR_FOR_FN (cfun
), next_bb
)
4037 fprintf (f
, "%4i %i ", bb
->index
, bitmap_bit_p (final_bbs
, bb
->index
));
4038 if (bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
4041 for (i
= 0; i
< func_param_count
; i
++)
4043 int idx
= bb
->index
* func_param_count
+ i
;
4044 fprintf (f
, " %4" HOST_WIDE_INT_PRINT
"d", table
[idx
]);
4049 fprintf (dump_file
, "\n");
4052 /* Determine what (parts of) parameters passed by reference that are not
4053 assigned to are not certainly dereferenced in this function and thus the
4054 dereferencing cannot be safely moved to the caller without potentially
4055 introducing a segfault. Mark such REPRESENTATIVES as
4056 grp_not_necessarilly_dereferenced.
4058 The dereferenced maximum "distance," i.e. the offset + size of the accessed
4059 part is calculated rather than simple booleans are calculated for each
4060 pointer parameter to handle cases when only a fraction of the whole
4061 aggregate is allocated (see testsuite/gcc.c-torture/execute/ipa-sra-2.c for
4064 The maximum dereference distances for each pointer parameter and BB are
4065 already stored in bb_dereference. This routine simply propagates these
4066 values upwards by propagate_dereference_distances and then compares the
4067 distances of individual parameters in the ENTRY BB to the equivalent
4068 distances of each representative of a (fraction of a) parameter. */
4071 analyze_caller_dereference_legality (vec
<access_p
> representatives
)
4075 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
4076 dump_dereferences_table (dump_file
,
4077 "Dereference table before propagation:\n",
4080 propagate_dereference_distances ();
4082 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
4083 dump_dereferences_table (dump_file
,
4084 "Dereference table after propagation:\n",
4087 for (i
= 0; i
< func_param_count
; i
++)
4089 struct access
*repr
= representatives
[i
];
4090 int idx
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
* func_param_count
+ i
;
4092 if (!repr
|| no_accesses_p (repr
))
4097 if ((repr
->offset
+ repr
->size
) > bb_dereferences
[idx
])
4098 repr
->grp_not_necessarilly_dereferenced
= 1;
4099 repr
= repr
->next_grp
;
4105 /* Return the representative access for the parameter declaration PARM if it is
4106 a scalar passed by reference which is not written to and the pointer value
4107 is not used directly. Thus, if it is legal to dereference it in the caller
4108 and we can rule out modifications through aliases, such parameter should be
4109 turned into one passed by value. Return NULL otherwise. */
4111 static struct access
*
4112 unmodified_by_ref_scalar_representative (tree parm
)
4114 int i
, access_count
;
4115 struct access
*repr
;
4116 vec
<access_p
> *access_vec
;
4118 access_vec
= get_base_access_vector (parm
);
4119 gcc_assert (access_vec
);
4120 repr
= (*access_vec
)[0];
4123 repr
->group_representative
= repr
;
4125 access_count
= access_vec
->length ();
4126 for (i
= 1; i
< access_count
; i
++)
4128 struct access
*access
= (*access_vec
)[i
];
4131 access
->group_representative
= repr
;
4132 access
->next_sibling
= repr
->next_sibling
;
4133 repr
->next_sibling
= access
;
4137 repr
->grp_scalar_ptr
= 1;
4141 /* Return true iff this ACCESS precludes IPA-SRA of the parameter it is
4142 associated with. REQ_ALIGN is the minimum required alignment. */
4145 access_precludes_ipa_sra_p (struct access
*access
, unsigned int req_align
)
4147 unsigned int exp_align
;
4148 /* Avoid issues such as the second simple testcase in PR 42025. The problem
4149 is incompatible assign in a call statement (and possibly even in asm
4150 statements). This can be relaxed by using a new temporary but only for
4151 non-TREE_ADDRESSABLE types and is probably not worth the complexity. (In
4152 intraprocedural SRA we deal with this by keeping the old aggregate around,
4153 something we cannot do in IPA-SRA.) */
4155 && (is_gimple_call (access
->stmt
)
4156 || gimple_code (access
->stmt
) == GIMPLE_ASM
))
4159 exp_align
= get_object_alignment (access
->expr
);
4160 if (exp_align
< req_align
)
4167 /* Sort collected accesses for parameter PARM, identify representatives for
4168 each accessed region and link them together. Return NULL if there are
4169 different but overlapping accesses, return the special ptr value meaning
4170 there are no accesses for this parameter if that is the case and return the
4171 first representative otherwise. Set *RO_GRP if there is a group of accesses
4172 with only read (i.e. no write) accesses. */
4174 static struct access
*
4175 splice_param_accesses (tree parm
, bool *ro_grp
)
4177 int i
, j
, access_count
, group_count
;
4178 int agg_size
, total_size
= 0;
4179 struct access
*access
, *res
, **prev_acc_ptr
= &res
;
4180 vec
<access_p
> *access_vec
;
4182 access_vec
= get_base_access_vector (parm
);
4184 return &no_accesses_representant
;
4185 access_count
= access_vec
->length ();
4187 access_vec
->qsort (compare_access_positions
);
4192 while (i
< access_count
)
4196 access
= (*access_vec
)[i
];
4197 modification
= access
->write
;
4198 if (access_precludes_ipa_sra_p (access
, TYPE_ALIGN (access
->type
)))
4200 a1_alias_type
= reference_alias_ptr_type (access
->expr
);
4202 /* Access is about to become group representative unless we find some
4203 nasty overlap which would preclude us from breaking this parameter
4207 while (j
< access_count
)
4209 struct access
*ac2
= (*access_vec
)[j
];
4210 if (ac2
->offset
!= access
->offset
)
4212 /* All or nothing law for parameters. */
4213 if (access
->offset
+ access
->size
> ac2
->offset
)
4218 else if (ac2
->size
!= access
->size
)
4221 if (access_precludes_ipa_sra_p (ac2
, TYPE_ALIGN (access
->type
))
4222 || (ac2
->type
!= access
->type
4223 && (TREE_ADDRESSABLE (ac2
->type
)
4224 || TREE_ADDRESSABLE (access
->type
)))
4225 || (reference_alias_ptr_type (ac2
->expr
) != a1_alias_type
))
4228 modification
|= ac2
->write
;
4229 ac2
->group_representative
= access
;
4230 ac2
->next_sibling
= access
->next_sibling
;
4231 access
->next_sibling
= ac2
;
4236 access
->grp_maybe_modified
= modification
;
4239 *prev_acc_ptr
= access
;
4240 prev_acc_ptr
= &access
->next_grp
;
4241 total_size
+= access
->size
;
4245 if (POINTER_TYPE_P (TREE_TYPE (parm
)))
4246 agg_size
= tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm
))));
4248 agg_size
= tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm
)));
4249 if (total_size
>= agg_size
)
4252 gcc_assert (group_count
> 0);
4256 /* Decide whether parameters with representative accesses given by REPR should
4257 be reduced into components. */
4260 decide_one_param_reduction (struct access
*repr
)
4262 int total_size
, cur_parm_size
, agg_size
, new_param_count
, parm_size_limit
;
4267 cur_parm_size
= tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm
)));
4268 gcc_assert (cur_parm_size
> 0);
4270 if (POINTER_TYPE_P (TREE_TYPE (parm
)))
4273 agg_size
= tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (parm
))));
4278 agg_size
= cur_parm_size
;
4284 fprintf (dump_file
, "Evaluating PARAM group sizes for ");
4285 print_generic_expr (dump_file
, parm
, 0);
4286 fprintf (dump_file
, " (UID: %u): \n", DECL_UID (parm
));
4287 for (acc
= repr
; acc
; acc
= acc
->next_grp
)
4288 dump_access (dump_file
, acc
, true);
4292 new_param_count
= 0;
4294 for (; repr
; repr
= repr
->next_grp
)
4296 gcc_assert (parm
== repr
->base
);
4298 /* Taking the address of a non-addressable field is verboten. */
4299 if (by_ref
&& repr
->non_addressable
)
4302 /* Do not decompose a non-BLKmode param in a way that would
4303 create BLKmode params. Especially for by-reference passing
4304 (thus, pointer-type param) this is hardly worthwhile. */
4305 if (DECL_MODE (parm
) != BLKmode
4306 && TYPE_MODE (repr
->type
) == BLKmode
)
4309 if (!by_ref
|| (!repr
->grp_maybe_modified
4310 && !repr
->grp_not_necessarilly_dereferenced
))
4311 total_size
+= repr
->size
;
4313 total_size
+= cur_parm_size
;
4318 gcc_assert (new_param_count
> 0);
4320 if (optimize_function_for_size_p (cfun
))
4321 parm_size_limit
= cur_parm_size
;
4323 parm_size_limit
= (PARAM_VALUE (PARAM_IPA_SRA_PTR_GROWTH_FACTOR
)
4326 if (total_size
< agg_size
4327 && total_size
<= parm_size_limit
)
4330 fprintf (dump_file
, " ....will be split into %i components\n",
4332 return new_param_count
;
4338 /* The order of the following enums is important, we need to do extra work for
4339 UNUSED_PARAMS, BY_VAL_ACCESSES and UNMODIF_BY_REF_ACCESSES. */
4340 enum ipa_splicing_result
{ NO_GOOD_ACCESS
, UNUSED_PARAMS
, BY_VAL_ACCESSES
,
4341 MODIF_BY_REF_ACCESSES
, UNMODIF_BY_REF_ACCESSES
};
4343 /* Identify representatives of all accesses to all candidate parameters for
4344 IPA-SRA. Return result based on what representatives have been found. */
4346 static enum ipa_splicing_result
4347 splice_all_param_accesses (vec
<access_p
> &representatives
)
4349 enum ipa_splicing_result result
= NO_GOOD_ACCESS
;
4351 struct access
*repr
;
4353 representatives
.create (func_param_count
);
4355 for (parm
= DECL_ARGUMENTS (current_function_decl
);
4357 parm
= DECL_CHAIN (parm
))
4359 if (is_unused_scalar_param (parm
))
4361 representatives
.quick_push (&no_accesses_representant
);
4362 if (result
== NO_GOOD_ACCESS
)
4363 result
= UNUSED_PARAMS
;
4365 else if (POINTER_TYPE_P (TREE_TYPE (parm
))
4366 && is_gimple_reg_type (TREE_TYPE (TREE_TYPE (parm
)))
4367 && bitmap_bit_p (candidate_bitmap
, DECL_UID (parm
)))
4369 repr
= unmodified_by_ref_scalar_representative (parm
);
4370 representatives
.quick_push (repr
);
4372 result
= UNMODIF_BY_REF_ACCESSES
;
4374 else if (bitmap_bit_p (candidate_bitmap
, DECL_UID (parm
)))
4376 bool ro_grp
= false;
4377 repr
= splice_param_accesses (parm
, &ro_grp
);
4378 representatives
.quick_push (repr
);
4380 if (repr
&& !no_accesses_p (repr
))
4382 if (POINTER_TYPE_P (TREE_TYPE (parm
)))
4385 result
= UNMODIF_BY_REF_ACCESSES
;
4386 else if (result
< MODIF_BY_REF_ACCESSES
)
4387 result
= MODIF_BY_REF_ACCESSES
;
4389 else if (result
< BY_VAL_ACCESSES
)
4390 result
= BY_VAL_ACCESSES
;
4392 else if (no_accesses_p (repr
) && (result
== NO_GOOD_ACCESS
))
4393 result
= UNUSED_PARAMS
;
4396 representatives
.quick_push (NULL
);
4399 if (result
== NO_GOOD_ACCESS
)
4401 representatives
.release ();
4402 return NO_GOOD_ACCESS
;
4408 /* Return the index of BASE in PARMS. Abort if it is not found. */
4411 get_param_index (tree base
, vec
<tree
> parms
)
4415 len
= parms
.length ();
4416 for (i
= 0; i
< len
; i
++)
4417 if (parms
[i
] == base
)
4422 /* Convert the decisions made at the representative level into compact
4423 parameter adjustments. REPRESENTATIVES are pointers to first
4424 representatives of each param accesses, ADJUSTMENTS_COUNT is the expected
4425 final number of adjustments. */
4427 static ipa_parm_adjustment_vec
4428 turn_representatives_into_adjustments (vec
<access_p
> representatives
,
4429 int adjustments_count
)
4432 ipa_parm_adjustment_vec adjustments
;
4436 gcc_assert (adjustments_count
> 0);
4437 parms
= ipa_get_vector_of_formal_parms (current_function_decl
);
4438 adjustments
.create (adjustments_count
);
4439 parm
= DECL_ARGUMENTS (current_function_decl
);
4440 for (i
= 0; i
< func_param_count
; i
++, parm
= DECL_CHAIN (parm
))
4442 struct access
*repr
= representatives
[i
];
4444 if (!repr
|| no_accesses_p (repr
))
4446 struct ipa_parm_adjustment adj
;
4448 memset (&adj
, 0, sizeof (adj
));
4449 adj
.base_index
= get_param_index (parm
, parms
);
4452 adj
.op
= IPA_PARM_OP_COPY
;
4454 adj
.op
= IPA_PARM_OP_REMOVE
;
4455 adj
.arg_prefix
= "ISRA";
4456 adjustments
.quick_push (adj
);
4460 struct ipa_parm_adjustment adj
;
4461 int index
= get_param_index (parm
, parms
);
4463 for (; repr
; repr
= repr
->next_grp
)
4465 memset (&adj
, 0, sizeof (adj
));
4466 gcc_assert (repr
->base
== parm
);
4467 adj
.base_index
= index
;
4468 adj
.base
= repr
->base
;
4469 adj
.type
= repr
->type
;
4470 adj
.alias_ptr_type
= reference_alias_ptr_type (repr
->expr
);
4471 adj
.offset
= repr
->offset
;
4472 adj
.by_ref
= (POINTER_TYPE_P (TREE_TYPE (repr
->base
))
4473 && (repr
->grp_maybe_modified
4474 || repr
->grp_not_necessarilly_dereferenced
));
4475 adj
.arg_prefix
= "ISRA";
4476 adjustments
.quick_push (adj
);
4484 /* Analyze the collected accesses and produce a plan what to do with the
4485 parameters in the form of adjustments, NULL meaning nothing. */
4487 static ipa_parm_adjustment_vec
4488 analyze_all_param_acesses (void)
4490 enum ipa_splicing_result repr_state
;
4491 bool proceed
= false;
4492 int i
, adjustments_count
= 0;
4493 vec
<access_p
> representatives
;
4494 ipa_parm_adjustment_vec adjustments
;
4496 repr_state
= splice_all_param_accesses (representatives
);
4497 if (repr_state
== NO_GOOD_ACCESS
)
4498 return ipa_parm_adjustment_vec ();
4500 /* If there are any parameters passed by reference which are not modified
4501 directly, we need to check whether they can be modified indirectly. */
4502 if (repr_state
== UNMODIF_BY_REF_ACCESSES
)
4504 analyze_caller_dereference_legality (representatives
);
4505 analyze_modified_params (representatives
);
4508 for (i
= 0; i
< func_param_count
; i
++)
4510 struct access
*repr
= representatives
[i
];
4512 if (repr
&& !no_accesses_p (repr
))
4514 if (repr
->grp_scalar_ptr
)
4516 adjustments_count
++;
4517 if (repr
->grp_not_necessarilly_dereferenced
4518 || repr
->grp_maybe_modified
)
4519 representatives
[i
] = NULL
;
4523 sra_stats
.scalar_by_ref_to_by_val
++;
4528 int new_components
= decide_one_param_reduction (repr
);
4530 if (new_components
== 0)
4532 representatives
[i
] = NULL
;
4533 adjustments_count
++;
4537 adjustments_count
+= new_components
;
4538 sra_stats
.aggregate_params_reduced
++;
4539 sra_stats
.param_reductions_created
+= new_components
;
4546 if (no_accesses_p (repr
))
4549 sra_stats
.deleted_unused_parameters
++;
4551 adjustments_count
++;
4555 if (!proceed
&& dump_file
)
4556 fprintf (dump_file
, "NOT proceeding to change params.\n");
4559 adjustments
= turn_representatives_into_adjustments (representatives
,
4562 adjustments
= ipa_parm_adjustment_vec ();
4564 representatives
.release ();
4568 /* If a parameter replacement identified by ADJ does not yet exist in the form
4569 of declaration, create it and record it, otherwise return the previously
4573 get_replaced_param_substitute (struct ipa_parm_adjustment
*adj
)
4576 if (!adj
->new_ssa_base
)
4578 char *pretty_name
= make_fancy_name (adj
->base
);
4580 repl
= create_tmp_reg (TREE_TYPE (adj
->base
), "ISR");
4581 DECL_NAME (repl
) = get_identifier (pretty_name
);
4582 obstack_free (&name_obstack
, pretty_name
);
4584 adj
->new_ssa_base
= repl
;
4587 repl
= adj
->new_ssa_base
;
4591 /* Find the first adjustment for a particular parameter BASE in a vector of
4592 ADJUSTMENTS which is not a copy_param. Return NULL if there is no such
4595 static struct ipa_parm_adjustment
*
4596 get_adjustment_for_base (ipa_parm_adjustment_vec adjustments
, tree base
)
4600 len
= adjustments
.length ();
4601 for (i
= 0; i
< len
; i
++)
4603 struct ipa_parm_adjustment
*adj
;
4605 adj
= &adjustments
[i
];
4606 if (adj
->op
!= IPA_PARM_OP_COPY
&& adj
->base
== base
)
4613 /* If OLD_NAME, which is being defined by statement STMT, is an SSA_NAME of a
4614 parameter which is to be removed because its value is not used, create a new
4615 SSA_NAME relating to a replacement VAR_DECL, replace all uses of the
4616 original with it and return it. If there is no need to re-map, return NULL.
4617 ADJUSTMENTS is a pointer to a vector of IPA-SRA adjustments. */
4620 replace_removed_params_ssa_names (tree old_name
, gimple
*stmt
,
4621 ipa_parm_adjustment_vec adjustments
)
4623 struct ipa_parm_adjustment
*adj
;
4624 tree decl
, repl
, new_name
;
4626 if (TREE_CODE (old_name
) != SSA_NAME
)
4629 decl
= SSA_NAME_VAR (old_name
);
4630 if (decl
== NULL_TREE
4631 || TREE_CODE (decl
) != PARM_DECL
)
4634 adj
= get_adjustment_for_base (adjustments
, decl
);
4638 repl
= get_replaced_param_substitute (adj
);
4639 new_name
= make_ssa_name (repl
, stmt
);
4643 fprintf (dump_file
, "replacing an SSA name of a removed param ");
4644 print_generic_expr (dump_file
, old_name
, 0);
4645 fprintf (dump_file
, " with ");
4646 print_generic_expr (dump_file
, new_name
, 0);
4647 fprintf (dump_file
, "\n");
4650 replace_uses_by (old_name
, new_name
);
4654 /* If the statement STMT contains any expressions that need to replaced with a
4655 different one as noted by ADJUSTMENTS, do so. Handle any potential type
4656 incompatibilities (GSI is used to accommodate conversion statements and must
4657 point to the statement). Return true iff the statement was modified. */
4660 sra_ipa_modify_assign (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4661 ipa_parm_adjustment_vec adjustments
)
4663 tree
*lhs_p
, *rhs_p
;
4666 if (!gimple_assign_single_p (stmt
))
4669 rhs_p
= gimple_assign_rhs1_ptr (stmt
);
4670 lhs_p
= gimple_assign_lhs_ptr (stmt
);
4672 any
= ipa_modify_expr (rhs_p
, false, adjustments
);
4673 any
|= ipa_modify_expr (lhs_p
, false, adjustments
);
4676 tree new_rhs
= NULL_TREE
;
4678 if (!useless_type_conversion_p (TREE_TYPE (*lhs_p
), TREE_TYPE (*rhs_p
)))
4680 if (TREE_CODE (*rhs_p
) == CONSTRUCTOR
)
4682 /* V_C_Es of constructors can cause trouble (PR 42714). */
4683 if (is_gimple_reg_type (TREE_TYPE (*lhs_p
)))
4684 *rhs_p
= build_zero_cst (TREE_TYPE (*lhs_p
));
4686 *rhs_p
= build_constructor (TREE_TYPE (*lhs_p
),
4690 new_rhs
= fold_build1_loc (gimple_location (stmt
),
4691 VIEW_CONVERT_EXPR
, TREE_TYPE (*lhs_p
),
4694 else if (REFERENCE_CLASS_P (*rhs_p
)
4695 && is_gimple_reg_type (TREE_TYPE (*lhs_p
))
4696 && !is_gimple_reg (*lhs_p
))
4697 /* This can happen when an assignment in between two single field
4698 structures is turned into an assignment in between two pointers to
4699 scalars (PR 42237). */
4704 tree tmp
= force_gimple_operand_gsi (gsi
, new_rhs
, true, NULL_TREE
,
4705 true, GSI_SAME_STMT
);
4707 gimple_assign_set_rhs_from_tree (gsi
, tmp
);
4716 /* Traverse the function body and all modifications as described in
4717 ADJUSTMENTS. Return true iff the CFG has been changed. */
4720 ipa_sra_modify_function_body (ipa_parm_adjustment_vec adjustments
)
4722 bool cfg_changed
= false;
4725 FOR_EACH_BB_FN (bb
, cfun
)
4727 gimple_stmt_iterator gsi
;
4729 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4731 gphi
*phi
= as_a
<gphi
*> (gsi_stmt (gsi
));
4732 tree new_lhs
, old_lhs
= gimple_phi_result (phi
);
4733 new_lhs
= replace_removed_params_ssa_names (old_lhs
, phi
, adjustments
);
4736 gimple_phi_set_result (phi
, new_lhs
);
4737 release_ssa_name (old_lhs
);
4741 gsi
= gsi_start_bb (bb
);
4742 while (!gsi_end_p (gsi
))
4744 gimple
*stmt
= gsi_stmt (gsi
);
4745 bool modified
= false;
4749 switch (gimple_code (stmt
))
4752 t
= gimple_return_retval_ptr (as_a
<greturn
*> (stmt
));
4753 if (*t
!= NULL_TREE
)
4754 modified
|= ipa_modify_expr (t
, true, adjustments
);
4758 modified
|= sra_ipa_modify_assign (stmt
, &gsi
, adjustments
);
4762 /* Operands must be processed before the lhs. */
4763 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
4765 t
= gimple_call_arg_ptr (stmt
, i
);
4766 modified
|= ipa_modify_expr (t
, true, adjustments
);
4769 if (gimple_call_lhs (stmt
))
4771 t
= gimple_call_lhs_ptr (stmt
);
4772 modified
|= ipa_modify_expr (t
, false, adjustments
);
4778 gasm
*asm_stmt
= as_a
<gasm
*> (stmt
);
4779 for (i
= 0; i
< gimple_asm_ninputs (asm_stmt
); i
++)
4781 t
= &TREE_VALUE (gimple_asm_input_op (asm_stmt
, i
));
4782 modified
|= ipa_modify_expr (t
, true, adjustments
);
4784 for (i
= 0; i
< gimple_asm_noutputs (asm_stmt
); i
++)
4786 t
= &TREE_VALUE (gimple_asm_output_op (asm_stmt
, i
));
4787 modified
|= ipa_modify_expr (t
, false, adjustments
);
4798 FOR_EACH_SSA_DEF_OPERAND (defp
, stmt
, iter
, SSA_OP_DEF
)
4800 tree old_def
= DEF_FROM_PTR (defp
);
4801 if (tree new_def
= replace_removed_params_ssa_names (old_def
, stmt
,
4804 SET_DEF (defp
, new_def
);
4805 release_ssa_name (old_def
);
4813 if (maybe_clean_eh_stmt (stmt
)
4814 && gimple_purge_dead_eh_edges (gimple_bb (stmt
)))
4824 /* Call gimple_debug_bind_reset_value on all debug statements describing
4825 gimple register parameters that are being removed or replaced. */
4828 sra_ipa_reset_debug_stmts (ipa_parm_adjustment_vec adjustments
)
4831 gimple_stmt_iterator
*gsip
= NULL
, gsi
;
4833 if (MAY_HAVE_DEBUG_STMTS
&& single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun
)))
4835 gsi
= gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4838 len
= adjustments
.length ();
4839 for (i
= 0; i
< len
; i
++)
4841 struct ipa_parm_adjustment
*adj
;
4842 imm_use_iterator ui
;
4845 tree name
, vexpr
, copy
= NULL_TREE
;
4846 use_operand_p use_p
;
4848 adj
= &adjustments
[i
];
4849 if (adj
->op
== IPA_PARM_OP_COPY
|| !is_gimple_reg (adj
->base
))
4851 name
= ssa_default_def (cfun
, adj
->base
);
4854 FOR_EACH_IMM_USE_STMT (stmt
, ui
, name
)
4856 if (gimple_clobber_p (stmt
))
4858 gimple_stmt_iterator cgsi
= gsi_for_stmt (stmt
);
4859 unlink_stmt_vdef (stmt
);
4860 gsi_remove (&cgsi
, true);
4861 release_defs (stmt
);
4864 /* All other users must have been removed by
4865 ipa_sra_modify_function_body. */
4866 gcc_assert (is_gimple_debug (stmt
));
4867 if (vexpr
== NULL
&& gsip
!= NULL
)
4869 gcc_assert (TREE_CODE (adj
->base
) == PARM_DECL
);
4870 vexpr
= make_node (DEBUG_EXPR_DECL
);
4871 def_temp
= gimple_build_debug_source_bind (vexpr
, adj
->base
,
4873 DECL_ARTIFICIAL (vexpr
) = 1;
4874 TREE_TYPE (vexpr
) = TREE_TYPE (name
);
4875 DECL_MODE (vexpr
) = DECL_MODE (adj
->base
);
4876 gsi_insert_before (gsip
, def_temp
, GSI_SAME_STMT
);
4880 FOR_EACH_IMM_USE_ON_STMT (use_p
, ui
)
4881 SET_USE (use_p
, vexpr
);
4884 gimple_debug_bind_reset_value (stmt
);
4887 /* Create a VAR_DECL for debug info purposes. */
4888 if (!DECL_IGNORED_P (adj
->base
))
4890 copy
= build_decl (DECL_SOURCE_LOCATION (current_function_decl
),
4891 VAR_DECL
, DECL_NAME (adj
->base
),
4892 TREE_TYPE (adj
->base
));
4893 if (DECL_PT_UID_SET_P (adj
->base
))
4894 SET_DECL_PT_UID (copy
, DECL_PT_UID (adj
->base
));
4895 TREE_ADDRESSABLE (copy
) = TREE_ADDRESSABLE (adj
->base
);
4896 TREE_READONLY (copy
) = TREE_READONLY (adj
->base
);
4897 TREE_THIS_VOLATILE (copy
) = TREE_THIS_VOLATILE (adj
->base
);
4898 DECL_GIMPLE_REG_P (copy
) = DECL_GIMPLE_REG_P (adj
->base
);
4899 DECL_ARTIFICIAL (copy
) = DECL_ARTIFICIAL (adj
->base
);
4900 DECL_IGNORED_P (copy
) = DECL_IGNORED_P (adj
->base
);
4901 DECL_ABSTRACT_ORIGIN (copy
) = DECL_ORIGIN (adj
->base
);
4902 DECL_SEEN_IN_BIND_EXPR_P (copy
) = 1;
4903 SET_DECL_RTL (copy
, 0);
4904 TREE_USED (copy
) = 1;
4905 DECL_CONTEXT (copy
) = current_function_decl
;
4906 add_local_decl (cfun
, copy
);
4908 BLOCK_VARS (DECL_INITIAL (current_function_decl
));
4909 BLOCK_VARS (DECL_INITIAL (current_function_decl
)) = copy
;
4911 if (gsip
!= NULL
&& copy
&& target_for_debug_bind (adj
->base
))
4913 gcc_assert (TREE_CODE (adj
->base
) == PARM_DECL
);
4915 def_temp
= gimple_build_debug_bind (copy
, vexpr
, NULL
);
4917 def_temp
= gimple_build_debug_source_bind (copy
, adj
->base
,
4919 gsi_insert_before (gsip
, def_temp
, GSI_SAME_STMT
);
4924 /* Return false if all callers have at least as many actual arguments as there
4925 are formal parameters in the current function and that their types
4929 some_callers_have_mismatched_arguments_p (struct cgraph_node
*node
,
4930 void *data ATTRIBUTE_UNUSED
)
4932 struct cgraph_edge
*cs
;
4933 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
4934 if (!cs
->call_stmt
|| !callsite_arguments_match_p (cs
->call_stmt
))
4940 /* Return false if all callers have vuse attached to a call statement. */
4943 some_callers_have_no_vuse_p (struct cgraph_node
*node
,
4944 void *data ATTRIBUTE_UNUSED
)
4946 struct cgraph_edge
*cs
;
4947 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
4948 if (!cs
->call_stmt
|| !gimple_vuse (cs
->call_stmt
))
4954 /* Convert all callers of NODE. */
4957 convert_callers_for_node (struct cgraph_node
*node
,
4960 ipa_parm_adjustment_vec
*adjustments
= (ipa_parm_adjustment_vec
*) data
;
4961 bitmap recomputed_callers
= BITMAP_ALLOC (NULL
);
4962 struct cgraph_edge
*cs
;
4964 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
4966 push_cfun (DECL_STRUCT_FUNCTION (cs
->caller
->decl
));
4969 fprintf (dump_file
, "Adjusting call %s/%i -> %s/%i\n",
4970 xstrdup (cs
->caller
->name ()),
4972 xstrdup (cs
->callee
->name ()),
4975 ipa_modify_call_arguments (cs
, cs
->call_stmt
, *adjustments
);
4980 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
4981 if (bitmap_set_bit (recomputed_callers
, cs
->caller
->uid
)
4982 && gimple_in_ssa_p (DECL_STRUCT_FUNCTION (cs
->caller
->decl
)))
4983 compute_inline_parameters (cs
->caller
, true);
4984 BITMAP_FREE (recomputed_callers
);
4989 /* Convert all callers of NODE to pass parameters as given in ADJUSTMENTS. */
4992 convert_callers (struct cgraph_node
*node
, tree old_decl
,
4993 ipa_parm_adjustment_vec adjustments
)
4995 basic_block this_block
;
4997 node
->call_for_symbol_and_aliases (convert_callers_for_node
,
4998 &adjustments
, false);
5000 if (!encountered_recursive_call
)
5003 FOR_EACH_BB_FN (this_block
, cfun
)
5005 gimple_stmt_iterator gsi
;
5007 for (gsi
= gsi_start_bb (this_block
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5011 stmt
= dyn_cast
<gcall
*> (gsi_stmt (gsi
));
5014 call_fndecl
= gimple_call_fndecl (stmt
);
5015 if (call_fndecl
== old_decl
)
5018 fprintf (dump_file
, "Adjusting recursive call");
5019 gimple_call_set_fndecl (stmt
, node
->decl
);
5020 ipa_modify_call_arguments (NULL
, stmt
, adjustments
);
5028 /* Perform all the modification required in IPA-SRA for NODE to have parameters
5029 as given in ADJUSTMENTS. Return true iff the CFG has been changed. */
5032 modify_function (struct cgraph_node
*node
, ipa_parm_adjustment_vec adjustments
)
5034 struct cgraph_node
*new_node
;
5037 cgraph_edge::rebuild_edges ();
5038 free_dominance_info (CDI_DOMINATORS
);
5041 /* This must be done after rebuilding cgraph edges for node above.
5042 Otherwise any recursive calls to node that are recorded in
5043 redirect_callers will be corrupted. */
5044 vec
<cgraph_edge
*> redirect_callers
= node
->collect_callers ();
5045 new_node
= node
->create_version_clone_with_body (redirect_callers
, NULL
,
5046 NULL
, false, NULL
, NULL
,
5048 redirect_callers
.release ();
5050 push_cfun (DECL_STRUCT_FUNCTION (new_node
->decl
));
5051 ipa_modify_formal_parameters (current_function_decl
, adjustments
);
5052 cfg_changed
= ipa_sra_modify_function_body (adjustments
);
5053 sra_ipa_reset_debug_stmts (adjustments
);
5054 convert_callers (new_node
, node
->decl
, adjustments
);
5055 new_node
->make_local ();
5059 /* Means of communication between ipa_sra_check_caller and
5060 ipa_sra_preliminary_function_checks. */
5062 struct ipa_sra_check_caller_data
5065 bool bad_arg_alignment
;
5069 /* If NODE has a caller, mark that fact in DATA which is pointer to
5070 ipa_sra_check_caller_data. Also check all aggregate arguments in all known
5071 calls if they are unit aligned and if not, set the appropriate flag in DATA
5075 ipa_sra_check_caller (struct cgraph_node
*node
, void *data
)
5080 struct ipa_sra_check_caller_data
*iscc
;
5081 iscc
= (struct ipa_sra_check_caller_data
*) data
;
5082 iscc
->has_callers
= true;
5084 for (cgraph_edge
*cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
5086 if (cs
->caller
->thunk
.thunk_p
)
5088 iscc
->has_thunk
= true;
5091 gimple
*call_stmt
= cs
->call_stmt
;
5092 unsigned count
= gimple_call_num_args (call_stmt
);
5093 for (unsigned i
= 0; i
< count
; i
++)
5095 tree arg
= gimple_call_arg (call_stmt
, i
);
5096 if (is_gimple_reg (arg
))
5100 HOST_WIDE_INT bitsize
, bitpos
;
5102 int unsignedp
, volatilep
= 0;
5103 get_inner_reference (arg
, &bitsize
, &bitpos
, &offset
, &mode
,
5104 &unsignedp
, &volatilep
, false);
5105 if (bitpos
% BITS_PER_UNIT
)
5107 iscc
->bad_arg_alignment
= true;
5116 /* Return false the function is apparently unsuitable for IPA-SRA based on it's
5117 attributes, return true otherwise. NODE is the cgraph node of the current
5121 ipa_sra_preliminary_function_checks (struct cgraph_node
*node
)
5123 if (!node
->can_be_local_p ())
5126 fprintf (dump_file
, "Function not local to this compilation unit.\n");
5130 if (!node
->local
.can_change_signature
)
5133 fprintf (dump_file
, "Function can not change signature.\n");
5137 if (!tree_versionable_function_p (node
->decl
))
5140 fprintf (dump_file
, "Function is not versionable.\n");
5144 if (!opt_for_fn (node
->decl
, optimize
)
5145 || !opt_for_fn (node
->decl
, flag_ipa_sra
))
5148 fprintf (dump_file
, "Function not optimized.\n");
5152 if (DECL_VIRTUAL_P (current_function_decl
))
5155 fprintf (dump_file
, "Function is a virtual method.\n");
5159 if ((DECL_ONE_ONLY (node
->decl
) || DECL_EXTERNAL (node
->decl
))
5160 && inline_summaries
->get (node
)->size
>= MAX_INLINE_INSNS_AUTO
)
5163 fprintf (dump_file
, "Function too big to be made truly local.\n");
5170 fprintf (dump_file
, "Function uses stdarg. \n");
5174 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
5177 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
5180 fprintf (dump_file
, "Always inline function will be inlined "
5185 struct ipa_sra_check_caller_data iscc
;
5186 memset (&iscc
, 0, sizeof(iscc
));
5187 node
->call_for_symbol_and_aliases (ipa_sra_check_caller
, &iscc
, true);
5188 if (!iscc
.has_callers
)
5192 "Function has no callers in this compilation unit.\n");
5196 if (iscc
.bad_arg_alignment
)
5200 "A function call has an argument with non-unit alignment.\n");
5215 /* Perform early interprocedural SRA. */
5218 ipa_early_sra (void)
5220 struct cgraph_node
*node
= cgraph_node::get (current_function_decl
);
5221 ipa_parm_adjustment_vec adjustments
;
5224 if (!ipa_sra_preliminary_function_checks (node
))
5228 sra_mode
= SRA_MODE_EARLY_IPA
;
5230 if (!find_param_candidates ())
5233 fprintf (dump_file
, "Function has no IPA-SRA candidates.\n");
5237 if (node
->call_for_symbol_and_aliases
5238 (some_callers_have_mismatched_arguments_p
, NULL
, true))
5241 fprintf (dump_file
, "There are callers with insufficient number of "
5242 "arguments or arguments with type mismatches.\n");
5246 if (node
->call_for_symbol_and_aliases
5247 (some_callers_have_no_vuse_p
, NULL
, true))
5250 fprintf (dump_file
, "There are callers with no VUSE attached "
5251 "to a call stmt.\n");
5255 bb_dereferences
= XCNEWVEC (HOST_WIDE_INT
,
5257 * last_basic_block_for_fn (cfun
));
5258 final_bbs
= BITMAP_ALLOC (NULL
);
5261 if (encountered_apply_args
)
5264 fprintf (dump_file
, "Function calls __builtin_apply_args().\n");
5268 if (encountered_unchangable_recursive_call
)
5271 fprintf (dump_file
, "Function calls itself with insufficient "
5272 "number of arguments.\n");
5276 adjustments
= analyze_all_param_acesses ();
5277 if (!adjustments
.exists ())
5280 ipa_dump_param_adjustments (dump_file
, adjustments
, current_function_decl
);
5282 if (modify_function (node
, adjustments
))
5283 ret
= TODO_update_ssa
| TODO_cleanup_cfg
;
5285 ret
= TODO_update_ssa
;
5286 adjustments
.release ();
5288 statistics_counter_event (cfun
, "Unused parameters deleted",
5289 sra_stats
.deleted_unused_parameters
);
5290 statistics_counter_event (cfun
, "Scalar parameters converted to by-value",
5291 sra_stats
.scalar_by_ref_to_by_val
);
5292 statistics_counter_event (cfun
, "Aggregate parameters broken up",
5293 sra_stats
.aggregate_params_reduced
);
5294 statistics_counter_event (cfun
, "Aggregate parameter components created",
5295 sra_stats
.param_reductions_created
);
5298 BITMAP_FREE (final_bbs
);
5299 free (bb_dereferences
);
5301 sra_deinitialize ();
5307 const pass_data pass_data_early_ipa_sra
=
5309 GIMPLE_PASS
, /* type */
5310 "eipa_sra", /* name */
5311 OPTGROUP_NONE
, /* optinfo_flags */
5312 TV_IPA_SRA
, /* tv_id */
5313 0, /* properties_required */
5314 0, /* properties_provided */
5315 0, /* properties_destroyed */
5316 0, /* todo_flags_start */
5317 TODO_dump_symtab
, /* todo_flags_finish */
5320 class pass_early_ipa_sra
: public gimple_opt_pass
5323 pass_early_ipa_sra (gcc::context
*ctxt
)
5324 : gimple_opt_pass (pass_data_early_ipa_sra
, ctxt
)
5327 /* opt_pass methods: */
5328 virtual bool gate (function
*) { return flag_ipa_sra
&& dbg_cnt (eipa_sra
); }
5329 virtual unsigned int execute (function
*) { return ipa_early_sra (); }
5331 }; // class pass_early_ipa_sra
5336 make_pass_early_ipa_sra (gcc::context
*ctxt
)
5338 return new pass_early_ipa_sra (ctxt
);