2 Copyright (C) 2003-2021 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
69 #include "coretypes.h"
75 #include "stringpool.h"
77 #include "lto-streamer.h"
80 #include "tree-inline.h"
82 #include "gimple-pretty-print.h"
83 #include "alloc-pool.h"
84 #include "symbol-summary.h"
87 #include "ipa-fnsummary.h"
88 #include "symtab-thunks.h"
89 #include "symtab-clones.h"
91 /* Create clone of edge in the node N represented by CALL_EXPR
95 cgraph_edge::clone (cgraph_node
*n
, gcall
*call_stmt
, unsigned stmt_uid
,
96 profile_count num
, profile_count den
,
99 cgraph_edge
*new_edge
;
100 profile_count::adjust_for_ipa_scaling (&num
, &den
);
101 profile_count prof_count
= count
.apply_scale (num
, den
);
103 if (indirect_unknown_callee
)
107 if (call_stmt
&& (decl
= gimple_call_fndecl (call_stmt
))
108 /* When the call is speculative, we need to resolve it
109 via cgraph_resolve_speculation and not here. */
112 cgraph_node
*callee
= cgraph_node::get (decl
);
113 gcc_checking_assert (callee
);
114 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
118 new_edge
= n
->create_indirect_edge (call_stmt
,
119 indirect_info
->ecf_flags
,
121 *new_edge
->indirect_info
= *indirect_info
;
126 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
129 new_edge
->indirect_info
130 = ggc_cleared_alloc
<cgraph_indirect_call_info
> ();
131 *new_edge
->indirect_info
= *indirect_info
;
135 new_edge
->inline_failed
= inline_failed
;
136 new_edge
->indirect_inlining_edge
= indirect_inlining_edge
;
138 new_edge
->lto_stmt_uid
= stmt_uid
;
139 new_edge
->speculative_id
= speculative_id
;
140 /* Clone flags that depend on call_stmt availability manually. */
141 new_edge
->can_throw_external
= can_throw_external
;
142 new_edge
->call_stmt_cannot_inline_p
= call_stmt_cannot_inline_p
;
143 new_edge
->speculative
= speculative
;
144 new_edge
->in_polymorphic_cdtor
= in_polymorphic_cdtor
;
146 /* Update IPA profile. Local profiles need no updating in original. */
148 count
= count
.combine_with_ipa_count_within (count
.ipa ()
149 - new_edge
->count
.ipa (),
151 symtab
->call_edge_duplication_hooks (this, new_edge
);
155 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
156 clone or its thunk. */
159 set_new_clone_decl_and_node_flags (cgraph_node
*new_node
)
161 DECL_EXTERNAL (new_node
->decl
) = 0;
162 TREE_PUBLIC (new_node
->decl
) = 0;
163 DECL_COMDAT (new_node
->decl
) = 0;
164 DECL_WEAK (new_node
->decl
) = 0;
165 DECL_VIRTUAL_P (new_node
->decl
) = 0;
166 DECL_STATIC_CONSTRUCTOR (new_node
->decl
) = 0;
167 DECL_STATIC_DESTRUCTOR (new_node
->decl
) = 0;
168 DECL_SET_IS_OPERATOR_NEW (new_node
->decl
, 0);
169 DECL_SET_IS_OPERATOR_DELETE (new_node
->decl
, 0);
170 DECL_IS_REPLACEABLE_OPERATOR (new_node
->decl
) = 0;
172 new_node
->externally_visible
= 0;
174 new_node
->lowered
= true;
175 new_node
->semantic_interposition
= 0;
178 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
179 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
180 Function can return NODE if no thunk is necessary, which can happen when
181 thunk is this_adjusting but we are removing this parameter. */
184 duplicate_thunk_for_node (cgraph_node
*thunk
, cgraph_node
*node
)
186 cgraph_node
*new_thunk
, *thunk_of
;
187 thunk_of
= thunk
->callees
->callee
->ultimate_alias_target ();
190 node
= duplicate_thunk_for_node (thunk_of
, node
);
192 if (!DECL_ARGUMENTS (thunk
->decl
))
193 thunk
->get_untransformed_body ();
195 thunk_info
*i
= thunk_info::get (thunk
);
197 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
198 if (cs
->caller
->thunk
)
200 thunk_info
*i2
= thunk_info::get (cs
->caller
);
206 clone_info
*info
= clone_info::get (node
);
207 if (info
&& info
->param_adjustments
)
209 /* We do not need to duplicate this_adjusting thunks if we have removed
211 if (i
->this_adjusting
212 && !info
->param_adjustments
->first_param_intact_p ())
215 new_decl
= copy_node (thunk
->decl
);
216 ipa_param_body_adjustments
body_adj (info
->param_adjustments
,
218 body_adj
.modify_formal_parameters ();
221 new_decl
= copy_node (thunk
->decl
);
223 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl
));
224 gcc_checking_assert (!DECL_INITIAL (new_decl
));
225 gcc_checking_assert (!DECL_RESULT (new_decl
));
226 gcc_checking_assert (!DECL_RTL_SET_P (new_decl
));
228 DECL_NAME (new_decl
) = clone_function_name_numbered (thunk
->decl
,
230 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
232 /* We need to force DECL_IGNORED_P because the new thunk is created after
233 early debug was run. */
234 DECL_IGNORED_P (new_decl
) = 1;
236 new_thunk
= cgraph_node::create (new_decl
);
237 set_new_clone_decl_and_node_flags (new_thunk
);
238 new_thunk
->definition
= true;
239 new_thunk
->can_change_signature
= node
->can_change_signature
;
240 new_thunk
->thunk
= thunk
->thunk
;
241 new_thunk
->unique_name
= in_lto_p
;
242 new_thunk
->former_clone_of
= thunk
->decl
;
243 if (info
&& info
->param_adjustments
)
244 clone_info::get_create (new_thunk
)->param_adjustments
245 = info
->param_adjustments
;
246 new_thunk
->unit_id
= thunk
->unit_id
;
247 new_thunk
->merged_comdat
= thunk
->merged_comdat
;
248 new_thunk
->merged_extern_inline
= thunk
->merged_extern_inline
;
250 cgraph_edge
*e
= new_thunk
->create_edge (node
, NULL
, new_thunk
->count
);
251 symtab
->call_edge_duplication_hooks (thunk
->callees
, e
);
252 symtab
->call_cgraph_duplication_hooks (thunk
, new_thunk
);
256 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
257 one or more equivalent thunks for N and redirect E to the first in the
258 chain. Note that it is then necessary to call
259 n->expand_all_artificial_thunks once all callers are redirected. */
262 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node
*n
)
264 cgraph_node
*orig_to
= callee
->ultimate_alias_target ();
266 n
= duplicate_thunk_for_node (orig_to
, n
);
271 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
272 that were expanded. */
275 cgraph_node::expand_all_artificial_thunks ()
278 for (e
= callers
; e
;)
279 if (e
->caller
->thunk
)
281 cgraph_node
*thunk
= e
->caller
;
284 if (expand_thunk (thunk
, false, false))
286 thunk
->thunk
= false;
288 ipa_analyze_node (thunk
);
289 inline_analyze_function (thunk
);
291 thunk
->expand_all_artificial_thunks ();
298 dump_callgraph_transformation (const cgraph_node
*original
,
299 const cgraph_node
*clone
,
302 if (symtab
->ipa_clones_dump_file
)
304 fprintf (symtab
->ipa_clones_dump_file
,
305 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
306 original
->asm_name (), original
->order
,
307 DECL_SOURCE_FILE (original
->decl
),
308 DECL_SOURCE_LINE (original
->decl
),
309 DECL_SOURCE_COLUMN (original
->decl
), clone
->asm_name (),
310 clone
->order
, DECL_SOURCE_FILE (clone
->decl
),
311 DECL_SOURCE_LINE (clone
->decl
), DECL_SOURCE_COLUMN (clone
->decl
),
314 symtab
->cloned_nodes
.add (original
);
315 symtab
->cloned_nodes
.add (clone
);
319 /* Turn profile of N to local profile. */
322 localize_profile (cgraph_node
*n
)
324 n
->count
= n
->count
.guessed_local ();
325 for (cgraph_edge
*e
= n
->callees
; e
; e
=e
->next_callee
)
327 e
->count
= e
->count
.guessed_local ();
328 if (!e
->inline_failed
)
329 localize_profile (e
->callee
);
331 for (cgraph_edge
*e
= n
->indirect_calls
; e
; e
=e
->next_callee
)
332 e
->count
= e
->count
.guessed_local ();
335 /* Create node representing clone of N executed COUNT times. Decrease
336 the execution counts from original node too.
337 The new clone will have decl set to DECL that may or may not be the same
340 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
341 function's profile to reflect the fact that part of execution is handled
343 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
344 the new clone. Otherwise the caller is responsible for doing so later.
346 If the new node is being inlined into another one, NEW_INLINED_TO should be
347 the outline function the new one is (even indirectly) inlined to. All hooks
348 will see this in node's inlined_to, when invoked. Can be NULL if the
351 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
352 will be overwritten by the new structure. Otherwise the new node will
353 share parameter manipulation information with the original node. */
356 cgraph_node::create_clone (tree new_decl
, profile_count prof_count
,
357 bool update_original
,
358 vec
<cgraph_edge
*> redirect_callers
,
359 bool call_duplication_hook
,
360 cgraph_node
*new_inlined_to
,
361 ipa_param_adjustments
*param_adjustments
,
364 cgraph_node
*new_node
= symtab
->create_empty ();
367 profile_count old_count
= count
;
368 bool nonzero
= count
.ipa ().nonzero_p ();
371 dump_callgraph_transformation (this, new_inlined_to
, "inlining to");
373 /* When inlining we scale precisely to prof_count, when cloning we can
374 preserve local profile. */
376 prof_count
= count
.combine_with_ipa_count (prof_count
);
377 new_node
->count
= prof_count
;
379 /* Update IPA profile. Local profiles need no updating in original. */
383 count
= count
.combine_with_ipa_count_within (count
.ipa ()
387 count
= count
.combine_with_ipa_count (count
.ipa () - prof_count
.ipa ());
389 new_node
->decl
= new_decl
;
390 new_node
->register_symbol ();
391 new_node
->lto_file_data
= lto_file_data
;
392 new_node
->analyzed
= analyzed
;
393 new_node
->definition
= definition
;
394 new_node
->versionable
= versionable
;
395 new_node
->can_change_signature
= can_change_signature
;
396 new_node
->redefined_extern_inline
= redefined_extern_inline
;
397 new_node
->tm_may_enter_irr
= tm_may_enter_irr
;
398 new_node
->externally_visible
= false;
399 new_node
->no_reorder
= no_reorder
;
400 new_node
->local
= true;
401 new_node
->inlined_to
= new_inlined_to
;
403 new_node
->frequency
= frequency
;
404 new_node
->tp_first_run
= tp_first_run
;
405 new_node
->tm_clone
= tm_clone
;
406 new_node
->icf_merged
= icf_merged
;
407 new_node
->thunk
= thunk
;
408 new_node
->unit_id
= unit_id
;
409 new_node
->merged_comdat
= merged_comdat
;
410 new_node
->merged_extern_inline
= merged_extern_inline
;
411 clone_info
*info
= clone_info::get (this);
413 if (param_adjustments
)
414 clone_info::get_create (new_node
)->param_adjustments
= param_adjustments
;
415 else if (info
&& info
->param_adjustments
)
416 clone_info::get_create (new_node
)->param_adjustments
417 = info
->param_adjustments
;
418 new_node
->split_part
= split_part
;
420 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
422 /* Redirect calls to the old version node to point to its new
423 version. The only exception is when the edge was proved to
424 be unreachable during the cloning procedure. */
426 || !fndecl_built_in_p (e
->callee
->decl
, BUILT_IN_UNREACHABLE
))
427 e
->redirect_callee_duplicating_thunks (new_node
);
429 new_node
->expand_all_artificial_thunks ();
431 for (e
= callees
;e
; e
=e
->next_callee
)
432 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
, new_node
->count
, old_count
,
435 for (e
= indirect_calls
; e
; e
= e
->next_callee
)
436 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
437 new_node
->count
, old_count
, update_original
);
438 new_node
->clone_references (this);
440 new_node
->next_sibling_clone
= clones
;
442 clones
->prev_sibling_clone
= new_node
;
444 new_node
->clone_of
= this;
446 if (call_duplication_hook
)
447 symtab
->call_cgraph_duplication_hooks (this, new_node
);
448 /* With partial train run we do not want to assume that original's
449 count is zero whenever we redurect all executed edges to clone.
450 Simply drop profile to local one in this case. */
452 && opt_for_fn (decl
, flag_profile_partial_training
)
455 && !count
.ipa ().nonzero_p ()
457 localize_profile (this);
460 dump_callgraph_transformation (this, new_node
, suffix
);
465 static GTY(()) hash_map
<const char *, unsigned> *clone_fn_ids
;
467 /* Return a new assembler name for a clone of decl named NAME. Apart
468 from the string SUFFIX, the new name will end with a unique (for
469 each NAME) unspecified number. If clone numbering is not needed
470 then the two argument clone_function_name should be used instead.
471 Should not be called directly except for by
472 lto-partition.c:privatize_symbol_name_1. */
475 clone_function_name_numbered (const char *name
, const char *suffix
)
477 /* Initialize the function->counter mapping the first time it's
480 clone_fn_ids
= hash_map
<const char *, unsigned int>::create_ggc (64);
481 unsigned int &suffix_counter
= clone_fn_ids
->get_or_insert (
482 IDENTIFIER_POINTER (get_identifier (name
)));
483 return clone_function_name (name
, suffix
, suffix_counter
++);
486 /* Return a new assembler name for a clone of DECL. Apart from string
487 SUFFIX, the new name will end with a unique (for each DECL
488 assembler name) unspecified number. If clone numbering is not
489 needed then the two argument clone_function_name should be used
493 clone_function_name_numbered (tree decl
, const char *suffix
)
495 tree name
= DECL_ASSEMBLER_NAME (decl
);
496 return clone_function_name_numbered (IDENTIFIER_POINTER (name
),
500 /* Return a new assembler name for a clone of decl named NAME. Apart
501 from the string SUFFIX, the new name will end with the specified
502 NUMBER. If clone numbering is not needed then the two argument
503 clone_function_name should be used instead. */
506 clone_function_name (const char *name
, const char *suffix
,
507 unsigned long number
)
509 size_t len
= strlen (name
);
510 char *tmp_name
, *prefix
;
512 prefix
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
513 memcpy (prefix
, name
, len
);
514 strcpy (prefix
+ len
+ 1, suffix
);
515 prefix
[len
] = symbol_table::symbol_suffix_separator ();
516 ASM_FORMAT_PRIVATE_NAME (tmp_name
, prefix
, number
);
517 return get_identifier (tmp_name
);
520 /* Return a new assembler name for a clone of DECL. Apart from the
521 string SUFFIX, the new name will end with the specified NUMBER. If
522 clone numbering is not needed then the two argument
523 clone_function_name should be used instead. */
526 clone_function_name (tree decl
, const char *suffix
,
527 unsigned long number
)
529 return clone_function_name (
530 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)), suffix
, number
);
533 /* Return a new assembler name ending with the string SUFFIX for a
537 clone_function_name (tree decl
, const char *suffix
)
539 tree identifier
= DECL_ASSEMBLER_NAME (decl
);
540 /* For consistency this needs to behave the same way as
541 ASM_FORMAT_PRIVATE_NAME does, but without the final number
543 char *separator
= XALLOCAVEC (char, 2);
544 separator
[0] = symbol_table::symbol_suffix_separator ();
546 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
547 const char *prefix
= "__";
549 const char *prefix
= "";
551 char *result
= ACONCAT ((prefix
,
552 IDENTIFIER_POINTER (identifier
),
556 return get_identifier (result
);
560 /* Create callgraph node clone with new declaration. The actual body will be
561 copied later at compilation stage. The name of the new clone will be
562 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
564 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
568 cgraph_node::create_virtual_clone (const vec
<cgraph_edge
*> &redirect_callers
,
569 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
570 ipa_param_adjustments
*param_adjustments
,
571 const char * suffix
, unsigned num_suffix
)
573 tree old_decl
= decl
;
574 cgraph_node
*new_node
= NULL
;
577 ipa_replace_map
*map
;
580 gcc_checking_assert (versionable
);
581 /* TODO: It would be nice if we could recognize that param_adjustments do not
582 actually perform any changes, but at the moment let's require it simply
584 gcc_assert (can_change_signature
|| !param_adjustments
);
586 /* Make a new FUNCTION_DECL tree node */
587 if (!param_adjustments
)
588 new_decl
= copy_node (old_decl
);
590 new_decl
= param_adjustments
->adjust_decl (old_decl
);
592 /* These pointers represent function body and will be populated only when clone
594 gcc_assert (new_decl
!= old_decl
);
595 DECL_STRUCT_FUNCTION (new_decl
) = NULL
;
596 DECL_ARGUMENTS (new_decl
) = NULL
;
597 DECL_INITIAL (new_decl
) = NULL
;
598 DECL_RESULT (new_decl
) = NULL
;
599 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
600 sometimes storing only clone decl instead of original. */
602 /* Generate a new name for the new version. */
603 len
= IDENTIFIER_LENGTH (DECL_NAME (old_decl
));
604 name
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
605 memcpy (name
, IDENTIFIER_POINTER (DECL_NAME (old_decl
)), len
);
606 strcpy (name
+ len
+ 1, suffix
);
608 DECL_NAME (new_decl
) = get_identifier (name
);
609 SET_DECL_ASSEMBLER_NAME (new_decl
,
610 clone_function_name (old_decl
, suffix
, num_suffix
));
611 SET_DECL_RTL (new_decl
, NULL
);
613 new_node
= create_clone (new_decl
, count
, false,
614 redirect_callers
, false, NULL
, param_adjustments
,
617 /* Update the properties.
618 Make clone visible only within this translation unit. Make sure
619 that is not weak also.
620 ??? We cannot use COMDAT linkage because there is no
621 ABI support for this. */
622 set_new_clone_decl_and_node_flags (new_node
);
623 new_node
->ipcp_clone
= ipcp_clone
;
625 clone_info::get_create (new_node
)->tree_map
= tree_map
;
626 if (!implicit_section
)
627 new_node
->set_section (*this);
629 /* Clones of global symbols or symbols with unique names are unique. */
630 if ((TREE_PUBLIC (old_decl
)
631 && !DECL_EXTERNAL (old_decl
)
632 && !DECL_WEAK (old_decl
)
633 && !DECL_COMDAT (old_decl
))
635 new_node
->unique_name
= true;
636 FOR_EACH_VEC_SAFE_ELT (tree_map
, i
, map
)
638 tree repl
= map
->new_tree
;
639 if (map
->force_load_ref
)
641 gcc_assert (TREE_CODE (repl
) == ADDR_EXPR
);
642 repl
= get_base_address (TREE_OPERAND (repl
, 0));
644 new_node
->maybe_create_reference (repl
, NULL
);
647 if (ipa_transforms_to_apply
.exists ())
648 new_node
->ipa_transforms_to_apply
649 = ipa_transforms_to_apply
.copy ();
651 symtab
->call_cgraph_duplication_hooks (this, new_node
);
656 /* callgraph node being removed from symbol table; see if its entry can be
657 replaced by other inline clone.
658 INFO is clone info to attach to the new root. */
660 cgraph_node::find_replacement (clone_info
*info
)
662 cgraph_node
*next_inline_clone
, *replacement
;
664 for (next_inline_clone
= clones
;
666 && next_inline_clone
->decl
!= decl
;
667 next_inline_clone
= next_inline_clone
->next_sibling_clone
)
670 /* If there is inline clone of the node being removed, we need
671 to put it into the position of removed node and reorganize all
672 other clones to be based on it. */
673 if (next_inline_clone
)
676 cgraph_node
*new_clones
;
678 replacement
= next_inline_clone
;
680 /* Unlink inline clone from the list of clones of removed node. */
681 if (next_inline_clone
->next_sibling_clone
)
682 next_inline_clone
->next_sibling_clone
->prev_sibling_clone
683 = next_inline_clone
->prev_sibling_clone
;
684 if (next_inline_clone
->prev_sibling_clone
)
686 gcc_assert (clones
!= next_inline_clone
);
687 next_inline_clone
->prev_sibling_clone
->next_sibling_clone
688 = next_inline_clone
->next_sibling_clone
;
692 gcc_assert (clones
== next_inline_clone
);
693 clones
= next_inline_clone
->next_sibling_clone
;
699 /* Copy clone info. */
701 *clone_info::get_create (next_inline_clone
) = *info
;
703 /* Now place it into clone tree at same level at NODE. */
704 next_inline_clone
->clone_of
= clone_of
;
705 next_inline_clone
->prev_sibling_clone
= NULL
;
706 next_inline_clone
->next_sibling_clone
= NULL
;
709 if (clone_of
->clones
)
710 clone_of
->clones
->prev_sibling_clone
= next_inline_clone
;
711 next_inline_clone
->next_sibling_clone
= clone_of
->clones
;
712 clone_of
->clones
= next_inline_clone
;
715 /* Merge the clone list. */
718 if (!next_inline_clone
->clones
)
719 next_inline_clone
->clones
= new_clones
;
722 n
= next_inline_clone
->clones
;
723 while (n
->next_sibling_clone
)
724 n
= n
->next_sibling_clone
;
725 n
->next_sibling_clone
= new_clones
;
726 new_clones
->prev_sibling_clone
= n
;
730 /* Update clone_of pointers. */
734 n
->clone_of
= next_inline_clone
;
735 n
= n
->next_sibling_clone
;
738 /* Update order in order to be able to find a LTO section
739 with function body. */
740 replacement
->order
= order
;
748 /* Like cgraph_set_call_stmt but walk the clone tree and update all
749 clones sharing the same function body.
750 When WHOLE_SPECULATIVE_EDGES is true, all three components of
751 speculative edge gets updated. Otherwise we update only direct
755 cgraph_node::set_call_stmt_including_clones (gimple
*old_stmt
,
757 bool update_speculative
)
760 cgraph_edge
*master_edge
= get_edge (old_stmt
);
763 cgraph_edge::set_call_stmt (master_edge
, new_stmt
, update_speculative
);
769 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
772 edge
= cgraph_edge::set_call_stmt (edge
, new_stmt
,
774 /* If UPDATE_SPECULATIVE is false, it means that we are turning
775 speculative call into a real code sequence. Update the
777 if (edge
->speculative
&& !update_speculative
)
779 cgraph_edge
*indirect
= edge
->speculative_call_indirect_edge ();
781 for (cgraph_edge
*next
, *direct
782 = edge
->first_speculative_call_target ();
786 next
= direct
->next_speculative_call_target ();
787 direct
->speculative_call_target_ref ()->speculative
= false;
788 direct
->speculative
= false;
790 indirect
->speculative
= false;
795 else if (node
->next_sibling_clone
)
796 node
= node
->next_sibling_clone
;
799 while (node
!= this && !node
->next_sibling_clone
)
800 node
= node
->clone_of
;
802 node
= node
->next_sibling_clone
;
807 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
808 same function body. If clones already have edge for OLD_STMT; only
809 update the edge same way as cgraph_set_call_stmt_including_clones does.
811 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
812 frequencies of the clones. */
815 cgraph_node::create_edge_including_clones (cgraph_node
*callee
,
816 gimple
*old_stmt
, gcall
*stmt
,
818 cgraph_inline_failed_t reason
)
822 if (!get_edge (stmt
))
824 cgraph_edge
*edge
= create_edge (callee
, stmt
, count
);
825 edge
->inline_failed
= reason
;
831 /* Thunk clones do not get updated while copying inline function body. */
834 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
836 /* It is possible that clones already contain the edge while
837 master didn't. Either we promoted indirect call into direct
838 call in the clone or we are processing clones of unreachable
839 master where edges has been removed. */
841 edge
= cgraph_edge::set_call_stmt (edge
, stmt
);
842 else if (! node
->get_edge (stmt
))
844 edge
= node
->create_edge (callee
, stmt
, count
);
845 edge
->inline_failed
= reason
;
850 else if (node
->next_sibling_clone
)
851 node
= node
->next_sibling_clone
;
854 while (node
!= this && !node
->next_sibling_clone
)
855 node
= node
->clone_of
;
857 node
= node
->next_sibling_clone
;
862 /* Remove the node from cgraph and all inline clones inlined into it.
863 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
864 removed. This allows to call the function from outer loop walking clone
868 cgraph_node::remove_symbol_and_inline_clones (cgraph_node
*forbidden_node
)
870 cgraph_edge
*e
, *next
;
873 if (this == forbidden_node
)
875 cgraph_edge::remove (callers
);
878 for (e
= callees
; e
; e
= next
)
880 next
= e
->next_callee
;
881 if (!e
->inline_failed
)
882 found
|= e
->callee
->remove_symbol_and_inline_clones (forbidden_node
);
888 /* The edges representing the callers of the NEW_VERSION node were
889 fixed by cgraph_function_versioning (), now the call_expr in their
890 respective tree code should be updated to call the NEW_VERSION. */
893 update_call_expr (cgraph_node
*new_version
)
897 gcc_assert (new_version
);
899 /* Update the call expr on the edges to call the new version. */
900 for (e
= new_version
->callers
; e
; e
= e
->next_caller
)
902 function
*inner_function
= DECL_STRUCT_FUNCTION (e
->caller
->decl
);
903 gimple_call_set_fndecl (e
->call_stmt
, new_version
->decl
);
904 maybe_clean_eh_stmt_fn (inner_function
, e
->call_stmt
);
909 /* Create a new cgraph node which is the new version of
910 callgraph node. REDIRECT_CALLERS holds the callers
911 edges which should be redirected to point to
912 NEW_VERSION. ALL the callees edges of the node
913 are cloned to the new version node. Return the new
916 If non-NULL BLOCK_TO_COPY determine what basic blocks
917 was copied to prevent duplications of calls that are dead
921 cgraph_node::create_version_clone (tree new_decl
,
922 vec
<cgraph_edge
*> redirect_callers
,
926 cgraph_node
*new_version
;
930 new_version
= cgraph_node::create (new_decl
);
932 new_version
->analyzed
= analyzed
;
933 new_version
->definition
= definition
;
934 new_version
->local
= local
;
935 new_version
->externally_visible
= false;
936 new_version
->no_reorder
= no_reorder
;
937 new_version
->local
= new_version
->definition
;
938 new_version
->inlined_to
= inlined_to
;
939 new_version
->rtl
= rtl
;
940 new_version
->count
= count
;
941 new_version
->unit_id
= unit_id
;
942 new_version
->merged_comdat
= merged_comdat
;
943 new_version
->merged_extern_inline
= merged_extern_inline
;
945 for (e
= callees
; e
; e
=e
->next_callee
)
947 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
948 e
->clone (new_version
, e
->call_stmt
,
949 e
->lto_stmt_uid
, count
, count
,
951 for (e
= indirect_calls
; e
; e
=e
->next_callee
)
953 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
954 e
->clone (new_version
, e
->call_stmt
,
955 e
->lto_stmt_uid
, count
, count
,
957 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
959 /* Redirect calls to the old version node to point to its new
961 e
->redirect_callee (new_version
);
964 dump_callgraph_transformation (this, new_version
, suffix
);
969 /* Perform function versioning.
970 Function versioning includes copying of the tree and
971 a callgraph update (creating a new cgraph node and updating
972 its callees and callers).
974 REDIRECT_CALLERS varray includes the edges to be redirected
977 TREE_MAP is a mapping of tree nodes we want to replace with
978 new ones (according to results of prior analysis).
980 If non-NULL ARGS_TO_SKIP determine function parameters to remove
982 If SKIP_RETURN is true, the new version will return void.
983 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
984 If non_NULL NEW_ENTRY determine new entry BB of the clone.
986 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
987 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
988 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
991 If VERSION_DECL is set true, use clone_function_name_numbered for the
992 function clone. Otherwise, use clone_function_name.
994 Return the new version's cgraph node. */
997 cgraph_node::create_version_clone_with_body
998 (vec
<cgraph_edge
*> redirect_callers
,
999 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
1000 ipa_param_adjustments
*param_adjustments
,
1001 bitmap bbs_to_copy
, basic_block new_entry_block
, const char *suffix
,
1002 tree target_attributes
, bool version_decl
)
1004 tree old_decl
= decl
;
1005 cgraph_node
*new_version_node
= NULL
;
1008 if (!tree_versionable_function_p (old_decl
))
1011 /* TODO: Restore an assert that we do not change signature if
1012 can_change_signature is false. We cannot just check that
1013 param_adjustments is NULL because unfortunately ipa-split removes return
1014 values from such functions. */
1016 /* Make a new FUNCTION_DECL tree node for the new version. */
1017 if (param_adjustments
)
1018 new_decl
= param_adjustments
->adjust_decl (old_decl
);
1020 new_decl
= copy_node (old_decl
);
1022 /* Generate a new name for the new version. */
1023 tree fnname
= (version_decl
? clone_function_name_numbered (old_decl
, suffix
)
1024 : clone_function_name (old_decl
, suffix
));
1025 DECL_NAME (new_decl
) = fnname
;
1026 SET_DECL_ASSEMBLER_NAME (new_decl
, fnname
);
1027 SET_DECL_RTL (new_decl
, NULL
);
1029 DECL_VIRTUAL_P (new_decl
) = 0;
1031 if (target_attributes
)
1033 DECL_ATTRIBUTES (new_decl
) = target_attributes
;
1035 location_t saved_loc
= input_location
;
1036 tree v
= TREE_VALUE (target_attributes
);
1037 input_location
= DECL_SOURCE_LOCATION (new_decl
);
1038 bool r
= targetm
.target_option
.valid_attribute_p (new_decl
, NULL
, v
, 1);
1039 input_location
= saved_loc
;
1044 /* When the old decl was a con-/destructor make sure the clone isn't. */
1045 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
1046 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
1047 DECL_SET_IS_OPERATOR_NEW (new_decl
, 0);
1048 DECL_SET_IS_OPERATOR_DELETE (new_decl
, 0);
1049 DECL_IS_REPLACEABLE_OPERATOR (new_decl
) = 0;
1051 /* Create the new version's call-graph node.
1052 and update the edges of the new node. */
1053 new_version_node
= create_version_clone (new_decl
, redirect_callers
,
1054 bbs_to_copy
, suffix
);
1056 if (ipa_transforms_to_apply
.exists ())
1057 new_version_node
->ipa_transforms_to_apply
1058 = ipa_transforms_to_apply
.copy ();
1059 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1060 tree_function_versioning (old_decl
, new_decl
, tree_map
, param_adjustments
,
1061 false, bbs_to_copy
, new_entry_block
);
1063 /* Update the new version's properties.
1064 Make The new version visible only within this translation unit. Make sure
1065 that is not weak also.
1066 ??? We cannot use COMDAT linkage because there is no
1067 ABI support for this. */
1068 new_version_node
->make_decl_local ();
1069 DECL_VIRTUAL_P (new_version_node
->decl
) = 0;
1070 new_version_node
->externally_visible
= 0;
1071 new_version_node
->local
= 1;
1072 new_version_node
->lowered
= true;
1073 if (!implicit_section
)
1074 new_version_node
->set_section (*this);
1075 /* Clones of global symbols or symbols with unique names are unique. */
1076 if ((TREE_PUBLIC (old_decl
)
1077 && !DECL_EXTERNAL (old_decl
)
1078 && !DECL_WEAK (old_decl
)
1079 && !DECL_COMDAT (old_decl
))
1081 new_version_node
->unique_name
= true;
1083 /* Update the call_expr on the edges to call the new version node. */
1084 update_call_expr (new_version_node
);
1086 symtab
->call_cgraph_insertion_hooks (new_version_node
);
1087 return new_version_node
;
1090 /* Remove the node from the tree of virtual and inline clones and make it a
1091 standalone node - not a clone any more. */
1093 void cgraph_node::remove_from_clone_tree ()
1095 if (next_sibling_clone
)
1096 next_sibling_clone
->prev_sibling_clone
= prev_sibling_clone
;
1097 if (prev_sibling_clone
)
1098 prev_sibling_clone
->next_sibling_clone
= next_sibling_clone
;
1100 clone_of
->clones
= next_sibling_clone
;
1101 next_sibling_clone
= NULL
;
1102 prev_sibling_clone
= NULL
;
1106 /* Given virtual clone, turn it into actual clone. */
1109 cgraph_node::materialize_clone ()
1111 clone_info
*info
= clone_info::get (this);
1112 clone_of
->get_untransformed_body ();
1113 former_clone_of
= clone_of
->decl
;
1114 if (clone_of
->former_clone_of
)
1115 former_clone_of
= clone_of
->former_clone_of
;
1116 if (symtab
->dump_file
)
1118 fprintf (symtab
->dump_file
, "cloning %s to %s\n",
1119 clone_of
->dump_name (),
1121 if (info
&& info
->tree_map
)
1123 fprintf (symtab
->dump_file
, " replace map:");
1124 for (unsigned int i
= 0;
1125 i
< vec_safe_length (info
->tree_map
);
1128 ipa_replace_map
*replace_info
;
1129 replace_info
= (*info
->tree_map
)[i
];
1130 fprintf (symtab
->dump_file
, "%s %i -> ",
1131 i
? "," : "", replace_info
->parm_num
);
1132 print_generic_expr (symtab
->dump_file
,
1133 replace_info
->new_tree
);
1135 fprintf (symtab
->dump_file
, "\n");
1137 if (info
&& info
->param_adjustments
)
1138 info
->param_adjustments
->dump (symtab
->dump_file
);
1140 clear_stmts_in_references ();
1141 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1142 tree_function_versioning (clone_of
->decl
, decl
,
1143 info
? info
->tree_map
: NULL
,
1144 info
? info
->param_adjustments
: NULL
,
1146 if (symtab
->dump_file
)
1148 dump_function_to_file (clone_of
->decl
, symtab
->dump_file
,
1150 dump_function_to_file (decl
, symtab
->dump_file
, dump_flags
);
1153 cgraph_node
*this_clone_of
= clone_of
;
1154 /* Function is no longer clone. */
1155 remove_from_clone_tree ();
1156 if (!this_clone_of
->analyzed
&& !this_clone_of
->clones
)
1157 this_clone_of
->release_body ();
1160 #include "gt-cgraphclones.h"