2 Copyright (C) 2003-2023 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
69 #include "coretypes.h"
75 #include "stringpool.h"
77 #include "lto-streamer.h"
80 #include "tree-inline.h"
82 #include "gimple-pretty-print.h"
83 #include "alloc-pool.h"
84 #include "symbol-summary.h"
87 #include "ipa-fnsummary.h"
88 #include "symtab-thunks.h"
89 #include "symtab-clones.h"
91 /* Create clone of edge in the node N represented by CALL_EXPR
95 cgraph_edge::clone (cgraph_node
*n
, gcall
*call_stmt
, unsigned stmt_uid
,
96 profile_count num
, profile_count den
,
99 cgraph_edge
*new_edge
;
100 profile_count::adjust_for_ipa_scaling (&num
, &den
);
101 profile_count prof_count
= count
.apply_scale (num
, den
);
103 if (indirect_unknown_callee
)
107 if (call_stmt
&& (decl
= gimple_call_fndecl (call_stmt
))
108 /* When the call is speculative, we need to resolve it
109 via cgraph_resolve_speculation and not here. */
112 cgraph_node
*callee
= cgraph_node::get (decl
);
113 gcc_checking_assert (callee
);
114 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
118 new_edge
= n
->create_indirect_edge (call_stmt
,
119 indirect_info
->ecf_flags
,
121 *new_edge
->indirect_info
= *indirect_info
;
126 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
129 new_edge
->indirect_info
130 = ggc_cleared_alloc
<cgraph_indirect_call_info
> ();
131 *new_edge
->indirect_info
= *indirect_info
;
135 new_edge
->inline_failed
= inline_failed
;
136 new_edge
->indirect_inlining_edge
= indirect_inlining_edge
;
138 new_edge
->lto_stmt_uid
= stmt_uid
;
139 new_edge
->speculative_id
= speculative_id
;
140 /* Clone flags that depend on call_stmt availability manually. */
141 new_edge
->can_throw_external
= can_throw_external
;
142 new_edge
->call_stmt_cannot_inline_p
= call_stmt_cannot_inline_p
;
143 new_edge
->speculative
= speculative
;
144 new_edge
->in_polymorphic_cdtor
= in_polymorphic_cdtor
;
146 /* Update IPA profile. Local profiles need no updating in original. */
148 count
= count
.combine_with_ipa_count_within (count
.ipa ()
149 - new_edge
->count
.ipa (),
151 symtab
->call_edge_duplication_hooks (this, new_edge
);
155 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
156 clone or its thunk. */
159 set_new_clone_decl_and_node_flags (cgraph_node
*new_node
)
161 DECL_EXTERNAL (new_node
->decl
) = 0;
162 TREE_PUBLIC (new_node
->decl
) = 0;
163 DECL_COMDAT (new_node
->decl
) = 0;
164 DECL_WEAK (new_node
->decl
) = 0;
165 DECL_VIRTUAL_P (new_node
->decl
) = 0;
166 DECL_STATIC_CONSTRUCTOR (new_node
->decl
) = 0;
167 DECL_STATIC_DESTRUCTOR (new_node
->decl
) = 0;
168 DECL_SET_IS_OPERATOR_NEW (new_node
->decl
, 0);
169 DECL_SET_IS_OPERATOR_DELETE (new_node
->decl
, 0);
170 DECL_IS_REPLACEABLE_OPERATOR (new_node
->decl
) = 0;
172 new_node
->externally_visible
= 0;
174 new_node
->lowered
= true;
175 new_node
->semantic_interposition
= 0;
178 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
179 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
180 Function can return NODE if no thunk is necessary, which can happen when
181 thunk is this_adjusting but we are removing this parameter. */
184 duplicate_thunk_for_node (cgraph_node
*thunk
, cgraph_node
*node
)
186 cgraph_node
*new_thunk
, *thunk_of
;
187 thunk_of
= thunk
->callees
->callee
->ultimate_alias_target ();
190 node
= duplicate_thunk_for_node (thunk_of
, node
);
192 if (!DECL_ARGUMENTS (thunk
->decl
))
193 thunk
->get_untransformed_body ();
195 thunk_info
*i
= thunk_info::get (thunk
);
197 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
198 if (cs
->caller
->thunk
)
200 thunk_info
*i2
= thunk_info::get (cs
->caller
);
206 clone_info
*info
= clone_info::get (node
);
207 if (info
&& info
->param_adjustments
)
209 /* We do not need to duplicate this_adjusting thunks if we have removed
211 if (i
->this_adjusting
212 && !info
->param_adjustments
->first_param_intact_p ())
215 new_decl
= copy_node (thunk
->decl
);
216 ipa_param_body_adjustments
body_adj (info
->param_adjustments
,
218 body_adj
.modify_formal_parameters ();
222 new_decl
= copy_node (thunk
->decl
);
223 for (tree
*arg
= &DECL_ARGUMENTS (new_decl
);
224 *arg
; arg
= &DECL_CHAIN (*arg
))
226 tree next
= DECL_CHAIN (*arg
);
227 *arg
= copy_node (*arg
);
228 DECL_CONTEXT (*arg
) = new_decl
;
229 DECL_CHAIN (*arg
) = next
;
233 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl
));
234 gcc_checking_assert (!DECL_INITIAL (new_decl
));
235 gcc_checking_assert (!DECL_RESULT (new_decl
));
236 gcc_checking_assert (!DECL_RTL_SET_P (new_decl
));
238 DECL_NAME (new_decl
) = clone_function_name_numbered (thunk
->decl
,
240 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
242 /* We need to force DECL_IGNORED_P because the new thunk is created after
243 early debug was run. */
244 DECL_IGNORED_P (new_decl
) = 1;
246 new_thunk
= cgraph_node::create (new_decl
);
247 set_new_clone_decl_and_node_flags (new_thunk
);
248 new_thunk
->definition
= true;
249 new_thunk
->can_change_signature
= node
->can_change_signature
;
250 new_thunk
->thunk
= thunk
->thunk
;
251 new_thunk
->unique_name
= in_lto_p
;
252 new_thunk
->former_clone_of
= thunk
->decl
;
253 if (info
&& info
->param_adjustments
)
254 clone_info::get_create (new_thunk
)->param_adjustments
255 = info
->param_adjustments
;
256 new_thunk
->unit_id
= thunk
->unit_id
;
257 new_thunk
->merged_comdat
= thunk
->merged_comdat
;
258 new_thunk
->merged_extern_inline
= thunk
->merged_extern_inline
;
260 cgraph_edge
*e
= new_thunk
->create_edge (node
, NULL
, new_thunk
->count
);
261 symtab
->call_edge_duplication_hooks (thunk
->callees
, e
);
262 symtab
->call_cgraph_duplication_hooks (thunk
, new_thunk
);
266 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
267 one or more equivalent thunks for N and redirect E to the first in the
268 chain. Note that it is then necessary to call
269 n->expand_all_artificial_thunks once all callers are redirected. */
272 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node
*n
)
274 cgraph_node
*orig_to
= callee
->ultimate_alias_target ();
276 n
= duplicate_thunk_for_node (orig_to
, n
);
281 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
282 that were expanded. */
285 cgraph_node::expand_all_artificial_thunks ()
288 for (e
= callers
; e
;)
289 if (e
->caller
->thunk
)
291 cgraph_node
*thunk
= e
->caller
;
294 if (expand_thunk (thunk
, false, false))
296 thunk
->thunk
= false;
298 ipa_analyze_node (thunk
);
299 inline_analyze_function (thunk
);
301 thunk
->expand_all_artificial_thunks ();
308 dump_callgraph_transformation (const cgraph_node
*original
,
309 const cgraph_node
*clone
,
312 if (symtab
->ipa_clones_dump_file
)
314 fprintf (symtab
->ipa_clones_dump_file
,
315 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
316 original
->asm_name (), original
->order
,
317 DECL_SOURCE_FILE (original
->decl
),
318 DECL_SOURCE_LINE (original
->decl
),
319 DECL_SOURCE_COLUMN (original
->decl
), clone
->asm_name (),
320 clone
->order
, DECL_SOURCE_FILE (clone
->decl
),
321 DECL_SOURCE_LINE (clone
->decl
), DECL_SOURCE_COLUMN (clone
->decl
),
324 symtab
->cloned_nodes
.add (original
);
325 symtab
->cloned_nodes
.add (clone
);
329 /* Turn profile of N to local profile. */
332 localize_profile (cgraph_node
*n
)
334 n
->count
= n
->count
.guessed_local ();
335 for (cgraph_edge
*e
= n
->callees
; e
; e
=e
->next_callee
)
337 e
->count
= e
->count
.guessed_local ();
338 if (!e
->inline_failed
)
339 localize_profile (e
->callee
);
341 for (cgraph_edge
*e
= n
->indirect_calls
; e
; e
=e
->next_callee
)
342 e
->count
= e
->count
.guessed_local ();
345 /* Create node representing clone of N executed COUNT times. Decrease
346 the execution counts from original node too.
347 The new clone will have decl set to DECL that may or may not be the same
350 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
351 function's profile to reflect the fact that part of execution is handled
353 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
354 the new clone. Otherwise the caller is responsible for doing so later.
356 If the new node is being inlined into another one, NEW_INLINED_TO should be
357 the outline function the new one is (even indirectly) inlined to. All hooks
358 will see this in node's inlined_to, when invoked. Can be NULL if the
361 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
362 will be overwritten by the new structure. Otherwise the new node will
363 share parameter manipulation information with the original node. */
366 cgraph_node::create_clone (tree new_decl
, profile_count prof_count
,
367 bool update_original
,
368 vec
<cgraph_edge
*> redirect_callers
,
369 bool call_duplication_hook
,
370 cgraph_node
*new_inlined_to
,
371 ipa_param_adjustments
*param_adjustments
,
374 cgraph_node
*new_node
= symtab
->create_empty ();
377 profile_count old_count
= count
;
378 bool nonzero
= count
.ipa ().nonzero_p ();
381 dump_callgraph_transformation (this, new_inlined_to
, "inlining to");
383 /* When inlining we scale precisely to prof_count, when cloning we can
384 preserve local profile. */
386 prof_count
= count
.combine_with_ipa_count (prof_count
);
387 new_node
->count
= prof_count
;
388 new_node
->calls_declare_variant_alt
= this->calls_declare_variant_alt
;
390 /* Update IPA profile. Local profiles need no updating in original. */
394 count
= count
.combine_with_ipa_count_within (count
.ipa ()
398 count
= count
.combine_with_ipa_count (count
.ipa () - prof_count
.ipa ());
400 new_node
->decl
= new_decl
;
401 new_node
->register_symbol ();
402 new_node
->lto_file_data
= lto_file_data
;
403 new_node
->analyzed
= analyzed
;
404 new_node
->definition
= definition
;
405 new_node
->versionable
= versionable
;
406 new_node
->can_change_signature
= can_change_signature
;
407 new_node
->redefined_extern_inline
= redefined_extern_inline
;
408 new_node
->semantic_interposition
= semantic_interposition
;
409 new_node
->tm_may_enter_irr
= tm_may_enter_irr
;
410 new_node
->externally_visible
= false;
411 new_node
->no_reorder
= no_reorder
;
412 new_node
->local
= true;
413 new_node
->inlined_to
= new_inlined_to
;
415 new_node
->frequency
= frequency
;
416 new_node
->tp_first_run
= tp_first_run
;
417 new_node
->tm_clone
= tm_clone
;
418 new_node
->icf_merged
= icf_merged
;
419 new_node
->thunk
= thunk
;
420 new_node
->unit_id
= unit_id
;
421 new_node
->merged_comdat
= merged_comdat
;
422 new_node
->merged_extern_inline
= merged_extern_inline
;
423 clone_info
*info
= clone_info::get (this);
425 if (param_adjustments
)
426 clone_info::get_create (new_node
)->param_adjustments
= param_adjustments
;
427 else if (info
&& info
->param_adjustments
)
428 clone_info::get_create (new_node
)->param_adjustments
429 = info
->param_adjustments
;
430 new_node
->split_part
= split_part
;
432 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
434 /* Redirect calls to the old version node to point to its new
435 version. The only exception is when the edge was proved to
436 be unreachable during the cloning procedure. */
438 || !(fndecl_built_in_p (e
->callee
->decl
, BUILT_IN_UNREACHABLE
)
439 || fndecl_built_in_p (e
->callee
->decl
,
440 BUILT_IN_UNREACHABLE_TRAP
)))
441 e
->redirect_callee_duplicating_thunks (new_node
);
443 new_node
->expand_all_artificial_thunks ();
445 for (e
= callees
;e
; e
=e
->next_callee
)
446 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
, new_node
->count
, old_count
,
449 for (e
= indirect_calls
; e
; e
= e
->next_callee
)
450 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
451 new_node
->count
, old_count
, update_original
);
452 new_node
->clone_references (this);
454 new_node
->next_sibling_clone
= clones
;
456 clones
->prev_sibling_clone
= new_node
;
458 new_node
->clone_of
= this;
460 if (call_duplication_hook
)
461 symtab
->call_cgraph_duplication_hooks (this, new_node
);
462 /* With partial train run we do not want to assume that original's
463 count is zero whenever we redurect all executed edges to clone.
464 Simply drop profile to local one in this case. */
466 && opt_for_fn (decl
, flag_profile_partial_training
)
469 && !count
.ipa ().nonzero_p ()
471 localize_profile (this);
474 dump_callgraph_transformation (this, new_node
, suffix
);
479 static GTY(()) hash_map
<const char *, unsigned> *clone_fn_ids
;
481 /* Return a new assembler name for a clone of decl named NAME. Apart
482 from the string SUFFIX, the new name will end with a unique (for
483 each NAME) unspecified number. If clone numbering is not needed
484 then the two argument clone_function_name should be used instead.
485 Should not be called directly except for by
486 lto-partition.cc:privatize_symbol_name_1. */
489 clone_function_name_numbered (const char *name
, const char *suffix
)
491 /* Initialize the function->counter mapping the first time it's
494 clone_fn_ids
= hash_map
<const char *, unsigned int>::create_ggc (64);
495 unsigned int &suffix_counter
= clone_fn_ids
->get_or_insert (
496 IDENTIFIER_POINTER (get_identifier (name
)));
497 return clone_function_name (name
, suffix
, suffix_counter
++);
500 /* Return a new assembler name for a clone of DECL. Apart from string
501 SUFFIX, the new name will end with a unique (for each DECL
502 assembler name) unspecified number. If clone numbering is not
503 needed then the two argument clone_function_name should be used
507 clone_function_name_numbered (tree decl
, const char *suffix
)
509 tree name
= DECL_ASSEMBLER_NAME (decl
);
510 return clone_function_name_numbered (IDENTIFIER_POINTER (name
),
514 /* Return a new assembler name for a clone of decl named NAME. Apart
515 from the string SUFFIX, the new name will end with the specified
516 NUMBER. If clone numbering is not needed then the two argument
517 clone_function_name should be used instead. */
520 clone_function_name (const char *name
, const char *suffix
,
521 unsigned long number
)
523 size_t len
= strlen (name
);
524 char *tmp_name
, *prefix
;
526 prefix
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
527 memcpy (prefix
, name
, len
);
528 strcpy (prefix
+ len
+ 1, suffix
);
529 prefix
[len
] = symbol_table::symbol_suffix_separator ();
530 ASM_FORMAT_PRIVATE_NAME (tmp_name
, prefix
, number
);
531 return get_identifier (tmp_name
);
534 /* Return a new assembler name for a clone of DECL. Apart from the
535 string SUFFIX, the new name will end with the specified NUMBER. If
536 clone numbering is not needed then the two argument
537 clone_function_name should be used instead. */
540 clone_function_name (tree decl
, const char *suffix
,
541 unsigned long number
)
543 return clone_function_name (
544 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)), suffix
, number
);
547 /* Return a new assembler name ending with the string SUFFIX for a
551 clone_function_name (tree decl
, const char *suffix
)
553 tree identifier
= DECL_ASSEMBLER_NAME (decl
);
554 /* For consistency this needs to behave the same way as
555 ASM_FORMAT_PRIVATE_NAME does, but without the final number
557 char *separator
= XALLOCAVEC (char, 2);
558 separator
[0] = symbol_table::symbol_suffix_separator ();
560 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
561 const char *prefix
= "__";
563 const char *prefix
= "";
565 char *result
= ACONCAT ((prefix
,
566 IDENTIFIER_POINTER (identifier
),
570 return get_identifier (result
);
574 /* Create callgraph node clone with new declaration. The actual body will be
575 copied later at compilation stage. The name of the new clone will be
576 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
578 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
582 cgraph_node::create_virtual_clone (const vec
<cgraph_edge
*> &redirect_callers
,
583 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
584 ipa_param_adjustments
*param_adjustments
,
585 const char * suffix
, unsigned num_suffix
)
587 tree old_decl
= decl
;
588 cgraph_node
*new_node
= NULL
;
591 ipa_replace_map
*map
;
594 gcc_checking_assert (versionable
);
595 /* TODO: It would be nice if we could recognize that param_adjustments do not
596 actually perform any changes, but at the moment let's require it simply
598 gcc_assert (can_change_signature
|| !param_adjustments
);
600 /* Make a new FUNCTION_DECL tree node */
601 if (!param_adjustments
)
602 new_decl
= copy_node (old_decl
);
604 new_decl
= param_adjustments
->adjust_decl (old_decl
);
606 /* These pointers represent function body and will be populated only when clone
608 gcc_assert (new_decl
!= old_decl
);
609 DECL_STRUCT_FUNCTION (new_decl
) = NULL
;
610 DECL_ARGUMENTS (new_decl
) = NULL
;
611 DECL_INITIAL (new_decl
) = NULL
;
612 DECL_RESULT (new_decl
) = NULL
;
613 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
614 sometimes storing only clone decl instead of original. */
616 /* Generate a new name for the new version. */
617 len
= IDENTIFIER_LENGTH (DECL_NAME (old_decl
));
618 name
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
619 memcpy (name
, IDENTIFIER_POINTER (DECL_NAME (old_decl
)), len
);
620 strcpy (name
+ len
+ 1, suffix
);
622 DECL_NAME (new_decl
) = get_identifier (name
);
623 SET_DECL_ASSEMBLER_NAME (new_decl
,
624 clone_function_name (old_decl
, suffix
, num_suffix
));
625 SET_DECL_RTL (new_decl
, NULL
);
627 new_node
= create_clone (new_decl
, count
, false,
628 redirect_callers
, false, NULL
, param_adjustments
,
631 /* Update the properties.
632 Make clone visible only within this translation unit. Make sure
633 that is not weak also.
634 ??? We cannot use COMDAT linkage because there is no
635 ABI support for this. */
636 set_new_clone_decl_and_node_flags (new_node
);
637 new_node
->ipcp_clone
= ipcp_clone
;
639 clone_info::get_create (new_node
)->tree_map
= tree_map
;
640 if (!implicit_section
)
641 new_node
->set_section (*this);
643 /* Clones of global symbols or symbols with unique names are unique. */
644 if ((TREE_PUBLIC (old_decl
)
645 && !DECL_EXTERNAL (old_decl
)
646 && !DECL_WEAK (old_decl
)
647 && !DECL_COMDAT (old_decl
))
649 new_node
->unique_name
= true;
650 FOR_EACH_VEC_SAFE_ELT (tree_map
, i
, map
)
652 tree repl
= map
->new_tree
;
653 if (map
->force_load_ref
)
655 gcc_assert (TREE_CODE (repl
) == ADDR_EXPR
);
656 repl
= get_base_address (TREE_OPERAND (repl
, 0));
658 new_node
->maybe_create_reference (repl
, NULL
);
661 if (ipa_transforms_to_apply
.exists ())
662 new_node
->ipa_transforms_to_apply
663 = ipa_transforms_to_apply
.copy ();
665 symtab
->call_cgraph_duplication_hooks (this, new_node
);
670 /* callgraph node being removed from symbol table; see if its entry can be
671 replaced by other inline clone.
672 INFO is clone info to attach to the new root. */
674 cgraph_node::find_replacement (clone_info
*info
)
676 cgraph_node
*next_inline_clone
, *replacement
;
678 for (next_inline_clone
= clones
;
680 && next_inline_clone
->decl
!= decl
;
681 next_inline_clone
= next_inline_clone
->next_sibling_clone
)
684 /* If there is inline clone of the node being removed, we need
685 to put it into the position of removed node and reorganize all
686 other clones to be based on it. */
687 if (next_inline_clone
)
690 cgraph_node
*new_clones
;
692 replacement
= next_inline_clone
;
694 /* Unlink inline clone from the list of clones of removed node. */
695 if (next_inline_clone
->next_sibling_clone
)
696 next_inline_clone
->next_sibling_clone
->prev_sibling_clone
697 = next_inline_clone
->prev_sibling_clone
;
698 if (next_inline_clone
->prev_sibling_clone
)
700 gcc_assert (clones
!= next_inline_clone
);
701 next_inline_clone
->prev_sibling_clone
->next_sibling_clone
702 = next_inline_clone
->next_sibling_clone
;
706 gcc_assert (clones
== next_inline_clone
);
707 clones
= next_inline_clone
->next_sibling_clone
;
713 /* Copy clone info. */
715 *clone_info::get_create (next_inline_clone
) = *info
;
717 /* Now place it into clone tree at same level at NODE. */
718 next_inline_clone
->clone_of
= clone_of
;
719 next_inline_clone
->prev_sibling_clone
= NULL
;
720 next_inline_clone
->next_sibling_clone
= NULL
;
723 if (clone_of
->clones
)
724 clone_of
->clones
->prev_sibling_clone
= next_inline_clone
;
725 next_inline_clone
->next_sibling_clone
= clone_of
->clones
;
726 clone_of
->clones
= next_inline_clone
;
729 /* Merge the clone list. */
732 if (!next_inline_clone
->clones
)
733 next_inline_clone
->clones
= new_clones
;
736 n
= next_inline_clone
->clones
;
737 while (n
->next_sibling_clone
)
738 n
= n
->next_sibling_clone
;
739 n
->next_sibling_clone
= new_clones
;
740 new_clones
->prev_sibling_clone
= n
;
744 /* Update clone_of pointers. */
748 n
->clone_of
= next_inline_clone
;
749 n
= n
->next_sibling_clone
;
752 /* Update order in order to be able to find a LTO section
753 with function body. */
754 replacement
->order
= order
;
762 /* Like cgraph_set_call_stmt but walk the clone tree and update all
763 clones sharing the same function body.
764 When WHOLE_SPECULATIVE_EDGES is true, all three components of
765 speculative edge gets updated. Otherwise we update only direct
769 cgraph_node::set_call_stmt_including_clones (gimple
*old_stmt
,
771 bool update_speculative
)
774 cgraph_edge
*master_edge
= get_edge (old_stmt
);
777 cgraph_edge::set_call_stmt (master_edge
, new_stmt
, update_speculative
);
783 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
786 edge
= cgraph_edge::set_call_stmt (edge
, new_stmt
,
788 /* If UPDATE_SPECULATIVE is false, it means that we are turning
789 speculative call into a real code sequence. Update the
791 if (edge
->speculative
&& !update_speculative
)
793 cgraph_edge
*indirect
= edge
->speculative_call_indirect_edge ();
795 for (cgraph_edge
*next
, *direct
796 = edge
->first_speculative_call_target ();
800 next
= direct
->next_speculative_call_target ();
801 direct
->speculative_call_target_ref ()->speculative
= false;
802 direct
->speculative
= false;
804 indirect
->speculative
= false;
809 else if (node
->next_sibling_clone
)
810 node
= node
->next_sibling_clone
;
813 while (node
!= this && !node
->next_sibling_clone
)
814 node
= node
->clone_of
;
816 node
= node
->next_sibling_clone
;
821 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
822 same function body. If clones already have edge for OLD_STMT; only
823 update the edge same way as cgraph_set_call_stmt_including_clones does.
825 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
826 frequencies of the clones. */
829 cgraph_node::create_edge_including_clones (cgraph_node
*callee
,
830 gimple
*old_stmt
, gcall
*stmt
,
832 cgraph_inline_failed_t reason
)
836 if (!get_edge (stmt
))
838 cgraph_edge
*edge
= create_edge (callee
, stmt
, count
);
839 edge
->inline_failed
= reason
;
845 /* Thunk clones do not get updated while copying inline function body. */
848 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
850 /* It is possible that clones already contain the edge while
851 master didn't. Either we promoted indirect call into direct
852 call in the clone or we are processing clones of unreachable
853 master where edges has been removed. */
855 edge
= cgraph_edge::set_call_stmt (edge
, stmt
);
856 else if (! node
->get_edge (stmt
))
858 edge
= node
->create_edge (callee
, stmt
, count
);
859 edge
->inline_failed
= reason
;
864 else if (node
->next_sibling_clone
)
865 node
= node
->next_sibling_clone
;
868 while (node
!= this && !node
->next_sibling_clone
)
869 node
= node
->clone_of
;
871 node
= node
->next_sibling_clone
;
876 /* Remove the node from cgraph and all inline clones inlined into it.
877 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
878 removed. This allows to call the function from outer loop walking clone
882 cgraph_node::remove_symbol_and_inline_clones (cgraph_node
*forbidden_node
)
884 cgraph_edge
*e
, *next
;
887 if (this == forbidden_node
)
889 cgraph_edge::remove (callers
);
892 for (e
= callees
; e
; e
= next
)
894 next
= e
->next_callee
;
895 if (!e
->inline_failed
)
896 found
|= e
->callee
->remove_symbol_and_inline_clones (forbidden_node
);
902 /* The edges representing the callers of the NEW_VERSION node were
903 fixed by cgraph_function_versioning (), now the call_expr in their
904 respective tree code should be updated to call the NEW_VERSION. */
907 update_call_expr (cgraph_node
*new_version
)
911 gcc_assert (new_version
);
913 /* Update the call expr on the edges to call the new version. */
914 for (e
= new_version
->callers
; e
; e
= e
->next_caller
)
916 function
*inner_function
= DECL_STRUCT_FUNCTION (e
->caller
->decl
);
917 gimple_call_set_fndecl (e
->call_stmt
, new_version
->decl
);
918 maybe_clean_eh_stmt_fn (inner_function
, e
->call_stmt
);
923 /* Create a new cgraph node which is the new version of
924 callgraph node. REDIRECT_CALLERS holds the callers
925 edges which should be redirected to point to
926 NEW_VERSION. ALL the callees edges of the node
927 are cloned to the new version node. Return the new
930 If non-NULL BLOCK_TO_COPY determine what basic blocks
931 was copied to prevent duplications of calls that are dead
935 cgraph_node::create_version_clone (tree new_decl
,
936 vec
<cgraph_edge
*> redirect_callers
,
940 cgraph_node
*new_version
;
944 new_version
= cgraph_node::create (new_decl
);
946 new_version
->analyzed
= analyzed
;
947 new_version
->definition
= definition
;
948 new_version
->local
= local
;
949 new_version
->externally_visible
= false;
950 new_version
->no_reorder
= no_reorder
;
951 new_version
->local
= new_version
->definition
;
952 new_version
->inlined_to
= inlined_to
;
953 new_version
->rtl
= rtl
;
954 new_version
->count
= count
;
955 new_version
->unit_id
= unit_id
;
956 new_version
->merged_comdat
= merged_comdat
;
957 new_version
->merged_extern_inline
= merged_extern_inline
;
959 for (e
= callees
; e
; e
=e
->next_callee
)
961 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
962 e
->clone (new_version
, e
->call_stmt
,
963 e
->lto_stmt_uid
, count
, count
,
965 for (e
= indirect_calls
; e
; e
=e
->next_callee
)
967 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
968 e
->clone (new_version
, e
->call_stmt
,
969 e
->lto_stmt_uid
, count
, count
,
971 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
973 /* Redirect calls to the old version node to point to its new
975 e
->redirect_callee (new_version
);
978 dump_callgraph_transformation (this, new_version
, suffix
);
983 /* Perform function versioning.
984 Function versioning includes copying of the tree and
985 a callgraph update (creating a new cgraph node and updating
986 its callees and callers).
988 REDIRECT_CALLERS varray includes the edges to be redirected
991 TREE_MAP is a mapping of tree nodes we want to replace with
992 new ones (according to results of prior analysis).
994 If non-NULL ARGS_TO_SKIP determine function parameters to remove
996 If SKIP_RETURN is true, the new version will return void.
997 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
998 If non_NULL NEW_ENTRY determine new entry BB of the clone.
1000 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
1001 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
1002 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
1005 If VERSION_DECL is set true, use clone_function_name_numbered for the
1006 function clone. Otherwise, use clone_function_name.
1008 Return the new version's cgraph node. */
1011 cgraph_node::create_version_clone_with_body
1012 (vec
<cgraph_edge
*> redirect_callers
,
1013 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
1014 ipa_param_adjustments
*param_adjustments
,
1015 bitmap bbs_to_copy
, basic_block new_entry_block
, const char *suffix
,
1016 tree target_attributes
, bool version_decl
)
1018 tree old_decl
= decl
;
1019 cgraph_node
*new_version_node
= NULL
;
1022 if (!tree_versionable_function_p (old_decl
))
1025 /* TODO: Restore an assert that we do not change signature if
1026 can_change_signature is false. We cannot just check that
1027 param_adjustments is NULL because unfortunately ipa-split removes return
1028 values from such functions. */
1030 /* Make a new FUNCTION_DECL tree node for the new version. */
1031 if (param_adjustments
)
1032 new_decl
= param_adjustments
->adjust_decl (old_decl
);
1034 new_decl
= copy_node (old_decl
);
1036 /* Generate a new name for the new version. */
1037 tree fnname
= (version_decl
? clone_function_name_numbered (old_decl
, suffix
)
1038 : clone_function_name (old_decl
, suffix
));
1039 DECL_NAME (new_decl
) = fnname
;
1040 SET_DECL_ASSEMBLER_NAME (new_decl
, fnname
);
1041 SET_DECL_RTL (new_decl
, NULL
);
1043 DECL_VIRTUAL_P (new_decl
) = 0;
1045 if (target_attributes
)
1047 DECL_ATTRIBUTES (new_decl
) = target_attributes
;
1049 location_t saved_loc
= input_location
;
1050 tree v
= TREE_VALUE (target_attributes
);
1051 input_location
= DECL_SOURCE_LOCATION (new_decl
);
1052 bool r
= targetm
.target_option
.valid_attribute_p (new_decl
, NULL
, v
, 1);
1053 input_location
= saved_loc
;
1058 /* When the old decl was a con-/destructor make sure the clone isn't. */
1059 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
1060 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
1061 DECL_SET_IS_OPERATOR_NEW (new_decl
, 0);
1062 DECL_SET_IS_OPERATOR_DELETE (new_decl
, 0);
1063 DECL_IS_REPLACEABLE_OPERATOR (new_decl
) = 0;
1065 /* Create the new version's call-graph node.
1066 and update the edges of the new node. */
1067 new_version_node
= create_version_clone (new_decl
, redirect_callers
,
1068 bbs_to_copy
, suffix
);
1070 if (ipa_transforms_to_apply
.exists ())
1071 new_version_node
->ipa_transforms_to_apply
1072 = ipa_transforms_to_apply
.copy ();
1073 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1074 tree_function_versioning (old_decl
, new_decl
, tree_map
, param_adjustments
,
1075 false, bbs_to_copy
, new_entry_block
);
1077 /* Update the new version's properties.
1078 Make The new version visible only within this translation unit. Make sure
1079 that is not weak also.
1080 ??? We cannot use COMDAT linkage because there is no
1081 ABI support for this. */
1082 new_version_node
->make_decl_local ();
1083 DECL_VIRTUAL_P (new_version_node
->decl
) = 0;
1084 new_version_node
->externally_visible
= 0;
1085 new_version_node
->local
= 1;
1086 new_version_node
->lowered
= true;
1087 if (!implicit_section
)
1088 new_version_node
->set_section (*this);
1089 /* Clones of global symbols or symbols with unique names are unique. */
1090 if ((TREE_PUBLIC (old_decl
)
1091 && !DECL_EXTERNAL (old_decl
)
1092 && !DECL_WEAK (old_decl
)
1093 && !DECL_COMDAT (old_decl
))
1095 new_version_node
->unique_name
= true;
1097 /* Update the call_expr on the edges to call the new version node. */
1098 update_call_expr (new_version_node
);
1100 symtab
->call_cgraph_insertion_hooks (new_version_node
);
1101 return new_version_node
;
1104 /* Remove the node from the tree of virtual and inline clones and make it a
1105 standalone node - not a clone any more. */
1107 void cgraph_node::remove_from_clone_tree ()
1109 if (next_sibling_clone
)
1110 next_sibling_clone
->prev_sibling_clone
= prev_sibling_clone
;
1111 if (prev_sibling_clone
)
1112 prev_sibling_clone
->next_sibling_clone
= next_sibling_clone
;
1114 clone_of
->clones
= next_sibling_clone
;
1115 next_sibling_clone
= NULL
;
1116 prev_sibling_clone
= NULL
;
1120 /* Given virtual clone, turn it into actual clone. */
1123 cgraph_node::materialize_clone ()
1125 clone_info
*info
= clone_info::get (this);
1126 clone_of
->get_untransformed_body ();
1127 former_clone_of
= clone_of
->decl
;
1128 if (clone_of
->former_clone_of
)
1129 former_clone_of
= clone_of
->former_clone_of
;
1130 if (symtab
->dump_file
)
1132 fprintf (symtab
->dump_file
, "cloning %s to %s\n",
1133 clone_of
->dump_name (),
1135 if (info
&& info
->tree_map
)
1137 fprintf (symtab
->dump_file
, " replace map:");
1138 for (unsigned int i
= 0;
1139 i
< vec_safe_length (info
->tree_map
);
1142 ipa_replace_map
*replace_info
;
1143 replace_info
= (*info
->tree_map
)[i
];
1144 fprintf (symtab
->dump_file
, "%s %i -> ",
1145 i
? "," : "", replace_info
->parm_num
);
1146 print_generic_expr (symtab
->dump_file
,
1147 replace_info
->new_tree
);
1149 fprintf (symtab
->dump_file
, "\n");
1151 if (info
&& info
->param_adjustments
)
1152 info
->param_adjustments
->dump (symtab
->dump_file
);
1154 clear_stmts_in_references ();
1155 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1156 tree_function_versioning (clone_of
->decl
, decl
,
1157 info
? info
->tree_map
: NULL
,
1158 info
? info
->param_adjustments
: NULL
,
1160 if (symtab
->dump_file
)
1162 dump_function_to_file (clone_of
->decl
, symtab
->dump_file
,
1164 dump_function_to_file (decl
, symtab
->dump_file
, dump_flags
);
1167 cgraph_node
*this_clone_of
= clone_of
;
1168 /* Function is no longer clone. */
1169 remove_from_clone_tree ();
1170 if (!this_clone_of
->analyzed
&& !this_clone_of
->clones
)
1171 this_clone_of
->release_body ();
1174 #include "gt-cgraphclones.h"