2 Copyright (C) 2003-2021 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
69 #include "coretypes.h"
75 #include "stringpool.h"
77 #include "lto-streamer.h"
80 #include "tree-inline.h"
82 #include "gimple-pretty-print.h"
83 #include "alloc-pool.h"
84 #include "symbol-summary.h"
87 #include "ipa-fnsummary.h"
88 #include "symtab-thunks.h"
89 #include "symtab-clones.h"
91 /* Create clone of edge in the node N represented by CALL_EXPR
95 cgraph_edge::clone (cgraph_node
*n
, gcall
*call_stmt
, unsigned stmt_uid
,
96 profile_count num
, profile_count den
,
99 cgraph_edge
*new_edge
;
100 profile_count::adjust_for_ipa_scaling (&num
, &den
);
101 profile_count prof_count
= count
.apply_scale (num
, den
);
103 if (indirect_unknown_callee
)
107 if (call_stmt
&& (decl
= gimple_call_fndecl (call_stmt
))
108 /* When the call is speculative, we need to resolve it
109 via cgraph_resolve_speculation and not here. */
112 cgraph_node
*callee
= cgraph_node::get (decl
);
113 gcc_checking_assert (callee
);
114 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
118 new_edge
= n
->create_indirect_edge (call_stmt
,
119 indirect_info
->ecf_flags
,
121 *new_edge
->indirect_info
= *indirect_info
;
126 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
129 new_edge
->indirect_info
130 = ggc_cleared_alloc
<cgraph_indirect_call_info
> ();
131 *new_edge
->indirect_info
= *indirect_info
;
135 new_edge
->inline_failed
= inline_failed
;
136 new_edge
->indirect_inlining_edge
= indirect_inlining_edge
;
138 new_edge
->lto_stmt_uid
= stmt_uid
;
139 new_edge
->speculative_id
= speculative_id
;
140 /* Clone flags that depend on call_stmt availability manually. */
141 new_edge
->can_throw_external
= can_throw_external
;
142 new_edge
->call_stmt_cannot_inline_p
= call_stmt_cannot_inline_p
;
143 new_edge
->speculative
= speculative
;
144 new_edge
->in_polymorphic_cdtor
= in_polymorphic_cdtor
;
146 /* Update IPA profile. Local profiles need no updating in original. */
148 count
= count
.combine_with_ipa_count_within (count
.ipa ()
149 - new_edge
->count
.ipa (),
151 symtab
->call_edge_duplication_hooks (this, new_edge
);
155 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
156 clone or its thunk. */
159 set_new_clone_decl_and_node_flags (cgraph_node
*new_node
)
161 DECL_EXTERNAL (new_node
->decl
) = 0;
162 TREE_PUBLIC (new_node
->decl
) = 0;
163 DECL_COMDAT (new_node
->decl
) = 0;
164 DECL_WEAK (new_node
->decl
) = 0;
165 DECL_VIRTUAL_P (new_node
->decl
) = 0;
166 DECL_STATIC_CONSTRUCTOR (new_node
->decl
) = 0;
167 DECL_STATIC_DESTRUCTOR (new_node
->decl
) = 0;
168 DECL_SET_IS_OPERATOR_NEW (new_node
->decl
, 0);
169 DECL_SET_IS_OPERATOR_DELETE (new_node
->decl
, 0);
170 DECL_IS_REPLACEABLE_OPERATOR (new_node
->decl
) = 0;
172 new_node
->externally_visible
= 0;
174 new_node
->lowered
= true;
177 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
178 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
179 Function can return NODE if no thunk is necessary, which can happen when
180 thunk is this_adjusting but we are removing this parameter. */
183 duplicate_thunk_for_node (cgraph_node
*thunk
, cgraph_node
*node
)
185 cgraph_node
*new_thunk
, *thunk_of
;
186 thunk_of
= thunk
->callees
->callee
->ultimate_alias_target ();
189 node
= duplicate_thunk_for_node (thunk_of
, node
);
191 if (!DECL_ARGUMENTS (thunk
->decl
))
192 thunk
->get_untransformed_body ();
194 thunk_info
*i
= thunk_info::get (thunk
);
196 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
197 if (cs
->caller
->thunk
)
199 thunk_info
*i2
= thunk_info::get (cs
->caller
);
205 clone_info
*info
= clone_info::get (node
);
206 if (info
&& info
->param_adjustments
)
208 /* We do not need to duplicate this_adjusting thunks if we have removed
210 if (i
->this_adjusting
211 && !info
->param_adjustments
->first_param_intact_p ())
214 new_decl
= copy_node (thunk
->decl
);
215 ipa_param_body_adjustments
body_adj (info
->param_adjustments
,
217 body_adj
.modify_formal_parameters ();
220 new_decl
= copy_node (thunk
->decl
);
222 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl
));
223 gcc_checking_assert (!DECL_INITIAL (new_decl
));
224 gcc_checking_assert (!DECL_RESULT (new_decl
));
225 gcc_checking_assert (!DECL_RTL_SET_P (new_decl
));
227 DECL_NAME (new_decl
) = clone_function_name_numbered (thunk
->decl
,
229 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
231 /* We need to force DECL_IGNORED_P because the new thunk is created after
232 early debug was run. */
233 DECL_IGNORED_P (new_decl
) = 1;
235 new_thunk
= cgraph_node::create (new_decl
);
236 set_new_clone_decl_and_node_flags (new_thunk
);
237 new_thunk
->definition
= true;
238 new_thunk
->can_change_signature
= node
->can_change_signature
;
239 new_thunk
->thunk
= thunk
->thunk
;
240 new_thunk
->unique_name
= in_lto_p
;
241 new_thunk
->former_clone_of
= thunk
->decl
;
242 if (info
&& info
->param_adjustments
)
243 clone_info::get_create (new_thunk
)->param_adjustments
244 = info
->param_adjustments
;
245 new_thunk
->unit_id
= thunk
->unit_id
;
246 new_thunk
->merged_comdat
= thunk
->merged_comdat
;
247 new_thunk
->merged_extern_inline
= thunk
->merged_extern_inline
;
249 cgraph_edge
*e
= new_thunk
->create_edge (node
, NULL
, new_thunk
->count
);
250 symtab
->call_edge_duplication_hooks (thunk
->callees
, e
);
251 symtab
->call_cgraph_duplication_hooks (thunk
, new_thunk
);
255 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
256 one or more equivalent thunks for N and redirect E to the first in the
257 chain. Note that it is then necessary to call
258 n->expand_all_artificial_thunks once all callers are redirected. */
261 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node
*n
)
263 cgraph_node
*orig_to
= callee
->ultimate_alias_target ();
265 n
= duplicate_thunk_for_node (orig_to
, n
);
270 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
271 that were expanded. */
274 cgraph_node::expand_all_artificial_thunks ()
277 for (e
= callers
; e
;)
278 if (e
->caller
->thunk
)
280 cgraph_node
*thunk
= e
->caller
;
283 if (expand_thunk (thunk
, false, false))
285 thunk
->thunk
= false;
287 ipa_analyze_node (thunk
);
288 inline_analyze_function (thunk
);
290 thunk
->expand_all_artificial_thunks ();
297 dump_callgraph_transformation (const cgraph_node
*original
,
298 const cgraph_node
*clone
,
301 if (symtab
->ipa_clones_dump_file
)
303 fprintf (symtab
->ipa_clones_dump_file
,
304 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
305 original
->asm_name (), original
->order
,
306 DECL_SOURCE_FILE (original
->decl
),
307 DECL_SOURCE_LINE (original
->decl
),
308 DECL_SOURCE_COLUMN (original
->decl
), clone
->asm_name (),
309 clone
->order
, DECL_SOURCE_FILE (clone
->decl
),
310 DECL_SOURCE_LINE (clone
->decl
), DECL_SOURCE_COLUMN (clone
->decl
),
313 symtab
->cloned_nodes
.add (original
);
314 symtab
->cloned_nodes
.add (clone
);
318 /* Turn profile of N to local profile. */
321 localize_profile (cgraph_node
*n
)
323 n
->count
= n
->count
.guessed_local ();
324 for (cgraph_edge
*e
= n
->callees
; e
; e
=e
->next_callee
)
326 e
->count
= e
->count
.guessed_local ();
327 if (!e
->inline_failed
)
328 localize_profile (e
->callee
);
330 for (cgraph_edge
*e
= n
->indirect_calls
; e
; e
=e
->next_callee
)
331 e
->count
= e
->count
.guessed_local ();
334 /* Create node representing clone of N executed COUNT times. Decrease
335 the execution counts from original node too.
336 The new clone will have decl set to DECL that may or may not be the same
339 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
340 function's profile to reflect the fact that part of execution is handled
342 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
343 the new clone. Otherwise the caller is responsible for doing so later.
345 If the new node is being inlined into another one, NEW_INLINED_TO should be
346 the outline function the new one is (even indirectly) inlined to. All hooks
347 will see this in node's inlined_to, when invoked. Can be NULL if the
350 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
351 will be overwritten by the new structure. Otherwise the new node will
352 share parameter manipulation information with the original node. */
355 cgraph_node::create_clone (tree new_decl
, profile_count prof_count
,
356 bool update_original
,
357 vec
<cgraph_edge
*> redirect_callers
,
358 bool call_duplication_hook
,
359 cgraph_node
*new_inlined_to
,
360 ipa_param_adjustments
*param_adjustments
,
363 cgraph_node
*new_node
= symtab
->create_empty ();
366 profile_count old_count
= count
;
367 bool nonzero
= count
.ipa ().nonzero_p ();
370 dump_callgraph_transformation (this, new_inlined_to
, "inlining to");
372 /* When inlining we scale precisely to prof_count, when cloning we can
373 preserve local profile. */
375 prof_count
= count
.combine_with_ipa_count (prof_count
);
376 new_node
->count
= prof_count
;
378 /* Update IPA profile. Local profiles need no updating in original. */
382 count
= count
.combine_with_ipa_count_within (count
.ipa ()
386 count
= count
.combine_with_ipa_count (count
.ipa () - prof_count
.ipa ());
388 new_node
->decl
= new_decl
;
389 new_node
->register_symbol ();
390 new_node
->lto_file_data
= lto_file_data
;
391 new_node
->analyzed
= analyzed
;
392 new_node
->definition
= definition
;
393 new_node
->versionable
= versionable
;
394 new_node
->can_change_signature
= can_change_signature
;
395 new_node
->redefined_extern_inline
= redefined_extern_inline
;
396 new_node
->tm_may_enter_irr
= tm_may_enter_irr
;
397 new_node
->externally_visible
= false;
398 new_node
->no_reorder
= no_reorder
;
399 new_node
->local
= true;
400 new_node
->inlined_to
= new_inlined_to
;
402 new_node
->frequency
= frequency
;
403 new_node
->tp_first_run
= tp_first_run
;
404 new_node
->tm_clone
= tm_clone
;
405 new_node
->icf_merged
= icf_merged
;
406 new_node
->thunk
= thunk
;
407 new_node
->unit_id
= unit_id
;
408 new_node
->merged_comdat
= merged_comdat
;
409 new_node
->merged_extern_inline
= merged_extern_inline
;
410 clone_info
*info
= clone_info::get (this);
412 if (param_adjustments
)
413 clone_info::get_create (new_node
)->param_adjustments
= param_adjustments
;
414 else if (info
&& info
->param_adjustments
)
415 clone_info::get_create (new_node
)->param_adjustments
416 = info
->param_adjustments
;
417 new_node
->split_part
= split_part
;
419 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
421 /* Redirect calls to the old version node to point to its new
422 version. The only exception is when the edge was proved to
423 be unreachable during the cloning procedure. */
425 || !fndecl_built_in_p (e
->callee
->decl
, BUILT_IN_UNREACHABLE
))
426 e
->redirect_callee_duplicating_thunks (new_node
);
428 new_node
->expand_all_artificial_thunks ();
430 for (e
= callees
;e
; e
=e
->next_callee
)
431 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
, new_node
->count
, old_count
,
434 for (e
= indirect_calls
; e
; e
= e
->next_callee
)
435 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
436 new_node
->count
, old_count
, update_original
);
437 new_node
->clone_references (this);
439 new_node
->next_sibling_clone
= clones
;
441 clones
->prev_sibling_clone
= new_node
;
443 new_node
->clone_of
= this;
445 if (call_duplication_hook
)
446 symtab
->call_cgraph_duplication_hooks (this, new_node
);
447 /* With partial train run we do not want to assume that original's
448 count is zero whenever we redurect all executed edges to clone.
449 Simply drop profile to local one in this case. */
451 && opt_for_fn (decl
, flag_profile_partial_training
)
454 && !count
.ipa ().nonzero_p ()
456 localize_profile (this);
459 dump_callgraph_transformation (this, new_node
, suffix
);
464 static GTY(()) hash_map
<const char *, unsigned> *clone_fn_ids
;
466 /* Return a new assembler name for a clone of decl named NAME. Apart
467 from the string SUFFIX, the new name will end with a unique (for
468 each NAME) unspecified number. If clone numbering is not needed
469 then the two argument clone_function_name should be used instead.
470 Should not be called directly except for by
471 lto-partition.c:privatize_symbol_name_1. */
474 clone_function_name_numbered (const char *name
, const char *suffix
)
476 /* Initialize the function->counter mapping the first time it's
479 clone_fn_ids
= hash_map
<const char *, unsigned int>::create_ggc (64);
480 unsigned int &suffix_counter
= clone_fn_ids
->get_or_insert (
481 IDENTIFIER_POINTER (get_identifier (name
)));
482 return clone_function_name (name
, suffix
, suffix_counter
++);
485 /* Return a new assembler name for a clone of DECL. Apart from string
486 SUFFIX, the new name will end with a unique (for each DECL
487 assembler name) unspecified number. If clone numbering is not
488 needed then the two argument clone_function_name should be used
492 clone_function_name_numbered (tree decl
, const char *suffix
)
494 tree name
= DECL_ASSEMBLER_NAME (decl
);
495 return clone_function_name_numbered (IDENTIFIER_POINTER (name
),
499 /* Return a new assembler name for a clone of decl named NAME. Apart
500 from the string SUFFIX, the new name will end with the specified
501 NUMBER. If clone numbering is not needed then the two argument
502 clone_function_name should be used instead. */
505 clone_function_name (const char *name
, const char *suffix
,
506 unsigned long number
)
508 size_t len
= strlen (name
);
509 char *tmp_name
, *prefix
;
511 prefix
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
512 memcpy (prefix
, name
, len
);
513 strcpy (prefix
+ len
+ 1, suffix
);
514 prefix
[len
] = symbol_table::symbol_suffix_separator ();
515 ASM_FORMAT_PRIVATE_NAME (tmp_name
, prefix
, number
);
516 return get_identifier (tmp_name
);
519 /* Return a new assembler name for a clone of DECL. Apart from the
520 string SUFFIX, the new name will end with the specified NUMBER. If
521 clone numbering is not needed then the two argument
522 clone_function_name should be used instead. */
525 clone_function_name (tree decl
, const char *suffix
,
526 unsigned long number
)
528 return clone_function_name (
529 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)), suffix
, number
);
532 /* Return a new assembler name ending with the string SUFFIX for a
536 clone_function_name (tree decl
, const char *suffix
)
538 tree identifier
= DECL_ASSEMBLER_NAME (decl
);
539 /* For consistency this needs to behave the same way as
540 ASM_FORMAT_PRIVATE_NAME does, but without the final number
542 char *separator
= XALLOCAVEC (char, 2);
543 separator
[0] = symbol_table::symbol_suffix_separator ();
545 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
546 const char *prefix
= "__";
548 const char *prefix
= "";
550 char *result
= ACONCAT ((prefix
,
551 IDENTIFIER_POINTER (identifier
),
555 return get_identifier (result
);
559 /* Create callgraph node clone with new declaration. The actual body will be
560 copied later at compilation stage. The name of the new clone will be
561 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
563 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
567 cgraph_node::create_virtual_clone (const vec
<cgraph_edge
*> &redirect_callers
,
568 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
569 ipa_param_adjustments
*param_adjustments
,
570 const char * suffix
, unsigned num_suffix
)
572 tree old_decl
= decl
;
573 cgraph_node
*new_node
= NULL
;
576 ipa_replace_map
*map
;
579 gcc_checking_assert (versionable
);
580 /* TODO: It would be nice if we could recognize that param_adjustments do not
581 actually perform any changes, but at the moment let's require it simply
583 gcc_assert (can_change_signature
|| !param_adjustments
);
585 /* Make a new FUNCTION_DECL tree node */
586 if (!param_adjustments
)
587 new_decl
= copy_node (old_decl
);
589 new_decl
= param_adjustments
->adjust_decl (old_decl
);
591 /* These pointers represent function body and will be populated only when clone
593 gcc_assert (new_decl
!= old_decl
);
594 DECL_STRUCT_FUNCTION (new_decl
) = NULL
;
595 DECL_ARGUMENTS (new_decl
) = NULL
;
596 DECL_INITIAL (new_decl
) = NULL
;
597 DECL_RESULT (new_decl
) = NULL
;
598 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
599 sometimes storing only clone decl instead of original. */
601 /* Generate a new name for the new version. */
602 len
= IDENTIFIER_LENGTH (DECL_NAME (old_decl
));
603 name
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
604 memcpy (name
, IDENTIFIER_POINTER (DECL_NAME (old_decl
)), len
);
605 strcpy (name
+ len
+ 1, suffix
);
607 DECL_NAME (new_decl
) = get_identifier (name
);
608 SET_DECL_ASSEMBLER_NAME (new_decl
,
609 clone_function_name (old_decl
, suffix
, num_suffix
));
610 SET_DECL_RTL (new_decl
, NULL
);
612 new_node
= create_clone (new_decl
, count
, false,
613 redirect_callers
, false, NULL
, param_adjustments
,
616 /* Update the properties.
617 Make clone visible only within this translation unit. Make sure
618 that is not weak also.
619 ??? We cannot use COMDAT linkage because there is no
620 ABI support for this. */
621 set_new_clone_decl_and_node_flags (new_node
);
622 new_node
->ipcp_clone
= ipcp_clone
;
624 clone_info::get_create (new_node
)->tree_map
= tree_map
;
625 if (!implicit_section
)
626 new_node
->set_section (*this);
628 /* Clones of global symbols or symbols with unique names are unique. */
629 if ((TREE_PUBLIC (old_decl
)
630 && !DECL_EXTERNAL (old_decl
)
631 && !DECL_WEAK (old_decl
)
632 && !DECL_COMDAT (old_decl
))
634 new_node
->unique_name
= true;
635 FOR_EACH_VEC_SAFE_ELT (tree_map
, i
, map
)
637 tree repl
= map
->new_tree
;
638 if (map
->force_load_ref
)
640 gcc_assert (TREE_CODE (repl
) == ADDR_EXPR
);
641 repl
= get_base_address (TREE_OPERAND (repl
, 0));
643 new_node
->maybe_create_reference (repl
, NULL
);
646 if (ipa_transforms_to_apply
.exists ())
647 new_node
->ipa_transforms_to_apply
648 = ipa_transforms_to_apply
.copy ();
650 symtab
->call_cgraph_duplication_hooks (this, new_node
);
655 /* callgraph node being removed from symbol table; see if its entry can be
656 replaced by other inline clone.
657 INFO is clone info to attach to the new root. */
659 cgraph_node::find_replacement (clone_info
*info
)
661 cgraph_node
*next_inline_clone
, *replacement
;
663 for (next_inline_clone
= clones
;
665 && next_inline_clone
->decl
!= decl
;
666 next_inline_clone
= next_inline_clone
->next_sibling_clone
)
669 /* If there is inline clone of the node being removed, we need
670 to put it into the position of removed node and reorganize all
671 other clones to be based on it. */
672 if (next_inline_clone
)
675 cgraph_node
*new_clones
;
677 replacement
= next_inline_clone
;
679 /* Unlink inline clone from the list of clones of removed node. */
680 if (next_inline_clone
->next_sibling_clone
)
681 next_inline_clone
->next_sibling_clone
->prev_sibling_clone
682 = next_inline_clone
->prev_sibling_clone
;
683 if (next_inline_clone
->prev_sibling_clone
)
685 gcc_assert (clones
!= next_inline_clone
);
686 next_inline_clone
->prev_sibling_clone
->next_sibling_clone
687 = next_inline_clone
->next_sibling_clone
;
691 gcc_assert (clones
== next_inline_clone
);
692 clones
= next_inline_clone
->next_sibling_clone
;
698 /* Copy clone info. */
700 *clone_info::get_create (next_inline_clone
) = *info
;
702 /* Now place it into clone tree at same level at NODE. */
703 next_inline_clone
->clone_of
= clone_of
;
704 next_inline_clone
->prev_sibling_clone
= NULL
;
705 next_inline_clone
->next_sibling_clone
= NULL
;
708 if (clone_of
->clones
)
709 clone_of
->clones
->prev_sibling_clone
= next_inline_clone
;
710 next_inline_clone
->next_sibling_clone
= clone_of
->clones
;
711 clone_of
->clones
= next_inline_clone
;
714 /* Merge the clone list. */
717 if (!next_inline_clone
->clones
)
718 next_inline_clone
->clones
= new_clones
;
721 n
= next_inline_clone
->clones
;
722 while (n
->next_sibling_clone
)
723 n
= n
->next_sibling_clone
;
724 n
->next_sibling_clone
= new_clones
;
725 new_clones
->prev_sibling_clone
= n
;
729 /* Update clone_of pointers. */
733 n
->clone_of
= next_inline_clone
;
734 n
= n
->next_sibling_clone
;
737 /* Update order in order to be able to find a LTO section
738 with function body. */
739 replacement
->order
= order
;
747 /* Like cgraph_set_call_stmt but walk the clone tree and update all
748 clones sharing the same function body.
749 When WHOLE_SPECULATIVE_EDGES is true, all three components of
750 speculative edge gets updated. Otherwise we update only direct
754 cgraph_node::set_call_stmt_including_clones (gimple
*old_stmt
,
756 bool update_speculative
)
759 cgraph_edge
*master_edge
= get_edge (old_stmt
);
762 cgraph_edge::set_call_stmt (master_edge
, new_stmt
, update_speculative
);
768 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
771 edge
= cgraph_edge::set_call_stmt (edge
, new_stmt
,
773 /* If UPDATE_SPECULATIVE is false, it means that we are turning
774 speculative call into a real code sequence. Update the
776 if (edge
->speculative
&& !update_speculative
)
778 cgraph_edge
*indirect
= edge
->speculative_call_indirect_edge ();
780 for (cgraph_edge
*next
, *direct
781 = edge
->first_speculative_call_target ();
785 next
= direct
->next_speculative_call_target ();
786 direct
->speculative_call_target_ref ()->speculative
= false;
787 direct
->speculative
= false;
789 indirect
->speculative
= false;
794 else if (node
->next_sibling_clone
)
795 node
= node
->next_sibling_clone
;
798 while (node
!= this && !node
->next_sibling_clone
)
799 node
= node
->clone_of
;
801 node
= node
->next_sibling_clone
;
806 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
807 same function body. If clones already have edge for OLD_STMT; only
808 update the edge same way as cgraph_set_call_stmt_including_clones does.
810 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
811 frequencies of the clones. */
814 cgraph_node::create_edge_including_clones (cgraph_node
*callee
,
815 gimple
*old_stmt
, gcall
*stmt
,
817 cgraph_inline_failed_t reason
)
821 if (!get_edge (stmt
))
823 cgraph_edge
*edge
= create_edge (callee
, stmt
, count
);
824 edge
->inline_failed
= reason
;
830 /* Thunk clones do not get updated while copying inline function body. */
833 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
835 /* It is possible that clones already contain the edge while
836 master didn't. Either we promoted indirect call into direct
837 call in the clone or we are processing clones of unreachable
838 master where edges has been removed. */
840 edge
= cgraph_edge::set_call_stmt (edge
, stmt
);
841 else if (! node
->get_edge (stmt
))
843 edge
= node
->create_edge (callee
, stmt
, count
);
844 edge
->inline_failed
= reason
;
849 else if (node
->next_sibling_clone
)
850 node
= node
->next_sibling_clone
;
853 while (node
!= this && !node
->next_sibling_clone
)
854 node
= node
->clone_of
;
856 node
= node
->next_sibling_clone
;
861 /* Remove the node from cgraph and all inline clones inlined into it.
862 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
863 removed. This allows to call the function from outer loop walking clone
867 cgraph_node::remove_symbol_and_inline_clones (cgraph_node
*forbidden_node
)
869 cgraph_edge
*e
, *next
;
872 if (this == forbidden_node
)
874 cgraph_edge::remove (callers
);
877 for (e
= callees
; e
; e
= next
)
879 next
= e
->next_callee
;
880 if (!e
->inline_failed
)
881 found
|= e
->callee
->remove_symbol_and_inline_clones (forbidden_node
);
887 /* The edges representing the callers of the NEW_VERSION node were
888 fixed by cgraph_function_versioning (), now the call_expr in their
889 respective tree code should be updated to call the NEW_VERSION. */
892 update_call_expr (cgraph_node
*new_version
)
896 gcc_assert (new_version
);
898 /* Update the call expr on the edges to call the new version. */
899 for (e
= new_version
->callers
; e
; e
= e
->next_caller
)
901 function
*inner_function
= DECL_STRUCT_FUNCTION (e
->caller
->decl
);
902 gimple_call_set_fndecl (e
->call_stmt
, new_version
->decl
);
903 maybe_clean_eh_stmt_fn (inner_function
, e
->call_stmt
);
908 /* Create a new cgraph node which is the new version of
909 callgraph node. REDIRECT_CALLERS holds the callers
910 edges which should be redirected to point to
911 NEW_VERSION. ALL the callees edges of the node
912 are cloned to the new version node. Return the new
915 If non-NULL BLOCK_TO_COPY determine what basic blocks
916 was copied to prevent duplications of calls that are dead
920 cgraph_node::create_version_clone (tree new_decl
,
921 vec
<cgraph_edge
*> redirect_callers
,
925 cgraph_node
*new_version
;
929 new_version
= cgraph_node::create (new_decl
);
931 new_version
->analyzed
= analyzed
;
932 new_version
->definition
= definition
;
933 new_version
->local
= local
;
934 new_version
->externally_visible
= false;
935 new_version
->no_reorder
= no_reorder
;
936 new_version
->local
= new_version
->definition
;
937 new_version
->inlined_to
= inlined_to
;
938 new_version
->rtl
= rtl
;
939 new_version
->count
= count
;
940 new_version
->unit_id
= unit_id
;
941 new_version
->merged_comdat
= merged_comdat
;
942 new_version
->merged_extern_inline
= merged_extern_inline
;
944 for (e
= callees
; e
; e
=e
->next_callee
)
946 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
947 e
->clone (new_version
, e
->call_stmt
,
948 e
->lto_stmt_uid
, count
, count
,
950 for (e
= indirect_calls
; e
; e
=e
->next_callee
)
952 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
953 e
->clone (new_version
, e
->call_stmt
,
954 e
->lto_stmt_uid
, count
, count
,
956 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
958 /* Redirect calls to the old version node to point to its new
960 e
->redirect_callee (new_version
);
963 dump_callgraph_transformation (this, new_version
, suffix
);
968 /* Perform function versioning.
969 Function versioning includes copying of the tree and
970 a callgraph update (creating a new cgraph node and updating
971 its callees and callers).
973 REDIRECT_CALLERS varray includes the edges to be redirected
976 TREE_MAP is a mapping of tree nodes we want to replace with
977 new ones (according to results of prior analysis).
979 If non-NULL ARGS_TO_SKIP determine function parameters to remove
981 If SKIP_RETURN is true, the new version will return void.
982 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
983 If non_NULL NEW_ENTRY determine new entry BB of the clone.
985 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
986 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
987 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
990 Return the new version's cgraph node. */
993 cgraph_node::create_version_clone_with_body
994 (vec
<cgraph_edge
*> redirect_callers
,
995 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
996 ipa_param_adjustments
*param_adjustments
,
997 bitmap bbs_to_copy
, basic_block new_entry_block
, const char *suffix
,
998 tree target_attributes
)
1000 tree old_decl
= decl
;
1001 cgraph_node
*new_version_node
= NULL
;
1004 if (!tree_versionable_function_p (old_decl
))
1007 /* TODO: Restore an assert that we do not change signature if
1008 can_change_signature is false. We cannot just check that
1009 param_adjustments is NULL because unfortunately ipa-split removes return
1010 values from such functions. */
1012 /* Make a new FUNCTION_DECL tree node for the new version. */
1013 if (param_adjustments
)
1014 new_decl
= param_adjustments
->adjust_decl (old_decl
);
1016 new_decl
= copy_node (old_decl
);
1018 /* Generate a new name for the new version. */
1019 DECL_NAME (new_decl
) = clone_function_name_numbered (old_decl
, suffix
);
1020 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
1021 SET_DECL_RTL (new_decl
, NULL
);
1023 DECL_VIRTUAL_P (new_decl
) = 0;
1025 if (target_attributes
)
1027 DECL_ATTRIBUTES (new_decl
) = target_attributes
;
1029 location_t saved_loc
= input_location
;
1030 tree v
= TREE_VALUE (target_attributes
);
1031 input_location
= DECL_SOURCE_LOCATION (new_decl
);
1032 bool r
= targetm
.target_option
.valid_attribute_p (new_decl
, NULL
, v
, 1);
1033 input_location
= saved_loc
;
1038 /* When the old decl was a con-/destructor make sure the clone isn't. */
1039 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
1040 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
1041 DECL_SET_IS_OPERATOR_NEW (new_decl
, 0);
1042 DECL_SET_IS_OPERATOR_DELETE (new_decl
, 0);
1043 DECL_IS_REPLACEABLE_OPERATOR (new_decl
) = 0;
1045 /* Create the new version's call-graph node.
1046 and update the edges of the new node. */
1047 new_version_node
= create_version_clone (new_decl
, redirect_callers
,
1048 bbs_to_copy
, suffix
);
1050 if (ipa_transforms_to_apply
.exists ())
1051 new_version_node
->ipa_transforms_to_apply
1052 = ipa_transforms_to_apply
.copy ();
1053 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1054 tree_function_versioning (old_decl
, new_decl
, tree_map
, param_adjustments
,
1055 false, bbs_to_copy
, new_entry_block
);
1057 /* Update the new version's properties.
1058 Make The new version visible only within this translation unit. Make sure
1059 that is not weak also.
1060 ??? We cannot use COMDAT linkage because there is no
1061 ABI support for this. */
1062 new_version_node
->make_decl_local ();
1063 DECL_VIRTUAL_P (new_version_node
->decl
) = 0;
1064 new_version_node
->externally_visible
= 0;
1065 new_version_node
->local
= 1;
1066 new_version_node
->lowered
= true;
1067 if (!implicit_section
)
1068 new_version_node
->set_section (*this);
1069 /* Clones of global symbols or symbols with unique names are unique. */
1070 if ((TREE_PUBLIC (old_decl
)
1071 && !DECL_EXTERNAL (old_decl
)
1072 && !DECL_WEAK (old_decl
)
1073 && !DECL_COMDAT (old_decl
))
1075 new_version_node
->unique_name
= true;
1077 /* Update the call_expr on the edges to call the new version node. */
1078 update_call_expr (new_version_node
);
1080 symtab
->call_cgraph_insertion_hooks (new_version_node
);
1081 return new_version_node
;
1084 /* Remove the node from the tree of virtual and inline clones and make it a
1085 standalone node - not a clone any more. */
1087 void cgraph_node::remove_from_clone_tree ()
1089 if (next_sibling_clone
)
1090 next_sibling_clone
->prev_sibling_clone
= prev_sibling_clone
;
1091 if (prev_sibling_clone
)
1092 prev_sibling_clone
->next_sibling_clone
= next_sibling_clone
;
1094 clone_of
->clones
= next_sibling_clone
;
1095 next_sibling_clone
= NULL
;
1096 prev_sibling_clone
= NULL
;
1100 /* Given virtual clone, turn it into actual clone. */
1103 cgraph_node::materialize_clone ()
1105 clone_info
*info
= clone_info::get (this);
1106 clone_of
->get_untransformed_body ();
1107 former_clone_of
= clone_of
->decl
;
1108 if (clone_of
->former_clone_of
)
1109 former_clone_of
= clone_of
->former_clone_of
;
1110 if (symtab
->dump_file
)
1112 fprintf (symtab
->dump_file
, "cloning %s to %s\n",
1113 clone_of
->dump_name (),
1115 if (info
&& info
->tree_map
)
1117 fprintf (symtab
->dump_file
, " replace map:");
1118 for (unsigned int i
= 0;
1119 i
< vec_safe_length (info
->tree_map
);
1122 ipa_replace_map
*replace_info
;
1123 replace_info
= (*info
->tree_map
)[i
];
1124 fprintf (symtab
->dump_file
, "%s %i -> ",
1125 i
? "," : "", replace_info
->parm_num
);
1126 print_generic_expr (symtab
->dump_file
,
1127 replace_info
->new_tree
);
1129 fprintf (symtab
->dump_file
, "\n");
1131 if (info
&& info
->param_adjustments
)
1132 info
->param_adjustments
->dump (symtab
->dump_file
);
1134 clear_stmts_in_references ();
1135 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1136 tree_function_versioning (clone_of
->decl
, decl
,
1137 info
? info
->tree_map
: NULL
,
1138 info
? info
->param_adjustments
: NULL
,
1140 if (symtab
->dump_file
)
1142 dump_function_to_file (clone_of
->decl
, symtab
->dump_file
,
1144 dump_function_to_file (decl
, symtab
->dump_file
, dump_flags
);
1147 cgraph_node
*this_clone_of
= clone_of
;
1148 /* Function is no longer clone. */
1149 remove_from_clone_tree ();
1150 if (!this_clone_of
->analyzed
&& !this_clone_of
->clones
)
1151 this_clone_of
->release_body ();
1154 #include "gt-cgraphclones.h"