2 Copyright (C) 2003-2014 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* This module provide facilities for clonning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
69 #include "coretypes.h"
73 #include "stringpool.h"
76 #include "basic-block.h"
77 #include "tree-ssa-alias.h"
78 #include "internal-fn.h"
80 #include "gimple-expr.h"
85 #include "tree-inline.h"
86 #include "langhooks.h"
91 #include "diagnostic.h"
96 #include "tree-iterator.h"
97 #include "tree-dump.h"
98 #include "gimple-pretty-print.h"
100 #include "ipa-inline.h"
101 #include "ipa-utils.h"
102 #include "lto-streamer.h"
106 /* Create clone of E in the node N represented by CALL_EXPR the callgraph. */
108 cgraph_clone_edge (struct cgraph_edge
*e
, struct cgraph_node
*n
,
109 gimple call_stmt
, unsigned stmt_uid
, gcov_type count_scale
,
110 int freq_scale
, bool update_original
)
112 struct cgraph_edge
*new_edge
;
113 gcov_type count
= apply_probability (e
->count
, count_scale
);
116 /* We do not want to ignore loop nest after frequency drops to 0. */
119 freq
= e
->frequency
* (gcov_type
) freq_scale
/ CGRAPH_FREQ_BASE
;
120 if (freq
> CGRAPH_FREQ_MAX
)
121 freq
= CGRAPH_FREQ_MAX
;
123 if (e
->indirect_unknown_callee
)
127 if (call_stmt
&& (decl
= gimple_call_fndecl (call_stmt
))
128 /* When the call is speculative, we need to resolve it
129 via cgraph_resolve_speculation and not here. */
132 struct cgraph_node
*callee
;
133 if (L_IPO_COMP_MODE
&& cgraph_pre_profiling_inlining_done
)
134 callee
= cgraph_lipo_get_resolved_node (decl
);
136 callee
= cgraph_get_node (decl
);
137 gcc_checking_assert (callee
);
138 new_edge
= cgraph_create_edge (n
, callee
, call_stmt
, count
, freq
);
142 new_edge
= cgraph_create_indirect_edge (n
, call_stmt
,
143 e
->indirect_info
->ecf_flags
,
145 *new_edge
->indirect_info
= *e
->indirect_info
;
150 new_edge
= cgraph_create_edge (n
, e
->callee
, call_stmt
, count
, freq
);
151 if (e
->indirect_info
)
153 new_edge
->indirect_info
154 = ggc_alloc_cleared_cgraph_indirect_call_info ();
155 *new_edge
->indirect_info
= *e
->indirect_info
;
159 new_edge
->inline_failed
= e
->inline_failed
;
160 new_edge
->indirect_inlining_edge
= e
->indirect_inlining_edge
;
161 new_edge
->lto_stmt_uid
= stmt_uid
;
162 /* Clone flags that depend on call_stmt availability manually. */
163 new_edge
->can_throw_external
= e
->can_throw_external
;
164 new_edge
->call_stmt_cannot_inline_p
= e
->call_stmt_cannot_inline_p
;
165 new_edge
->speculative
= e
->speculative
;
168 e
->count
-= new_edge
->count
;
172 cgraph_call_edge_duplication_hooks (e
, new_edge
);
176 /* Build variant of function type ORIG_TYPE skipping ARGS_TO_SKIP and the
177 return value if SKIP_RETURN is true. */
180 build_function_type_skip_args (tree orig_type
, bitmap args_to_skip
,
183 tree new_type
= NULL
;
184 tree args
, new_args
= NULL
, t
;
188 for (args
= TYPE_ARG_TYPES (orig_type
); args
&& args
!= void_list_node
;
189 args
= TREE_CHAIN (args
), i
++)
190 if (!args_to_skip
|| !bitmap_bit_p (args_to_skip
, i
))
191 new_args
= tree_cons (NULL_TREE
, TREE_VALUE (args
), new_args
);
193 new_reversed
= nreverse (new_args
);
197 TREE_CHAIN (new_args
) = void_list_node
;
199 new_reversed
= void_list_node
;
202 /* Use copy_node to preserve as much as possible from original type
203 (debug info, attribute lists etc.)
204 Exception is METHOD_TYPEs must have THIS argument.
205 When we are asked to remove it, we need to build new FUNCTION_TYPE
207 if (TREE_CODE (orig_type
) != METHOD_TYPE
209 || !bitmap_bit_p (args_to_skip
, 0))
211 new_type
= build_distinct_type_copy (orig_type
);
212 TYPE_ARG_TYPES (new_type
) = new_reversed
;
217 = build_distinct_type_copy (build_function_type (TREE_TYPE (orig_type
),
219 TYPE_CONTEXT (new_type
) = TYPE_CONTEXT (orig_type
);
223 TREE_TYPE (new_type
) = void_type_node
;
225 /* This is a new type, not a copy of an old type. Need to reassociate
226 variants. We can handle everything except the main variant lazily. */
227 t
= TYPE_MAIN_VARIANT (orig_type
);
230 t
= build_function_type_skip_args (t
, args_to_skip
, skip_return
);
231 TYPE_MAIN_VARIANT (new_type
) = t
;
232 TYPE_NEXT_VARIANT (new_type
) = TYPE_NEXT_VARIANT (t
);
233 TYPE_NEXT_VARIANT (t
) = new_type
;
237 TYPE_MAIN_VARIANT (new_type
) = new_type
;
238 TYPE_NEXT_VARIANT (new_type
) = NULL
;
244 /* Build variant of function decl ORIG_DECL skipping ARGS_TO_SKIP and the
245 return value if SKIP_RETURN is true.
247 Arguments from DECL_ARGUMENTS list can't be removed now, since they are
248 linked by TREE_CHAIN directly. The caller is responsible for eliminating
249 them when they are being duplicated (i.e. copy_arguments_for_versioning). */
252 build_function_decl_skip_args (tree orig_decl
, bitmap args_to_skip
,
255 tree new_decl
= copy_node (orig_decl
);
258 new_type
= TREE_TYPE (orig_decl
);
259 if (prototype_p (new_type
)
260 || (skip_return
&& !VOID_TYPE_P (TREE_TYPE (new_type
))))
262 = build_function_type_skip_args (new_type
, args_to_skip
, skip_return
);
263 TREE_TYPE (new_decl
) = new_type
;
265 /* For declarations setting DECL_VINDEX (i.e. methods)
266 we expect first argument to be THIS pointer. */
267 if (args_to_skip
&& bitmap_bit_p (args_to_skip
, 0))
268 DECL_VINDEX (new_decl
) = NULL_TREE
;
270 /* When signature changes, we need to clear builtin info. */
271 if (DECL_BUILT_IN (new_decl
)
273 && !bitmap_empty_p (args_to_skip
))
275 DECL_BUILT_IN_CLASS (new_decl
) = NOT_BUILT_IN
;
276 DECL_FUNCTION_CODE (new_decl
) = (enum built_in_function
) 0;
278 /* The FE might have information and assumptions about the other
280 DECL_LANG_SPECIFIC (new_decl
) = NULL
;
284 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
285 clone or its thunk. */
288 set_new_clone_decl_and_node_flags (cgraph_node
*new_node
)
290 DECL_EXTERNAL (new_node
->decl
) = 0;
291 DECL_COMDAT_GROUP (new_node
->decl
) = 0;
292 TREE_PUBLIC (new_node
->decl
) = 0;
293 DECL_COMDAT (new_node
->decl
) = 0;
294 DECL_WEAK (new_node
->decl
) = 0;
295 DECL_VIRTUAL_P (new_node
->decl
) = 0;
296 DECL_STATIC_CONSTRUCTOR (new_node
->decl
) = 0;
297 DECL_STATIC_DESTRUCTOR (new_node
->decl
) = 0;
299 new_node
->externally_visible
= 0;
300 new_node
->local
.local
= 1;
301 new_node
->lowered
= true;
304 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
305 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
306 Function can return NODE if no thunk is necessary, which can happen when
307 thunk is this_adjusting but we are removing this parameter. */
310 duplicate_thunk_for_node (cgraph_node
*thunk
, cgraph_node
*node
,
313 cgraph_node
*new_thunk
, *thunk_of
;
314 thunk_of
= cgraph_function_or_thunk_node (thunk
->callees
->callee
);
316 if (thunk_of
->thunk
.thunk_p
)
317 node
= duplicate_thunk_for_node (thunk_of
, node
, args_to_skip
);
319 struct cgraph_edge
*cs
;
320 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
321 if (cs
->caller
->thunk
.thunk_p
322 && cs
->caller
->thunk
.this_adjusting
== thunk
->thunk
.this_adjusting
323 && cs
->caller
->thunk
.fixed_offset
== thunk
->thunk
.fixed_offset
324 && cs
->caller
->thunk
.virtual_offset_p
== thunk
->thunk
.virtual_offset_p
325 && cs
->caller
->thunk
.virtual_value
== thunk
->thunk
.virtual_value
)
330 new_decl
= copy_node (thunk
->decl
);
333 /* We do not need to duplicate this_adjusting thunks if we have removed
335 if (thunk
->thunk
.this_adjusting
336 && bitmap_bit_p (args_to_skip
, 0))
339 new_decl
= build_function_decl_skip_args (thunk
->decl
, args_to_skip
,
342 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl
));
343 gcc_checking_assert (!DECL_INITIAL (new_decl
));
344 gcc_checking_assert (!DECL_RESULT (new_decl
));
345 gcc_checking_assert (!DECL_RTL_SET_P (new_decl
));
347 DECL_NAME (new_decl
) = clone_function_name (thunk
->decl
, "artificial_thunk");
348 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
349 DECL_SECTION_NAME (new_decl
) = NULL
;
351 new_thunk
= cgraph_create_node (new_decl
);
352 set_new_clone_decl_and_node_flags (new_thunk
);
353 new_thunk
->definition
= true;
354 new_thunk
->thunk
= thunk
->thunk
;
355 new_thunk
->unique_name
= in_lto_p
;
356 new_thunk
->former_clone_of
= thunk
->decl
;
358 struct cgraph_edge
*e
= cgraph_create_edge (new_thunk
, node
, NULL
, 0,
360 e
->call_stmt_cannot_inline_p
= true;
361 cgraph_call_edge_duplication_hooks (thunk
->callees
, e
);
362 if (!expand_thunk (new_thunk
, false))
363 new_thunk
->analyzed
= true;
364 cgraph_call_node_duplication_hooks (thunk
, new_thunk
);
368 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
369 one or more equivalent thunks for N and redirect E to the first in the
373 redirect_edge_duplicating_thunks (struct cgraph_edge
*e
, struct cgraph_node
*n
,
376 cgraph_node
*orig_to
= cgraph_function_or_thunk_node (e
->callee
);
377 if (orig_to
->thunk
.thunk_p
)
378 n
= duplicate_thunk_for_node (orig_to
, n
, args_to_skip
);
380 cgraph_redirect_edge_callee (e
, n
);
383 /* Create node representing clone of N executed COUNT times. Decrease
384 the execution counts from original node too.
385 The new clone will have decl set to DECL that may or may not be the same
388 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
389 function's profile to reflect the fact that part of execution is handled
391 When CALL_DUPLICATOIN_HOOK is true, the ipa passes are acknowledged about
392 the new clone. Otherwise the caller is responsible for doing so later.
394 If the new node is being inlined into another one, NEW_INLINED_TO should be
395 the outline function the new one is (even indirectly) inlined to. All hooks
396 will see this in node's global.inlined_to, when invoked. Can be NULL if the
397 node is not inlined. */
400 cgraph_clone_node (struct cgraph_node
*n
, tree decl
, gcov_type count
, int freq
,
401 bool update_original
,
402 vec
<cgraph_edge_p
> redirect_callers
,
403 bool call_duplication_hook
,
404 struct cgraph_node
*new_inlined_to
,
407 struct cgraph_node
*new_node
= cgraph_create_empty_node ();
408 struct cgraph_edge
*e
;
409 gcov_type count_scale
;
412 new_node
->decl
= decl
;
413 symtab_register_node (new_node
);
414 new_node
->origin
= n
->origin
;
415 new_node
->lto_file_data
= n
->lto_file_data
;
416 if (new_node
->origin
)
418 new_node
->next_nested
= new_node
->origin
->nested
;
419 new_node
->origin
->nested
= new_node
;
421 new_node
->analyzed
= n
->analyzed
;
422 new_node
->definition
= n
->definition
;
423 new_node
->local
= n
->local
;
424 new_node
->externally_visible
= false;
425 new_node
->local
.local
= true;
426 new_node
->global
= n
->global
;
427 new_node
->global
.inlined_to
= new_inlined_to
;
428 new_node
->rtl
= n
->rtl
;
429 new_node
->count
= count
;
430 new_node
->max_bb_count
= count
;
432 new_node
->max_bb_count
= ((n
->max_bb_count
+ n
->count
/ 2)
434 new_node
->frequency
= n
->frequency
;
435 new_node
->clone
= n
->clone
;
436 new_node
->clone
.tree_map
= NULL
;
437 new_node
->tp_first_run
= n
->tp_first_run
;
440 if (new_node
->count
> n
->count
)
441 count_scale
= REG_BR_PROB_BASE
;
443 count_scale
= GCOV_COMPUTE_SCALE (new_node
->count
, n
->count
);
447 /* In AutoFDO, if edge count is larger than callee's entry block
448 count, we will not update the original callee because it may
449 mistakenly mark some hot function as cold. */
450 if (flag_auto_profile
&& count
>= n
->count
)
451 update_original
= false;
457 n
->max_bb_count
-= new_node
->max_bb_count
;
458 if (n
->max_bb_count
< 0)
462 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
464 /* Redirect calls to the old version node to point to its new
465 version. The only exception is when the edge was proved to
466 be unreachable during the clonning procedure. */
468 || DECL_BUILT_IN_CLASS (e
->callee
->decl
) != BUILT_IN_NORMAL
469 || DECL_FUNCTION_CODE (e
->callee
->decl
) != BUILT_IN_UNREACHABLE
)
470 redirect_edge_duplicating_thunks (e
, new_node
, args_to_skip
);
474 for (e
= n
->callees
;e
; e
=e
->next_callee
)
475 cgraph_clone_edge (e
, new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
476 count_scale
, freq
, update_original
);
478 for (e
= n
->indirect_calls
; e
; e
= e
->next_callee
)
479 cgraph_clone_edge (e
, new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
480 count_scale
, freq
, update_original
);
481 ipa_clone_references (new_node
, &n
->ref_list
);
483 new_node
->next_sibling_clone
= n
->clones
;
485 n
->clones
->prev_sibling_clone
= new_node
;
486 n
->clones
= new_node
;
487 new_node
->clone_of
= n
;
489 if (call_duplication_hook
)
490 cgraph_call_node_duplication_hooks (n
, new_node
);
494 /* Return a new assembler name for a clone of DECL with SUFFIX. */
496 static GTY(()) unsigned int clone_fn_id_num
;
499 clone_function_name (tree decl
, const char *suffix
)
501 tree name
= DECL_ASSEMBLER_NAME (decl
);
502 size_t len
= IDENTIFIER_LENGTH (name
);
503 char *tmp_name
, *prefix
;
505 prefix
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
506 memcpy (prefix
, IDENTIFIER_POINTER (name
), len
);
507 strcpy (prefix
+ len
+ 1, suffix
);
508 #ifndef NO_DOT_IN_LABEL
510 #elif !defined NO_DOLLAR_IN_LABEL
515 ASM_FORMAT_PRIVATE_NAME (tmp_name
, prefix
, clone_fn_id_num
++);
516 return get_identifier (tmp_name
);
519 /* Create callgraph node clone with new declaration. The actual body will
520 be copied later at compilation stage.
522 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
526 cgraph_create_virtual_clone (struct cgraph_node
*old_node
,
527 vec
<cgraph_edge_p
> redirect_callers
,
528 vec
<ipa_replace_map_p
, va_gc
> *tree_map
,
532 tree old_decl
= old_node
->decl
;
533 struct cgraph_node
*new_node
= NULL
;
536 struct ipa_replace_map
*map
;
540 gcc_checking_assert (tree_versionable_function_p (old_decl
));
542 gcc_assert (old_node
->local
.can_change_signature
|| !args_to_skip
);
544 /* Make a new FUNCTION_DECL tree node */
546 new_decl
= copy_node (old_decl
);
548 new_decl
= build_function_decl_skip_args (old_decl
, args_to_skip
, false);
550 /* These pointers represent function body and will be populated only when clone
552 gcc_assert (new_decl
!= old_decl
);
553 DECL_STRUCT_FUNCTION (new_decl
) = NULL
;
554 DECL_ARGUMENTS (new_decl
) = NULL
;
555 DECL_INITIAL (new_decl
) = NULL
;
556 DECL_RESULT (new_decl
) = NULL
;
557 /* We can not do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
558 sometimes storing only clone decl instead of original. */
560 /* Generate a new name for the new version. */
561 len
= IDENTIFIER_LENGTH (DECL_NAME (old_decl
));
562 name
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
563 memcpy (name
, IDENTIFIER_POINTER (DECL_NAME (old_decl
)), len
);
564 strcpy (name
+ len
+ 1, suffix
);
566 DECL_NAME (new_decl
) = get_identifier (name
);
567 SET_DECL_ASSEMBLER_NAME (new_decl
, clone_function_name (old_decl
, suffix
));
568 SET_DECL_RTL (new_decl
, NULL
);
570 new_node
= cgraph_clone_node (old_node
, new_decl
, old_node
->count
,
571 CGRAPH_FREQ_BASE
, false,
572 redirect_callers
, false, NULL
, args_to_skip
);
573 /* Update the properties.
574 Make clone visible only within this translation unit. Make sure
575 that is not weak also.
576 ??? We cannot use COMDAT linkage because there is no
577 ABI support for this. */
578 if (DECL_ONE_ONLY (old_decl
))
579 DECL_SECTION_NAME (new_node
->decl
) = NULL
;
580 set_new_clone_decl_and_node_flags (new_node
);
581 new_node
->clone
.tree_map
= tree_map
;
582 new_node
->clone
.args_to_skip
= args_to_skip
;
584 /* Clones of global symbols or symbols with unique names are unique. */
585 if ((TREE_PUBLIC (old_decl
)
586 && !DECL_EXTERNAL (old_decl
)
587 && !DECL_WEAK (old_decl
)
588 && !DECL_COMDAT (old_decl
))
590 new_node
->unique_name
= true;
591 FOR_EACH_VEC_SAFE_ELT (tree_map
, i
, map
)
592 ipa_maybe_record_reference (new_node
, map
->new_tree
,
595 new_node
->clone
.combined_args_to_skip
= old_node
->clone
.combined_args_to_skip
;
596 else if (old_node
->clone
.combined_args_to_skip
)
598 int newi
= 0, oldi
= 0;
600 bitmap new_args_to_skip
= BITMAP_GGC_ALLOC ();
601 struct cgraph_node
*orig_node
;
602 for (orig_node
= old_node
; orig_node
->clone_of
; orig_node
= orig_node
->clone_of
)
604 for (arg
= DECL_ARGUMENTS (orig_node
->decl
);
605 arg
; arg
= DECL_CHAIN (arg
), oldi
++)
607 if (bitmap_bit_p (old_node
->clone
.combined_args_to_skip
, oldi
))
609 bitmap_set_bit (new_args_to_skip
, oldi
);
612 if (bitmap_bit_p (args_to_skip
, newi
))
613 bitmap_set_bit (new_args_to_skip
, oldi
);
616 new_node
->clone
.combined_args_to_skip
= new_args_to_skip
;
619 new_node
->clone
.combined_args_to_skip
= args_to_skip
;
621 cgraph_call_node_duplication_hooks (old_node
, new_node
);
627 /* NODE is being removed from symbol table; see if its entry can be replaced by
628 other inline clone. */
630 cgraph_find_replacement_node (struct cgraph_node
*node
)
632 struct cgraph_node
*next_inline_clone
, *replacement
;
634 for (next_inline_clone
= node
->clones
;
636 && next_inline_clone
->decl
!= node
->decl
;
637 next_inline_clone
= next_inline_clone
->next_sibling_clone
)
640 /* If there is inline clone of the node being removed, we need
641 to put it into the position of removed node and reorganize all
642 other clones to be based on it. */
643 if (next_inline_clone
)
645 struct cgraph_node
*n
;
646 struct cgraph_node
*new_clones
;
648 replacement
= next_inline_clone
;
650 /* Unlink inline clone from the list of clones of removed node. */
651 if (next_inline_clone
->next_sibling_clone
)
652 next_inline_clone
->next_sibling_clone
->prev_sibling_clone
653 = next_inline_clone
->prev_sibling_clone
;
654 if (next_inline_clone
->prev_sibling_clone
)
656 gcc_assert (node
->clones
!= next_inline_clone
);
657 next_inline_clone
->prev_sibling_clone
->next_sibling_clone
658 = next_inline_clone
->next_sibling_clone
;
662 gcc_assert (node
->clones
== next_inline_clone
);
663 node
->clones
= next_inline_clone
->next_sibling_clone
;
666 new_clones
= node
->clones
;
669 /* Copy clone info. */
670 next_inline_clone
->clone
= node
->clone
;
672 /* Now place it into clone tree at same level at NODE. */
673 next_inline_clone
->clone_of
= node
->clone_of
;
674 next_inline_clone
->prev_sibling_clone
= NULL
;
675 next_inline_clone
->next_sibling_clone
= NULL
;
678 if (node
->clone_of
->clones
)
679 node
->clone_of
->clones
->prev_sibling_clone
= next_inline_clone
;
680 next_inline_clone
->next_sibling_clone
= node
->clone_of
->clones
;
681 node
->clone_of
->clones
= next_inline_clone
;
684 /* Merge the clone list. */
687 if (!next_inline_clone
->clones
)
688 next_inline_clone
->clones
= new_clones
;
691 n
= next_inline_clone
->clones
;
692 while (n
->next_sibling_clone
)
693 n
= n
->next_sibling_clone
;
694 n
->next_sibling_clone
= new_clones
;
695 new_clones
->prev_sibling_clone
= n
;
699 /* Update clone_of pointers. */
703 n
->clone_of
= next_inline_clone
;
704 n
= n
->next_sibling_clone
;
712 /* Like cgraph_set_call_stmt but walk the clone tree and update all
713 clones sharing the same function body.
714 When WHOLE_SPECULATIVE_EDGES is true, all three components of
715 speculative edge gets updated. Otherwise we update only direct
719 cgraph_set_call_stmt_including_clones (struct cgraph_node
*orig
,
720 gimple old_stmt
, gimple new_stmt
,
721 bool update_speculative
)
723 struct cgraph_node
*node
;
724 struct cgraph_edge
*edge
= cgraph_edge (orig
, old_stmt
);
727 cgraph_set_call_stmt (edge
, new_stmt
, update_speculative
);
733 struct cgraph_edge
*edge
= cgraph_edge (node
, old_stmt
);
736 cgraph_set_call_stmt (edge
, new_stmt
, update_speculative
);
737 /* If UPDATE_SPECULATIVE is false, it means that we are turning
738 speculative call into a real code sequence. Update the
740 if (edge
->speculative
&& !update_speculative
)
742 struct cgraph_edge
*direct
, *indirect
;
745 gcc_assert (!edge
->indirect_unknown_callee
);
746 cgraph_speculative_call_info (edge
, direct
, indirect
, ref
);
747 direct
->speculative
= false;
748 indirect
->speculative
= false;
749 ref
->speculative
= false;
754 else if (node
->next_sibling_clone
)
755 node
= node
->next_sibling_clone
;
758 while (node
!= orig
&& !node
->next_sibling_clone
)
759 node
= node
->clone_of
;
761 node
= node
->next_sibling_clone
;
766 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
767 same function body. If clones already have edge for OLD_STMT; only
768 update the edge same way as cgraph_set_call_stmt_including_clones does.
770 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
771 frequencies of the clones. */
774 cgraph_create_edge_including_clones (struct cgraph_node
*orig
,
775 struct cgraph_node
*callee
,
777 gimple stmt
, gcov_type count
,
779 cgraph_inline_failed_t reason
)
781 struct cgraph_node
*node
;
782 struct cgraph_edge
*edge
;
784 if (!cgraph_edge (orig
, stmt
))
786 edge
= cgraph_create_edge (orig
, callee
, stmt
, count
, freq
);
787 edge
->inline_failed
= reason
;
794 struct cgraph_edge
*edge
= cgraph_edge (node
, old_stmt
);
796 /* It is possible that clones already contain the edge while
797 master didn't. Either we promoted indirect call into direct
798 call in the clone or we are processing clones of unreachable
799 master where edges has been removed. */
801 cgraph_set_call_stmt (edge
, stmt
);
802 else if (!cgraph_edge (node
, stmt
))
804 edge
= cgraph_create_edge (node
, callee
, stmt
, count
,
806 edge
->inline_failed
= reason
;
811 else if (node
->next_sibling_clone
)
812 node
= node
->next_sibling_clone
;
815 while (node
!= orig
&& !node
->next_sibling_clone
)
816 node
= node
->clone_of
;
818 node
= node
->next_sibling_clone
;
823 /* Remove the node from cgraph and all inline clones inlined into it.
824 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
825 removed. This allows to call the function from outer loop walking clone
829 cgraph_remove_node_and_inline_clones (struct cgraph_node
*node
, struct cgraph_node
*forbidden_node
)
831 struct cgraph_edge
*e
, *next
;
834 if (node
== forbidden_node
)
836 cgraph_remove_edge (node
->callers
);
839 for (e
= node
->callees
; e
; e
= next
)
841 next
= e
->next_callee
;
842 if (!e
->inline_failed
)
843 found
|= cgraph_remove_node_and_inline_clones (e
->callee
, forbidden_node
);
845 cgraph_remove_node (node
);
849 /* The edges representing the callers of the NEW_VERSION node were
850 fixed by cgraph_function_versioning (), now the call_expr in their
851 respective tree code should be updated to call the NEW_VERSION. */
854 update_call_expr (struct cgraph_node
*new_version
)
856 struct cgraph_edge
*e
;
858 gcc_assert (new_version
);
860 /* Update the call expr on the edges to call the new version. */
861 for (e
= new_version
->callers
; e
; e
= e
->next_caller
)
863 struct function
*inner_function
= DECL_STRUCT_FUNCTION (e
->caller
->decl
);
864 gimple_call_set_fndecl (e
->call_stmt
, new_version
->decl
);
865 maybe_clean_eh_stmt_fn (inner_function
, e
->call_stmt
);
870 /* Create a new cgraph node which is the new version of
871 OLD_VERSION node. REDIRECT_CALLERS holds the callers
872 edges which should be redirected to point to
873 NEW_VERSION. ALL the callees edges of OLD_VERSION
874 are cloned to the new version node. Return the new
877 If non-NULL BLOCK_TO_COPY determine what basic blocks
878 was copied to prevent duplications of calls that are dead
882 cgraph_copy_node_for_versioning (struct cgraph_node
*old_version
,
884 vec
<cgraph_edge_p
> redirect_callers
,
887 struct cgraph_node
*new_version
;
888 struct cgraph_edge
*e
;
891 gcc_assert (old_version
);
893 new_version
= cgraph_create_node (new_decl
);
895 new_version
->analyzed
= old_version
->analyzed
;
896 new_version
->definition
= old_version
->definition
;
897 new_version
->local
= old_version
->local
;
898 new_version
->externally_visible
= false;
899 new_version
->local
.local
= new_version
->definition
;
900 new_version
->global
= old_version
->global
;
901 new_version
->rtl
= old_version
->rtl
;
902 new_version
->count
= old_version
->count
;
903 new_version
->max_bb_count
= old_version
->max_bb_count
;
905 for (e
= old_version
->callees
; e
; e
=e
->next_callee
)
907 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
908 cgraph_clone_edge (e
, new_version
, e
->call_stmt
,
909 e
->lto_stmt_uid
, REG_BR_PROB_BASE
,
912 for (e
= old_version
->indirect_calls
; e
; e
=e
->next_callee
)
914 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
915 cgraph_clone_edge (e
, new_version
, e
->call_stmt
,
916 e
->lto_stmt_uid
, REG_BR_PROB_BASE
,
919 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
921 /* Redirect calls to the old version node to point to its new
923 cgraph_redirect_edge_callee (e
, new_version
);
926 cgraph_call_node_duplication_hooks (old_version
, new_version
);
931 /* Perform function versioning.
932 Function versioning includes copying of the tree and
933 a callgraph update (creating a new cgraph node and updating
934 its callees and callers).
936 REDIRECT_CALLERS varray includes the edges to be redirected
939 TREE_MAP is a mapping of tree nodes we want to replace with
940 new ones (according to results of prior analysis).
941 OLD_VERSION_NODE is the node that is versioned.
943 If non-NULL ARGS_TO_SKIP determine function parameters to remove
945 If SKIP_RETURN is true, the new version will return void.
946 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
947 If non_NULL NEW_ENTRY determine new entry BB of the clone.
949 Return the new version's cgraph node. */
952 cgraph_function_versioning (struct cgraph_node
*old_version_node
,
953 vec
<cgraph_edge_p
> redirect_callers
,
954 vec
<ipa_replace_map_p
, va_gc
> *tree_map
,
958 basic_block new_entry_block
,
959 const char *clone_name
)
961 tree old_decl
= old_version_node
->decl
;
962 struct cgraph_node
*new_version_node
= NULL
;
965 if (!tree_versionable_function_p (old_decl
))
968 gcc_assert (old_version_node
->local
.can_change_signature
|| !args_to_skip
);
970 /* Make a new FUNCTION_DECL tree node for the new version. */
971 if (!args_to_skip
&& !skip_return
)
972 new_decl
= copy_node (old_decl
);
975 = build_function_decl_skip_args (old_decl
, args_to_skip
, skip_return
);
977 /* Generate a new name for the new version. */
978 DECL_NAME (new_decl
) = clone_function_name (old_decl
, clone_name
);
979 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
980 SET_DECL_RTL (new_decl
, NULL
);
982 /* When the old decl was a con-/destructor make sure the clone isn't. */
983 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
984 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
986 /* Create the new version's call-graph node.
987 and update the edges of the new node. */
989 cgraph_copy_node_for_versioning (old_version_node
, new_decl
,
990 redirect_callers
, bbs_to_copy
);
992 /* Copy the OLD_VERSION_NODE function tree to the new version. */
993 tree_function_versioning (old_decl
, new_decl
, tree_map
, false, args_to_skip
,
994 skip_return
, bbs_to_copy
, new_entry_block
);
996 /* Update the new version's properties.
997 Make The new version visible only within this translation unit. Make sure
998 that is not weak also.
999 ??? We cannot use COMDAT linkage because there is no
1000 ABI support for this. */
1001 symtab_make_decl_local (new_version_node
->decl
);
1002 DECL_VIRTUAL_P (new_version_node
->decl
) = 0;
1003 new_version_node
->externally_visible
= 0;
1004 new_version_node
->local
.local
= 1;
1005 new_version_node
->lowered
= true;
1006 /* Clones of global symbols or symbols with unique names are unique. */
1007 if ((TREE_PUBLIC (old_decl
)
1008 && !DECL_EXTERNAL (old_decl
)
1009 && !DECL_WEAK (old_decl
)
1010 && !DECL_COMDAT (old_decl
))
1012 new_version_node
->unique_name
= true;
1014 /* Update the call_expr on the edges to call the new version node. */
1015 update_call_expr (new_version_node
);
1017 cgraph_call_function_insertion_hooks (new_version_node
);
1018 return new_version_node
;
1021 /* Given virtual clone, turn it into actual clone. */
1024 cgraph_materialize_clone (struct cgraph_node
*node
)
1026 bitmap_obstack_initialize (NULL
);
1027 node
->former_clone_of
= node
->clone_of
->decl
;
1028 if (node
->clone_of
->former_clone_of
)
1029 node
->former_clone_of
= node
->clone_of
->former_clone_of
;
1030 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1031 tree_function_versioning (node
->clone_of
->decl
, node
->decl
,
1032 node
->clone
.tree_map
, true,
1033 node
->clone
.args_to_skip
, false,
1035 if (cgraph_dump_file
)
1037 dump_function_to_file (node
->clone_of
->decl
, cgraph_dump_file
, dump_flags
);
1038 dump_function_to_file (node
->decl
, cgraph_dump_file
, dump_flags
);
1041 /* Function is no longer clone. */
1042 if (node
->next_sibling_clone
)
1043 node
->next_sibling_clone
->prev_sibling_clone
= node
->prev_sibling_clone
;
1044 if (node
->prev_sibling_clone
)
1045 node
->prev_sibling_clone
->next_sibling_clone
= node
->next_sibling_clone
;
1047 node
->clone_of
->clones
= node
->next_sibling_clone
;
1048 node
->next_sibling_clone
= NULL
;
1049 node
->prev_sibling_clone
= NULL
;
1050 if (!node
->clone_of
->analyzed
&& !node
->clone_of
->clones
)
1052 cgraph_release_function_body (node
->clone_of
);
1053 cgraph_node_remove_callees (node
->clone_of
);
1054 ipa_remove_all_references (&node
->clone_of
->ref_list
);
1056 node
->clone_of
= NULL
;
1057 bitmap_obstack_release (NULL
);
1060 /* Once all functions from compilation unit are in memory, produce all clones
1061 and update all calls. We might also do this on demand if we don't want to
1062 bring all functions to memory prior compilation, but current WHOPR
1063 implementation does that and it is is bit easier to keep everything right in
1067 cgraph_materialize_all_clones (void)
1069 struct cgraph_node
*node
;
1070 bool stabilized
= false;
1073 if (cgraph_dump_file
)
1074 fprintf (cgraph_dump_file
, "Materializing clones\n");
1075 #ifdef ENABLE_CHECKING
1079 /* We can also do topological order, but number of iterations should be
1080 bounded by number of IPA passes since single IPA pass is probably not
1081 going to create clones of clones it created itself. */
1085 FOR_EACH_FUNCTION (node
)
1087 if (node
->clone_of
&& node
->decl
!= node
->clone_of
->decl
1088 && !gimple_has_body_p (node
->decl
))
1090 if (!node
->clone_of
->clone_of
)
1091 cgraph_get_body (node
->clone_of
);
1092 if (gimple_has_body_p (node
->clone_of
->decl
))
1094 if (cgraph_dump_file
)
1096 fprintf (cgraph_dump_file
, "cloning %s to %s\n",
1097 xstrdup (node
->clone_of
->name ()),
1098 xstrdup (node
->name ()));
1099 if (node
->clone
.tree_map
)
1102 fprintf (cgraph_dump_file
, " replace map: ");
1104 i
< vec_safe_length (node
->clone
.tree_map
);
1107 struct ipa_replace_map
*replace_info
;
1108 replace_info
= (*node
->clone
.tree_map
)[i
];
1109 print_generic_expr (cgraph_dump_file
, replace_info
->old_tree
, 0);
1110 fprintf (cgraph_dump_file
, " -> ");
1111 print_generic_expr (cgraph_dump_file
, replace_info
->new_tree
, 0);
1112 fprintf (cgraph_dump_file
, "%s%s;",
1113 replace_info
->replace_p
? "(replace)":"",
1114 replace_info
->ref_p
? "(ref)":"");
1116 fprintf (cgraph_dump_file
, "\n");
1118 if (node
->clone
.args_to_skip
)
1120 fprintf (cgraph_dump_file
, " args_to_skip: ");
1121 dump_bitmap (cgraph_dump_file
, node
->clone
.args_to_skip
);
1123 if (node
->clone
.args_to_skip
)
1125 fprintf (cgraph_dump_file
, " combined_args_to_skip:");
1126 dump_bitmap (cgraph_dump_file
, node
->clone
.combined_args_to_skip
);
1129 cgraph_materialize_clone (node
);
1135 FOR_EACH_FUNCTION (node
)
1136 if (!node
->analyzed
&& node
->callees
)
1138 cgraph_node_remove_callees (node
);
1139 ipa_remove_all_references (&node
->ref_list
);
1142 ipa_clear_stmts_in_references (node
);
1143 if (cgraph_dump_file
)
1144 fprintf (cgraph_dump_file
, "Materialization Call site updates done.\n");
1145 #ifdef ENABLE_CHECKING
1148 symtab_remove_unreachable_nodes (false, cgraph_dump_file
);
1151 #include "gt-cgraphclones.h"