target/112280 - properly guard permute query
[official-gcc.git] / gcc / ipa-strub.cc
bloba3c5318202e2840d710e2f8c6bce7a86db037c94
1 /* strub (stack scrubbing) support.
2 Copyright (C) 2021-2024 Free Software Foundation, Inc.
3 Contributed by Alexandre Oliva <oliva@adacore.com>.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "tree.h"
26 #include "gimple.h"
27 #include "gimplify.h"
28 #include "tree-pass.h"
29 #include "ssa.h"
30 #include "gimple-iterator.h"
31 #include "gimplify-me.h"
32 #include "tree-into-ssa.h"
33 #include "tree-ssa.h"
34 #include "tree-cfg.h"
35 #include "cfghooks.h"
36 #include "cfgloop.h"
37 #include "cfgcleanup.h"
38 #include "tree-eh.h"
39 #include "except.h"
40 #include "builtins.h"
41 #include "attribs.h"
42 #include "tree-inline.h"
43 #include "cgraph.h"
44 #include "alloc-pool.h"
45 #include "symbol-summary.h"
46 #include "ipa-prop.h"
47 #include "ipa-fnsummary.h"
48 #include "gimple-fold.h"
49 #include "fold-const.h"
50 #include "gimple-walk.h"
51 #include "tree-dfa.h"
52 #include "langhooks.h"
53 #include "calls.h"
54 #include "vec.h"
55 #include "stor-layout.h"
56 #include "varasm.h"
57 #include "alias.h"
58 #include "diagnostic.h"
59 #include "intl.h"
60 #include "ipa-strub.h"
61 #include "symtab-thunks.h"
62 #include "attr-fnspec.h"
63 #include "target.h"
65 /* This file introduces two passes that, together, implement
66 machine-independent stack scrubbing, strub for short. It arranges
67 for stack frames that have strub enabled to be zeroed-out after
68 relinquishing control to a caller, whether by returning or by
69 propagating an exception. This admittedly unusual design decision
70 was driven by exception support (one needs a stack frame to be
71 active to propagate exceptions out of it), and it enabled an
72 implementation that is entirely machine-independent (no custom
73 epilogue code is required).
75 Strub modes can be selected for stack frames by attaching attribute
76 strub to functions or to variables (to their types, actually).
77 Different strub modes, with different implementation details, are
78 available, and they can be selected by an argument to the strub
79 attribute. When enabled by strub-enabled variables, whether by
80 accessing (as in reading from) statically-allocated ones, or by
81 introducing (as in declaring) automatically-allocated ones, a
82 suitable mode is selected automatically.
84 At-calls mode modifies the interface of a function, adding a stack
85 watermark argument, that callers use to clean up the stack frame of
86 the called function. Because of the interface change, it can only
87 be used when explicitly selected, or when a function is internal to
88 a translation unit. Strub-at-calls function types are distinct
89 from their original types (they're not modified in-place), and they
90 are not interchangeable with other function types.
92 Internal mode, in turn, does not modify the type or the interface
93 of a function. It is currently implemented by turning the function
94 into a wrapper, moving the function body to a separate wrapped
95 function, and scrubbing the wrapped body's stack in the wrapper.
96 Internal-strub function types are mostly interface-compatible with
97 other strub modes, namely callable (from strub functions, though
98 not strub-enabled) and disabled (not callable from strub
99 functions).
101 Always_inline functions can be strub functions, but they can only
102 be called from other strub functions, because strub functions must
103 never be inlined into non-strub functions. Internal and at-calls
104 modes are indistinguishable when it comes to always_inline
105 functions: they will necessarily be inlined into another strub
106 function, and will thus be integrated into the caller's stack
107 frame, whatever the mode. (Contrast with non-always_inline strub
108 functions: an at-calls function can be called from other strub
109 functions, ensuring no discontinuity in stack erasing, whereas an
110 internal-strub function can only be called from other strub
111 functions if it happens to be inlined, or if -fstrub=relaxed mode
112 is in effect (that's the default). In -fstrub=strict mode,
113 internal-strub functions are not callable from strub functions,
114 because the wrapper itself is not strubbed.
116 The implementation involves two simple-IPA passes. The earliest
117 one, strub-mode, assigns strub modes to functions. It needs to run
118 before any inlining, so that we can prevent inlining of strub
119 functions into non-strub functions. It notes explicit strub mode
120 requests, enables strub in response to strub variables and testing
121 options, and flags unsatisfiable requests.
123 Three possibilities of unsatisfiable requests come to mind: (a)
124 when a strub mode is explicitly selected, but the function uses
125 features that make it ineligible for that mode (e.g. at-calls rules
126 out calling __builtin_apply_args, because of the interface changes,
127 and internal mode rules out noclone or otherwise non-versionable
128 functions, non-default varargs, non-local or forced labels, and
129 functions with far too many arguments); (b) when some strub mode
130 must be enabled because of a strub variable, but the function is
131 not eligible or not viable for any mode; and (c) when
132 -fstrub=strict is enabled, and calls are found in strub functions
133 to functions that are not callable from strub contexts.
134 compute_strub_mode implements (a) and (b), and verify_strub
135 implements (c).
137 The second IPA pass modifies interfaces of at-calls-strub functions
138 and types, introduces strub calls in and around them. and splits
139 internal-strub functions. It is placed after early inlining, so
140 that even internal-strub functions get a chance of being inlined
141 into other strub functions, but before non-early inlining, so that
142 internal-strub wrapper functions still get a chance of inlining
143 after splitting.
145 Wrappers avoid duplicating the copying of large arguments again by
146 passing them by reference to the wrapped bodies. This involves
147 occasional SSA rewriting of address computations, because of the
148 additional indirection. Besides these changes, and the
149 introduction of the stack watermark parameter, wrappers and wrapped
150 functions cooperate to handle variable argument lists (performing
151 va_start in the wrapper, passing the list as an argument, and
152 replacing va_start calls in the wrapped body with va_copy), and
153 __builtin_apply_args (also called in the wrapper and passed to the
154 wrapped body as an argument).
156 Strub bodies (both internal-mode wrapped bodies, and at-calls
157 functions) always start by adjusting the watermark parameter, by
158 calling __builtin___strub_update. The compiler inserts them in the
159 main strub pass. Allocations of additional stack space for the
160 frame (__builtin_alloca) are also followed by watermark updates.
161 Stack space temporarily allocated to pass arguments to other
162 functions, released right after the call, is not regarded as part
163 of the frame. Around calls to them, i.e., in internal-mode
164 wrappers and at-calls callers (even calls through pointers), calls
165 to __builtin___strub_enter and __builtin___strub_leave are
166 inserted, the latter as a __finally block, so that it runs at
167 regular and exceptional exit paths. strub_enter only initializes
168 the stack watermark, and strub_leave is where the scrubbing takes
169 place, overwriting with zeros the stack space from the top of the
170 stack to the watermark.
172 These calls can be optimized in various cases. In
173 pass_ipa_strub::adjust_at_calls_call, for example, we enable
174 tail-calling and other optimized calls from one strub body to
175 another by passing on the watermark parameter. The builtins
176 themselves may undergo inline substitution during expansion,
177 dependign on optimization levels. This involves dealing with stack
178 red zones (when the builtins are called out-of-line, the red zone
179 cannot be used) and other ugly details related with inlining strub
180 bodies into other strub bodies (see expand_builtin_strub_update).
181 expand_builtin_strub_leave may even perform partial inline
182 substitution. */
184 /* Const and pure functions that gain a watermark parameter for strub purposes
185 are still regarded as such, which may cause the inline expansions of the
186 __strub builtins to malfunction. Ideally, attribute "fn spec" would enable
187 us to inform the backend about requirements and side effects of the call, but
188 call_fusage building in calls.c:expand_call does not even look at
189 attr_fnspec, so we resort to asm loads and updates to attain an equivalent
190 effect. Once expand_call gains the ability to issue extra memory uses and
191 clobbers based on pure/const function's fnspec, we can define this to 1. */
192 #define ATTR_FNSPEC_DECONST_WATERMARK 0
194 enum strub_mode {
195 /* This mode denotes a regular function, that does not require stack
196 scrubbing (strubbing). It may call any other functions, but if
197 it calls AT_CALLS (or WRAPPED) ones, strubbing logic is
198 automatically introduced around those calls (the latter, by
199 inlining INTERNAL wrappers). */
200 STRUB_DISABLED = 0,
202 /* This denotes a function whose signature is (to be) modified to
203 take an extra parameter, for stack use annotation, and its
204 callers must initialize and pass that argument, and perform the
205 strubbing. Functions that are explicitly marked with attribute
206 strub must have the mark visible wherever the function is,
207 including aliases, and overriders and overriding methods.
208 Functions that are implicitly marked for strubbing, for accessing
209 variables explicitly marked as such, will only select this
210 strubbing method if they are internal to a translation unit. It
211 can only be inlined into other strubbing functions, i.e.,
212 STRUB_AT_CALLS or STRUB_WRAPPED. */
213 STRUB_AT_CALLS = 1,
215 /* This denotes a function that is to perform strubbing internally,
216 without any changes to its interface (the function is turned into
217 a strubbing wrapper, and its original body is moved to a separate
218 STRUB_WRAPPED function, with a modified interface). Functions
219 may be explicitly marked with attribute strub(2), and the
220 attribute must be visible at the point of definition. Functions
221 that are explicitly marked for strubbing, for accessing variables
222 explicitly marked as such, may select this strubbing mode if
223 their interface cannot change, e.g. because its interface is
224 visible to other translation units, directly, by indirection
225 (having its address taken), inheritance, etc. Functions that use
226 this method must not have the noclone attribute, nor the noipa
227 one. Functions marked as always_inline may select this mode, but
228 they are NOT wrapped, they remain unchanged, and are only inlined
229 into strubbed contexts. Once non-always_inline functions are
230 wrapped, the wrapper becomes STRUB_WRAPPER, and the wrapped becomes
231 STRUB_WRAPPED. */
232 STRUB_INTERNAL = 2,
234 /* This denotes a function whose stack is not strubbed, but that is
235 nevertheless explicitly or implicitly marked as callable from strubbing
236 functions. Normally, only STRUB_AT_CALLS (and STRUB_INTERNAL ->
237 STRUB_WRAPPED) functions can be called from strubbing contexts (bodies of
238 STRUB_AT_CALLS, STRUB_INTERNAL and STRUB_WRAPPED functions), but attribute
239 strub(3) enables other functions to be (indirectly) called from these
240 contexts. Some builtins and internal functions may be implicitly marked as
241 STRUB_CALLABLE. */
242 STRUB_CALLABLE = 3,
244 /* This denotes the function that took over the body of a
245 STRUB_INTERNAL function. At first, it's only called by its
246 wrapper, but the wrapper may be inlined. The wrapped function,
247 in turn, can only be inlined into other functions whose stack
248 frames are strubbed, i.e., that are STRUB_WRAPPED or
249 STRUB_AT_CALLS. */
250 STRUB_WRAPPED = -1,
252 /* This denotes the wrapper function that replaced the STRUB_INTERNAL
253 function. This mode overrides the STRUB_INTERNAL mode at the time the
254 internal to-be-wrapped function becomes a wrapper, so that inlining logic
255 can tell one from the other. */
256 STRUB_WRAPPER = -2,
258 /* This denotes an always_inline function that requires strubbing. It can
259 only be called from, and inlined into, other strubbing contexts. */
260 STRUB_INLINABLE = -3,
262 /* This denotes a function that accesses strub variables, so it would call for
263 internal strubbing (whether or not it's eligible for that), but since
264 at-calls strubbing is viable, that's selected as an optimization. This
265 mode addresses the inconvenience that such functions may have different
266 modes selected depending on optimization flags, and get a different
267 callable status depending on that choice: if we assigned them
268 STRUB_AT_CALLS mode, they would be callable when optimizing, whereas
269 STRUB_INTERNAL would not be callable. */
270 STRUB_AT_CALLS_OPT = -4,
274 /* Look up a strub attribute in TYPE, and return it. */
276 static tree
277 get_strub_attr_from_type (tree type)
279 return lookup_attribute ("strub", TYPE_ATTRIBUTES (type));
282 /* Look up a strub attribute in DECL or in its type, and return it. */
284 static tree
285 get_strub_attr_from_decl (tree decl)
287 tree ret = lookup_attribute ("strub", DECL_ATTRIBUTES (decl));
288 if (ret)
289 return ret;
290 return get_strub_attr_from_type (TREE_TYPE (decl));
293 #define STRUB_ID_COUNT 8
294 #define STRUB_IDENT_COUNT 3
295 #define STRUB_TYPE_COUNT 5
297 #define STRUB_ID_BASE 0
298 #define STRUB_IDENT_BASE (STRUB_ID_BASE + STRUB_ID_COUNT)
299 #define STRUB_TYPE_BASE (STRUB_IDENT_BASE + STRUB_IDENT_COUNT)
300 #define STRUB_CACHE_SIZE (STRUB_TYPE_BASE + STRUB_TYPE_COUNT)
302 /* Keep the strub mode and temp identifiers and types from being GC'd. */
303 static GTY((deletable)) tree strub_cache[STRUB_CACHE_SIZE];
305 /* Define a function to cache identifier ID, to be used as a strub attribute
306 parameter for a strub mode named after NAME. */
307 #define DEF_STRUB_IDS(IDX, NAME, ID) \
308 static inline tree get_strub_mode_id_ ## NAME () { \
309 int idx = STRUB_ID_BASE + IDX; \
310 tree identifier = strub_cache[idx]; \
311 if (!identifier) \
312 strub_cache[idx] = identifier = get_identifier (ID); \
313 return identifier; \
315 /* Same as DEF_STRUB_IDS, but use the string expansion of NAME as ID. */
316 #define DEF_STRUB_ID(IDX, NAME) \
317 DEF_STRUB_IDS (IDX, NAME, #NAME)
319 /* Define functions for each of the strub mode identifiers.
320 Expose dashes rather than underscores. */
321 DEF_STRUB_ID (0, disabled)
322 DEF_STRUB_IDS (1, at_calls, "at-calls")
323 DEF_STRUB_ID (2, internal)
324 DEF_STRUB_ID (3, callable)
325 DEF_STRUB_ID (4, wrapped)
326 DEF_STRUB_ID (5, wrapper)
327 DEF_STRUB_ID (6, inlinable)
328 DEF_STRUB_IDS (7, at_calls_opt, "at-calls-opt")
330 /* Release the temporary macro names. */
331 #undef DEF_STRUB_IDS
332 #undef DEF_STRUB_ID
334 /* Return the identifier corresponding to strub MODE. */
336 static tree
337 get_strub_mode_attr_parm (enum strub_mode mode)
339 switch (mode)
341 case STRUB_DISABLED:
342 return get_strub_mode_id_disabled ();
344 case STRUB_AT_CALLS:
345 return get_strub_mode_id_at_calls ();
347 case STRUB_INTERNAL:
348 return get_strub_mode_id_internal ();
350 case STRUB_CALLABLE:
351 return get_strub_mode_id_callable ();
353 case STRUB_WRAPPED:
354 return get_strub_mode_id_wrapped ();
356 case STRUB_WRAPPER:
357 return get_strub_mode_id_wrapper ();
359 case STRUB_INLINABLE:
360 return get_strub_mode_id_inlinable ();
362 case STRUB_AT_CALLS_OPT:
363 return get_strub_mode_id_at_calls_opt ();
365 default:
366 gcc_unreachable ();
370 /* Return the parmeters (TREE_VALUE) for a strub attribute of MODE.
371 We know we use a single parameter, so we bypass the creation of a
372 tree list. */
374 static tree
375 get_strub_mode_attr_value (enum strub_mode mode)
377 return get_strub_mode_attr_parm (mode);
380 /* Determine whether ID is a well-formed strub mode-specifying attribute
381 parameter for a function (type). Only user-visible modes are accepted, and
382 ID must be non-NULL.
384 For unacceptable parms, return 0, otherwise a nonzero value as below.
386 If the parm enables strub, return positive, otherwise negative.
388 If the affected type must be a distinct, incompatible type,return an integer
389 of absolute value 2, otherwise 1. */
392 strub_validate_fn_attr_parm (tree id)
394 int ret;
395 const char *s = NULL;
396 size_t len = 0;
398 /* do NOT test for NULL. This is only to be called with non-NULL arguments.
399 We assume that the strub parameter applies to a function, because only
400 functions accept an explicit argument. If we accepted NULL, and we
401 happened to be called to verify the argument for a variable, our return
402 values would be wrong. */
403 if (TREE_CODE (id) == STRING_CST)
405 s = TREE_STRING_POINTER (id);
406 len = TREE_STRING_LENGTH (id) - 1;
408 else if (TREE_CODE (id) == IDENTIFIER_NODE)
410 s = IDENTIFIER_POINTER (id);
411 len = IDENTIFIER_LENGTH (id);
413 else
414 return 0;
416 enum strub_mode mode;
418 if (len != 8)
419 return 0;
421 switch (s[0])
423 case 'd':
424 mode = STRUB_DISABLED;
425 ret = -1;
426 break;
428 case 'a':
429 mode = STRUB_AT_CALLS;
430 ret = 2;
431 break;
433 case 'i':
434 mode = STRUB_INTERNAL;
435 ret = 1;
436 break;
438 case 'c':
439 mode = STRUB_CALLABLE;
440 ret = -2;
441 break;
443 default:
444 /* Other parms are for internal use only. */
445 return 0;
448 tree mode_id = get_strub_mode_attr_parm (mode);
450 if (TREE_CODE (id) == IDENTIFIER_NODE
451 ? id != mode_id
452 : strncmp (s, IDENTIFIER_POINTER (mode_id), len) != 0)
453 return 0;
455 return ret;
458 /* Return the strub mode from STRUB_ATTR. VAR_P should be TRUE if the attribute
459 is taken from a variable, rather than from a function, or a type thereof. */
461 static enum strub_mode
462 get_strub_mode_from_attr (tree strub_attr, bool var_p = false)
464 enum strub_mode mode = STRUB_DISABLED;
466 if (strub_attr)
468 if (!TREE_VALUE (strub_attr))
469 mode = !var_p ? STRUB_AT_CALLS : STRUB_INTERNAL;
470 else
472 gcc_checking_assert (!var_p);
473 tree id = TREE_VALUE (strub_attr);
474 if (TREE_CODE (id) == TREE_LIST)
475 id = TREE_VALUE (id);
476 const char *s = (TREE_CODE (id) == STRING_CST
477 ? TREE_STRING_POINTER (id)
478 : IDENTIFIER_POINTER (id));
479 size_t len = (TREE_CODE (id) == STRING_CST
480 ? TREE_STRING_LENGTH (id) - 1
481 : IDENTIFIER_LENGTH (id));
483 switch (len)
485 case 7:
486 switch (s[6])
488 case 'r':
489 mode = STRUB_WRAPPER;
490 break;
492 case 'd':
493 mode = STRUB_WRAPPED;
494 break;
496 default:
497 gcc_unreachable ();
499 break;
501 case 8:
502 switch (s[0])
504 case 'd':
505 mode = STRUB_DISABLED;
506 break;
508 case 'a':
509 mode = STRUB_AT_CALLS;
510 break;
512 case 'i':
513 mode = STRUB_INTERNAL;
514 break;
516 case 'c':
517 mode = STRUB_CALLABLE;
518 break;
520 default:
521 gcc_unreachable ();
523 break;
525 case 9:
526 mode = STRUB_INLINABLE;
527 break;
529 case 12:
530 mode = STRUB_AT_CALLS_OPT;
531 break;
533 default:
534 gcc_unreachable ();
537 gcc_checking_assert (TREE_CODE (id) == IDENTIFIER_NODE
538 ? id == get_strub_mode_attr_parm (mode)
539 : strncmp (IDENTIFIER_POINTER
540 (get_strub_mode_attr_parm (mode)),
541 s, len) == 0);
545 return mode;
548 /* Look up, decode and return the strub mode associated with FNDECL. */
550 static enum strub_mode
551 get_strub_mode_from_fndecl (tree fndecl)
553 return get_strub_mode_from_attr (get_strub_attr_from_decl (fndecl));
556 /* Look up, decode and return the strub mode associated with NODE. */
558 static enum strub_mode
559 get_strub_mode (cgraph_node *node)
561 return get_strub_mode_from_fndecl (node->decl);
564 /* Look up, decode and return the strub mode associated with TYPE. */
566 static enum strub_mode
567 get_strub_mode_from_type (tree type)
569 bool var_p = !FUNC_OR_METHOD_TYPE_P (type);
570 tree attr = get_strub_attr_from_type (type);
572 if (attr)
573 return get_strub_mode_from_attr (attr, var_p);
575 if (flag_strub >= -1 && !var_p)
576 return STRUB_CALLABLE;
578 return STRUB_DISABLED;
582 /* Return TRUE iff NODE calls builtin va_start. */
584 static bool
585 calls_builtin_va_start_p (cgraph_node *node)
587 bool result = false;
589 for (cgraph_edge *e = node->callees; e; e = e->next_callee)
591 tree cdecl = e->callee->decl;
592 if (fndecl_built_in_p (cdecl, BUILT_IN_VA_START))
593 return true;
596 return result;
599 /* Return TRUE iff NODE calls builtin apply_args, and optionally REPORT it. */
601 static bool
602 calls_builtin_apply_args_p (cgraph_node *node, bool report = false)
604 bool result = false;
606 for (cgraph_edge *e = node->callees; e; e = e->next_callee)
608 tree cdecl = e->callee->decl;
609 if (!fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS))
610 continue;
612 result = true;
614 if (!report)
615 break;
617 sorry_at (e->call_stmt
618 ? gimple_location (e->call_stmt)
619 : DECL_SOURCE_LOCATION (node->decl),
620 "at-calls %<strub%> does not support call to %qD",
621 cdecl);
624 return result;
627 /* Return TRUE iff NODE carries the always_inline attribute. */
629 static inline bool
630 strub_always_inline_p (cgraph_node *node)
632 return lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl));
635 /* Return TRUE iff the target has strub support for T, a function
636 decl, or a type used in an indirect call, and optionally REPORT the
637 reasons for ineligibility. If T is a type and error REPORTing is
638 enabled, the LOCation (of the indirect call) should be provided. */
639 static inline bool
640 strub_target_support_p (tree t, bool report = false,
641 location_t loc = UNKNOWN_LOCATION)
643 bool result = true;
645 if (!targetm.have_strub_support_for (t))
647 result = false;
649 if (!report)
650 return result;
652 if (DECL_P (t))
653 sorry_at (DECL_SOURCE_LOCATION (t),
654 "%qD is not eligible for %<strub%>"
655 " on the target system", t);
656 else
657 sorry_at (loc,
658 "unsupported %<strub%> call"
659 " on the target system");
662 return result;
665 /* Return TRUE iff NODE is potentially eligible for any strub-enabled mode, and
666 optionally REPORT the reasons for ineligibility. */
668 static inline bool
669 can_strub_p (cgraph_node *node, bool report = false)
671 bool result = strub_target_support_p (node->decl, report);
673 if (!report && (!result || strub_always_inline_p (node)))
674 return result;
676 if (flag_split_stack)
678 result = false;
680 if (!report)
681 return result;
683 sorry_at (DECL_SOURCE_LOCATION (node->decl),
684 "%qD is not eligible for %<strub%>"
685 " because %<-fsplit-stack%> is enabled",
686 node->decl);
689 if (lookup_attribute ("noipa", DECL_ATTRIBUTES (node->decl)))
691 result = false;
693 if (!report)
694 return result;
696 sorry_at (DECL_SOURCE_LOCATION (node->decl),
697 "%qD is not eligible for %<strub%>"
698 " because of attribute %<noipa%>",
699 node->decl);
702 /* We can't, and don't want to vectorize the watermark and other
703 strub-introduced parms. */
704 if (lookup_attribute ("simd", DECL_ATTRIBUTES (node->decl)))
706 result = false;
708 if (!report)
709 return result;
711 sorry_at (DECL_SOURCE_LOCATION (node->decl),
712 "%qD is not eligible for %<strub%>"
713 " because of attribute %<simd%>",
714 node->decl);
717 return result;
720 /* Return TRUE iff NODE is eligible for at-calls strub, and optionally REPORT
721 the reasons for ineligibility. Besides general non-eligibility for
722 strub-enabled modes, at-calls rules out calling builtin apply_args. */
724 static bool
725 can_strub_at_calls_p (cgraph_node *node, bool report = false)
727 bool result = !report || can_strub_p (node, report);
729 if (!result && !report)
730 return result;
732 return !calls_builtin_apply_args_p (node, report);
735 /* Return TRUE iff the called function (pointer or, if available,
736 decl) undergoes a significant type conversion for the call. Strub
737 mode changes between function types, and other non-useless type
738 conversions, are regarded as significant. When the function type
739 is overridden, the effective strub mode for the call is that of the
740 call fntype, rather than that of the pointer or of the decl.
741 Functions called with type overrides cannot undergo type changes;
742 it's as if their address was taken, so they're considered
743 non-viable for implicit at-calls strub mode. */
745 static inline bool
746 strub_call_fntype_override_p (const gcall *gs)
748 if (gimple_call_internal_p (gs))
749 return false;
750 tree fn_type = TREE_TYPE (TREE_TYPE (gimple_call_fn (gs)));
751 if (tree decl = gimple_call_fndecl (gs))
752 fn_type = TREE_TYPE (decl);
754 /* We do NOT want to take the mode from the decl here. This
755 function is used to tell whether we can change the strub mode of
756 a function, and whether the effective mode for the call is to be
757 taken from the decl or from an overrider type. When the strub
758 mode is explicitly declared, or overridden with a type cast, the
759 difference will be noticed in function types. However, if the
760 strub mode is implicit due to e.g. strub variables or -fstrub=*
761 command-line flags, we will adjust call types along with function
762 types. In either case, the presence of type or strub mode
763 overriders in calls will prevent a function from having its strub
764 modes changed in ways that would imply type changes, but taking
765 strub modes from decls would defeat this, since we set strub
766 modes and then call this function to tell whether the original
767 type was overridden to decide whether to adjust the call. We
768 need the answer to be about the type, not the decl. */
769 enum strub_mode mode = get_strub_mode_from_type (fn_type);
770 return (get_strub_mode_from_type (gs->u.fntype) != mode
771 || !useless_type_conversion_p (gs->u.fntype, fn_type));
774 /* Return TRUE iff NODE is called directly with a type override. */
776 static bool
777 called_directly_with_type_override_p (cgraph_node *node, void *)
779 for (cgraph_edge *e = node->callers; e; e = e->next_caller)
780 if (e->call_stmt && strub_call_fntype_override_p (e->call_stmt))
781 return true;
783 return false;
786 /* Return TRUE iff NODE or any other nodes aliased to it are called
787 with type overrides. We can't safely change the type of such
788 functions. */
790 static bool
791 called_with_type_override_p (cgraph_node *node)
793 return (node->call_for_symbol_thunks_and_aliases
794 (called_directly_with_type_override_p, NULL, true, true));
797 /* Symbolic macro for the max number of arguments that internal strub may add to
798 a function. */
800 #define STRUB_INTERNAL_MAX_EXTRA_ARGS 3
802 /* We can't perform internal strubbing if the function body involves certain
803 features:
805 - a non-default __builtin_va_start (e.g. x86's __builtin_ms_va_start) is
806 currently unsupported because we can't discover the corresponding va_copy and
807 va_end decls in the wrapper, and we don't convey the alternate variable
808 arguments ABI to the modified wrapped function. The default
809 __builtin_va_start is supported by calling va_start/va_end at the wrapper,
810 that takes variable arguments, passing a pointer to the va_list object to the
811 wrapped function, that runs va_copy from it where the original function ran
812 va_start.
814 __builtin_next_arg is currently unsupported because the wrapped function
815 won't be a variable argument function. We could process it in the wrapper,
816 that remains a variable argument function, and replace calls in the wrapped
817 body, but we currently don't.
819 __builtin_return_address is rejected because it's generally used when the
820 actual caller matters, and introducing a wrapper breaks such uses as those in
821 the unwinder. */
823 static bool
824 can_strub_internally_p (cgraph_node *node, bool report = false)
826 bool result = !report || can_strub_p (node, report);
828 if (!result && !report)
829 return result;
831 if (!report && strub_always_inline_p (node))
832 return result;
834 /* Since we're not changing the function identity proper, just
835 moving its full implementation, we *could* disable
836 fun->cannot_be_copied_reason and/or temporarily drop a noclone
837 attribute, but we'd have to prevent remapping of the labels. */
838 if (lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
840 result = false;
842 if (!report)
843 return result;
845 sorry_at (DECL_SOURCE_LOCATION (node->decl),
846 "%qD is not eligible for internal %<strub%>"
847 " because of attribute %<noclone%>",
848 node->decl);
851 if (node->has_gimple_body_p ())
853 for (cgraph_edge *e = node->callees; e; e = e->next_callee)
855 tree cdecl = e->callee->decl;
856 if (!((fndecl_built_in_p (cdecl, BUILT_IN_VA_START)
857 && cdecl != builtin_decl_explicit (BUILT_IN_VA_START))
858 || fndecl_built_in_p (cdecl, BUILT_IN_NEXT_ARG)
859 || fndecl_built_in_p (cdecl, BUILT_IN_RETURN_ADDRESS)))
860 continue;
862 result = false;
864 if (!report)
865 return result;
867 sorry_at (e->call_stmt
868 ? gimple_location (e->call_stmt)
869 : DECL_SOURCE_LOCATION (node->decl),
870 "%qD is not eligible for internal %<strub%> "
871 "because it calls %qD",
872 node->decl, cdecl);
875 struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
876 if (fun->has_nonlocal_label)
878 result = false;
880 if (!report)
881 return result;
883 sorry_at (DECL_SOURCE_LOCATION (node->decl),
884 "%qD is not eligible for internal %<strub%> "
885 "because it contains a non-local goto target",
886 node->decl);
889 if (fun->has_forced_label_in_static)
891 result = false;
893 if (!report)
894 return result;
896 sorry_at (DECL_SOURCE_LOCATION (node->decl),
897 "%qD is not eligible for internal %<strub%> "
898 "because the address of a local label escapes",
899 node->decl);
902 /* Catch any other case that would prevent versioning/cloning
903 so as to also have it covered above. */
904 gcc_checking_assert (!result /* || !node->has_gimple_body_p () */
905 || tree_versionable_function_p (node->decl));
908 /* Label values references are not preserved when copying. If referenced
909 in nested functions, as in 920415-1.c and 920721-4.c their decls get
910 remapped independently. The exclusion below might be too broad, in
911 that we might be able to support correctly cases in which the labels
912 are only used internally in a function, but disconnecting forced labels
913 from their original declarations is undesirable in general. */
914 basic_block bb;
915 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
916 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
917 !gsi_end_p (gsi); gsi_next (&gsi))
919 glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (gsi));
920 tree target;
922 if (!label_stmt)
923 break;
925 target = gimple_label_label (label_stmt);
927 if (!FORCED_LABEL (target))
928 continue;
930 result = false;
932 if (!report)
933 return result;
935 sorry_at (gimple_location (label_stmt),
936 "internal %<strub%> does not support forced labels");
940 if (list_length (TYPE_ARG_TYPES (TREE_TYPE (node->decl)))
941 >= (((HOST_WIDE_INT) 1 << IPA_PARAM_MAX_INDEX_BITS)
942 - STRUB_INTERNAL_MAX_EXTRA_ARGS))
944 result = false;
946 if (!report)
947 return result;
949 sorry_at (DECL_SOURCE_LOCATION (node->decl),
950 "%qD has too many arguments for internal %<strub%>",
951 node->decl);
954 return result;
957 /* Return TRUE iff NODE has any strub-requiring local variable, or accesses (as
958 in reading) any variable through a strub-requiring type. */
960 static bool
961 strub_from_body_p (cgraph_node *node)
963 if (!node->has_gimple_body_p ())
964 return false;
966 /* If any local variable is marked for strub... */
967 unsigned i;
968 tree var;
969 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (node->decl),
970 i, var)
971 if (get_strub_mode_from_type (TREE_TYPE (var))
972 != STRUB_DISABLED)
973 return true;
975 /* Now scan the body for loads with strub-requiring types.
976 ??? Compound types don't propagate the strub requirement to
977 component types. */
978 basic_block bb;
979 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
980 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
981 !gsi_end_p (gsi); gsi_next (&gsi))
983 gimple *stmt = gsi_stmt (gsi);
985 if (!gimple_assign_load_p (stmt))
986 continue;
988 tree rhs = gimple_assign_rhs1 (stmt);
989 if (get_strub_mode_from_type (TREE_TYPE (rhs))
990 != STRUB_DISABLED)
991 return true;
994 return false;
997 /* Return TRUE iff node is associated with a builtin that should be callable
998 from strub contexts. */
1000 static inline bool
1001 strub_callable_builtin_p (cgraph_node *node)
1003 if (DECL_BUILT_IN_CLASS (node->decl) != BUILT_IN_NORMAL)
1004 return false;
1006 enum built_in_function fcode = DECL_FUNCTION_CODE (node->decl);
1008 switch (fcode)
1010 case BUILT_IN_NONE:
1011 gcc_unreachable ();
1013 /* This temporarily allocates stack for the call, and we can't reasonably
1014 update the watermark for that. Besides, we don't check the actual call
1015 target, nor its signature, and it seems to be overkill to as much as
1016 try to do so. */
1017 case BUILT_IN_APPLY:
1018 return false;
1020 /* Conversely, this shouldn't be called from within strub contexts, since
1021 the caller may have had its signature modified. STRUB_INTERNAL is ok,
1022 the call will remain in the STRUB_WRAPPER, and removed from the
1023 STRUB_WRAPPED clone. */
1024 case BUILT_IN_APPLY_ARGS:
1025 return false;
1027 /* ??? Make all other builtins callable. We wish to make any builtin call
1028 the compiler might introduce on its own callable. Anything that is
1029 predictable enough as to be known not to allow stack data that should
1030 be strubbed to unintentionally escape to non-strub contexts can be
1031 allowed, and pretty much every builtin appears to fit this description.
1032 The exceptions to this rule seem to be rare, and only available as
1033 explicit __builtin calls, so let's keep it simple and allow all of
1034 them... */
1035 default:
1036 return true;
1040 /* Compute the strub mode to be used for NODE. STRUB_ATTR should be the strub
1041 attribute,found for NODE, if any. */
1043 static enum strub_mode
1044 compute_strub_mode (cgraph_node *node, tree strub_attr)
1046 enum strub_mode req_mode = get_strub_mode_from_attr (strub_attr);
1048 gcc_checking_assert (flag_strub >= -2 && flag_strub <= 3);
1050 /* Symbolic encodings of the -fstrub-* flags. */
1051 /* Enable strub when explicitly requested through attributes to functions or
1052 variables, reporting errors if the requests cannot be satisfied. */
1053 const bool strub_flag_auto = flag_strub < 0;
1054 /* strub_flag_auto with strub call verification; without this, functions are
1055 implicitly callable. */
1056 const bool strub_flag_strict = flag_strub < -1;
1057 /* Disable strub altogether, ignore attributes entirely. */
1058 const bool strub_flag_disabled = flag_strub == 0;
1059 /* On top of _auto, also enable strub implicitly for functions that can
1060 safely undergo at-calls strubbing. Internal mode will still be used in
1061 functions that request it explicitly with attribute strub(2), or when the
1062 function body requires strubbing and at-calls strubbing is not viable. */
1063 const bool strub_flag_at_calls = flag_strub == 1;
1064 /* On top of default, also enable strub implicitly for functions that can
1065 safely undergo internal strubbing. At-calls mode will still be used in
1066 functions that requiest it explicitly with attribute strub() or strub(1),
1067 or when the function body requires strubbing and internal strubbing is not
1068 viable. */
1069 const bool strub_flag_internal = flag_strub == 2;
1070 /* On top of default, also enable strub implicitly for functions that can
1071 safely undergo strubbing in either mode. When both modes are viable,
1072 at-calls is preferred. */
1073 const bool strub_flag_either = flag_strub == 3;
1074 /* Besides the default behavior, enable strub implicitly for all viable
1075 functions. */
1076 const bool strub_flag_viable = flag_strub > 0;
1078 /* The consider_* variables should be TRUE if selecting the corresponding
1079 strub modes would be consistent with requests from attributes and command
1080 line flags. Attributes associated with functions pretty much mandate a
1081 selection, and should report an error if not satisfied; strub_flag_auto
1082 implicitly enables some viable strub mode if that's required by references
1083 to variables marked for strub; strub_flag_viable enables strub if viable
1084 (even when favoring one mode, body-requested strub can still be satisfied
1085 by either mode), and falls back to callable, silently unless variables
1086 require strubbing. */
1088 const bool consider_at_calls
1089 = (!strub_flag_disabled
1090 && (strub_attr
1091 ? req_mode == STRUB_AT_CALLS
1092 : true));
1093 const bool consider_internal
1094 = (!strub_flag_disabled
1095 && (strub_attr
1096 ? req_mode == STRUB_INTERNAL
1097 : true));
1099 const bool consider_callable
1100 = (!strub_flag_disabled
1101 && (strub_attr
1102 ? req_mode == STRUB_CALLABLE
1103 : (!strub_flag_strict
1104 || strub_callable_builtin_p (node))));
1106 /* This is a shorthand for either strub-enabled mode. */
1107 const bool consider_strub
1108 = (consider_at_calls || consider_internal);
1110 /* We can cope with always_inline functions even with noipa and noclone,
1111 because we just leave them alone. */
1112 const bool is_always_inline
1113 = strub_always_inline_p (node);
1115 /* Strubbing in general, and each specific strub mode, may have its own set of
1116 requirements. We require noipa for strubbing, either because of cloning
1117 required for internal strub, or because of caller enumeration required for
1118 at-calls strub. We don't consider the at-calls mode eligible if it's not
1119 even considered, it has no further requirements. Internal mode requires
1120 cloning and the absence of certain features in the body and, like at-calls,
1121 it's not eligible if it's not even under consideration.
1123 ??? Do we need target hooks for further constraints? E.g., x86's
1124 "interrupt" attribute breaks internal strubbing because the wrapped clone
1125 carries the attribute and thus isn't callable; in this case, we could use a
1126 target hook to adjust the clone instead. */
1127 const bool strub_eligible
1128 = (consider_strub
1129 && (is_always_inline || can_strub_p (node)));
1130 const bool at_calls_eligible
1131 = (consider_at_calls && strub_eligible
1132 && can_strub_at_calls_p (node));
1133 const bool internal_eligible
1134 = (consider_internal && strub_eligible
1135 && (is_always_inline
1136 || can_strub_internally_p (node)));
1138 /* In addition to the strict eligibility requirements, some additional
1139 constraints are placed on implicit selection of certain modes. These do
1140 not prevent the selection of a mode if explicitly specified as part of a
1141 function interface (the strub attribute), but they may prevent modes from
1142 being selected by the command line or by function bodies. The only actual
1143 constraint is on at-calls mode: since we change the function's exposed
1144 signature, we won't do it implicitly if the function can possibly be used
1145 in ways that do not expect the signature change, e.g., if the function is
1146 available to or interposable by other units, if its address is taken,
1147 etc. */
1148 const bool at_calls_viable
1149 = (at_calls_eligible
1150 && (strub_attr
1151 || (node->has_gimple_body_p ()
1152 && (!node->externally_visible
1153 || (node->binds_to_current_def_p ()
1154 && node->can_be_local_p ()))
1155 && node->only_called_directly_p ()
1156 && !called_with_type_override_p (node))));
1157 const bool internal_viable
1158 = (internal_eligible);
1160 /* Shorthand. */
1161 const bool strub_viable
1162 = (at_calls_viable || internal_viable);
1164 /* We wish to analyze the body, to look for implicit requests for strub, both
1165 to implicitly enable it when the body calls for it, and to report errors if
1166 the body calls for it but neither mode is viable (even if that follows from
1167 non-eligibility because of the explicit specification of some non-strubbing
1168 mode). We can refrain from scanning the body only in rare circumstances:
1169 when strub is enabled by a function attribute (scanning might be redundant
1170 in telling us to also enable it), and when we are enabling strub implicitly
1171 but there are non-viable modes: we want to know whether strubbing is
1172 required, to fallback to another mode, even if we're only enabling a
1173 certain mode, or, when either mode would do, to report an error if neither
1174 happens to be viable. */
1175 const bool analyze_body
1176 = (strub_attr
1177 ? !consider_strub
1178 : (strub_flag_auto
1179 || (strub_flag_viable && (!at_calls_viable && !internal_viable))
1180 || (strub_flag_either && !strub_viable)));
1182 /* Cases in which strubbing is enabled or disabled by strub_flag_auto.
1183 Unsatisfiable requests ought to be reported. */
1184 const bool strub_required
1185 = ((strub_attr && consider_strub)
1186 || (analyze_body && strub_from_body_p (node)));
1188 /* Besides the required cases, we want to abide by the requests to enabling on
1189 an if-viable basis. */
1190 const bool strub_enable
1191 = (strub_required
1192 || (strub_flag_at_calls && at_calls_viable)
1193 || (strub_flag_internal && internal_viable)
1194 || (strub_flag_either && strub_viable));
1196 /* And now we're finally ready to select a mode that abides by the viability
1197 and eligibility constraints, and that satisfies the strubbing requirements
1198 and requests, subject to the constraints. If both modes are viable and
1199 strub is to be enabled, pick STRUB_AT_CALLS unless STRUB_INTERNAL was named
1200 as preferred. */
1201 const enum strub_mode mode
1202 = ((strub_enable && is_always_inline)
1203 ? (strub_required ? STRUB_INLINABLE : STRUB_CALLABLE)
1204 : (strub_enable && internal_viable
1205 && (strub_flag_internal || !at_calls_viable))
1206 ? STRUB_INTERNAL
1207 : (strub_enable && at_calls_viable)
1208 ? (strub_required && !strub_attr
1209 ? STRUB_AT_CALLS_OPT
1210 : STRUB_AT_CALLS)
1211 : consider_callable
1212 ? STRUB_CALLABLE
1213 : STRUB_DISABLED);
1215 switch (mode)
1217 case STRUB_CALLABLE:
1218 if (is_always_inline)
1219 break;
1220 /* Fall through. */
1222 case STRUB_DISABLED:
1223 if (strub_enable && !strub_attr)
1225 gcc_checking_assert (analyze_body);
1226 error_at (DECL_SOURCE_LOCATION (node->decl),
1227 "%qD requires %<strub%>,"
1228 " but no viable %<strub%> mode was found",
1229 node->decl);
1230 break;
1232 /* Fall through. */
1234 case STRUB_AT_CALLS:
1235 case STRUB_INTERNAL:
1236 case STRUB_INLINABLE:
1237 /* Differences from an mode requested through a function attribute are
1238 reported in set_strub_mode_to. */
1239 break;
1241 case STRUB_AT_CALLS_OPT:
1242 /* Functions that select this mode do so because of references to strub
1243 variables. Even if we choose at-calls as an optimization, the
1244 requirements for internal strub must still be satisfied. Optimization
1245 options may render implicit at-calls strub not viable (-O0 sets
1246 force_output for static non-inline functions), and it would not be good
1247 if changing optimization options turned a well-formed into an
1248 ill-formed one. */
1249 if (!internal_viable)
1250 can_strub_internally_p (node, true);
1251 break;
1253 case STRUB_WRAPPED:
1254 case STRUB_WRAPPER:
1255 default:
1256 gcc_unreachable ();
1259 return mode;
1262 /* Set FNDT's strub mode to MODE; FNDT may be a function decl or
1263 function type. If OVERRIDE, do not check whether a mode is already
1264 set. */
1266 static void
1267 strub_set_fndt_mode_to (tree fndt, enum strub_mode mode, bool override)
1269 gcc_checking_assert (override
1270 || !(DECL_P (fndt)
1271 ? get_strub_attr_from_decl (fndt)
1272 : get_strub_attr_from_type (fndt)));
1274 tree attr = tree_cons (get_identifier ("strub"),
1275 get_strub_mode_attr_value (mode),
1276 NULL_TREE);
1277 tree *attrp = NULL;
1278 if (DECL_P (fndt))
1280 gcc_checking_assert (FUNC_OR_METHOD_TYPE_P (TREE_TYPE (fndt)));
1281 attrp = &DECL_ATTRIBUTES (fndt);
1283 else if (FUNC_OR_METHOD_TYPE_P (fndt))
1284 attrp = &TYPE_ATTRIBUTES (fndt);
1285 else
1286 gcc_unreachable ();
1288 TREE_CHAIN (attr) = *attrp;
1289 *attrp = attr;
1292 /* Set FNDT's strub mode to callable.
1293 FNDT may be a function decl or a function type. */
1295 void
1296 strub_make_callable (tree fndt)
1298 strub_set_fndt_mode_to (fndt, STRUB_CALLABLE, false);
1301 /* Set NODE to strub MODE. Report incompatibilities between MODE and the mode
1302 requested through explicit attributes, and cases of non-eligibility. */
1304 static void
1305 set_strub_mode_to (cgraph_node *node, enum strub_mode mode)
1307 tree attr = get_strub_attr_from_decl (node->decl);
1308 enum strub_mode req_mode = get_strub_mode_from_attr (attr);
1310 if (attr)
1312 /* Check for and report incompatible mode changes. */
1313 if (mode != req_mode
1314 && !(req_mode == STRUB_INTERNAL
1315 && (mode == STRUB_WRAPPED
1316 || mode == STRUB_WRAPPER))
1317 && !((req_mode == STRUB_INTERNAL
1318 || req_mode == STRUB_AT_CALLS
1319 || req_mode == STRUB_CALLABLE)
1320 && mode == STRUB_INLINABLE))
1322 error_at (DECL_SOURCE_LOCATION (node->decl),
1323 "%<strub%> mode %qE selected for %qD, when %qE was requested",
1324 get_strub_mode_attr_parm (mode),
1325 node->decl,
1326 get_strub_mode_attr_parm (req_mode));
1327 if (node->alias)
1329 cgraph_node *target = node->ultimate_alias_target ();
1330 if (target != node)
1331 error_at (DECL_SOURCE_LOCATION (target->decl),
1332 "the incompatible selection was determined"
1333 " by ultimate alias target %qD",
1334 target->decl);
1337 /* Report any incompatibilities with explicitly-requested strub. */
1338 switch (req_mode)
1340 case STRUB_AT_CALLS:
1341 can_strub_at_calls_p (node, true);
1342 break;
1344 case STRUB_INTERNAL:
1345 can_strub_internally_p (node, true);
1346 break;
1348 default:
1349 break;
1353 /* Drop any incompatible strub attributes leading the decl attribute
1354 chain. Return if we find one with the mode we need. */
1355 for (;;)
1357 if (mode == req_mode)
1358 return;
1360 if (DECL_ATTRIBUTES (node->decl) != attr)
1361 break;
1363 DECL_ATTRIBUTES (node->decl) = TREE_CHAIN (attr);
1364 attr = get_strub_attr_from_decl (node->decl);
1365 if (!attr)
1366 break;
1368 req_mode = get_strub_mode_from_attr (attr);
1371 else if (mode == req_mode)
1372 return;
1374 strub_set_fndt_mode_to (node->decl, mode, attr);
1377 /* Compute and set NODE's strub mode. */
1379 static void
1380 set_strub_mode (cgraph_node *node)
1382 tree attr = get_strub_attr_from_decl (node->decl);
1384 if (attr)
1385 switch (get_strub_mode_from_attr (attr))
1387 /* These can't have been requested through user attributes, so we must
1388 have already gone through them. */
1389 case STRUB_WRAPPER:
1390 case STRUB_WRAPPED:
1391 case STRUB_INLINABLE:
1392 case STRUB_AT_CALLS_OPT:
1393 return;
1395 case STRUB_DISABLED:
1396 case STRUB_AT_CALLS:
1397 case STRUB_INTERNAL:
1398 case STRUB_CALLABLE:
1399 break;
1401 default:
1402 gcc_unreachable ();
1405 cgraph_node *xnode = node;
1406 if (node->alias)
1407 xnode = node->ultimate_alias_target ();
1408 /* Weakrefs may remain unresolved (the above will return node) if
1409 their targets are not defined, so make sure we compute a strub
1410 mode for them, instead of defaulting to STRUB_DISABLED and
1411 rendering them uncallable. */
1412 enum strub_mode mode = (xnode != node && !xnode->alias
1413 ? get_strub_mode (xnode)
1414 : compute_strub_mode (node, attr));
1416 set_strub_mode_to (node, mode);
1420 /* Non-strub functions shouldn't be called from within strub contexts,
1421 except through callable ones. Always inline strub functions can
1422 only be called from strub functions. */
1424 static bool
1425 strub_callable_from_p (strub_mode caller_mode, strub_mode callee_mode)
1427 switch (caller_mode)
1429 case STRUB_WRAPPED:
1430 case STRUB_AT_CALLS_OPT:
1431 case STRUB_AT_CALLS:
1432 case STRUB_INTERNAL:
1433 case STRUB_INLINABLE:
1434 break;
1436 case STRUB_WRAPPER:
1437 case STRUB_DISABLED:
1438 case STRUB_CALLABLE:
1439 return callee_mode != STRUB_INLINABLE;
1441 default:
1442 gcc_unreachable ();
1445 switch (callee_mode)
1447 case STRUB_WRAPPED:
1448 case STRUB_AT_CALLS:
1449 case STRUB_INLINABLE:
1450 break;
1452 case STRUB_AT_CALLS_OPT:
1453 case STRUB_INTERNAL:
1454 case STRUB_WRAPPER:
1455 return (flag_strub >= -1);
1457 case STRUB_DISABLED:
1458 return false;
1460 case STRUB_CALLABLE:
1461 break;
1463 default:
1464 gcc_unreachable ();
1467 return true;
1470 /* Return TRUE iff CALLEE can be inlined into CALLER. We wish to avoid inlining
1471 WRAPPED functions back into their WRAPPERs. More generally, we wish to avoid
1472 inlining strubbed functions into non-strubbed ones. CALLER doesn't have to
1473 be an immediate caller of CALLEE: the immediate caller may have already been
1474 cloned for inlining, and then CALLER may be further up the original call
1475 chain. ??? It would be nice if our own caller would retry inlining callee
1476 if caller gets inlined. */
1478 bool
1479 strub_inlinable_to_p (cgraph_node *callee, cgraph_node *caller)
1481 strub_mode callee_mode = get_strub_mode (callee);
1483 switch (callee_mode)
1485 case STRUB_WRAPPED:
1486 case STRUB_AT_CALLS:
1487 case STRUB_INTERNAL:
1488 case STRUB_INLINABLE:
1489 case STRUB_AT_CALLS_OPT:
1490 break;
1492 case STRUB_WRAPPER:
1493 case STRUB_DISABLED:
1494 case STRUB_CALLABLE:
1495 /* When we consider inlining, we've already verified callability, so we
1496 can even inline callable and then disabled into a strub context. That
1497 will get strubbed along with the context, so it's hopefully not a
1498 problem. */
1499 return true;
1501 default:
1502 gcc_unreachable ();
1505 strub_mode caller_mode = get_strub_mode (caller);
1507 switch (caller_mode)
1509 case STRUB_WRAPPED:
1510 case STRUB_AT_CALLS:
1511 case STRUB_INTERNAL:
1512 case STRUB_INLINABLE:
1513 case STRUB_AT_CALLS_OPT:
1514 return true;
1516 case STRUB_WRAPPER:
1517 case STRUB_DISABLED:
1518 case STRUB_CALLABLE:
1519 break;
1521 default:
1522 gcc_unreachable ();
1525 return false;
1528 /* Check that types T1 and T2 are strub-compatible. Return 1 if the strub modes
1529 are the same, 2 if they are interchangeable, and 0 otherwise. */
1532 strub_comptypes (tree t1, tree t2)
1534 if (TREE_CODE (t1) != TREE_CODE (t2))
1535 return 0;
1537 enum strub_mode m1 = get_strub_mode_from_type (t1);
1538 enum strub_mode m2 = get_strub_mode_from_type (t2);
1540 if (m1 == m2)
1541 return 1;
1543 /* We're dealing with types, so only strub modes that can be selected by
1544 attributes in the front end matter. If either mode is at-calls (for
1545 functions) or internal (for variables), the conversion is not
1546 compatible. */
1547 bool var_p = !FUNC_OR_METHOD_TYPE_P (t1);
1548 enum strub_mode mr = var_p ? STRUB_INTERNAL : STRUB_AT_CALLS;
1549 if (m1 == mr || m2 == mr)
1550 return 0;
1552 return 2;
1555 /* Return the effective strub mode used for CALL, and set *TYPEP to
1556 the effective type used for the call. The effective type and mode
1557 are those of the callee, unless the call involves a typecast. */
1559 static enum strub_mode
1560 effective_strub_mode_for_call (gcall *call, tree *typep)
1562 tree type;
1563 enum strub_mode mode;
1565 if (strub_call_fntype_override_p (call))
1567 type = gimple_call_fntype (call);
1568 mode = get_strub_mode_from_type (type);
1570 else
1572 type = TREE_TYPE (TREE_TYPE (gimple_call_fn (call)));
1573 tree decl = gimple_call_fndecl (call);
1574 if (decl)
1575 mode = get_strub_mode_from_fndecl (decl);
1576 else
1577 mode = get_strub_mode_from_type (type);
1580 if (typep)
1581 *typep = type;
1583 return mode;
1586 /* Create a distinct copy of the type of NODE's function, and change
1587 the fntype of all calls to it with the same main type to the new
1588 type. */
1590 static void
1591 distinctify_node_type (cgraph_node *node)
1593 tree old_type = TREE_TYPE (node->decl);
1594 tree new_type = build_distinct_type_copy (old_type);
1595 tree new_ptr_type = NULL_TREE;
1597 /* Remap any calls to node->decl that use old_type, or a variant
1598 thereof, to new_type as well. We don't look for aliases, their
1599 declarations will have their types changed independently, and
1600 we'll adjust their fntypes then. */
1601 for (cgraph_edge *e = node->callers; e; e = e->next_caller)
1603 if (!e->call_stmt)
1604 continue;
1605 tree fnaddr = gimple_call_fn (e->call_stmt);
1606 gcc_checking_assert (TREE_CODE (fnaddr) == ADDR_EXPR
1607 && TREE_OPERAND (fnaddr, 0) == node->decl);
1608 if (strub_call_fntype_override_p (e->call_stmt))
1609 continue;
1610 if (!new_ptr_type)
1611 new_ptr_type = build_pointer_type (new_type);
1612 TREE_TYPE (fnaddr) = new_ptr_type;
1613 gimple_call_set_fntype (e->call_stmt, new_type);
1616 TREE_TYPE (node->decl) = new_type;
1619 /* Return TRUE iff TYPE and any variants have the same strub mode. */
1621 static bool
1622 same_strub_mode_in_variants_p (tree type)
1624 enum strub_mode mode = get_strub_mode_from_type (type);
1626 for (tree other = TYPE_MAIN_VARIANT (type);
1627 other != NULL_TREE; other = TYPE_NEXT_VARIANT (other))
1628 if (type != other && mode != get_strub_mode_from_type (other))
1629 return false;
1631 /* Check that the canonical type, if set, either is in the same
1632 variant chain, or has the same strub mode as type. Also check
1633 the variants of the canonical type. */
1634 if (TYPE_CANONICAL (type)
1635 && (TYPE_MAIN_VARIANT (TYPE_CANONICAL (type))
1636 != TYPE_MAIN_VARIANT (type)))
1638 if (mode != get_strub_mode_from_type (TYPE_CANONICAL (type)))
1639 return false;
1640 else
1641 return same_strub_mode_in_variants_p (TYPE_CANONICAL (type));
1644 return true;
1647 /* Check that strub functions don't call non-strub functions, and that
1648 always_inline strub functions are only called by strub
1649 functions. */
1651 static void
1652 verify_strub ()
1654 cgraph_node *node;
1656 /* It's expected that check strub-wise pointer type compatibility of variables
1657 and of functions is already taken care of by front-ends, on account of the
1658 attribute's being marked as affecting type identity and of the creation of
1659 distinct types. */
1661 /* Check that call targets in strub contexts have strub-callable types. */
1663 FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
1665 enum strub_mode caller_mode = get_strub_mode (node);
1667 for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
1669 gcc_checking_assert (e->indirect_unknown_callee);
1671 if (!e->call_stmt)
1672 continue;
1674 enum strub_mode callee_mode
1675 = effective_strub_mode_for_call (e->call_stmt, NULL);
1677 if (!strub_callable_from_p (caller_mode, callee_mode))
1678 error_at (gimple_location (e->call_stmt),
1679 "indirect non-%<strub%> call in %<strub%> context %qD",
1680 node->decl);
1683 for (cgraph_edge *e = node->callees; e; e = e->next_callee)
1685 gcc_checking_assert (!e->indirect_unknown_callee);
1687 if (!e->call_stmt)
1688 continue;
1690 tree callee_fntype;
1691 enum strub_mode callee_mode
1692 = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
1694 if (!strub_callable_from_p (caller_mode, callee_mode))
1696 if (callee_mode == STRUB_INLINABLE)
1697 error_at (gimple_location (e->call_stmt),
1698 "calling %<always_inline%> %<strub%> %qD"
1699 " in non-%<strub%> context %qD",
1700 e->callee->decl, node->decl);
1701 else if (fndecl_built_in_p (e->callee->decl, BUILT_IN_APPLY_ARGS)
1702 && caller_mode == STRUB_INTERNAL)
1703 /* This is ok, it will be kept in the STRUB_WRAPPER, and removed
1704 from the STRUB_WRAPPED's strub context. */
1705 continue;
1706 else if (!strub_call_fntype_override_p (e->call_stmt))
1707 error_at (gimple_location (e->call_stmt),
1708 "calling non-%<strub%> %qD in %<strub%> context %qD",
1709 e->callee->decl, node->decl);
1710 else
1711 error_at (gimple_location (e->call_stmt),
1712 "calling %qD using non-%<strub%> type %qT"
1713 " in %<strub%> context %qD",
1714 e->callee->decl, callee_fntype, node->decl);
1720 namespace {
1722 /* Define a pass to compute strub modes. */
1723 const pass_data pass_data_ipa_strub_mode = {
1724 SIMPLE_IPA_PASS,
1725 "strubm",
1726 OPTGROUP_NONE,
1727 TV_NONE,
1728 PROP_cfg, // properties_required
1729 0, // properties_provided
1730 0, // properties_destroyed
1731 0, // properties_start
1732 0, // properties_finish
1735 class pass_ipa_strub_mode : public simple_ipa_opt_pass
1737 public:
1738 pass_ipa_strub_mode (gcc::context *ctxt)
1739 : simple_ipa_opt_pass (pass_data_ipa_strub_mode, ctxt)
1741 opt_pass *clone () { return new pass_ipa_strub_mode (m_ctxt); }
1742 virtual bool gate (function *) {
1743 /* In relaxed (-3) and strict (-4) settings, that only enable strub at a
1744 function or variable attribute's request, the attribute handler changes
1745 flag_strub to -1 or -2, respectively, if any strub-enabling occurence of
1746 the attribute is found. Therefore, if it remains at -3 or -4, nothing
1747 that would enable strub was found, so we can disable it and avoid the
1748 overhead. */
1749 if (flag_strub < -2)
1750 flag_strub = 0;
1751 return flag_strub;
1753 virtual unsigned int execute (function *);
1756 /* Define a pass to introduce strub transformations. */
1757 const pass_data pass_data_ipa_strub = {
1758 SIMPLE_IPA_PASS,
1759 "strub",
1760 OPTGROUP_NONE,
1761 TV_NONE,
1762 PROP_cfg | PROP_ssa, // properties_required
1763 0, // properties_provided
1764 0, // properties_destroyed
1765 0, // properties_start
1766 TODO_update_ssa
1767 | TODO_cleanup_cfg
1768 | TODO_rebuild_cgraph_edges
1769 | TODO_verify_il, // properties_finish
1772 class pass_ipa_strub : public simple_ipa_opt_pass
1774 public:
1775 pass_ipa_strub (gcc::context *ctxt)
1776 : simple_ipa_opt_pass (pass_data_ipa_strub, ctxt)
1778 opt_pass *clone () { return new pass_ipa_strub (m_ctxt); }
1779 virtual bool gate (function *) { return flag_strub && !seen_error (); }
1780 virtual unsigned int execute (function *);
1782 /* Define on demand and cache some types we use often. */
1783 #define DEF_TYPE(IDX, NAME, INIT) \
1784 static inline tree get_ ## NAME () { \
1785 int idx = STRUB_TYPE_BASE + IDX; \
1786 static tree type = strub_cache[idx]; \
1787 if (!type) \
1788 strub_cache[idx] = type = (INIT); \
1789 return type; \
1792 /* Use a distinct ptr_type_node to denote the watermark, so that we can
1793 recognize it in arg lists and avoid modifying types twice. */
1794 DEF_TYPE (0, wmt, build_variant_type_copy (ptr_type_node))
1796 DEF_TYPE (1, pwmt, build_reference_type (get_wmt ()))
1798 DEF_TYPE (2, qpwmt,
1799 build_qualified_type (get_pwmt (),
1800 TYPE_QUAL_RESTRICT
1801 /* | TYPE_QUAL_CONST */))
1803 DEF_TYPE (3, qptr,
1804 build_qualified_type (ptr_type_node,
1805 TYPE_QUAL_RESTRICT
1806 | TYPE_QUAL_CONST))
1808 DEF_TYPE (4, qpvalst,
1809 build_qualified_type (build_reference_type
1810 (va_list_type_node),
1811 TYPE_QUAL_RESTRICT
1812 /* | TYPE_QUAL_CONST */))
1814 #undef DEF_TYPE
1816 /* Define non-strub builtins on demand. */
1817 #define DEF_NM_BUILTIN(NAME, CODE, FNTYPELIST) \
1818 static tree get_ ## NAME () { \
1819 tree decl = builtin_decl_explicit (CODE); \
1820 if (!decl) \
1822 tree type = build_function_type_list FNTYPELIST; \
1823 decl = add_builtin_function \
1824 ("__builtin_" #NAME, \
1825 type, CODE, BUILT_IN_NORMAL, \
1826 NULL, NULL); \
1827 TREE_NOTHROW (decl) = true; \
1828 set_builtin_decl ((CODE), decl, true); \
1830 return decl; \
1833 DEF_NM_BUILTIN (stack_address,
1834 BUILT_IN_STACK_ADDRESS,
1835 (ptr_type_node, NULL))
1837 #undef DEF_NM_BUILTIN
1839 /* Define strub builtins on demand. */
1840 #define DEF_SS_BUILTIN(NAME, FNSPEC, CODE, FNTYPELIST) \
1841 static tree get_ ## NAME () { \
1842 tree decl = builtin_decl_explicit (CODE); \
1843 if (!decl) \
1845 tree type = build_function_type_list FNTYPELIST; \
1846 tree attrs = NULL; \
1847 if (FNSPEC) \
1848 attrs = tree_cons (get_identifier ("fn spec"), \
1849 build_tree_list \
1850 (NULL_TREE, \
1851 build_string (strlen (FNSPEC), \
1852 (FNSPEC))), \
1853 attrs); \
1854 decl = add_builtin_function_ext_scope \
1855 ("__builtin___strub_" #NAME, \
1856 type, CODE, BUILT_IN_NORMAL, \
1857 "__strub_" #NAME, attrs); \
1858 TREE_NOTHROW (decl) = true; \
1859 set_builtin_decl ((CODE), decl, true); \
1861 return decl; \
1864 DEF_SS_BUILTIN (enter, ". Ot",
1865 BUILT_IN___STRUB_ENTER,
1866 (void_type_node, get_qpwmt (), NULL))
1867 DEF_SS_BUILTIN (update, ". Wt",
1868 BUILT_IN___STRUB_UPDATE,
1869 (void_type_node, get_qpwmt (), NULL))
1870 DEF_SS_BUILTIN (leave, ". w ",
1871 BUILT_IN___STRUB_LEAVE,
1872 (void_type_node, get_qpwmt (), NULL))
1874 #undef DEF_SS_BUILTIN
1876 /* Define strub identifiers on demand. */
1877 #define DEF_IDENT(IDX, NAME) \
1878 static inline tree get_ ## NAME () { \
1879 int idx = STRUB_IDENT_BASE + IDX; \
1880 tree identifier = strub_cache[idx]; \
1881 if (!identifier) \
1882 strub_cache[idx] = identifier = get_identifier (".strub." #NAME); \
1883 return identifier; \
1886 DEF_IDENT (0, watermark_ptr)
1887 DEF_IDENT (1, va_list_ptr)
1888 DEF_IDENT (2, apply_args)
1890 #undef DEF_IDENT
1892 static inline int adjust_at_calls_type (tree);
1893 static inline void adjust_at_calls_call (cgraph_edge *, int, tree);
1894 static inline void adjust_at_calls_calls (cgraph_node *);
1896 /* Add to SEQ a call to the strub watermark update builtin, taking NODE's
1897 location if given. Optionally add the corresponding edge from NODE, with
1898 execution frequency COUNT. Return the modified SEQ. */
1900 static inline gimple_seq
1901 call_update_watermark (tree wmptr, cgraph_node *node, profile_count count,
1902 gimple_seq seq = NULL)
1904 tree uwm = get_update ();
1905 gcall *update = gimple_build_call (uwm, 1, wmptr);
1906 if (node)
1907 gimple_set_location (update, DECL_SOURCE_LOCATION (node->decl));
1908 gimple_seq_add_stmt (&seq, update);
1909 if (node)
1910 node->create_edge (cgraph_node::get_create (uwm), update, count, false);
1911 return seq;
1916 } // anon namespace
1918 /* Gather with this type a collection of parameters that we're turning into
1919 explicit references. */
1921 typedef hash_set<tree> indirect_parms_t;
1923 /* Dereference OP's incoming turned-into-reference parm if it's an
1924 INDIRECT_PARMS or an ADDR_EXPR thereof. Set *REC and return according to
1925 gimple-walking expectations. */
1927 static tree
1928 maybe_make_indirect (indirect_parms_t &indirect_parms, tree op, int *rec)
1930 if (DECL_P (op))
1932 *rec = 0;
1933 if (indirect_parms.contains (op))
1935 tree ret = gimple_fold_indirect_ref (op);
1936 if (!ret)
1937 ret = build2 (MEM_REF,
1938 TREE_TYPE (TREE_TYPE (op)),
1940 build_int_cst (TREE_TYPE (op), 0));
1941 return ret;
1944 else if (TREE_CODE (op) == ADDR_EXPR
1945 && DECL_P (TREE_OPERAND (op, 0)))
1947 *rec = 0;
1948 if (indirect_parms.contains (TREE_OPERAND (op, 0)))
1950 op = TREE_OPERAND (op, 0);
1951 return op;
1955 return NULL_TREE;
1958 /* A gimple-walking function that adds dereferencing to indirect parms. */
1960 static tree
1961 walk_make_indirect (tree *op, int *rec, void *arg)
1963 walk_stmt_info *wi = (walk_stmt_info *)arg;
1964 indirect_parms_t &indirect_parms = *(indirect_parms_t *)wi->info;
1966 if (!*op || TYPE_P (*op))
1968 *rec = 0;
1969 return NULL_TREE;
1972 if (tree repl = maybe_make_indirect (indirect_parms, *op, rec))
1974 *op = repl;
1975 wi->changed = true;
1978 return NULL_TREE;
1981 /* A gimple-walking function that turns any non-gimple-val ADDR_EXPRs into a
1982 separate SSA. Though addresses of e.g. parameters, and of members thereof,
1983 are gimple vals, turning parameters into references, with an extra layer of
1984 indirection and thus explicit dereferencing, need to be regimplified. */
1986 static tree
1987 walk_regimplify_addr_expr (tree *op, int *rec, void *arg)
1989 walk_stmt_info *wi = (walk_stmt_info *)arg;
1990 gimple_stmt_iterator &gsi = *(gimple_stmt_iterator *)wi->info;
1992 *rec = 0;
1994 if (!*op || TREE_CODE (*op) != ADDR_EXPR)
1995 return NULL_TREE;
1997 if (!is_gimple_val (*op))
1999 tree ret = force_gimple_operand_gsi (&gsi, *op, true,
2000 NULL_TREE, true, GSI_SAME_STMT);
2001 gcc_assert (ret != *op);
2002 *op = ret;
2003 wi->changed = true;
2006 return NULL_TREE;
2009 /* Turn STMT's PHI arg defs into separate SSA defs if they've become
2010 non-gimple_val. Return TRUE if any edge insertions need to be committed. */
2012 static bool
2013 walk_regimplify_phi (gphi *stmt)
2015 bool needs_commit = false;
2017 for (unsigned i = 0, n = gimple_phi_num_args (stmt); i < n; i++)
2019 tree op = gimple_phi_arg_def (stmt, i);
2020 if ((TREE_CODE (op) == ADDR_EXPR
2021 && !is_gimple_val (op))
2022 /* ??? A PARM_DECL that was addressable in the original function and
2023 had its address in PHI nodes, but that became a reference in the
2024 wrapped clone would NOT be updated by update_ssa in PHI nodes.
2025 Alas, if we were to create a default def for it now, update_ssa
2026 would complain that the symbol that needed rewriting already has
2027 SSA names associated with it. OTOH, leaving the PARM_DECL alone,
2028 it eventually causes errors because it remains unchanged in PHI
2029 nodes, but it gets rewritten as expected if it appears in other
2030 stmts. So we cheat a little here, and force the PARM_DECL out of
2031 the PHI node and into an assignment. It's a little expensive,
2032 because we insert it at the edge, which introduces a basic block
2033 that's entirely unnecessary, but it works, and the block will be
2034 removed as the default def gets propagated back into the PHI node,
2035 so the final optimized code looks just as expected. */
2036 || (TREE_CODE (op) == PARM_DECL
2037 && !TREE_ADDRESSABLE (op)))
2039 tree temp = make_ssa_name (TREE_TYPE (op), stmt);
2040 if (TREE_CODE (op) == PARM_DECL)
2041 SET_SSA_NAME_VAR_OR_IDENTIFIER (temp, DECL_NAME (op));
2042 SET_PHI_ARG_DEF (stmt, i, temp);
2044 gimple *assign = gimple_build_assign (temp, op);
2045 if (gimple_phi_arg_has_location (stmt, i))
2046 gimple_set_location (assign, gimple_phi_arg_location (stmt, i));
2047 gsi_insert_on_edge (gimple_phi_arg_edge (stmt, i), assign);
2048 needs_commit = true;
2052 return needs_commit;
2055 /* Create a reference type to use for PARM when turning it into a reference.
2056 NONALIASED causes the reference type to gain its own separate alias set, so
2057 that accessing the indirectly-passed parm won'will not add aliasing
2058 noise. */
2060 static tree
2061 build_ref_type_for (tree parm, bool nonaliased = true)
2063 gcc_checking_assert (TREE_CODE (parm) == PARM_DECL);
2065 tree ref_type = build_reference_type (TREE_TYPE (parm));
2067 if (!nonaliased)
2068 return ref_type;
2070 /* Each PARM turned indirect still points to the distinct memory area at the
2071 wrapper, and the reference in unchanging, so we might qualify it, but...
2072 const is not really important, since we're only using default defs for the
2073 reference parm anyway, and not introducing any defs, and restrict seems to
2074 cause trouble. E.g., libgnat/s-concat3.adb:str_concat_3 has memmoves that,
2075 if it's wrapped, the memmoves are deleted in dse1. Using a distinct alias
2076 set seems to not run afoul of this problem, and it hopefully enables the
2077 compiler to tell the pointers do point to objects that are not otherwise
2078 aliased. */
2079 tree qref_type = build_variant_type_copy (ref_type);
2081 TYPE_ALIAS_SET (qref_type) = new_alias_set ();
2082 record_alias_subset (TYPE_ALIAS_SET (qref_type), get_alias_set (ref_type));
2084 return qref_type;
2087 /* Add cgraph edges from current_function_decl to callees in SEQ with frequency
2088 COUNT, assuming all calls in SEQ are direct. */
2090 static void
2091 add_call_edges_for_seq (gimple_seq seq, profile_count count)
2093 cgraph_node *node = cgraph_node::get_create (current_function_decl);
2095 for (gimple_stmt_iterator gsi = gsi_start (seq);
2096 !gsi_end_p (gsi); gsi_next (&gsi))
2098 gimple *stmt = gsi_stmt (gsi);
2100 gcall *call = dyn_cast <gcall *> (stmt);
2101 if (!call)
2102 continue;
2104 tree callee = gimple_call_fndecl (call);
2105 gcc_checking_assert (callee);
2106 node->create_edge (cgraph_node::get_create (callee), call, count, false);
2110 /* Insert SEQ after the call at GSI, as if the call was in a try block with SEQ
2111 as finally, i.e., SEQ will run after the call whether it returns or
2112 propagates an exception. This handles block splitting, EH edge and block
2113 creation, noreturn and nothrow optimizations, and even throwing calls without
2114 preexisting local handlers. */
2116 static void
2117 gsi_insert_finally_seq_after_call (gimple_stmt_iterator gsi, gimple_seq seq)
2119 if (!seq)
2120 return;
2122 gimple *stmt = gsi_stmt (gsi);
2124 if (gimple_has_location (stmt))
2125 annotate_all_with_location (seq, gimple_location (stmt));
2127 gcall *call = dyn_cast <gcall *> (stmt);
2128 bool noreturn_p = call && gimple_call_noreturn_p (call);
2129 int eh_lp = lookup_stmt_eh_lp (stmt);
2130 bool must_not_throw_p = eh_lp < 0;
2131 bool nothrow_p = (must_not_throw_p
2132 || (call && gimple_call_nothrow_p (call))
2133 || (eh_lp <= 0
2134 && (TREE_NOTHROW (cfun->decl)
2135 || !opt_for_fn (cfun->decl, flag_exceptions))));
2137 if (noreturn_p && nothrow_p)
2138 return;
2140 /* Don't expect an EH edge if we're not to throw, or if we're not in an EH
2141 region yet. */
2142 bool no_eh_edge_p = (nothrow_p || !eh_lp);
2143 bool must_end_bb = stmt_ends_bb_p (stmt);
2145 edge eft = NULL, eeh = NULL;
2146 if (must_end_bb && !(noreturn_p && no_eh_edge_p))
2148 gcc_checking_assert (gsi_one_before_end_p (gsi));
2150 edge e;
2151 edge_iterator ei;
2152 FOR_EACH_EDGE (e, ei, gsi_bb (gsi)->succs)
2154 if ((e->flags & EDGE_EH))
2156 gcc_checking_assert (!eeh);
2157 eeh = e;
2158 #if !CHECKING_P
2159 if (eft || noreturn_p)
2160 break;
2161 #endif
2163 if ((e->flags & EDGE_FALLTHRU))
2165 gcc_checking_assert (!eft);
2166 eft = e;
2167 #if !CHECKING_P
2168 if (eeh || no_eh_edge_p)
2169 break;
2170 #endif
2174 gcc_checking_assert (!(eft && (eft->flags & EDGE_FALLTHRU))
2175 == noreturn_p);
2176 gcc_checking_assert (!(eeh && (eeh->flags & EDGE_EH))
2177 == no_eh_edge_p);
2178 gcc_checking_assert (eft != eeh);
2181 if (!noreturn_p)
2183 gimple_seq nseq = nothrow_p ? seq : gimple_seq_copy (seq);
2185 if (must_end_bb)
2187 gcc_checking_assert (gsi_one_before_end_p (gsi));
2188 add_call_edges_for_seq (nseq, eft->count ());
2189 gsi_insert_seq_on_edge_immediate (eft, nseq);
2191 else
2193 add_call_edges_for_seq (nseq, gsi_bb (gsi)->count);
2194 gsi_insert_seq_after (&gsi, nseq, GSI_SAME_STMT);
2198 if (nothrow_p)
2199 return;
2201 if (eh_lp)
2203 add_call_edges_for_seq (seq, eeh->count ());
2204 gsi_insert_seq_on_edge_immediate (eeh, seq);
2205 return;
2208 /* A throwing call may appear within a basic block in a function that doesn't
2209 have any EH regions. We're going to add a cleanup if so, therefore the
2210 block will have to be split. */
2211 basic_block bb = gsi_bb (gsi);
2212 if (!gsi_one_before_end_p (gsi))
2213 split_block (bb, stmt);
2215 /* Create a new block for the EH cleanup. */
2216 basic_block bb_eh_cleanup = create_empty_bb (bb);
2217 if (dom_info_available_p (CDI_DOMINATORS))
2218 set_immediate_dominator (CDI_DOMINATORS, bb_eh_cleanup, bb);
2219 if (current_loops)
2220 add_bb_to_loop (bb_eh_cleanup, current_loops->tree_root);
2222 /* Make the new block an EH cleanup for the call. */
2223 eh_region new_r = gen_eh_region_cleanup (NULL);
2224 eh_landing_pad lp = gen_eh_landing_pad (new_r);
2225 tree label = gimple_block_label (bb_eh_cleanup);
2226 lp->post_landing_pad = label;
2227 EH_LANDING_PAD_NR (label) = lp->index;
2228 add_stmt_to_eh_lp (stmt, lp->index);
2230 /* Add the cleanup code to the EH cleanup block. */
2231 gsi = gsi_after_labels (bb_eh_cleanup);
2232 gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2234 /* And then propagate the exception further. */
2235 gresx *resx = gimple_build_resx (new_r->index);
2236 if (gimple_has_location (stmt))
2237 gimple_set_location (resx, gimple_location (stmt));
2238 gsi_insert_before (&gsi, resx, GSI_SAME_STMT);
2240 /* Finally, wire the EH cleanup block into the CFG. */
2241 edge neeh = make_eh_edge (stmt);
2242 neeh->probability = profile_probability::never ();
2243 gcc_checking_assert (neeh->dest == bb_eh_cleanup);
2244 gcc_checking_assert (!neeh->dest->count.initialized_p ());
2245 neeh->dest->count = neeh->count ();
2246 add_call_edges_for_seq (seq, neeh->dest->count);
2249 /* Copy the attribute list at *ATTRS, minus any NAME attributes, leaving
2250 shareable trailing nodes alone. */
2252 static inline void
2253 remove_named_attribute_unsharing (const char *name, tree *attrs)
2255 while (tree found = lookup_attribute (name, *attrs))
2257 /* Copy nodes up to the next NAME attribute. */
2258 while (*attrs != found)
2260 *attrs = tree_cons (TREE_PURPOSE (*attrs),
2261 TREE_VALUE (*attrs),
2262 TREE_CHAIN (*attrs));
2263 attrs = &TREE_CHAIN (*attrs);
2265 /* Then drop it. */
2266 gcc_checking_assert (*attrs == found);
2267 *attrs = TREE_CHAIN (*attrs);
2271 /* Record the order of the last cgraph entry whose mode we've already set, so
2272 that we can perform mode setting incrementally without duplication. */
2273 static int last_cgraph_order;
2275 /* Set strub modes for functions introduced since the last call. */
2277 static void
2278 ipa_strub_set_mode_for_new_functions ()
2280 if (symtab->order == last_cgraph_order)
2281 return;
2283 cgraph_node *node;
2285 /* Go through the functions twice, once over non-aliases, and then over
2286 aliases, so that aliases can reuse the mode computation of their ultimate
2287 targets. */
2288 for (int aliases = 0; aliases <= 1; aliases++)
2289 FOR_EACH_FUNCTION (node)
2291 if (!node->alias != !aliases)
2292 continue;
2294 /* Already done. */
2295 if (node->order < last_cgraph_order)
2296 continue;
2298 set_strub_mode (node);
2301 last_cgraph_order = symtab->order;
2304 /* Return FALSE if NODE is a strub context, and TRUE otherwise. */
2306 bool
2307 strub_splittable_p (cgraph_node *node)
2309 switch (get_strub_mode (node))
2311 case STRUB_WRAPPED:
2312 case STRUB_AT_CALLS:
2313 case STRUB_AT_CALLS_OPT:
2314 case STRUB_INLINABLE:
2315 case STRUB_INTERNAL:
2316 case STRUB_WRAPPER:
2317 return false;
2319 case STRUB_CALLABLE:
2320 case STRUB_DISABLED:
2321 break;
2323 default:
2324 gcc_unreachable ();
2327 return true;
2330 /* Return the PARM_DECL of the incoming watermark pointer, if there is one. */
2332 tree
2333 strub_watermark_parm (tree fndecl)
2335 switch (get_strub_mode_from_fndecl (fndecl))
2337 case STRUB_WRAPPED:
2338 case STRUB_AT_CALLS:
2339 case STRUB_AT_CALLS_OPT:
2340 break;
2342 case STRUB_INTERNAL:
2343 case STRUB_WRAPPER:
2344 case STRUB_CALLABLE:
2345 case STRUB_DISABLED:
2346 case STRUB_INLINABLE:
2347 return NULL_TREE;
2349 default:
2350 gcc_unreachable ();
2353 for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
2354 /* The type (variant) compare finds the parameter even in a just-created
2355 clone, before we set its name, but the type-based compare doesn't work
2356 during builtin expansion within the lto compiler, because we'll have
2357 created a separate variant in that run. */
2358 if (TREE_TYPE (parm) == pass_ipa_strub::get_qpwmt ()
2359 || DECL_NAME (parm) == pass_ipa_strub::get_watermark_ptr ())
2360 return parm;
2362 gcc_unreachable ();
2365 /* Adjust a STRUB_AT_CALLS function TYPE, adding a watermark pointer if it
2366 hasn't been added yet. Return the named argument count. */
2369 pass_ipa_strub::adjust_at_calls_type (tree type)
2371 int named_args = 0;
2373 gcc_checking_assert (same_strub_mode_in_variants_p (type));
2375 if (!TYPE_ARG_TYPES (type))
2376 return named_args;
2378 tree *tlist = &TYPE_ARG_TYPES (type);
2379 tree qpwmptrt = get_qpwmt ();
2380 while (*tlist && TREE_VALUE (*tlist) != void_type_node)
2382 /* The type has already been adjusted. */
2383 if (TREE_VALUE (*tlist) == qpwmptrt)
2384 return named_args;
2385 named_args++;
2386 *tlist = tree_cons (TREE_PURPOSE (*tlist),
2387 TREE_VALUE (*tlist),
2388 TREE_CHAIN (*tlist));
2389 tlist = &TREE_CHAIN (*tlist);
2392 /* Add the new argument after all named arguments, so as to not mess with
2393 attributes that reference parameters. */
2394 *tlist = tree_cons (NULL_TREE, get_qpwmt (), *tlist);
2396 #if ATTR_FNSPEC_DECONST_WATERMARK
2397 if (!type_already_adjusted)
2399 int flags = flags_from_decl_or_type (type);
2400 tree fnspec = lookup_attribute ("fn spec", type);
2402 if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
2404 size_t xargs = 1;
2405 size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
2406 auto_vec<char> nspecv (tgtlen);
2407 char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
2408 if (fnspec)
2410 tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
2411 curlen = TREE_STRING_LENGTH (fnspecstr);
2412 memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
2414 if (!curlen)
2416 nspec[curlen++] = '.';
2417 nspec[curlen++] = ((flags & ECF_CONST)
2418 ? 'c'
2419 : (flags & ECF_PURE)
2420 ? 'p'
2421 : ' ');
2423 while (curlen < tgtlen - 2 * xargs)
2425 nspec[curlen++] = '.';
2426 nspec[curlen++] = ' ';
2428 nspec[curlen++] = 'W';
2429 nspec[curlen++] = 't';
2431 /* The type has already been copied, if needed, before adding
2432 parameters. */
2433 TYPE_ATTRIBUTES (type)
2434 = tree_cons (get_identifier ("fn spec"),
2435 build_tree_list (NULL_TREE,
2436 build_string (tgtlen, nspec)),
2437 TYPE_ATTRIBUTES (type));
2440 #endif
2442 return named_args;
2445 /* Adjust a call to an at-calls call target. Create a watermark local variable
2446 if needed, initialize it before, pass it to the callee according to the
2447 modified at-calls interface, and release the callee's stack space after the
2448 call, if not deferred. If the call is const or pure, arrange for the
2449 watermark to not be assumed unused or unchanged. */
2451 void
2452 pass_ipa_strub::adjust_at_calls_call (cgraph_edge *e, int named_args,
2453 tree callee_fntype)
2455 gcc_checking_assert (e->call_stmt);
2456 gcall *ocall = e->call_stmt;
2457 gimple_stmt_iterator gsi = gsi_for_stmt (ocall);
2459 /* Make sure we haven't modified this call yet. */
2460 gcc_checking_assert (!(int (gimple_call_num_args (ocall)) > named_args
2461 && (TREE_TYPE (gimple_call_arg (ocall, named_args))
2462 == get_pwmt ())));
2464 tree tsup;
2465 if (!(tsup = gimple_call_fndecl (ocall)))
2466 tsup = TREE_TYPE (TREE_TYPE (gimple_call_fn (ocall)));
2467 if (!strub_target_support_p (tsup, true, gimple_location (ocall)))
2468 return;
2470 /* If we're already within a strub context, pass on the incoming watermark
2471 pointer, and omit the enter and leave calls around the modified call, as an
2472 optimization, or as a means to satisfy a tail-call requirement. */
2473 tree swmp = ((opt_for_fn (e->caller->decl, optimize_size)
2474 || opt_for_fn (e->caller->decl, optimize) > 2
2475 || gimple_call_must_tail_p (ocall)
2476 || (opt_for_fn (e->caller->decl, optimize) == 2
2477 && gimple_call_tail_p (ocall)))
2478 ? strub_watermark_parm (e->caller->decl)
2479 : NULL_TREE);
2480 bool omit_own_watermark = swmp;
2481 tree swm = NULL_TREE;
2482 if (!omit_own_watermark)
2484 swm = create_tmp_var (get_wmt (), ".strub.watermark");
2485 TREE_ADDRESSABLE (swm) = true;
2486 swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
2488 /* Initialize the watermark before the call. */
2489 tree enter = get_enter ();
2490 gcall *stptr = gimple_build_call (enter, 1,
2491 unshare_expr (swmp));
2492 if (gimple_has_location (ocall))
2493 gimple_set_location (stptr, gimple_location (ocall));
2494 gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
2495 e->caller->create_edge (cgraph_node::get_create (enter),
2496 stptr, gsi_bb (gsi)->count, false);
2500 /* Replace the call with one that passes the swmp argument first. */
2501 gcall *wrcall;
2502 { gcall *stmt = ocall;
2503 // Mostly copied from gimple_call_copy_skip_args.
2504 int i = 0;
2505 int nargs = gimple_call_num_args (stmt);
2506 auto_vec<tree> vargs (MAX (nargs, named_args) + 1);
2507 gcall *new_stmt;
2509 /* pr71109.c calls a prototypeless function, then defines it with
2510 additional arguments. It's ill-formed, but after it's inlined,
2511 it somehow works out. */
2512 for (; i < named_args && i < nargs; i++)
2513 vargs.quick_push (gimple_call_arg (stmt, i));
2514 for (; i < named_args; i++)
2515 vargs.quick_push (null_pointer_node);
2517 vargs.quick_push (unshare_expr (swmp));
2519 for (; i < nargs; i++)
2520 vargs.quick_push (gimple_call_arg (stmt, i));
2522 if (gimple_call_internal_p (stmt))
2523 gcc_unreachable ();
2524 else
2525 new_stmt = gimple_build_call_vec (gimple_call_fn (stmt), vargs);
2526 gimple_call_set_fntype (new_stmt, callee_fntype);
2528 if (gimple_call_lhs (stmt))
2529 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2531 gimple_move_vops (new_stmt, stmt);
2533 if (gimple_has_location (stmt))
2534 gimple_set_location (new_stmt, gimple_location (stmt));
2535 gimple_call_copy_flags (new_stmt, stmt);
2536 gimple_call_set_chain (new_stmt, gimple_call_chain (stmt));
2538 gimple_set_modified (new_stmt, true);
2540 wrcall = new_stmt;
2543 update_stmt (wrcall);
2544 gsi_replace (&gsi, wrcall, true);
2545 cgraph_edge::set_call_stmt (e, wrcall, false);
2547 /* Insert the strub code after the call. */
2548 gimple_seq seq = NULL;
2550 #if !ATTR_FNSPEC_DECONST_WATERMARK
2551 /* If the call will be assumed to not modify or even read the
2552 watermark, make it read and modified ourselves. */
2553 if ((gimple_call_flags (wrcall)
2554 & (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
2556 if (!swm)
2557 swm = build2 (MEM_REF,
2558 TREE_TYPE (TREE_TYPE (swmp)),
2559 swmp,
2560 build_int_cst (TREE_TYPE (swmp), 0));
2562 vec<tree, va_gc> *inputs = NULL;
2563 vec<tree, va_gc> *outputs = NULL;
2564 vec_safe_push (outputs,
2565 build_tree_list
2566 (build_tree_list
2567 (NULL_TREE, build_string (2, "=m")),
2568 unshare_expr (swm)));
2569 vec_safe_push (inputs,
2570 build_tree_list
2571 (build_tree_list
2572 (NULL_TREE, build_string (1, "m")),
2573 unshare_expr (swm)));
2574 gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
2575 NULL, NULL);
2576 gimple_seq_add_stmt (&seq, forcemod);
2578 /* If the call will be assumed to not even read the watermark,
2579 make sure it is already in memory before the call. */
2580 if ((gimple_call_flags (wrcall) & ECF_CONST))
2582 vec<tree, va_gc> *inputs = NULL;
2583 vec_safe_push (inputs,
2584 build_tree_list
2585 (build_tree_list
2586 (NULL_TREE, build_string (1, "m")),
2587 unshare_expr (swm)));
2588 gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
2589 NULL, NULL);
2590 if (gimple_has_location (wrcall))
2591 gimple_set_location (force_store, gimple_location (wrcall));
2592 gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
2595 #endif
2597 if (!omit_own_watermark)
2599 gcall *sleave = gimple_build_call (get_leave (), 1,
2600 unshare_expr (swmp));
2601 gimple_seq_add_stmt (&seq, sleave);
2603 gassign *clobber = gimple_build_assign (swm,
2604 build_clobber
2605 (TREE_TYPE (swm)));
2606 gimple_seq_add_stmt (&seq, clobber);
2609 gsi_insert_finally_seq_after_call (gsi, seq);
2612 /* Adjust all at-calls calls in NODE. */
2614 void
2615 pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node)
2617 /* Adjust unknown-callee indirect calls with STRUB_AT_CALLS types within
2618 onode. */
2619 if (node->indirect_calls)
2621 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2622 for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2624 gcc_checking_assert (e->indirect_unknown_callee);
2626 if (!e->call_stmt)
2627 continue;
2629 tree callee_fntype;
2630 enum strub_mode callee_mode
2631 = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
2633 if (callee_mode != STRUB_AT_CALLS
2634 && callee_mode != STRUB_AT_CALLS_OPT)
2635 continue;
2637 int named_args = adjust_at_calls_type (callee_fntype);
2639 adjust_at_calls_call (e, named_args, callee_fntype);
2641 pop_cfun ();
2644 if (node->callees)
2646 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2647 for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2649 gcc_checking_assert (!e->indirect_unknown_callee);
2651 if (!e->call_stmt)
2652 continue;
2654 tree callee_fntype;
2655 enum strub_mode callee_mode
2656 = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
2658 if (callee_mode != STRUB_AT_CALLS
2659 && callee_mode != STRUB_AT_CALLS_OPT)
2660 continue;
2662 int named_args = adjust_at_calls_type (callee_fntype);
2664 adjust_at_calls_call (e, named_args, callee_fntype);
2666 pop_cfun ();
2670 /* The strubm (strub mode) pass computes a strub mode for each function in the
2671 call graph, and checks, before any inlining, that strub callability
2672 requirements in effect are satisfied. */
2674 unsigned int
2675 pass_ipa_strub_mode::execute (function *)
2677 last_cgraph_order = 0;
2678 ipa_strub_set_mode_for_new_functions ();
2680 /* Verify before any inlining or other transformations. */
2681 verify_strub ();
2683 return 0;
2686 /* Create a strub mode pass. */
2688 simple_ipa_opt_pass *
2689 make_pass_ipa_strub_mode (gcc::context *ctxt)
2691 return new pass_ipa_strub_mode (ctxt);
2694 /* The strub pass proper adjusts types, signatures, and at-calls calls, and
2695 splits internal-strub functions. */
2697 unsigned int
2698 pass_ipa_strub::execute (function *)
2700 cgraph_node *onode;
2702 ipa_strub_set_mode_for_new_functions ();
2704 /* First, adjust the signature of at-calls functions. We adjust types of
2705 at-calls functions first, so that we don't modify types in place unless
2706 strub is explicitly requested. */
2707 FOR_EACH_FUNCTION (onode)
2709 enum strub_mode mode = get_strub_mode (onode);
2711 if (mode == STRUB_AT_CALLS
2712 || mode == STRUB_AT_CALLS_OPT)
2714 /* Create a type variant if strubbing was not explicitly requested in
2715 the function type. */
2716 if (get_strub_mode_from_type (TREE_TYPE (onode->decl)) != mode)
2717 distinctify_node_type (onode);
2719 int named_args = adjust_at_calls_type (TREE_TYPE (onode->decl));
2721 /* An external function explicitly declared with strub won't have a
2722 body. Even with implicit at-calls strub, a function may have had its
2723 body removed after we selected the mode, and then we have nothing
2724 further to do. */
2725 if (!onode->has_gimple_body_p ())
2726 continue;
2728 tree *pargs = &DECL_ARGUMENTS (onode->decl);
2730 /* A noninterposable_alias reuses the same parm decl chain, don't add
2731 the parm twice. */
2732 bool aliased_parms = (onode->alias && *pargs
2733 && DECL_CONTEXT (*pargs) != onode->decl);
2735 if (aliased_parms)
2736 continue;
2738 for (int i = 0; i < named_args; i++)
2739 pargs = &DECL_CHAIN (*pargs);
2741 tree wmptr = build_decl (DECL_SOURCE_LOCATION (onode->decl),
2742 PARM_DECL,
2743 get_watermark_ptr (),
2744 get_qpwmt ());
2745 DECL_ARTIFICIAL (wmptr) = 1;
2746 DECL_ARG_TYPE (wmptr) = get_qpwmt ();
2747 DECL_CONTEXT (wmptr) = onode->decl;
2748 TREE_USED (wmptr) = 1;
2749 DECL_CHAIN (wmptr) = *pargs;
2750 *pargs = wmptr;
2752 if (onode->alias)
2753 continue;
2755 cgraph_node *nnode = onode;
2756 push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
2759 edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2760 gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count);
2761 gsi_insert_seq_on_edge_immediate (e, seq);
2764 if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca)
2766 basic_block bb;
2767 FOR_EACH_BB_FN (bb, cfun)
2768 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2769 !gsi_end_p (gsi); gsi_next (&gsi))
2771 gimple *stmt = gsi_stmt (gsi);
2773 gcall *call = dyn_cast <gcall *> (stmt);
2775 if (!call)
2776 continue;
2778 if (gimple_alloca_call_p (call))
2780 /* Capture stack growth. */
2781 gimple_seq seq = call_update_watermark (wmptr, NULL,
2782 gsi_bb (gsi)
2783 ->count);
2784 gsi_insert_finally_seq_after_call (gsi, seq);
2789 pop_cfun ();
2793 FOR_EACH_FUNCTION (onode)
2795 if (!onode->has_gimple_body_p ())
2796 continue;
2798 enum strub_mode mode = get_strub_mode (onode);
2800 if (mode != STRUB_INTERNAL)
2802 adjust_at_calls_calls (onode);
2803 continue;
2806 bool is_stdarg = calls_builtin_va_start_p (onode);;
2807 bool apply_args = calls_builtin_apply_args_p (onode);
2809 vec<ipa_adjusted_param, va_gc> *nparms = NULL;
2810 unsigned j = 0;
2812 // The following loop copied from ipa-split.c:split_function.
2813 for (tree parm = DECL_ARGUMENTS (onode->decl);
2814 parm; parm = DECL_CHAIN (parm), j++)
2816 ipa_adjusted_param adj = {};
2817 adj.op = IPA_PARAM_OP_COPY;
2818 adj.base_index = j;
2819 adj.prev_clone_index = j;
2820 vec_safe_push (nparms, adj);
2823 if (apply_args)
2825 ipa_adjusted_param aaadj = {};
2826 aaadj.op = IPA_PARAM_OP_NEW;
2827 aaadj.type = get_qptr ();
2828 vec_safe_push (nparms, aaadj);
2831 if (is_stdarg)
2833 ipa_adjusted_param vladj = {};
2834 vladj.op = IPA_PARAM_OP_NEW;
2835 vladj.type = get_qpvalst ();
2836 vec_safe_push (nparms, vladj);
2839 ipa_adjusted_param wmadj = {};
2840 wmadj.op = IPA_PARAM_OP_NEW;
2841 wmadj.type = get_qpwmt ();
2842 vec_safe_push (nparms, wmadj);
2844 ipa_param_adjustments adj (nparms, -1, false);
2846 cgraph_node *nnode = onode->create_version_clone_with_body
2847 (auto_vec<cgraph_edge *> (0),
2848 NULL, &adj, NULL, NULL, "strub", NULL);
2850 if (!nnode)
2852 error_at (DECL_SOURCE_LOCATION (onode->decl),
2853 "failed to split %qD for %<strub%>",
2854 onode->decl);
2855 continue;
2858 onode->split_part = true;
2859 if (onode->calls_comdat_local)
2860 nnode->add_to_same_comdat_group (onode);
2862 set_strub_mode_to (onode, STRUB_WRAPPER);
2863 set_strub_mode_to (nnode, STRUB_WRAPPED);
2865 adjust_at_calls_calls (nnode);
2867 /* Decide which of the wrapped function's parms we want to turn into
2868 references to the argument passed to the wrapper. In general, we want to
2869 copy small arguments, and avoid copying large ones. Variable-sized array
2870 lengths given by other arguments, as in 20020210-1.c, would lead to
2871 problems if passed by value, after resetting the original function and
2872 dropping the length computation; passing them by reference works.
2873 DECL_BY_REFERENCE is *not* a substitute for this: it involves copying
2874 anyway, but performed at the caller. */
2875 indirect_parms_t indirect_nparms (3, false);
2876 unsigned adjust_ftype = 0;
2877 unsigned named_args = 0;
2878 for (tree parm = DECL_ARGUMENTS (onode->decl),
2879 nparm = DECL_ARGUMENTS (nnode->decl),
2880 nparmt = TYPE_ARG_TYPES (TREE_TYPE (nnode->decl));
2881 parm;
2882 named_args++,
2883 parm = DECL_CHAIN (parm),
2884 nparm = DECL_CHAIN (nparm),
2885 nparmt = nparmt ? TREE_CHAIN (nparmt) : NULL_TREE)
2886 if (TREE_THIS_VOLATILE (parm)
2887 || !(0 /* DECL_BY_REFERENCE (narg) */
2888 || is_gimple_reg_type (TREE_TYPE (nparm))
2889 || VECTOR_TYPE_P (TREE_TYPE (nparm))
2890 || TREE_CODE (TREE_TYPE (nparm)) == COMPLEX_TYPE
2891 || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2892 && (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2893 <= 4 * UNITS_PER_WORD))))
2895 /* No point in indirecting pointer types. Presumably they
2896 won't ever pass the size-based test above, but check the
2897 assumption here, because getting this wrong would mess
2898 with attribute access and possibly others. We deal with
2899 fn spec below. */
2900 gcc_checking_assert (!POINTER_TYPE_P (TREE_TYPE (nparm)));
2902 indirect_nparms.add (nparm);
2904 /* ??? Is there any case in which it is not safe to suggest the parms
2905 turned indirect don't alias anything else? They are distinct,
2906 unaliased memory in the wrapper, and the wrapped can't possibly
2907 take pointers into them because none of the pointers passed to the
2908 wrapper can alias other incoming parameters passed by value, even
2909 if with transparent reference, and the wrapper doesn't take any
2910 extra parms that could point into wrapper's parms. So we can
2911 probably drop the TREE_ADDRESSABLE and keep the TRUE. */
2912 tree ref_type = build_ref_type_for (nparm,
2913 true
2914 || !TREE_ADDRESSABLE (parm));
2916 DECL_ARG_TYPE (nparm) = TREE_TYPE (nparm) = ref_type;
2917 relayout_decl (nparm);
2918 TREE_ADDRESSABLE (nparm) = 0;
2919 DECL_BY_REFERENCE (nparm) = 0;
2920 DECL_NOT_GIMPLE_REG_P (nparm) = 0;
2921 /* ??? This avoids mismatches in debug info bind stmts in
2922 e.g. a-chahan . */
2923 DECL_ABSTRACT_ORIGIN (nparm) = NULL;
2925 if (nparmt)
2926 adjust_ftype++;
2929 /* Also adjust the wrapped function type, if needed. */
2930 if (adjust_ftype)
2932 tree nftype = TREE_TYPE (nnode->decl);
2934 /* We always add at least one argument at the end of the signature, when
2935 cloning the function, so we don't expect to need to duplicate the
2936 type here. */
2937 gcc_checking_assert (TYPE_ARG_TYPES (nftype)
2938 != TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
2940 /* Check that fnspec still works for the modified function signature,
2941 and drop it otherwise. */
2942 bool drop_fnspec = false;
2943 tree fnspec = lookup_attribute ("fn spec", TYPE_ATTRIBUTES (nftype));
2944 attr_fnspec spec = fnspec ? attr_fnspec (fnspec) : attr_fnspec ("");
2946 unsigned retcopy;
2947 if (!(fnspec && spec.returns_arg (&retcopy)))
2948 retcopy = (unsigned) -1;
2950 unsigned i = 0;
2951 for (tree nparm = DECL_ARGUMENTS (nnode->decl),
2952 nparmt = TYPE_ARG_TYPES (nftype);
2953 adjust_ftype > 0;
2954 i++, nparm = DECL_CHAIN (nparm), nparmt = TREE_CHAIN (nparmt))
2955 if (indirect_nparms.contains (nparm))
2957 TREE_VALUE (nparmt) = TREE_TYPE (nparm);
2958 adjust_ftype--;
2960 if (fnspec && !drop_fnspec)
2962 if (i == retcopy)
2963 drop_fnspec = true;
2964 else if (spec.arg_specified_p (i))
2966 /* Properties that apply to pointers only must not be
2967 present, because we don't make pointers further
2968 indirect. */
2969 gcc_checking_assert
2970 (!spec.arg_max_access_size_given_by_arg_p (i, NULL));
2971 gcc_checking_assert (!spec.arg_copied_to_arg_p (i, NULL));
2973 /* Any claim of direct access only is invalidated by
2974 adding an indirection level. */
2975 if (spec.arg_direct_p (i))
2976 drop_fnspec = true;
2978 /* If there's a claim the argument is not read from, the
2979 added indirection invalidates it: if the argument is
2980 used at all, then the pointer will necessarily be
2981 read. */
2982 if (!spec.arg_maybe_read_p (i)
2983 && spec.arg_used_p (i))
2984 drop_fnspec = true;
2989 /* ??? Maybe we could adjust it instead. Note we don't need
2990 to mess with attribute access: pointer-typed parameters are
2991 not modified, so they can remain unchanged. */
2992 if (drop_fnspec)
2993 remove_named_attribute_unsharing ("fn spec",
2994 &TYPE_ATTRIBUTES (nftype));
2996 TREE_TYPE (nnode->decl) = nftype;
2999 #if ATTR_FNSPEC_DECONST_WATERMARK
3001 int flags = flags_from_decl_or_type (nnode->decl);
3002 tree fnspec = lookup_attribute ("fn spec", TREE_TYPE (nnode->decl));
3004 if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
3006 size_t xargs = 1 + int (is_stdarg) + int (apply_args);
3007 size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
3008 auto_vec<char> nspecv (tgtlen);
3009 char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
3010 bool no_writes_p = true;
3011 if (fnspec)
3013 tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
3014 curlen = TREE_STRING_LENGTH (fnspecstr);
3015 memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
3016 if (!(flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS))
3017 && curlen >= 2
3018 && nspec[1] != 'c' && nspec[1] != 'C'
3019 && nspec[1] != 'p' && nspec[1] != 'P')
3020 no_writes_p = false;
3022 if (!curlen)
3024 nspec[curlen++] = '.';
3025 nspec[curlen++] = ((flags & ECF_CONST)
3026 ? 'c'
3027 : (flags & ECF_PURE)
3028 ? 'p'
3029 : ' ');
3031 while (curlen < tgtlen - 2 * xargs)
3033 nspec[curlen++] = '.';
3034 nspec[curlen++] = ' ';
3037 /* These extra args are unlikely to be present in const or pure
3038 functions. It's conceivable that a function that takes variable
3039 arguments, or that passes its arguments on to another function,
3040 could be const or pure, but it would not modify the arguments, and,
3041 being pure or const, it couldn't possibly modify or even access
3042 memory referenced by them. But it can read from these internal
3043 data structures created by the wrapper, and from any
3044 argument-passing memory referenced by them, so we denote the
3045 possibility of reading from multiple levels of indirection, but
3046 only of reading because const/pure. */
3047 if (apply_args)
3049 nspec[curlen++] = 'r';
3050 nspec[curlen++] = ' ';
3052 if (is_stdarg)
3054 nspec[curlen++] = (no_writes_p ? 'r' : '.');
3055 nspec[curlen++] = (no_writes_p ? 't' : ' ');
3058 nspec[curlen++] = 'W';
3059 nspec[curlen++] = 't';
3061 /* The type has already been copied before adding parameters. */
3062 gcc_checking_assert (TYPE_ARG_TYPES (TREE_TYPE (nnode->decl))
3063 != TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
3064 TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl))
3065 = tree_cons (get_identifier ("fn spec"),
3066 build_tree_list (NULL_TREE,
3067 build_string (tgtlen, nspec)),
3068 TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)));
3071 #endif
3074 tree decl = onode->decl;
3075 cgraph_node *target = nnode;
3077 { // copied from create_wrapper
3079 /* Preserve DECL_RESULT so we get right by reference flag. */
3080 tree decl_result = DECL_RESULT (decl);
3082 /* Remove the function's body but keep arguments to be reused
3083 for thunk. */
3084 onode->release_body (true);
3085 onode->reset (/* unlike create_wrapper: preserve_comdat_group = */true);
3087 DECL_UNINLINABLE (decl) = false;
3088 DECL_RESULT (decl) = decl_result;
3089 DECL_INITIAL (decl) = NULL;
3090 allocate_struct_function (decl, false);
3091 set_cfun (NULL);
3093 /* Turn alias into thunk and expand it into GIMPLE representation. */
3094 onode->definition = true;
3096 thunk_info::get_create (onode);
3097 onode->thunk = true;
3098 onode->create_edge (target, NULL, onode->count);
3099 onode->callees->can_throw_external = !TREE_NOTHROW (target->decl);
3101 tree arguments = DECL_ARGUMENTS (decl);
3103 while (arguments)
3105 TREE_ADDRESSABLE (arguments) = false;
3106 arguments = TREE_CHAIN (arguments);
3110 tree alias = onode->callees->callee->decl;
3111 tree thunk_fndecl = decl;
3112 tree a;
3114 int nxargs = 1 + is_stdarg + apply_args;
3116 { // Simplified from expand_thunk.
3117 tree restype;
3118 basic_block bb, then_bb, else_bb, return_bb;
3119 gimple_stmt_iterator bsi;
3120 int nargs = 0;
3121 tree arg;
3122 int i;
3123 tree resdecl;
3124 tree restmp = NULL;
3126 gcall *call;
3127 greturn *ret;
3128 bool alias_is_noreturn = TREE_THIS_VOLATILE (alias);
3130 a = DECL_ARGUMENTS (thunk_fndecl);
3132 current_function_decl = thunk_fndecl;
3134 /* Ensure thunks are emitted in their correct sections. */
3135 resolve_unique_section (thunk_fndecl, 0,
3136 flag_function_sections);
3138 bitmap_obstack_initialize (NULL);
3140 /* Build the return declaration for the function. */
3141 restype = TREE_TYPE (TREE_TYPE (thunk_fndecl));
3142 if (DECL_RESULT (thunk_fndecl) == NULL_TREE)
3144 resdecl = build_decl (input_location, RESULT_DECL, 0, restype);
3145 DECL_ARTIFICIAL (resdecl) = 1;
3146 DECL_IGNORED_P (resdecl) = 1;
3147 DECL_CONTEXT (resdecl) = thunk_fndecl;
3148 DECL_RESULT (thunk_fndecl) = resdecl;
3150 else
3151 resdecl = DECL_RESULT (thunk_fndecl);
3153 profile_count cfg_count = onode->count;
3154 if (!cfg_count.initialized_p ())
3155 cfg_count = profile_count::from_gcov_type (BB_FREQ_MAX).guessed_local ();
3157 bb = then_bb = else_bb = return_bb
3158 = init_lowered_empty_function (thunk_fndecl, true, cfg_count);
3160 bsi = gsi_start_bb (bb);
3162 /* Build call to the function being thunked. */
3163 if (!VOID_TYPE_P (restype)
3164 && (!alias_is_noreturn
3165 || TREE_ADDRESSABLE (restype)
3166 || TREE_CODE (TYPE_SIZE_UNIT (restype)) != INTEGER_CST))
3168 if (DECL_BY_REFERENCE (resdecl))
3170 restmp = gimple_fold_indirect_ref (resdecl);
3171 if (!restmp)
3172 restmp = build2 (MEM_REF,
3173 TREE_TYPE (TREE_TYPE (resdecl)),
3174 resdecl,
3175 build_int_cst (TREE_TYPE (resdecl), 0));
3177 else if (!is_gimple_reg_type (restype))
3179 if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl)))
3181 restmp = resdecl;
3183 if (VAR_P (restmp))
3185 add_local_decl (cfun, restmp);
3186 BLOCK_VARS (DECL_INITIAL (current_function_decl))
3187 = restmp;
3190 else
3191 restmp = create_tmp_var (restype, "retval");
3193 else
3194 restmp = create_tmp_reg (restype, "retval");
3197 for (arg = a; arg; arg = DECL_CHAIN (arg))
3198 nargs++;
3199 auto_vec<tree> vargs (nargs + nxargs);
3200 i = 0;
3201 arg = a;
3203 if (nargs)
3204 for (tree nparm = DECL_ARGUMENTS (nnode->decl);
3205 i < nargs;
3206 i++, arg = DECL_CHAIN (arg), nparm = DECL_CHAIN (nparm))
3208 tree save_arg = arg;
3210 /* Arrange to pass indirectly the parms, if we decided to do
3211 so, and revert its type in the wrapper. */
3212 if (indirect_nparms.contains (nparm))
3214 tree ref_type = TREE_TYPE (nparm);
3215 TREE_ADDRESSABLE (arg) = true;
3216 arg = build1 (ADDR_EXPR, ref_type, arg);
3218 else if (!TREE_THIS_VOLATILE (arg))
3219 DECL_NOT_GIMPLE_REG_P (arg) = 0;
3221 /* Convert the argument back to the type used by the calling
3222 conventions, e.g. a non-prototyped float type is passed as
3223 double, as in 930603-1.c, and needs to be converted back to
3224 double to be passed on unchanged to the wrapped
3225 function. */
3226 if (TREE_TYPE (nparm) != DECL_ARG_TYPE (nparm))
3228 tree tmp = arg;
3229 /* If ARG is e.g. volatile, we must copy and
3230 convert in separate statements. */
3231 if (!is_gimple_val (arg))
3233 tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3234 (TREE_TYPE (arg)), "arg");
3235 gimple *stmt = gimple_build_assign (tmp, arg);
3236 gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3238 arg = fold_convert (DECL_ARG_TYPE (nparm), tmp);
3241 if (!is_gimple_val (arg))
3243 tree tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3244 (TREE_TYPE (arg)), "arg");
3245 gimple *stmt = gimple_build_assign (tmp, arg);
3246 gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3247 arg = tmp;
3249 vargs.quick_push (arg);
3250 arg = save_arg;
3252 /* These strub arguments are adjusted later. */
3253 if (apply_args)
3254 vargs.quick_push (null_pointer_node);
3255 if (is_stdarg)
3256 vargs.quick_push (null_pointer_node);
3257 vargs.quick_push (null_pointer_node);
3258 call = gimple_build_call_vec (build_fold_addr_expr_loc (0, alias),
3259 vargs);
3260 onode->callees->call_stmt = call;
3261 // gimple_call_set_from_thunk (call, true);
3262 if (DECL_STATIC_CHAIN (alias))
3264 tree p = DECL_STRUCT_FUNCTION (alias)->static_chain_decl;
3265 tree type = TREE_TYPE (p);
3266 tree decl = build_decl (DECL_SOURCE_LOCATION (thunk_fndecl),
3267 PARM_DECL, create_tmp_var_name ("CHAIN"),
3268 type);
3269 DECL_ARTIFICIAL (decl) = 1;
3270 DECL_IGNORED_P (decl) = 1;
3271 TREE_USED (decl) = 1;
3272 DECL_CONTEXT (decl) = thunk_fndecl;
3273 DECL_ARG_TYPE (decl) = type;
3274 TREE_READONLY (decl) = 1;
3276 struct function *sf = DECL_STRUCT_FUNCTION (thunk_fndecl);
3277 sf->static_chain_decl = decl;
3279 gimple_call_set_chain (call, decl);
3282 /* Return slot optimization is always possible and in fact required to
3283 return values with DECL_BY_REFERENCE. */
3284 if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl))
3285 && (!is_gimple_reg_type (TREE_TYPE (resdecl))
3286 || DECL_BY_REFERENCE (resdecl)))
3287 gimple_call_set_return_slot_opt (call, true);
3289 if (restmp)
3291 gimple_call_set_lhs (call, restmp);
3292 gcc_assert (useless_type_conversion_p (TREE_TYPE (restmp),
3293 TREE_TYPE (TREE_TYPE (alias))));
3295 gsi_insert_after (&bsi, call, GSI_NEW_STMT);
3296 if (!alias_is_noreturn)
3298 /* Build return value. */
3299 if (!DECL_BY_REFERENCE (resdecl))
3300 ret = gimple_build_return (restmp);
3301 else
3302 ret = gimple_build_return (resdecl);
3304 gsi_insert_after (&bsi, ret, GSI_NEW_STMT);
3306 else
3308 remove_edge (single_succ_edge (bb));
3311 cfun->gimple_df->in_ssa_p = true;
3312 update_max_bb_count ();
3313 profile_status_for_fn (cfun)
3314 = cfg_count.initialized_p () && cfg_count.ipa_p ()
3315 ? PROFILE_READ : PROFILE_GUESSED;
3316 /* FIXME: C++ FE should stop setting TREE_ASM_WRITTEN on thunks. */
3317 // TREE_ASM_WRITTEN (thunk_fndecl) = false;
3318 delete_unreachable_blocks ();
3319 update_ssa (TODO_update_ssa);
3320 checking_verify_flow_info ();
3321 free_dominance_info (CDI_DOMINATORS);
3323 /* Since we want to emit the thunk, we explicitly mark its name as
3324 referenced. */
3325 onode->thunk = false;
3326 onode->lowered = true;
3327 bitmap_obstack_release (NULL);
3329 current_function_decl = NULL;
3330 set_cfun (NULL);
3333 thunk_info::remove (onode);
3335 // some more of create_wrapper at the end of the next block.
3340 tree aaval = NULL_TREE;
3341 tree vaptr = NULL_TREE;
3342 tree wmptr = NULL_TREE;
3343 for (tree arg = DECL_ARGUMENTS (nnode->decl); arg; arg = DECL_CHAIN (arg))
3345 aaval = vaptr;
3346 vaptr = wmptr;
3347 wmptr = arg;
3350 if (!apply_args)
3351 aaval = NULL_TREE;
3352 /* The trailing args are [apply_args], [va_list_ptr], and
3353 watermark. If we don't have a va_list_ptr, the penultimate
3354 argument is apply_args.
3356 else if (!is_stdarg)
3357 aaval = vaptr;
3359 if (!is_stdarg)
3360 vaptr = NULL_TREE;
3362 DECL_NAME (wmptr) = get_watermark_ptr ();
3363 DECL_ARTIFICIAL (wmptr) = 1;
3364 DECL_IGNORED_P (wmptr) = 1;
3365 TREE_USED (wmptr) = 1;
3367 if (is_stdarg)
3369 DECL_NAME (vaptr) = get_va_list_ptr ();
3370 DECL_ARTIFICIAL (vaptr) = 1;
3371 DECL_IGNORED_P (vaptr) = 1;
3372 TREE_USED (vaptr) = 1;
3375 if (apply_args)
3377 DECL_NAME (aaval) = get_apply_args ();
3378 DECL_ARTIFICIAL (aaval) = 1;
3379 DECL_IGNORED_P (aaval) = 1;
3380 TREE_USED (aaval) = 1;
3383 push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
3386 edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
3387 gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count);
3388 gsi_insert_seq_on_edge_immediate (e, seq);
3391 bool any_indirect = !indirect_nparms.is_empty ();
3393 if (any_indirect)
3395 basic_block bb;
3396 bool needs_commit = false;
3397 FOR_EACH_BB_FN (bb, cfun)
3399 for (gphi_iterator gsi = gsi_start_nonvirtual_phis (bb);
3400 !gsi_end_p (gsi);
3401 gsi_next_nonvirtual_phi (&gsi))
3403 gphi *stmt = gsi.phi ();
3405 walk_stmt_info wi = {};
3406 wi.info = &indirect_nparms;
3407 walk_gimple_op (stmt, walk_make_indirect, &wi);
3408 if (wi.changed && !is_gimple_debug (gsi_stmt (gsi)))
3409 if (walk_regimplify_phi (stmt))
3410 needs_commit = true;
3413 for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
3414 !gsi_end_p (gsi); gsi_next (&gsi))
3416 gimple *stmt = gsi_stmt (gsi);
3418 walk_stmt_info wi = {};
3419 wi.info = &indirect_nparms;
3420 walk_gimple_op (stmt, walk_make_indirect, &wi);
3421 if (wi.changed)
3423 if (!is_gimple_debug (stmt))
3425 wi.info = &gsi;
3426 walk_gimple_op (stmt, walk_regimplify_addr_expr,
3427 &wi);
3429 update_stmt (stmt);
3433 if (needs_commit)
3434 gsi_commit_edge_inserts ();
3437 if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca
3438 || is_stdarg || apply_args)
3439 for (cgraph_edge *e = nnode->callees, *enext; e; e = enext)
3441 if (!e->call_stmt)
3442 continue;
3444 gcall *call = e->call_stmt;
3445 gimple_stmt_iterator gsi = gsi_for_stmt (call);
3446 tree fndecl = e->callee->decl;
3448 enext = e->next_callee;
3450 if (gimple_alloca_call_p (call))
3452 gimple_seq seq = call_update_watermark (wmptr, NULL,
3453 gsi_bb (gsi)->count);
3454 gsi_insert_finally_seq_after_call (gsi, seq);
3456 else if (fndecl && is_stdarg
3457 && fndecl_built_in_p (fndecl, BUILT_IN_VA_START))
3459 /* Using a non-default stdarg ABI makes the function ineligible
3460 for internal strub. */
3461 gcc_checking_assert (builtin_decl_explicit (BUILT_IN_VA_START)
3462 == fndecl);
3463 tree bvacopy = builtin_decl_explicit (BUILT_IN_VA_COPY);
3464 gimple_call_set_fndecl (call, bvacopy);
3465 tree arg = vaptr;
3466 /* The va_copy source must be dereferenced, unless it's an array
3467 type, that would have decayed to a pointer. */
3468 if (TREE_CODE (TREE_TYPE (TREE_TYPE (vaptr))) != ARRAY_TYPE)
3470 arg = gimple_fold_indirect_ref (vaptr);
3471 if (!arg)
3472 arg = build2 (MEM_REF,
3473 TREE_TYPE (TREE_TYPE (vaptr)),
3474 vaptr,
3475 build_int_cst (TREE_TYPE (vaptr), 0));
3476 if (!is_gimple_val (arg))
3477 arg = force_gimple_operand_gsi (&gsi, arg, true,
3478 NULL_TREE, true, GSI_SAME_STMT);
3480 gimple_call_set_arg (call, 1, arg);
3481 update_stmt (call);
3482 e->redirect_callee (cgraph_node::get_create (bvacopy));
3484 else if (fndecl && apply_args
3485 && fndecl_built_in_p (fndecl, BUILT_IN_APPLY_ARGS))
3487 tree lhs = gimple_call_lhs (call);
3488 gimple *assign = (lhs
3489 ? gimple_build_assign (lhs, aaval)
3490 : gimple_build_nop ());
3491 gsi_replace (&gsi, assign, true);
3492 cgraph_edge::remove (e);
3496 { // a little more copied from create_wrapper
3498 /* Inline summary set-up. */
3499 nnode->analyze ();
3500 // inline_analyze_function (nnode);
3503 pop_cfun ();
3507 push_cfun (DECL_STRUCT_FUNCTION (onode->decl));
3508 gimple_stmt_iterator gsi
3509 = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
3511 gcall *wrcall;
3512 while (!(wrcall = dyn_cast <gcall *> (gsi_stmt (gsi))))
3513 gsi_next (&gsi);
3515 tree swm = create_tmp_var (get_wmt (), ".strub.watermark");
3516 TREE_ADDRESSABLE (swm) = true;
3517 tree swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
3519 tree enter = get_enter ();
3520 gcall *stptr = gimple_build_call (enter, 1, unshare_expr (swmp));
3521 gimple_set_location (stptr, gimple_location (wrcall));
3522 gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
3523 onode->create_edge (cgraph_node::get_create (enter),
3524 stptr, gsi_bb (gsi)->count, false);
3526 int nargs = gimple_call_num_args (wrcall);
3528 gimple_seq seq = NULL;
3530 if (apply_args)
3532 tree aalst = create_tmp_var (ptr_type_node, ".strub.apply_args");
3533 tree bappargs = builtin_decl_explicit (BUILT_IN_APPLY_ARGS);
3534 gcall *appargs = gimple_build_call (bappargs, 0);
3535 gimple_call_set_lhs (appargs, aalst);
3536 gimple_set_location (appargs, gimple_location (wrcall));
3537 gsi_insert_before (&gsi, appargs, GSI_SAME_STMT);
3538 gimple_call_set_arg (wrcall, nargs - 2 - is_stdarg, aalst);
3539 onode->create_edge (cgraph_node::get_create (bappargs),
3540 appargs, gsi_bb (gsi)->count, false);
3543 if (is_stdarg)
3545 tree valst = create_tmp_var (va_list_type_node, ".strub.va_list");
3546 TREE_ADDRESSABLE (valst) = true;
3547 tree vaptr = build1 (ADDR_EXPR,
3548 build_pointer_type (va_list_type_node),
3549 valst);
3550 gimple_call_set_arg (wrcall, nargs - 2, unshare_expr (vaptr));
3552 tree bvastart = builtin_decl_explicit (BUILT_IN_VA_START);
3553 gcall *vastart = gimple_build_call (bvastart, 2,
3554 unshare_expr (vaptr),
3555 integer_zero_node);
3556 gimple_set_location (vastart, gimple_location (wrcall));
3557 gsi_insert_before (&gsi, vastart, GSI_SAME_STMT);
3558 onode->create_edge (cgraph_node::get_create (bvastart),
3559 vastart, gsi_bb (gsi)->count, false);
3561 tree bvaend = builtin_decl_explicit (BUILT_IN_VA_END);
3562 gcall *vaend = gimple_build_call (bvaend, 1, unshare_expr (vaptr));
3563 gimple_set_location (vaend, gimple_location (wrcall));
3564 gimple_seq_add_stmt (&seq, vaend);
3567 gimple_call_set_arg (wrcall, nargs - 1, unshare_expr (swmp));
3568 // gimple_call_set_tail (wrcall, false);
3569 update_stmt (wrcall);
3572 #if !ATTR_FNSPEC_DECONST_WATERMARK
3573 /* If the call will be assumed to not modify or even read the
3574 watermark, make it read and modified ourselves. */
3575 if ((gimple_call_flags (wrcall)
3576 & (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
3578 vec<tree, va_gc> *inputs = NULL;
3579 vec<tree, va_gc> *outputs = NULL;
3580 vec_safe_push (outputs,
3581 build_tree_list
3582 (build_tree_list
3583 (NULL_TREE, build_string (2, "=m")),
3584 swm));
3585 vec_safe_push (inputs,
3586 build_tree_list
3587 (build_tree_list
3588 (NULL_TREE, build_string (1, "m")),
3589 swm));
3590 gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
3591 NULL, NULL);
3592 gimple_seq_add_stmt (&seq, forcemod);
3594 /* If the call will be assumed to not even read the watermark,
3595 make sure it is already in memory before the call. */
3596 if ((gimple_call_flags (wrcall) & ECF_CONST))
3598 vec<tree, va_gc> *inputs = NULL;
3599 vec_safe_push (inputs,
3600 build_tree_list
3601 (build_tree_list
3602 (NULL_TREE, build_string (1, "m")),
3603 swm));
3604 gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
3605 NULL, NULL);
3606 gimple_set_location (force_store, gimple_location (wrcall));
3607 gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
3610 #endif
3612 gcall *sleave = gimple_build_call (get_leave (), 1,
3613 unshare_expr (swmp));
3614 gimple_seq_add_stmt (&seq, sleave);
3616 gassign *clobber = gimple_build_assign (swm,
3617 build_clobber
3618 (TREE_TYPE (swm)));
3619 gimple_seq_add_stmt (&seq, clobber);
3622 gsi_insert_finally_seq_after_call (gsi, seq);
3624 /* For nnode, we don't rebuild edges because we wish to retain
3625 any redirections copied to it from earlier passes, so we add
3626 call graph edges explicitly there, but for onode, we create a
3627 fresh function, so we may as well just issue the calls and
3628 then rebuild all cgraph edges. */
3629 // cgraph_edge::rebuild_edges ();
3630 onode->analyze ();
3631 // inline_analyze_function (onode);
3633 pop_cfun ();
3637 return 0;
3640 simple_ipa_opt_pass *
3641 make_pass_ipa_strub (gcc::context *ctxt)
3643 return new pass_ipa_strub (ctxt);
3646 #include "gt-ipa-strub.h"