1 /* Passes for transactional memory support.
2 Copyright (C) 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
25 #include "tree-flow.h"
26 #include "tree-pass.h"
27 #include "tree-inline.h"
28 #include "diagnostic-core.h"
31 #include "trans-mem.h"
34 #include "langhooks.h"
35 #include "gimple-pretty-print.h"
39 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 2000 - 1)
40 #define PROB_VERY_LIKELY (PROB_ALWAYS - PROB_VERY_UNLIKELY)
41 #define PROB_UNLIKELY (REG_BR_PROB_BASE / 5 - 1)
42 #define PROB_LIKELY (PROB_ALWAYS - PROB_VERY_LIKELY)
43 #define PROB_ALWAYS (REG_BR_PROB_BASE)
45 #define A_RUNINSTRUMENTEDCODE 0x0001
46 #define A_RUNUNINSTRUMENTEDCODE 0x0002
47 #define A_SAVELIVEVARIABLES 0x0004
48 #define A_RESTORELIVEVARIABLES 0x0008
49 #define A_ABORTTRANSACTION 0x0010
51 #define AR_USERABORT 0x0001
52 #define AR_USERRETRY 0x0002
53 #define AR_TMCONFLICT 0x0004
54 #define AR_EXCEPTIONBLOCKABORT 0x0008
55 #define AR_OUTERABORT 0x0010
57 #define MODE_SERIALIRREVOCABLE 0x0000
60 /* The representation of a transaction changes several times during the
61 lowering process. In the beginning, in the front-end we have the
62 GENERIC tree TRANSACTION_EXPR. For example,
70 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
71 trivially replaced with a GIMPLE_TRANSACTION node.
73 During pass_lower_tm, we examine the body of transactions looking
74 for aborts. Transactions that do not contain an abort may be
75 merged into an outer transaction. We also add a TRY-FINALLY node
76 to arrange for the transaction to be committed on any exit.
78 [??? Think about how this arrangement affects throw-with-commit
79 and throw-with-abort operations. In this case we want the TRY to
80 handle gotos, but not to catch any exceptions because the transaction
81 will already be closed.]
83 GIMPLE_TRANSACTION [label=NULL] {
90 __builtin___tm_abort ();
92 __builtin___tm_commit ();
96 During pass_lower_eh, we create EH regions for the transactions,
97 intermixed with the regular EH stuff. This gives us a nice persistent
98 mapping (all the way through rtl) from transactional memory operation
99 back to the transaction, which allows us to get the abnormal edges
100 correct to model transaction aborts and restarts:
102 GIMPLE_TRANSACTION [label=over]
108 __builtin___tm_abort ();
109 __builtin___tm_commit ();
112 This is the end of all_lowering_passes, and so is what is present
113 during the IPA passes, and through all of the optimization passes.
115 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
116 functions and mark functions for cloning.
118 At the end of gimple optimization, before exiting SSA form,
119 pass_tm_edges replaces statements that perform transactional
120 memory operations with the appropriate TM builtins, and swap
121 out function calls with their transactional clones. At this
122 point we introduce the abnormal transaction restart edges and
123 complete lowering of the GIMPLE_TRANSACTION node.
125 x = __builtin___tm_start (MAY_ABORT);
127 if (x & abort_transaction)
130 t0 = __builtin___tm_load (global);
132 __builtin___tm_store (&global, t1);
134 __builtin___tm_abort ();
135 __builtin___tm_commit ();
139 static void *expand_regions (struct tm_region
*,
140 void *(*callback
)(struct tm_region
*, void *),
144 /* Return the attributes we want to examine for X, or NULL if it's not
145 something we examine. We look at function types, but allow pointers
146 to function types and function decls and peek through. */
149 get_attrs_for (const_tree x
)
151 switch (TREE_CODE (x
))
154 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
161 if (TREE_CODE (x
) != POINTER_TYPE
)
167 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
173 return TYPE_ATTRIBUTES (x
);
177 /* Return true if X has been marked TM_PURE. */
180 is_tm_pure (const_tree x
)
184 switch (TREE_CODE (x
))
195 if (TREE_CODE (x
) != POINTER_TYPE
)
201 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
206 flags
= flags_from_decl_or_type (x
);
207 return (flags
& ECF_TM_PURE
) != 0;
210 /* Return true if X has been marked TM_IRREVOCABLE. */
213 is_tm_irrevocable (tree x
)
215 tree attrs
= get_attrs_for (x
);
217 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
220 /* A call to the irrevocable builtin is by definition,
222 if (TREE_CODE (x
) == ADDR_EXPR
)
223 x
= TREE_OPERAND (x
, 0);
224 if (TREE_CODE (x
) == FUNCTION_DECL
225 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
226 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
232 /* Return true if X has been marked TM_SAFE. */
235 is_tm_safe (const_tree x
)
239 tree attrs
= get_attrs_for (x
);
242 if (lookup_attribute ("transaction_safe", attrs
))
244 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
251 /* Return true if CALL is const, or tm_pure. */
254 is_tm_pure_call (gimple call
)
256 tree fn
= gimple_call_fn (call
);
258 if (TREE_CODE (fn
) == ADDR_EXPR
)
260 fn
= TREE_OPERAND (fn
, 0);
261 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
266 return is_tm_pure (fn
);
269 /* Return true if X has been marked TM_CALLABLE. */
272 is_tm_callable (tree x
)
274 tree attrs
= get_attrs_for (x
);
277 if (lookup_attribute ("transaction_callable", attrs
))
279 if (lookup_attribute ("transaction_safe", attrs
))
281 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
287 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
290 is_tm_may_cancel_outer (tree x
)
292 tree attrs
= get_attrs_for (x
);
294 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
298 /* Return true for built in functions that "end" a transaction. */
301 is_tm_ending_fndecl (tree fndecl
)
303 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
304 switch (DECL_FUNCTION_CODE (fndecl
))
306 case BUILT_IN_TM_COMMIT
:
307 case BUILT_IN_TM_COMMIT_EH
:
308 case BUILT_IN_TM_ABORT
:
309 case BUILT_IN_TM_IRREVOCABLE
:
318 /* Return true if STMT is a TM load. */
321 is_tm_load (gimple stmt
)
325 if (gimple_code (stmt
) != GIMPLE_CALL
)
328 fndecl
= gimple_call_fndecl (stmt
);
329 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
330 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
333 /* Same as above, but for simple TM loads, that is, not the
334 after-write, after-read, etc optimized variants. */
337 is_tm_simple_load (gimple stmt
)
341 if (gimple_code (stmt
) != GIMPLE_CALL
)
344 fndecl
= gimple_call_fndecl (stmt
);
345 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
347 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
348 return (fcode
== BUILT_IN_TM_LOAD_1
349 || fcode
== BUILT_IN_TM_LOAD_2
350 || fcode
== BUILT_IN_TM_LOAD_4
351 || fcode
== BUILT_IN_TM_LOAD_8
352 || fcode
== BUILT_IN_TM_LOAD_FLOAT
353 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
354 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
355 || fcode
== BUILT_IN_TM_LOAD_M64
356 || fcode
== BUILT_IN_TM_LOAD_M128
357 || fcode
== BUILT_IN_TM_LOAD_M256
);
362 /* Return true if STMT is a TM store. */
365 is_tm_store (gimple stmt
)
369 if (gimple_code (stmt
) != GIMPLE_CALL
)
372 fndecl
= gimple_call_fndecl (stmt
);
373 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
374 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
377 /* Same as above, but for simple TM stores, that is, not the
378 after-write, after-read, etc optimized variants. */
381 is_tm_simple_store (gimple stmt
)
385 if (gimple_code (stmt
) != GIMPLE_CALL
)
388 fndecl
= gimple_call_fndecl (stmt
);
389 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
391 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
392 return (fcode
== BUILT_IN_TM_STORE_1
393 || fcode
== BUILT_IN_TM_STORE_2
394 || fcode
== BUILT_IN_TM_STORE_4
395 || fcode
== BUILT_IN_TM_STORE_8
396 || fcode
== BUILT_IN_TM_STORE_FLOAT
397 || fcode
== BUILT_IN_TM_STORE_DOUBLE
398 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
399 || fcode
== BUILT_IN_TM_STORE_M64
400 || fcode
== BUILT_IN_TM_STORE_M128
401 || fcode
== BUILT_IN_TM_STORE_M256
);
406 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
409 is_tm_abort (tree fndecl
)
412 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
413 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
416 /* Build a GENERIC tree for a user abort. This is called by front ends
417 while transforming the __tm_abort statement. */
420 build_tm_abort_call (location_t loc
, bool is_outer
)
422 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
423 build_int_cst (integer_type_node
,
425 | (is_outer
? AR_OUTERABORT
: 0)));
428 /* Common gateing function for several of the TM passes. */
436 /* Map for aribtrary function replacement under TM, as created
437 by the tm_wrap attribute. */
439 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
443 record_tm_replacement (tree from
, tree to
)
445 struct tree_map
**slot
, *h
;
447 /* Do not inline wrapper functions that will get replaced in the TM
450 Suppose you have foo() that will get replaced into tmfoo(). Make
451 sure the inliner doesn't try to outsmart us and inline foo()
452 before we get a chance to do the TM replacement. */
453 DECL_UNINLINABLE (from
) = 1;
455 if (tm_wrap_map
== NULL
)
456 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
458 h
= ggc_alloc_tree_map ();
459 h
->hash
= htab_hash_pointer (from
);
463 slot
= (struct tree_map
**)
464 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
468 /* Return a TM-aware replacement function for DECL. */
471 find_tm_replacement_function (tree fndecl
)
475 struct tree_map
*h
, in
;
477 in
.base
.from
= fndecl
;
478 in
.hash
= htab_hash_pointer (fndecl
);
479 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
484 /* ??? We may well want TM versions of most of the common <string.h>
485 functions. For now, we've already these two defined. */
486 /* Adjust expand_call_tm() attributes as necessary for the cases
488 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
489 switch (DECL_FUNCTION_CODE (fndecl
))
491 case BUILT_IN_MEMCPY
:
492 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
493 case BUILT_IN_MEMMOVE
:
494 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
495 case BUILT_IN_MEMSET
:
496 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
504 /* When appropriate, record TM replacement for memory allocation functions.
506 FROM is the FNDECL to wrap. */
508 tm_malloc_replacement (tree from
)
513 if (TREE_CODE (from
) != FUNCTION_DECL
)
516 /* If we have a previous replacement, the user must be explicitly
517 wrapping malloc/calloc/free. They better know what they're
519 if (find_tm_replacement_function (from
))
522 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
524 if (!strcmp (str
, "malloc"))
525 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
526 else if (!strcmp (str
, "calloc"))
527 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
528 else if (!strcmp (str
, "free"))
529 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
533 TREE_NOTHROW (to
) = 0;
535 record_tm_replacement (from
, to
);
538 /* Diagnostics for tm_safe functions/regions. Called by the front end
539 once we've lowered the function to high-gimple. */
541 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
542 Process exactly one statement. WI->INFO is set to non-null when in
543 the context of a tm_safe function, and null for a __transaction block. */
545 #define DIAG_TM_OUTER 1
546 #define DIAG_TM_SAFE 2
547 #define DIAG_TM_RELAXED 4
551 unsigned int summary_flags
: 8;
552 unsigned int block_flags
: 8;
553 unsigned int func_flags
: 8;
554 unsigned int saw_volatile
: 1;
558 /* Return true if T is a volatile variable of some kind. */
561 volatile_var_p (tree t
)
563 return (SSA_VAR_P (t
)
564 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
567 /* Tree callback function for diagnose_tm pass. */
570 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
573 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
574 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
576 if (volatile_var_p (*tp
)
577 && d
->block_flags
& DIAG_TM_SAFE
581 error_at (gimple_location (d
->stmt
),
582 "invalid volatile use of %qD inside transaction",
590 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
591 struct walk_stmt_info
*wi
)
593 gimple stmt
= gsi_stmt (*gsi
);
594 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
596 /* Save stmt for use in leaf analysis. */
599 switch (gimple_code (stmt
))
603 tree fn
= gimple_call_fn (stmt
);
605 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
606 && is_tm_may_cancel_outer (fn
))
607 error_at (gimple_location (stmt
),
608 "%<transaction_may_cancel_outer%> function call not within"
609 " outer transaction or %<transaction_may_cancel_outer%>");
611 if (d
->summary_flags
& DIAG_TM_SAFE
)
613 bool is_safe
, direct_call_p
;
616 if (TREE_CODE (fn
) == ADDR_EXPR
617 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
619 direct_call_p
= true;
620 replacement
= TREE_OPERAND (fn
, 0);
621 replacement
= find_tm_replacement_function (replacement
);
627 direct_call_p
= false;
628 replacement
= NULL_TREE
;
631 if (is_tm_safe_or_pure (fn
))
633 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
635 /* A function explicitly marked transaction_callable as
636 opposed to transaction_safe is being defined to be
637 unsafe as part of its ABI, regardless of its contents. */
640 else if (direct_call_p
)
642 if (flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
644 else if (replacement
)
646 /* ??? At present we've been considering replacements
647 merely transaction_callable, and therefore might
648 enter irrevocable. The tm_wrap attribute has not
649 yet made it into the new language spec. */
654 /* ??? Diagnostics for unmarked direct calls moved into
655 the IPA pass. Section 3.2 of the spec details how
656 functions not marked should be considered "implicitly
657 safe" based on having examined the function body. */
663 /* An unmarked indirect call. Consider it unsafe even
664 though optimization may yet figure out how to inline. */
670 if (TREE_CODE (fn
) == ADDR_EXPR
)
671 fn
= TREE_OPERAND (fn
, 0);
672 if (d
->block_flags
& DIAG_TM_SAFE
)
675 error_at (gimple_location (stmt
),
676 "unsafe function call %qD within "
677 "atomic transaction", fn
);
680 if (!DECL_P (fn
) || DECL_NAME (fn
))
681 error_at (gimple_location (stmt
),
682 "unsafe function call %qE within "
683 "atomic transaction", fn
);
685 error_at (gimple_location (stmt
),
686 "unsafe indirect function call within "
687 "atomic transaction");
693 error_at (gimple_location (stmt
),
694 "unsafe function call %qD within "
695 "%<transaction_safe%> function", fn
);
698 if (!DECL_P (fn
) || DECL_NAME (fn
))
699 error_at (gimple_location (stmt
),
700 "unsafe function call %qE within "
701 "%<transaction_safe%> function", fn
);
703 error_at (gimple_location (stmt
),
704 "unsafe indirect function call within "
705 "%<transaction_safe%> function");
714 /* ??? We ought to come up with a way to add attributes to
715 asm statements, and then add "transaction_safe" to it.
716 Either that or get the language spec to resurrect __tm_waiver. */
717 if (d
->block_flags
& DIAG_TM_SAFE
)
718 error_at (gimple_location (stmt
),
719 "asm not allowed in atomic transaction");
720 else if (d
->func_flags
& DIAG_TM_SAFE
)
721 error_at (gimple_location (stmt
),
722 "asm not allowed in %<transaction_safe%> function");
725 case GIMPLE_TRANSACTION
:
727 unsigned char inner_flags
= DIAG_TM_SAFE
;
729 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
731 if (d
->block_flags
& DIAG_TM_SAFE
)
732 error_at (gimple_location (stmt
),
733 "relaxed transaction in atomic transaction");
734 else if (d
->func_flags
& DIAG_TM_SAFE
)
735 error_at (gimple_location (stmt
),
736 "relaxed transaction in %<transaction_safe%> function");
737 inner_flags
= DIAG_TM_RELAXED
;
739 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
742 error_at (gimple_location (stmt
),
743 "outer transaction in transaction");
744 else if (d
->func_flags
& DIAG_TM_OUTER
)
745 error_at (gimple_location (stmt
),
746 "outer transaction in "
747 "%<transaction_may_cancel_outer%> function");
748 else if (d
->func_flags
& DIAG_TM_SAFE
)
749 error_at (gimple_location (stmt
),
750 "outer transaction in %<transaction_safe%> function");
751 inner_flags
|= DIAG_TM_OUTER
;
754 *handled_ops_p
= true;
755 if (gimple_transaction_body (stmt
))
757 struct walk_stmt_info wi_inner
;
758 struct diagnose_tm d_inner
;
760 memset (&d_inner
, 0, sizeof (d_inner
));
761 d_inner
.func_flags
= d
->func_flags
;
762 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
763 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
765 memset (&wi_inner
, 0, sizeof (wi_inner
));
766 wi_inner
.info
= &d_inner
;
768 walk_gimple_seq (gimple_transaction_body (stmt
),
769 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
782 diagnose_tm_blocks (void)
784 struct walk_stmt_info wi
;
785 struct diagnose_tm d
;
787 memset (&d
, 0, sizeof (d
));
788 if (is_tm_may_cancel_outer (current_function_decl
))
789 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
790 else if (is_tm_safe (current_function_decl
))
791 d
.func_flags
= DIAG_TM_SAFE
;
792 d
.summary_flags
= d
.func_flags
;
794 memset (&wi
, 0, sizeof (wi
));
797 walk_gimple_seq (gimple_body (current_function_decl
),
798 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
803 struct gimple_opt_pass pass_diagnose_tm_blocks
=
807 "*diagnose_tm_blocks", /* name */
808 OPTGROUP_NONE
, /* optinfo_flags */
810 diagnose_tm_blocks
, /* execute */
813 0, /* static_pass_number */
814 TV_TRANS_MEM
, /* tv_id */
815 PROP_gimple_any
, /* properties_required */
816 0, /* properties_provided */
817 0, /* properties_destroyed */
818 0, /* todo_flags_start */
819 0, /* todo_flags_finish */
823 /* Instead of instrumenting thread private memory, we save the
824 addresses in a log which we later use to save/restore the addresses
825 upon transaction start/restart.
827 The log is keyed by address, where each element contains individual
828 statements among different code paths that perform the store.
830 This log is later used to generate either plain save/restore of the
831 addresses upon transaction start/restart, or calls to the ITM_L*
834 So for something like:
836 struct large { int x[1000]; };
837 struct large lala = { 0 };
843 We can either save/restore:
846 trxn = _ITM_startTransaction ();
847 if (trxn & a_saveLiveVariables)
848 tmp_lala1 = lala.x[i];
849 else if (a & a_restoreLiveVariables)
850 lala.x[i] = tmp_lala1;
852 or use the logging functions:
855 trxn = _ITM_startTransaction ();
856 _ITM_LU4 (&lala.x[i]);
858 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
859 far up the dominator tree to shadow all of the writes to a given
860 location (thus reducing the total number of logging calls), but not
861 so high as to be called on a path that does not perform a
864 /* One individual log entry. We may have multiple statements for the
865 same location if neither dominate each other (on different
867 typedef struct tm_log_entry
869 /* Address to save. */
871 /* Entry block for the transaction this address occurs in. */
872 basic_block entry_block
;
873 /* Dominating statements the store occurs in. */
875 /* Initially, while we are building the log, we place a nonzero
876 value here to mean that this address *will* be saved with a
877 save/restore sequence. Later, when generating the save sequence
878 we place the SSA temp generated here. */
882 /* The actual log. */
883 static htab_t tm_log
;
885 /* Addresses to log with a save/restore sequence. These should be in
887 static vec
<tree
> tm_log_save_addresses
;
889 /* Map for an SSA_NAME originally pointing to a non aliased new piece
890 of memory (malloc, alloc, etc). */
891 static htab_t tm_new_mem_hash
;
893 enum thread_memory_type
897 mem_transaction_local
,
901 typedef struct tm_new_mem_map
903 /* SSA_NAME being dereferenced. */
905 enum thread_memory_type local_new_memory
;
908 /* Htab support. Return hash value for a `tm_log_entry'. */
910 tm_log_hash (const void *p
)
912 const struct tm_log_entry
*log
= (const struct tm_log_entry
*) p
;
913 return iterative_hash_expr (log
->addr
, 0);
916 /* Htab support. Return true if two log entries are the same. */
918 tm_log_eq (const void *p1
, const void *p2
)
920 const struct tm_log_entry
*log1
= (const struct tm_log_entry
*) p1
;
921 const struct tm_log_entry
*log2
= (const struct tm_log_entry
*) p2
;
925 rth: I suggest that we get rid of the component refs etc.
926 I.e. resolve the reference to base + offset.
928 We may need to actually finish a merge with mainline for this,
929 since we'd like to be presented with Richi's MEM_REF_EXPRs more
930 often than not. But in the meantime your tm_log_entry could save
931 the results of get_inner_reference.
933 See: g++.dg/tm/pr46653.C
936 /* Special case plain equality because operand_equal_p() below will
937 return FALSE if the addresses are equal but they have
938 side-effects (e.g. a volatile address). */
939 if (log1
->addr
== log2
->addr
)
942 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
945 /* Htab support. Free one tm_log_entry. */
947 tm_log_free (void *p
)
949 struct tm_log_entry
*lp
= (struct tm_log_entry
*) p
;
950 lp
->stmts
.release ();
954 /* Initialize logging data structures. */
958 tm_log
= htab_create (10, tm_log_hash
, tm_log_eq
, tm_log_free
);
959 tm_new_mem_hash
= htab_create (5, struct_ptr_hash
, struct_ptr_eq
, free
);
960 tm_log_save_addresses
.create (5);
963 /* Free logging data structures. */
967 htab_delete (tm_log
);
968 htab_delete (tm_new_mem_hash
);
969 tm_log_save_addresses
.release ();
972 /* Return true if MEM is a transaction invariant memory for the TM
973 region starting at REGION_ENTRY_BLOCK. */
975 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
977 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
978 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
982 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
983 return def_bb
!= region_entry_block
984 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
987 mem
= strip_invariant_refs (mem
);
988 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
991 /* Given an address ADDR in STMT, find it in the memory log or add it,
992 making sure to keep only the addresses highest in the dominator
995 ENTRY_BLOCK is the entry_block for the transaction.
997 If we find the address in the log, make sure it's either the same
998 address, or an equivalent one that dominates ADDR.
1000 If we find the address, but neither ADDR dominates the found
1001 address, nor the found one dominates ADDR, we're on different
1002 execution paths. Add it.
1004 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1007 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1010 struct tm_log_entry l
, *lp
;
1013 slot
= htab_find_slot (tm_log
, &l
, INSERT
);
1016 tree type
= TREE_TYPE (addr
);
1018 lp
= XNEW (struct tm_log_entry
);
1022 /* Small invariant addresses can be handled as save/restores. */
1024 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1025 && TYPE_SIZE_UNIT (type
) != NULL
1026 && host_integerp (TYPE_SIZE_UNIT (type
), 1)
1027 && (tree_low_cst (TYPE_SIZE_UNIT (type
), 1)
1028 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1029 /* We must be able to copy this type normally. I.e., no
1030 special constructors and the like. */
1031 && !TREE_ADDRESSABLE (type
))
1033 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1034 lp
->stmts
.create (0);
1035 lp
->entry_block
= entry_block
;
1036 /* Save addresses separately in dominator order so we don't
1037 get confused by overlapping addresses in the save/restore
1039 tm_log_save_addresses
.safe_push (lp
->addr
);
1043 /* Use the logging functions. */
1044 lp
->stmts
.create (5);
1045 lp
->stmts
.quick_push (stmt
);
1046 lp
->save_var
= NULL
;
1054 lp
= (struct tm_log_entry
*) *slot
;
1056 /* If we're generating a save/restore sequence, we don't care
1057 about statements. */
1061 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1063 if (stmt
== oldstmt
)
1065 /* We already have a store to the same address, higher up the
1066 dominator tree. Nothing to do. */
1067 if (dominated_by_p (CDI_DOMINATORS
,
1068 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1070 /* We should be processing blocks in dominator tree order. */
1071 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1072 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1074 /* Store is on a different code path. */
1075 lp
->stmts
.safe_push (stmt
);
1079 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1080 result, insert the new statements before GSI. */
1083 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1085 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1086 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1088 x
= build_fold_addr_expr (x
);
1089 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1092 /* Instrument one address with the logging functions.
1093 ADDR is the address to save.
1094 STMT is the statement before which to place it. */
1096 tm_log_emit_stmt (tree addr
, gimple stmt
)
1098 tree type
= TREE_TYPE (addr
);
1099 tree size
= TYPE_SIZE_UNIT (type
);
1100 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1102 enum built_in_function code
= BUILT_IN_TM_LOG
;
1104 if (type
== float_type_node
)
1105 code
= BUILT_IN_TM_LOG_FLOAT
;
1106 else if (type
== double_type_node
)
1107 code
= BUILT_IN_TM_LOG_DOUBLE
;
1108 else if (type
== long_double_type_node
)
1109 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1110 else if (host_integerp (size
, 1))
1112 unsigned int n
= tree_low_cst (size
, 1);
1116 code
= BUILT_IN_TM_LOG_1
;
1119 code
= BUILT_IN_TM_LOG_2
;
1122 code
= BUILT_IN_TM_LOG_4
;
1125 code
= BUILT_IN_TM_LOG_8
;
1128 code
= BUILT_IN_TM_LOG
;
1129 if (TREE_CODE (type
) == VECTOR_TYPE
)
1131 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1132 code
= BUILT_IN_TM_LOG_M64
;
1133 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1134 code
= BUILT_IN_TM_LOG_M128
;
1135 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1136 code
= BUILT_IN_TM_LOG_M256
;
1142 addr
= gimplify_addr (&gsi
, addr
);
1143 if (code
== BUILT_IN_TM_LOG
)
1144 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1146 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1147 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1150 /* Go through the log and instrument address that must be instrumented
1151 with the logging functions. Leave the save/restore addresses for
1157 struct tm_log_entry
*lp
;
1159 FOR_EACH_HTAB_ELEMENT (tm_log
, lp
, tm_log_entry_t
, hi
)
1166 fprintf (dump_file
, "TM thread private mem logging: ");
1167 print_generic_expr (dump_file
, lp
->addr
, 0);
1168 fprintf (dump_file
, "\n");
1174 fprintf (dump_file
, "DUMPING to variable\n");
1180 fprintf (dump_file
, "DUMPING with logging functions\n");
1181 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1182 tm_log_emit_stmt (lp
->addr
, stmt
);
1187 /* Emit the save sequence for the corresponding addresses in the log.
1188 ENTRY_BLOCK is the entry block for the transaction.
1189 BB is the basic block to insert the code in. */
1191 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1194 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1196 struct tm_log_entry l
, *lp
;
1198 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1200 l
.addr
= tm_log_save_addresses
[i
];
1201 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1202 gcc_assert (lp
->save_var
!= NULL
);
1204 /* We only care about variables in the current transaction. */
1205 if (lp
->entry_block
!= entry_block
)
1208 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1210 /* Make sure we can create an SSA_NAME for this type. For
1211 instance, aggregates aren't allowed, in which case the system
1212 will create a VOP for us and everything will just work. */
1213 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1215 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1216 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1219 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1223 /* Emit the restore sequence for the corresponding addresses in the log.
1224 ENTRY_BLOCK is the entry block for the transaction.
1225 BB is the basic block to insert the code in. */
1227 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1230 struct tm_log_entry l
, *lp
;
1231 gimple_stmt_iterator gsi
;
1234 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1236 l
.addr
= tm_log_save_addresses
[i
];
1237 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1238 gcc_assert (lp
->save_var
!= NULL
);
1240 /* We only care about variables in the current transaction. */
1241 if (lp
->entry_block
!= entry_block
)
1244 /* Restores are in LIFO order from the saves in case we have
1246 gsi
= gsi_start_bb (bb
);
1248 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1249 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1254 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1255 struct walk_stmt_info
*);
1256 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1257 struct walk_stmt_info
*);
1259 /* Evaluate an address X being dereferenced and determine if it
1260 originally points to a non aliased new chunk of memory (malloc,
1263 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1264 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1265 Return MEM_NON_LOCAL otherwise.
1267 ENTRY_BLOCK is the entry block to the transaction containing the
1268 dereference of X. */
1269 static enum thread_memory_type
1270 thread_private_new_memory (basic_block entry_block
, tree x
)
1273 enum tree_code code
;
1275 tm_new_mem_map_t elt
, *elt_p
;
1277 enum thread_memory_type retval
= mem_transaction_local
;
1280 || TREE_CODE (x
) != SSA_NAME
1281 /* Possible uninitialized use, or a function argument. In
1282 either case, we don't care. */
1283 || SSA_NAME_IS_DEFAULT_DEF (x
))
1284 return mem_non_local
;
1286 /* Look in cache first. */
1288 slot
= htab_find_slot (tm_new_mem_hash
, &elt
, INSERT
);
1289 elt_p
= (tm_new_mem_map_t
*) *slot
;
1291 return elt_p
->local_new_memory
;
1293 /* Optimistically assume the memory is transaction local during
1294 processing. This catches recursion into this variable. */
1295 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1297 elt_p
->local_new_memory
= mem_transaction_local
;
1299 /* Search DEF chain to find the original definition of this address. */
1302 if (ptr_deref_may_alias_global_p (x
))
1304 /* Address escapes. This is not thread-private. */
1305 retval
= mem_non_local
;
1306 goto new_memory_ret
;
1309 stmt
= SSA_NAME_DEF_STMT (x
);
1311 /* If the malloc call is outside the transaction, this is
1313 if (retval
!= mem_thread_local
1314 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1315 retval
= mem_thread_local
;
1317 if (is_gimple_assign (stmt
))
1319 code
= gimple_assign_rhs_code (stmt
);
1320 /* x = foo ==> foo */
1321 if (code
== SSA_NAME
)
1322 x
= gimple_assign_rhs1 (stmt
);
1323 /* x = foo + n ==> foo */
1324 else if (code
== POINTER_PLUS_EXPR
)
1325 x
= gimple_assign_rhs1 (stmt
);
1326 /* x = (cast*) foo ==> foo */
1327 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1328 x
= gimple_assign_rhs1 (stmt
);
1329 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1330 else if (code
== COND_EXPR
)
1332 tree op1
= gimple_assign_rhs2 (stmt
);
1333 tree op2
= gimple_assign_rhs3 (stmt
);
1334 enum thread_memory_type mem
;
1335 retval
= thread_private_new_memory (entry_block
, op1
);
1336 if (retval
== mem_non_local
)
1337 goto new_memory_ret
;
1338 mem
= thread_private_new_memory (entry_block
, op2
);
1339 retval
= MIN (retval
, mem
);
1340 goto new_memory_ret
;
1344 retval
= mem_non_local
;
1345 goto new_memory_ret
;
1350 if (gimple_code (stmt
) == GIMPLE_PHI
)
1353 enum thread_memory_type mem
;
1354 tree phi_result
= gimple_phi_result (stmt
);
1356 /* If any of the ancestors are non-local, we are sure to
1357 be non-local. Otherwise we can avoid doing anything
1358 and inherit what has already been generated. */
1360 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1362 tree op
= PHI_ARG_DEF (stmt
, i
);
1364 /* Exclude self-assignment. */
1365 if (phi_result
== op
)
1368 mem
= thread_private_new_memory (entry_block
, op
);
1369 if (mem
== mem_non_local
)
1372 goto new_memory_ret
;
1374 retval
= MIN (retval
, mem
);
1376 goto new_memory_ret
;
1381 while (TREE_CODE (x
) == SSA_NAME
);
1383 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1384 /* Thread-local or transaction-local. */
1387 retval
= mem_non_local
;
1390 elt_p
->local_new_memory
= retval
;
1394 /* Determine whether X has to be instrumented using a read
1397 ENTRY_BLOCK is the entry block for the region where stmt resides
1398 in. NULL if unknown.
1400 STMT is the statement in which X occurs in. It is used for thread
1401 private memory instrumentation. If no TPM instrumentation is
1402 desired, STMT should be null. */
1404 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1407 while (handled_component_p (x
))
1408 x
= TREE_OPERAND (x
, 0);
1410 switch (TREE_CODE (x
))
1415 enum thread_memory_type ret
;
1417 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1418 if (ret
== mem_non_local
)
1420 if (stmt
&& ret
== mem_thread_local
)
1421 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1422 tm_log_add (entry_block
, orig
, stmt
);
1424 /* Transaction-locals require nothing at all. For malloc, a
1425 transaction restart frees the memory and we reallocate.
1426 For alloca, the stack pointer gets reset by the retry and
1431 case TARGET_MEM_REF
:
1432 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1434 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1435 if (TREE_CODE (x
) == PARM_DECL
)
1437 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1443 if (DECL_BY_REFERENCE (x
))
1445 /* ??? This value is a pointer, but aggregate_value_p has been
1446 jigged to return true which confuses needs_to_live_in_memory.
1447 This ought to be cleaned up generically.
1449 FIXME: Verify this still happens after the next mainline
1450 merge. Testcase ie g++.dg/tm/pr47554.C.
1455 if (is_global_var (x
))
1456 return !TREE_READONLY (x
);
1457 if (/* FIXME: This condition should actually go below in the
1458 tm_log_add() call, however is_call_clobbered() depends on
1459 aliasing info which is not available during
1460 gimplification. Since requires_barrier() gets called
1461 during lower_sequence_tm/gimplification, leave the call
1462 to needs_to_live_in_memory until we eliminate
1463 lower_sequence_tm altogether. */
1464 needs_to_live_in_memory (x
))
1468 /* For local memory that doesn't escape (aka thread private
1469 memory), we can either save the value at the beginning of
1470 the transaction and restore on restart, or call a tm
1471 function to dynamically save and restore on restart
1474 tm_log_add (entry_block
, orig
, stmt
);
1483 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1484 a transaction region. */
1487 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1489 gimple stmt
= gsi_stmt (*gsi
);
1491 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1492 *state
|= GTMA_HAVE_LOAD
;
1493 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1494 *state
|= GTMA_HAVE_STORE
;
1497 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1500 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1502 gimple stmt
= gsi_stmt (*gsi
);
1505 if (is_tm_pure_call (stmt
))
1508 /* Check if this call is a transaction abort. */
1509 fn
= gimple_call_fndecl (stmt
);
1510 if (is_tm_abort (fn
))
1511 *state
|= GTMA_HAVE_ABORT
;
1513 /* Note that something may happen. */
1514 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1517 /* Lower a GIMPLE_TRANSACTION statement. */
1520 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1522 gimple g
, stmt
= gsi_stmt (*gsi
);
1523 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1524 unsigned int this_state
= 0;
1525 struct walk_stmt_info this_wi
;
1527 /* First, lower the body. The scanning that we do inside gives
1528 us some idea of what we're dealing with. */
1529 memset (&this_wi
, 0, sizeof (this_wi
));
1530 this_wi
.info
= (void *) &this_state
;
1531 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1532 lower_sequence_tm
, NULL
, &this_wi
);
1534 /* If there was absolutely nothing transaction related inside the
1535 transaction, we may elide it. Likewise if this is a nested
1536 transaction and does not contain an abort. */
1538 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1541 *outer_state
|= this_state
;
1543 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1545 gimple_transaction_set_body (stmt
, NULL
);
1547 gsi_remove (gsi
, true);
1548 wi
->removed_stmt
= true;
1552 /* Wrap the body of the transaction in a try-finally node so that
1553 the commit call is always properly called. */
1554 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1555 if (flag_exceptions
)
1558 gimple_seq n_seq
, e_seq
;
1560 n_seq
= gimple_seq_alloc_with_stmt (g
);
1563 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1564 1, integer_zero_node
);
1565 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1566 gimple_call_set_lhs (g
, ptr
);
1567 gimple_seq_add_stmt (&e_seq
, g
);
1569 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1571 gimple_seq_add_stmt (&e_seq
, g
);
1573 g
= gimple_build_eh_else (n_seq
, e_seq
);
1576 g
= gimple_build_try (gimple_transaction_body (stmt
),
1577 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1578 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1580 gimple_transaction_set_body (stmt
, NULL
);
1582 /* If the transaction calls abort or if this is an outer transaction,
1583 add an "over" label afterwards. */
1584 if ((this_state
& (GTMA_HAVE_ABORT
))
1585 || (gimple_transaction_subcode(stmt
) & GTMA_IS_OUTER
))
1587 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1588 gimple_transaction_set_label (stmt
, label
);
1589 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1592 /* Record the set of operations found for use later. */
1593 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1594 gimple_transaction_set_subcode (stmt
, this_state
);
1597 /* Iterate through the statements in the sequence, lowering them all
1598 as appropriate for being in a transaction. */
1601 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1602 struct walk_stmt_info
*wi
)
1604 unsigned int *state
= (unsigned int *) wi
->info
;
1605 gimple stmt
= gsi_stmt (*gsi
);
1607 *handled_ops_p
= true;
1608 switch (gimple_code (stmt
))
1611 /* Only memory reads/writes need to be instrumented. */
1612 if (gimple_assign_single_p (stmt
))
1613 examine_assign_tm (state
, gsi
);
1617 examine_call_tm (state
, gsi
);
1621 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1624 case GIMPLE_TRANSACTION
:
1625 lower_transaction (gsi
, wi
);
1629 *handled_ops_p
= !gimple_has_substatements (stmt
);
1636 /* Iterate through the statements in the sequence, lowering them all
1637 as appropriate for being outside of a transaction. */
1640 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1641 struct walk_stmt_info
* wi
)
1643 gimple stmt
= gsi_stmt (*gsi
);
1645 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1647 *handled_ops_p
= true;
1648 lower_transaction (gsi
, wi
);
1651 *handled_ops_p
= !gimple_has_substatements (stmt
);
1656 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1657 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1658 been moved out, and all the data required for constructing a proper
1659 CFG has been recorded. */
1662 execute_lower_tm (void)
1664 struct walk_stmt_info wi
;
1667 /* Transactional clones aren't created until a later pass. */
1668 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1670 body
= gimple_body (current_function_decl
);
1671 memset (&wi
, 0, sizeof (wi
));
1672 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1673 gimple_set_body (current_function_decl
, body
);
1678 struct gimple_opt_pass pass_lower_tm
=
1682 "tmlower", /* name */
1683 OPTGROUP_NONE
, /* optinfo_flags */
1685 execute_lower_tm
, /* execute */
1688 0, /* static_pass_number */
1689 TV_TRANS_MEM
, /* tv_id */
1690 PROP_gimple_lcf
, /* properties_required */
1691 0, /* properties_provided */
1692 0, /* properties_destroyed */
1693 0, /* todo_flags_start */
1694 0, /* todo_flags_finish */
1698 /* Collect region information for each transaction. */
1702 /* Link to the next unnested transaction. */
1703 struct tm_region
*next
;
1705 /* Link to the next inner transaction. */
1706 struct tm_region
*inner
;
1708 /* Link to the next outer transaction. */
1709 struct tm_region
*outer
;
1711 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1712 After TM_MARK, this gets replaced by a call to
1713 BUILT_IN_TM_START. */
1714 gimple transaction_stmt
;
1716 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1717 BUILT_IN_TM_START, this field is true if the transaction is an
1718 outer transaction. */
1719 bool original_transaction_was_outer
;
1721 /* Return value from BUILT_IN_TM_START. */
1724 /* The entry block to this region. This will always be the first
1725 block of the body of the transaction. */
1726 basic_block entry_block
;
1728 /* The first block after an expanded call to _ITM_beginTransaction. */
1729 basic_block restart_block
;
1731 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1732 These blocks are still a part of the region (i.e., the border is
1733 inclusive). Note that this set is only complete for paths in the CFG
1734 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1735 the edge to the "over" label. */
1738 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1742 typedef struct tm_region
*tm_region_p
;
1744 /* True if there are pending edge statements to be committed for the
1745 current function being scanned in the tmmark pass. */
1746 bool pending_edge_inserts_p
;
1748 static struct tm_region
*all_tm_regions
;
1749 static bitmap_obstack tm_obstack
;
1752 /* A subroutine of tm_region_init. Record the existence of the
1753 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1755 static struct tm_region
*
1756 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1758 struct tm_region
*region
;
1760 region
= (struct tm_region
*)
1761 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1765 region
->next
= outer
->inner
;
1766 outer
->inner
= region
;
1770 region
->next
= all_tm_regions
;
1771 all_tm_regions
= region
;
1773 region
->inner
= NULL
;
1774 region
->outer
= outer
;
1776 region
->transaction_stmt
= stmt
;
1777 region
->original_transaction_was_outer
= false;
1778 region
->tm_state
= NULL
;
1780 /* There are either one or two edges out of the block containing
1781 the GIMPLE_TRANSACTION, one to the actual region and one to the
1782 "over" label if the region contains an abort. The former will
1783 always be the one marked FALLTHRU. */
1784 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1786 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1787 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1792 /* A subroutine of tm_region_init. Record all the exit and
1793 irrevocable blocks in BB into the region's exit_blocks and
1794 irr_blocks bitmaps. Returns the new region being scanned. */
1796 static struct tm_region
*
1797 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1799 gimple_stmt_iterator gsi
;
1803 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1806 /* Check to see if this is the end of a region by seeing if it
1807 contains a call to __builtin_tm_commit{,_eh}. Note that the
1808 outermost region for DECL_IS_TM_CLONE need not collect this. */
1809 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1812 if (gimple_code (g
) == GIMPLE_CALL
)
1814 tree fn
= gimple_call_fndecl (g
);
1815 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1817 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1818 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1819 && region
->exit_blocks
)
1821 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1822 region
= region
->outer
;
1825 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1826 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1833 /* Collect all of the transaction regions within the current function
1834 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1835 an "outermost" region for use by tm clones. */
1838 tm_region_init (struct tm_region
*region
)
1844 vec
<basic_block
> queue
= vec
<basic_block
>();
1845 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1846 struct tm_region
*old_region
;
1847 vec
<tm_region_p
> bb_regions
= vec
<tm_region_p
>();
1849 all_tm_regions
= region
;
1850 bb
= single_succ (ENTRY_BLOCK_PTR
);
1852 /* We could store this information in bb->aux, but we may get called
1853 through get_all_tm_blocks() from another pass that may be already
1855 bb_regions
.safe_grow_cleared (last_basic_block
);
1857 queue
.safe_push (bb
);
1858 bb_regions
[bb
->index
] = region
;
1862 region
= bb_regions
[bb
->index
];
1863 bb_regions
[bb
->index
] = NULL
;
1865 /* Record exit and irrevocable blocks. */
1866 region
= tm_region_init_1 (region
, bb
);
1868 /* Check for the last statement in the block beginning a new region. */
1870 old_region
= region
;
1871 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1872 region
= tm_region_init_0 (region
, bb
, g
);
1874 /* Process subsequent blocks. */
1875 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1876 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1878 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1879 queue
.safe_push (e
->dest
);
1881 /* If the current block started a new region, make sure that only
1882 the entry block of the new region is associated with this region.
1883 Other successors are still part of the old region. */
1884 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1885 bb_regions
[e
->dest
->index
] = old_region
;
1887 bb_regions
[e
->dest
->index
] = region
;
1890 while (!queue
.is_empty ());
1892 BITMAP_FREE (visited_blocks
);
1893 bb_regions
.release ();
1896 /* The "gate" function for all transactional memory expansion and optimization
1897 passes. We collect region information for each top-level transaction, and
1898 if we don't find any, we skip all of the TM passes. Each region will have
1899 all of the exit blocks recorded, and the originating statement. */
1907 calculate_dominance_info (CDI_DOMINATORS
);
1908 bitmap_obstack_initialize (&tm_obstack
);
1910 /* If the function is a TM_CLONE, then the entire function is the region. */
1911 if (decl_is_tm_clone (current_function_decl
))
1913 struct tm_region
*region
= (struct tm_region
*)
1914 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1915 memset (region
, 0, sizeof (*region
));
1916 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR
);
1917 /* For a clone, the entire function is the region. But even if
1918 we don't need to record any exit blocks, we may need to
1919 record irrevocable blocks. */
1920 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1922 tm_region_init (region
);
1926 tm_region_init (NULL
);
1928 /* If we didn't find any regions, cleanup and skip the whole tree
1929 of tm-related optimizations. */
1930 if (all_tm_regions
== NULL
)
1932 bitmap_obstack_release (&tm_obstack
);
1940 struct gimple_opt_pass pass_tm_init
=
1944 "*tminit", /* name */
1945 OPTGROUP_NONE
, /* optinfo_flags */
1946 gate_tm_init
, /* gate */
1950 0, /* static_pass_number */
1951 TV_TRANS_MEM
, /* tv_id */
1952 PROP_ssa
| PROP_cfg
, /* properties_required */
1953 0, /* properties_provided */
1954 0, /* properties_destroyed */
1955 0, /* todo_flags_start */
1956 0, /* todo_flags_finish */
1960 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
1961 represented by STATE. */
1964 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
1966 if (region
&& region
->transaction_stmt
)
1968 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
1969 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
1973 /* Construct a memory load in a transactional context. Return the
1974 gimple statement performing the load, or NULL if there is no
1975 TM_LOAD builtin of the appropriate size to do the load.
1977 LOC is the location to use for the new statement(s). */
1980 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
1982 enum built_in_function code
= END_BUILTINS
;
1983 tree t
, type
= TREE_TYPE (rhs
), decl
;
1986 if (type
== float_type_node
)
1987 code
= BUILT_IN_TM_LOAD_FLOAT
;
1988 else if (type
== double_type_node
)
1989 code
= BUILT_IN_TM_LOAD_DOUBLE
;
1990 else if (type
== long_double_type_node
)
1991 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
1992 else if (TYPE_SIZE_UNIT (type
) != NULL
1993 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
1995 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
1998 code
= BUILT_IN_TM_LOAD_1
;
2001 code
= BUILT_IN_TM_LOAD_2
;
2004 code
= BUILT_IN_TM_LOAD_4
;
2007 code
= BUILT_IN_TM_LOAD_8
;
2012 if (code
== END_BUILTINS
)
2014 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2019 decl
= builtin_decl_explicit (code
);
2021 t
= gimplify_addr (gsi
, rhs
);
2022 gcall
= gimple_build_call (decl
, 1, t
);
2023 gimple_set_location (gcall
, loc
);
2025 t
= TREE_TYPE (TREE_TYPE (decl
));
2026 if (useless_type_conversion_p (type
, t
))
2028 gimple_call_set_lhs (gcall
, lhs
);
2029 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2036 temp
= create_tmp_reg (t
, NULL
);
2037 gimple_call_set_lhs (gcall
, temp
);
2038 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2040 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2041 g
= gimple_build_assign (lhs
, t
);
2042 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2049 /* Similarly for storing TYPE in a transactional context. */
2052 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2054 enum built_in_function code
= END_BUILTINS
;
2055 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2058 if (type
== float_type_node
)
2059 code
= BUILT_IN_TM_STORE_FLOAT
;
2060 else if (type
== double_type_node
)
2061 code
= BUILT_IN_TM_STORE_DOUBLE
;
2062 else if (type
== long_double_type_node
)
2063 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2064 else if (TYPE_SIZE_UNIT (type
) != NULL
2065 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
2067 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2070 code
= BUILT_IN_TM_STORE_1
;
2073 code
= BUILT_IN_TM_STORE_2
;
2076 code
= BUILT_IN_TM_STORE_4
;
2079 code
= BUILT_IN_TM_STORE_8
;
2084 if (code
== END_BUILTINS
)
2086 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2091 fn
= builtin_decl_explicit (code
);
2093 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2095 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2097 /* Handle the easy initialization to zero. */
2098 if (!CONSTRUCTOR_ELTS (rhs
))
2099 rhs
= build_int_cst (simple_type
, 0);
2102 /* ...otherwise punt to the caller and probably use
2103 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2104 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2109 else if (!useless_type_conversion_p (simple_type
, type
))
2114 temp
= create_tmp_reg (simple_type
, NULL
);
2115 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2116 g
= gimple_build_assign (temp
, t
);
2117 gimple_set_location (g
, loc
);
2118 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2123 t
= gimplify_addr (gsi
, lhs
);
2124 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2125 gimple_set_location (gcall
, loc
);
2126 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2132 /* Expand an assignment statement into transactional builtins. */
2135 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2137 gimple stmt
= gsi_stmt (*gsi
);
2138 location_t loc
= gimple_location (stmt
);
2139 tree lhs
= gimple_assign_lhs (stmt
);
2140 tree rhs
= gimple_assign_rhs1 (stmt
);
2141 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2142 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2143 gimple gcall
= NULL
;
2145 if (!load_p
&& !store_p
)
2147 /* Add thread private addresses to log if applicable. */
2148 requires_barrier (region
->entry_block
, lhs
, stmt
);
2153 // Remove original load/store statement.
2154 gsi_remove (gsi
, true);
2156 if (load_p
&& !store_p
)
2158 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2159 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2161 else if (store_p
&& !load_p
)
2163 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2164 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2168 tree lhs_addr
, rhs_addr
, tmp
;
2171 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2173 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2175 /* ??? Figure out if there's any possible overlap between the LHS
2176 and the RHS and if not, use MEMCPY. */
2178 if (load_p
&& is_gimple_reg (lhs
))
2180 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2181 lhs_addr
= build_fold_addr_expr (tmp
);
2186 lhs_addr
= gimplify_addr (gsi
, lhs
);
2188 rhs_addr
= gimplify_addr (gsi
, rhs
);
2189 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2190 3, lhs_addr
, rhs_addr
,
2191 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2192 gimple_set_location (gcall
, loc
);
2193 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2197 gcall
= gimple_build_assign (lhs
, tmp
);
2198 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2202 /* Now that we have the load/store in its instrumented form, add
2203 thread private addresses to the log if applicable. */
2205 requires_barrier (region
->entry_block
, lhs
, gcall
);
2207 // The calls to build_tm_{store,load} above inserted the instrumented
2208 // call into the stream.
2209 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2213 /* Expand a call statement as appropriate for a transaction. That is,
2214 either verify that the call does not affect the transaction, or
2215 redirect the call to a clone that handles transactions, or change
2216 the transaction state to IRREVOCABLE. Return true if the call is
2217 one of the builtins that end a transaction. */
2220 expand_call_tm (struct tm_region
*region
,
2221 gimple_stmt_iterator
*gsi
)
2223 gimple stmt
= gsi_stmt (*gsi
);
2224 tree lhs
= gimple_call_lhs (stmt
);
2226 struct cgraph_node
*node
;
2227 bool retval
= false;
2229 fn_decl
= gimple_call_fndecl (stmt
);
2231 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2232 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2233 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2234 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2235 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2237 if (is_tm_pure_call (stmt
))
2241 retval
= is_tm_ending_fndecl (fn_decl
);
2244 /* Assume all non-const/pure calls write to memory, except
2245 transaction ending builtins. */
2246 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2249 /* For indirect calls, we already generated a call into the runtime. */
2252 tree fn
= gimple_call_fn (stmt
);
2254 /* We are guaranteed never to go irrevocable on a safe or pure
2255 call, and the pure call was handled above. */
2256 if (is_tm_safe (fn
))
2259 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2264 node
= cgraph_get_node (fn_decl
);
2265 /* All calls should have cgraph here. */
2268 /* We can have a nodeless call here if some pass after IPA-tm
2269 added uninstrumented calls. For example, loop distribution
2270 can transform certain loop constructs into __builtin_mem*
2271 calls. In this case, see if we have a suitable TM
2272 replacement and fill in the gaps. */
2273 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2274 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2275 gcc_assert (code
== BUILT_IN_MEMCPY
2276 || code
== BUILT_IN_MEMMOVE
2277 || code
== BUILT_IN_MEMSET
);
2279 tree repl
= find_tm_replacement_function (fn_decl
);
2282 gimple_call_set_fndecl (stmt
, repl
);
2284 node
= cgraph_create_node (repl
);
2285 node
->local
.tm_may_enter_irr
= false;
2286 return expand_call_tm (region
, gsi
);
2290 if (node
->local
.tm_may_enter_irr
)
2291 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2293 if (is_tm_abort (fn_decl
))
2295 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2299 /* Instrument the store if needed.
2301 If the assignment happens inside the function call (return slot
2302 optimization), there is no instrumentation to be done, since
2303 the callee should have done the right thing. */
2304 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2305 && !gimple_call_return_slot_opt_p (stmt
))
2307 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2308 location_t loc
= gimple_location (stmt
);
2309 edge fallthru_edge
= NULL
;
2311 /* Remember if the call was going to throw. */
2312 if (stmt_can_throw_internal (stmt
))
2316 basic_block bb
= gimple_bb (stmt
);
2318 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2319 if (e
->flags
& EDGE_FALLTHRU
)
2326 gimple_call_set_lhs (stmt
, tmp
);
2328 stmt
= gimple_build_assign (lhs
, tmp
);
2329 gimple_set_location (stmt
, loc
);
2331 /* We cannot throw in the middle of a BB. If the call was going
2332 to throw, place the instrumentation on the fallthru edge, so
2333 the call remains the last statement in the block. */
2336 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2337 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2338 expand_assign_tm (region
, &fallthru_gsi
);
2339 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2340 pending_edge_inserts_p
= true;
2344 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2345 expand_assign_tm (region
, gsi
);
2348 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2355 /* Expand all statements in BB as appropriate for being inside
2359 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2361 gimple_stmt_iterator gsi
;
2363 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2365 gimple stmt
= gsi_stmt (gsi
);
2366 switch (gimple_code (stmt
))
2369 /* Only memory reads/writes need to be instrumented. */
2370 if (gimple_assign_single_p (stmt
)
2371 && !gimple_clobber_p (stmt
))
2373 expand_assign_tm (region
, &gsi
);
2379 if (expand_call_tm (region
, &gsi
))
2389 if (!gsi_end_p (gsi
))
2394 /* Return the list of basic-blocks in REGION.
2396 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2397 following a TM_IRREVOCABLE call. */
2399 static vec
<basic_block
>
2400 get_tm_region_blocks (basic_block entry_block
,
2403 bitmap all_region_blocks
,
2404 bool stop_at_irrevocable_p
)
2406 vec
<basic_block
> bbs
= vec
<basic_block
>();
2410 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2413 bbs
.safe_push (entry_block
);
2414 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2418 basic_block bb
= bbs
[i
++];
2421 bitmap_bit_p (exit_blocks
, bb
->index
))
2424 if (stop_at_irrevocable_p
2426 && bitmap_bit_p (irr_blocks
, bb
->index
))
2429 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2430 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2432 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2433 bbs
.safe_push (e
->dest
);
2436 while (i
< bbs
.length ());
2438 if (all_region_blocks
)
2439 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2441 BITMAP_FREE (visited_blocks
);
2445 // Callback for expand_regions, collect innermost region data for each bb.
2447 collect_bb2reg (struct tm_region
*region
, void *data
)
2449 vec
<tm_region_p
> *bb2reg
= (vec
<tm_region_p
> *) data
;
2450 vec
<basic_block
> queue
;
2454 queue
= get_tm_region_blocks (region
->entry_block
,
2455 region
->exit_blocks
,
2458 /*stop_at_irr_p=*/true);
2460 // We expect expand_region to perform a post-order traversal of the region
2461 // tree. Therefore the last region seen for any bb is the innermost.
2462 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2463 (*bb2reg
)[bb
->index
] = region
;
2469 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2470 // which a basic block belongs. Note that we only consider the instrumented
2471 // code paths for the region; the uninstrumented code paths are ignored.
2473 // ??? This data is very similar to the bb_regions array that is collected
2474 // during tm_region_init. Or, rather, this data is similar to what could
2475 // be used within tm_region_init. The actual computation in tm_region_init
2476 // begins and ends with bb_regions entirely full of NULL pointers, due to
2477 // the way in which pointers are swapped in and out of the array.
2479 // ??? Our callers expect that blocks are not shared between transactions.
2480 // When the optimizers get too smart, and blocks are shared, then during
2481 // the tm_mark phase we'll add log entries to only one of the two transactions,
2482 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2483 // cycles. The symptom being SSA defs that do not dominate their uses.
2484 // Note that the optimizers were locally correct with their transformation,
2485 // as we have no info within the program that suggests that the blocks cannot
2488 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2489 // only known instance of this block sharing.
2491 static vec
<tm_region_p
>
2492 get_bb_regions_instrumented (bool traverse_clones
)
2494 unsigned n
= last_basic_block
;
2495 vec
<tm_region_p
> ret
;
2498 ret
.safe_grow_cleared (n
);
2499 expand_regions (all_tm_regions
, collect_bb2reg
, &ret
, traverse_clones
);
2504 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2508 compute_transaction_bits (void)
2510 struct tm_region
*region
;
2511 vec
<basic_block
> queue
;
2515 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2516 certainly don't need it to calculate CDI_DOMINATOR info. */
2520 bb
->flags
&= ~BB_IN_TRANSACTION
;
2522 for (region
= all_tm_regions
; region
; region
= region
->next
)
2524 queue
= get_tm_region_blocks (region
->entry_block
,
2525 region
->exit_blocks
,
2528 /*stop_at_irr_p=*/true);
2529 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2530 bb
->flags
|= BB_IN_TRANSACTION
;
2535 bitmap_obstack_release (&tm_obstack
);
2538 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2539 call to BUILT_IN_TM_START. */
2542 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2544 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2545 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2546 tree tm_state
= region
->tm_state
;
2547 tree tm_state_type
= TREE_TYPE (tm_state
);
2548 edge abort_edge
= NULL
;
2549 edge inst_edge
= NULL
;
2550 edge uninst_edge
= NULL
;
2551 edge fallthru_edge
= NULL
;
2553 // Identify the various successors of the transaction start.
2557 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2559 if (e
->flags
& EDGE_TM_ABORT
)
2561 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2565 if (e
->flags
& EDGE_FALLTHRU
)
2570 /* ??? There are plenty of bits here we're not computing. */
2572 int subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2574 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2575 flags
|= PR_DOESGOIRREVOCABLE
;
2576 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2577 flags
|= PR_HASNOIRREVOCABLE
;
2578 /* If the transaction does not have an abort in lexical scope and is not
2579 marked as an outer transaction, then it will never abort. */
2580 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2581 flags
|= PR_HASNOABORT
;
2582 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2583 flags
|= PR_READONLY
;
2585 flags
|= PR_INSTRUMENTEDCODE
;
2587 flags
|= PR_UNINSTRUMENTEDCODE
;
2588 if (subcode
& GTMA_IS_OUTER
)
2589 region
->original_transaction_was_outer
= true;
2590 tree t
= build_int_cst (tm_state_type
, flags
);
2591 gimple call
= gimple_build_call (tm_start
, 1, t
);
2592 gimple_call_set_lhs (call
, tm_state
);
2593 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2595 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2596 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2597 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2598 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2599 gsi_remove (&gsi
, true);
2600 region
->transaction_stmt
= call
;
2603 // Generate log saves.
2604 if (!tm_log_save_addresses
.is_empty ())
2605 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2607 // In the beginning, we've no tests to perform on transaction restart.
2608 // Note that after this point, transaction_bb becomes the "most recent
2609 // block containing tests for the transaction".
2610 region
->restart_block
= region
->entry_block
;
2612 // Generate log restores.
2613 if (!tm_log_save_addresses
.is_empty ())
2615 basic_block test_bb
= create_empty_bb (transaction_bb
);
2616 basic_block code_bb
= create_empty_bb (test_bb
);
2617 basic_block join_bb
= create_empty_bb (code_bb
);
2618 if (current_loops
&& transaction_bb
->loop_father
)
2620 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2621 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2622 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2624 if (region
->restart_block
== region
->entry_block
)
2625 region
->restart_block
= test_bb
;
2627 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2628 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2629 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2631 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2632 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2634 t2
= build_int_cst (tm_state_type
, 0);
2635 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2636 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2638 tm_log_emit_restores (region
->entry_block
, code_bb
);
2640 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2641 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2642 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2643 redirect_edge_pred (fallthru_edge
, join_bb
);
2645 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2646 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2648 ei
->probability
= PROB_ALWAYS
;
2649 et
->probability
= PROB_LIKELY
;
2650 ef
->probability
= PROB_UNLIKELY
;
2651 et
->count
= apply_probability(test_bb
->count
, et
->probability
);
2652 ef
->count
= apply_probability(test_bb
->count
, ef
->probability
);
2654 code_bb
->count
= et
->count
;
2655 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2657 transaction_bb
= join_bb
;
2660 // If we have an ABORT edge, create a test to perform the abort.
2663 basic_block test_bb
= create_empty_bb (transaction_bb
);
2664 if (current_loops
&& transaction_bb
->loop_father
)
2665 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2666 if (region
->restart_block
== region
->entry_block
)
2667 region
->restart_block
= test_bb
;
2669 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2670 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2671 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2673 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2674 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2676 t2
= build_int_cst (tm_state_type
, 0);
2677 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2678 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2680 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2681 test_bb
->frequency
= transaction_bb
->frequency
;
2682 test_bb
->count
= transaction_bb
->count
;
2683 ei
->probability
= PROB_ALWAYS
;
2685 // Not abort edge. If both are live, chose one at random as we'll
2686 // we'll be fixing that up below.
2687 redirect_edge_pred (fallthru_edge
, test_bb
);
2688 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2689 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2690 fallthru_edge
->count
2691 = apply_probability(test_bb
->count
, fallthru_edge
->probability
);
2694 redirect_edge_pred (abort_edge
, test_bb
);
2695 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2696 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2698 = apply_probability(test_bb
->count
, abort_edge
->probability
);
2700 transaction_bb
= test_bb
;
2703 // If we have both instrumented and uninstrumented code paths, select one.
2704 if (inst_edge
&& uninst_edge
)
2706 basic_block test_bb
= create_empty_bb (transaction_bb
);
2707 if (current_loops
&& transaction_bb
->loop_father
)
2708 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2709 if (region
->restart_block
== region
->entry_block
)
2710 region
->restart_block
= test_bb
;
2712 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2713 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2715 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2717 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2718 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2720 t2
= build_int_cst (tm_state_type
, 0);
2721 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2722 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2724 // Create the edge into test_bb first, as we want to copy values
2725 // out of the fallthru edge.
2726 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2727 e
->probability
= fallthru_edge
->probability
;
2728 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2729 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2731 // Now update the edges to the inst/uninist implementations.
2732 // For now assume that the paths are equally likely. When using HTM,
2733 // we'll try the uninst path first and fallback to inst path if htm
2734 // buffers are exceeded. Without HTM we start with the inst path and
2735 // use the uninst path when falling back to serial mode.
2736 redirect_edge_pred (inst_edge
, test_bb
);
2737 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2738 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2740 = apply_probability(test_bb
->count
, inst_edge
->probability
);
2742 redirect_edge_pred (uninst_edge
, test_bb
);
2743 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2744 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2746 = apply_probability(test_bb
->count
, uninst_edge
->probability
);
2749 // If we have no previous special cases, and we have PHIs at the beginning
2750 // of the atomic region, this means we have a loop at the beginning of the
2751 // atomic region that shares the first block. This can cause problems with
2752 // the transaction restart abnormal edges to be added in the tm_edges pass.
2753 // Solve this by adding a new empty block to receive the abnormal edges.
2754 if (region
->restart_block
== region
->entry_block
2755 && phi_nodes (region
->entry_block
))
2757 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2758 region
->restart_block
= empty_bb
;
2759 if (current_loops
&& transaction_bb
->loop_father
)
2760 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2762 redirect_edge_pred (fallthru_edge
, empty_bb
);
2763 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2769 /* Generate the temporary to be used for the return value of
2770 BUILT_IN_TM_START. */
2773 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2775 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2777 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2779 // Reset the subcode, post optimizations. We'll fill this in
2780 // again as we process blocks.
2781 if (region
->exit_blocks
)
2783 unsigned int subcode
2784 = gimple_transaction_subcode (region
->transaction_stmt
);
2786 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2787 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2788 | GTMA_MAY_ENTER_IRREVOCABLE
);
2790 subcode
&= GTMA_DECLARATION_MASK
;
2791 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2797 // Propagate flags from inner transactions outwards.
2799 propagate_tm_flags_out (struct tm_region
*region
)
2803 propagate_tm_flags_out (region
->inner
);
2805 if (region
->outer
&& region
->outer
->transaction_stmt
)
2807 unsigned s
= gimple_transaction_subcode (region
->transaction_stmt
);
2808 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2809 | GTMA_MAY_ENTER_IRREVOCABLE
);
2810 s
|= gimple_transaction_subcode (region
->outer
->transaction_stmt
);
2811 gimple_transaction_set_subcode (region
->outer
->transaction_stmt
, s
);
2814 propagate_tm_flags_out (region
->next
);
2817 /* Entry point to the MARK phase of TM expansion. Here we replace
2818 transactional memory statements with calls to builtins, and function
2819 calls with their transactional clones (if available). But we don't
2820 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2823 execute_tm_mark (void)
2825 pending_edge_inserts_p
= false;
2827 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2828 /*traverse_clones=*/true);
2832 vec
<tm_region_p
> bb_regions
2833 = get_bb_regions_instrumented (/*traverse_clones=*/true);
2834 struct tm_region
*r
;
2837 // Expand memory operations into calls into the runtime.
2838 // This collects log entries as well.
2839 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
2841 expand_block_tm (r
, BASIC_BLOCK (i
));
2843 bb_regions
.release ();
2845 // Propagate flags from inner transactions outwards.
2846 propagate_tm_flags_out (all_tm_regions
);
2848 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
2849 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
2850 /*traverse_clones=*/false);
2855 if (pending_edge_inserts_p
)
2856 gsi_commit_edge_inserts ();
2857 free_dominance_info (CDI_DOMINATORS
);
2861 struct gimple_opt_pass pass_tm_mark
=
2865 "tmmark", /* name */
2866 OPTGROUP_NONE
, /* optinfo_flags */
2868 execute_tm_mark
, /* execute */
2871 0, /* static_pass_number */
2872 TV_TRANS_MEM
, /* tv_id */
2873 PROP_ssa
| PROP_cfg
, /* properties_required */
2874 0, /* properties_provided */
2875 0, /* properties_destroyed */
2876 0, /* todo_flags_start */
2878 | TODO_verify_ssa
, /* todo_flags_finish */
2883 /* Create an abnormal edge from STMT at iter, splitting the block
2884 as necessary. Adjust *PNEXT as needed for the split block. */
2887 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
2888 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
2890 basic_block bb
= gimple_bb (stmt
);
2891 if (!gsi_one_before_end_p (iter
))
2893 edge e
= split_block (bb
, stmt
);
2894 *pnext
= gsi_start_bb (e
->dest
);
2896 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
2898 // Record the need for the edge for the benefit of the rtl passes.
2899 if (cfun
->gimple_df
->tm_restart
== NULL
)
2900 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
2901 struct_ptr_eq
, ggc_free
);
2903 struct tm_restart_node dummy
;
2905 dummy
.label_or_list
= gimple_block_label (dest_bb
);
2907 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
2908 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
2911 n
= ggc_alloc_tm_restart_node ();
2916 tree old
= n
->label_or_list
;
2917 if (TREE_CODE (old
) == LABEL_DECL
)
2918 old
= tree_cons (NULL
, old
, NULL
);
2919 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
2923 /* Split block BB as necessary for every builtin function we added, and
2924 wire up the abnormal back edges implied by the transaction restart. */
2927 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
2929 gimple_stmt_iterator gsi
, next_gsi
;
2931 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
2933 gimple stmt
= gsi_stmt (gsi
);
2936 gsi_next (&next_gsi
);
2938 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
2939 if (gimple_code (stmt
) != GIMPLE_CALL
2940 || (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) == 0)
2943 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
)) == BUILT_IN_TM_ABORT
)
2945 // If we have a ``_transaction_cancel [[outer]]'', there is only
2946 // one abnormal edge: to the transaction marked OUTER.
2947 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
2948 // constant argument, which we can examine here. Users invoking
2949 // TM_ABORT directly get what they deserve.
2950 tree arg
= gimple_call_arg (stmt
, 0);
2951 if (TREE_CODE (arg
) == INTEGER_CST
2952 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
2953 && !decl_is_tm_clone (current_function_decl
))
2955 // Find the GTMA_IS_OUTER transaction.
2956 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
2957 if (o
->original_transaction_was_outer
)
2959 split_bb_make_tm_edge (stmt
, o
->restart_block
,
2964 // Otherwise, the front-end should have semantically checked
2965 // outer aborts, but in either case the target region is not
2966 // within this function.
2970 // Non-outer, TM aborts have an abnormal edge to the inner-most
2971 // transaction, the one being aborted;
2972 split_bb_make_tm_edge (stmt
, region
->restart_block
, gsi
, &next_gsi
);
2975 // All TM builtins have an abnormal edge to the outer-most transaction.
2976 // We never restart inner transactions. For tm clones, we know a-priori
2977 // that the outer-most transaction is outside the function.
2978 if (decl_is_tm_clone (current_function_decl
))
2981 if (cfun
->gimple_df
->tm_restart
== NULL
)
2982 cfun
->gimple_df
->tm_restart
2983 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
2985 // All TM builtins have an abnormal edge to the outer-most transaction.
2986 // We never restart inner transactions.
2987 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
2990 split_bb_make_tm_edge (stmt
, o
->restart_block
, gsi
, &next_gsi
);
2994 // Delete any tail-call annotation that may have been added.
2995 // The tail-call pass may have mis-identified the commit as being
2996 // a candidate because we had not yet added this restart edge.
2997 gimple_call_set_tail (stmt
, false);
3001 /* Entry point to the final expansion of transactional nodes. */
3004 execute_tm_edges (void)
3006 vec
<tm_region_p
> bb_regions
3007 = get_bb_regions_instrumented (/*traverse_clones=*/false);
3008 struct tm_region
*r
;
3011 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3013 expand_block_edges (r
, BASIC_BLOCK (i
));
3015 bb_regions
.release ();
3017 /* We've got to release the dominance info now, to indicate that it
3018 must be rebuilt completely. Otherwise we'll crash trying to update
3019 the SSA web in the TODO section following this pass. */
3020 free_dominance_info (CDI_DOMINATORS
);
3021 bitmap_obstack_release (&tm_obstack
);
3022 all_tm_regions
= NULL
;
3027 struct gimple_opt_pass pass_tm_edges
=
3031 "tmedge", /* name */
3032 OPTGROUP_NONE
, /* optinfo_flags */
3034 execute_tm_edges
, /* execute */
3037 0, /* static_pass_number */
3038 TV_TRANS_MEM
, /* tv_id */
3039 PROP_ssa
| PROP_cfg
, /* properties_required */
3040 0, /* properties_provided */
3041 0, /* properties_destroyed */
3042 0, /* todo_flags_start */
3044 | TODO_verify_ssa
, /* todo_flags_finish */
3048 /* Helper function for expand_regions. Expand REGION and recurse to
3049 the inner region. Call CALLBACK on each region. CALLBACK returns
3050 NULL to continue the traversal, otherwise a non-null value which
3051 this function will return as well. TRAVERSE_CLONES is true if we
3052 should traverse transactional clones. */
3055 expand_regions_1 (struct tm_region
*region
,
3056 void *(*callback
)(struct tm_region
*, void *),
3058 bool traverse_clones
)
3060 void *retval
= NULL
;
3061 if (region
->exit_blocks
3062 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3064 retval
= callback (region
, data
);
3070 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3077 /* Traverse the regions enclosed and including REGION. Execute
3078 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3079 continue the traversal, otherwise a non-null value which this
3080 function will return as well. TRAVERSE_CLONES is true if we should
3081 traverse transactional clones. */
3084 expand_regions (struct tm_region
*region
,
3085 void *(*callback
)(struct tm_region
*, void *),
3087 bool traverse_clones
)
3089 void *retval
= NULL
;
3092 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3095 region
= region
->next
;
3101 /* A unique TM memory operation. */
3102 typedef struct tm_memop
3104 /* Unique ID that all memory operations to the same location have. */
3105 unsigned int value_id
;
3106 /* Address of load/store. */
3110 /* Sets for solving data flow equations in the memory optimization pass. */
3111 struct tm_memopt_bitmaps
3113 /* Stores available to this BB upon entry. Basically, stores that
3114 dominate this BB. */
3115 bitmap store_avail_in
;
3116 /* Stores available at the end of this BB. */
3117 bitmap store_avail_out
;
3118 bitmap store_antic_in
;
3119 bitmap store_antic_out
;
3120 /* Reads available to this BB upon entry. Basically, reads that
3121 dominate this BB. */
3122 bitmap read_avail_in
;
3123 /* Reads available at the end of this BB. */
3124 bitmap read_avail_out
;
3125 /* Reads performed in this BB. */
3127 /* Writes performed in this BB. */
3130 /* Temporary storage for pass. */
3131 /* Is the current BB in the worklist? */
3132 bool avail_in_worklist_p
;
3133 /* Have we visited this BB? */
3137 static bitmap_obstack tm_memopt_obstack
;
3139 /* Unique counter for TM loads and stores. Loads and stores of the
3140 same address get the same ID. */
3141 static unsigned int tm_memopt_value_id
;
3142 static htab_t tm_memopt_value_numbers
;
3144 #define STORE_AVAIL_IN(BB) \
3145 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3146 #define STORE_AVAIL_OUT(BB) \
3147 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3148 #define STORE_ANTIC_IN(BB) \
3149 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3150 #define STORE_ANTIC_OUT(BB) \
3151 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3152 #define READ_AVAIL_IN(BB) \
3153 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3154 #define READ_AVAIL_OUT(BB) \
3155 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3156 #define READ_LOCAL(BB) \
3157 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3158 #define STORE_LOCAL(BB) \
3159 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3160 #define AVAIL_IN_WORKLIST_P(BB) \
3161 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3162 #define BB_VISITED_P(BB) \
3163 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3165 /* Htab support. Return a hash value for a `tm_memop'. */
3167 tm_memop_hash (const void *p
)
3169 const struct tm_memop
*mem
= (const struct tm_memop
*) p
;
3170 tree addr
= mem
->addr
;
3171 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3172 actually done with operand_equal_p (see tm_memop_eq). */
3173 if (TREE_CODE (addr
) == ADDR_EXPR
)
3174 addr
= TREE_OPERAND (addr
, 0);
3175 return iterative_hash_expr (addr
, 0);
3178 /* Htab support. Return true if two tm_memop's are the same. */
3180 tm_memop_eq (const void *p1
, const void *p2
)
3182 const struct tm_memop
*mem1
= (const struct tm_memop
*) p1
;
3183 const struct tm_memop
*mem2
= (const struct tm_memop
*) p2
;
3185 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3188 /* Given a TM load/store in STMT, return the value number for the address
3192 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3194 struct tm_memop tmpmem
, *mem
;
3197 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3198 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3199 slot
= htab_find_slot (tm_memopt_value_numbers
, &tmpmem
, op
);
3201 mem
= (struct tm_memop
*) *slot
;
3202 else if (op
== INSERT
)
3204 mem
= XNEW (struct tm_memop
);
3206 mem
->value_id
= tm_memopt_value_id
++;
3207 mem
->addr
= tmpmem
.addr
;
3211 return mem
->value_id
;
3214 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3217 tm_memopt_accumulate_memops (basic_block bb
)
3219 gimple_stmt_iterator gsi
;
3221 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3223 gimple stmt
= gsi_stmt (gsi
);
3227 if (is_tm_store (stmt
))
3228 bits
= STORE_LOCAL (bb
);
3229 else if (is_tm_load (stmt
))
3230 bits
= READ_LOCAL (bb
);
3234 loc
= tm_memopt_value_number (stmt
, INSERT
);
3235 bitmap_set_bit (bits
, loc
);
3238 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3239 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3240 gimple_bb (stmt
)->index
);
3241 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3242 fprintf (dump_file
, "\n");
3247 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3250 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3254 const char *comma
= "";
3256 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3257 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3260 struct tm_memop
*mem
;
3262 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3263 FOR_EACH_HTAB_ELEMENT (tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3264 if (mem
->value_id
== i
)
3266 gcc_assert (mem
->value_id
== i
);
3267 fprintf (dump_file
, "%s", comma
);
3269 print_generic_expr (dump_file
, mem
->addr
, 0);
3271 fprintf (dump_file
, "]\n");
3274 /* Prettily dump all of the memopt sets in BLOCKS. */
3277 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3282 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3284 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3285 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3286 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3287 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3288 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3289 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3290 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3294 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3297 tm_memopt_compute_avin (basic_block bb
)
3302 /* Seed with the AVOUT of any predecessor. */
3303 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3305 e
= EDGE_PRED (bb
, ix
);
3306 /* Make sure we have already visited this BB, and is thus
3309 If e->src->aux is NULL, this predecessor is actually on an
3310 enclosing transaction. We only care about the current
3311 transaction, so ignore it. */
3312 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3314 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3315 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3320 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3322 e
= EDGE_PRED (bb
, ix
);
3323 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3325 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3326 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3330 BB_VISITED_P (bb
) = true;
3333 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3336 tm_memopt_compute_antin (basic_block bb
)
3341 /* Seed with the ANTIC_OUT of any successor. */
3342 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3344 e
= EDGE_SUCC (bb
, ix
);
3345 /* Make sure we have already visited this BB, and is thus
3347 if (BB_VISITED_P (e
->dest
))
3349 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3354 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3356 e
= EDGE_SUCC (bb
, ix
);
3357 if (BB_VISITED_P (e
->dest
))
3358 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3361 BB_VISITED_P (bb
) = true;
3364 /* Compute the AVAIL sets for every basic block in BLOCKS.
3366 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3368 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3369 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3371 This is basically what we do in lcm's compute_available(), but here
3372 we calculate two sets of sets (one for STOREs and one for READs),
3373 and we work on a region instead of the entire CFG.
3375 REGION is the TM region.
3376 BLOCKS are the basic blocks in the region. */
3379 tm_memopt_compute_available (struct tm_region
*region
,
3380 vec
<basic_block
> blocks
)
3383 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3384 unsigned int qlen
, i
;
3388 /* Allocate a worklist array/queue. Entries are only added to the
3389 list if they were not already on the list. So the size is
3390 bounded by the number of basic blocks in the region. */
3391 qlen
= blocks
.length () - 1;
3392 qin
= qout
= worklist
=
3393 XNEWVEC (basic_block
, qlen
);
3395 /* Put every block in the region on the worklist. */
3396 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3398 /* Seed AVAIL_OUT with the LOCAL set. */
3399 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3400 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3402 AVAIL_IN_WORKLIST_P (bb
) = true;
3403 /* No need to insert the entry block, since it has an AVIN of
3404 null, and an AVOUT that has already been seeded in. */
3405 if (bb
!= region
->entry_block
)
3409 /* The entry block has been initialized with the local sets. */
3410 BB_VISITED_P (region
->entry_block
) = true;
3413 qend
= &worklist
[qlen
];
3415 /* Iterate until the worklist is empty. */
3418 /* Take the first entry off the worklist. */
3425 /* This block can be added to the worklist again if necessary. */
3426 AVAIL_IN_WORKLIST_P (bb
) = false;
3427 tm_memopt_compute_avin (bb
);
3429 /* Note: We do not add the LOCAL sets here because we already
3430 seeded the AVAIL_OUT sets with them. */
3431 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3432 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3434 && (region
->exit_blocks
== NULL
3435 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3436 /* If the out state of this block changed, then we need to add
3437 its successors to the worklist if they are not already in. */
3438 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3439 if (!AVAIL_IN_WORKLIST_P (e
->dest
) && e
->dest
!= EXIT_BLOCK_PTR
)
3442 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3453 dump_tm_memopt_sets (blocks
);
3456 /* Compute ANTIC sets for every basic block in BLOCKS.
3458 We compute STORE_ANTIC_OUT as follows:
3460 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3461 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3463 REGION is the TM region.
3464 BLOCKS are the basic blocks in the region. */
3467 tm_memopt_compute_antic (struct tm_region
*region
,
3468 vec
<basic_block
> blocks
)
3471 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3476 /* Allocate a worklist array/queue. Entries are only added to the
3477 list if they were not already on the list. So the size is
3478 bounded by the number of basic blocks in the region. */
3479 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3481 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3485 /* Seed ANTIC_OUT with the LOCAL set. */
3486 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3488 /* Put every block in the region on the worklist. */
3489 AVAIL_IN_WORKLIST_P (bb
) = true;
3490 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3491 and their ANTIC_OUT has already been seeded in. */
3492 if (region
->exit_blocks
3493 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3500 /* The exit blocks have been initialized with the local sets. */
3501 if (region
->exit_blocks
)
3505 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3506 BB_VISITED_P (BASIC_BLOCK (i
)) = true;
3510 qend
= &worklist
[qlen
];
3512 /* Iterate until the worklist is empty. */
3515 /* Take the first entry off the worklist. */
3522 /* This block can be added to the worklist again if necessary. */
3523 AVAIL_IN_WORKLIST_P (bb
) = false;
3524 tm_memopt_compute_antin (bb
);
3526 /* Note: We do not add the LOCAL sets here because we already
3527 seeded the ANTIC_OUT sets with them. */
3528 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3529 && bb
!= region
->entry_block
)
3530 /* If the out state of this block changed, then we need to add
3531 its predecessors to the worklist if they are not already in. */
3532 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3533 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3536 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3547 dump_tm_memopt_sets (blocks
);
3550 /* Offsets of load variants from TM_LOAD. For example,
3551 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3552 See gtm-builtins.def. */
3553 #define TRANSFORM_RAR 1
3554 #define TRANSFORM_RAW 2
3555 #define TRANSFORM_RFW 3
3556 /* Offsets of store variants from TM_STORE. */
3557 #define TRANSFORM_WAR 1
3558 #define TRANSFORM_WAW 2
3560 /* Inform about a load/store optimization. */
3563 dump_tm_memopt_transform (gimple stmt
)
3567 fprintf (dump_file
, "TM memopt: transforming: ");
3568 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3569 fprintf (dump_file
, "\n");
3573 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3574 by a builtin that is OFFSET entries down in the builtins table in
3575 gtm-builtins.def. */
3578 tm_memopt_transform_stmt (unsigned int offset
,
3580 gimple_stmt_iterator
*gsi
)
3582 tree fn
= gimple_call_fn (stmt
);
3583 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3584 TREE_OPERAND (fn
, 0)
3585 = builtin_decl_explicit ((enum built_in_function
)
3586 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3588 gimple_call_set_fn (stmt
, fn
);
3589 gsi_replace (gsi
, stmt
, true);
3590 dump_tm_memopt_transform (stmt
);
3593 /* Perform the actual TM memory optimization transformations in the
3594 basic blocks in BLOCKS. */
3597 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3601 gimple_stmt_iterator gsi
;
3603 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3605 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3607 gimple stmt
= gsi_stmt (gsi
);
3608 bitmap read_avail
= READ_AVAIL_IN (bb
);
3609 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3610 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3613 if (is_tm_simple_load (stmt
))
3615 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3616 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3617 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3618 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3620 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3621 bitmap_set_bit (store_avail
, loc
);
3623 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3624 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3626 bitmap_set_bit (read_avail
, loc
);
3628 else if (is_tm_simple_store (stmt
))
3630 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3631 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3632 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3635 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3636 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3637 bitmap_set_bit (store_avail
, loc
);
3644 /* Return a new set of bitmaps for a BB. */
3646 static struct tm_memopt_bitmaps
*
3647 tm_memopt_init_sets (void)
3649 struct tm_memopt_bitmaps
*b
3650 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3651 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3652 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3653 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3654 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3655 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3656 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3657 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3658 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3659 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3663 /* Free sets computed for each BB. */
3666 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3671 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3675 /* Clear the visited bit for every basic block in BLOCKS. */
3678 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3683 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3684 BB_VISITED_P (bb
) = false;
3687 /* Replace TM load/stores with hints for the runtime. We handle
3688 things like read-after-write, write-after-read, read-after-read,
3689 read-for-write, etc. */
3692 execute_tm_memopt (void)
3694 struct tm_region
*region
;
3695 vec
<basic_block
> bbs
;
3697 tm_memopt_value_id
= 0;
3698 tm_memopt_value_numbers
= htab_create (10, tm_memop_hash
, tm_memop_eq
, free
);
3700 for (region
= all_tm_regions
; region
; region
= region
->next
)
3702 /* All the TM stores/loads in the current region. */
3706 bitmap_obstack_initialize (&tm_memopt_obstack
);
3708 /* Save all BBs for the current region. */
3709 bbs
= get_tm_region_blocks (region
->entry_block
,
3710 region
->exit_blocks
,
3715 /* Collect all the memory operations. */
3716 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3718 bb
->aux
= tm_memopt_init_sets ();
3719 tm_memopt_accumulate_memops (bb
);
3722 /* Solve data flow equations and transform each block accordingly. */
3723 tm_memopt_clear_visited (bbs
);
3724 tm_memopt_compute_available (region
, bbs
);
3725 tm_memopt_clear_visited (bbs
);
3726 tm_memopt_compute_antic (region
, bbs
);
3727 tm_memopt_transform_blocks (bbs
);
3729 tm_memopt_free_sets (bbs
);
3731 bitmap_obstack_release (&tm_memopt_obstack
);
3732 htab_empty (tm_memopt_value_numbers
);
3735 htab_delete (tm_memopt_value_numbers
);
3740 gate_tm_memopt (void)
3742 return flag_tm
&& optimize
> 0;
3745 struct gimple_opt_pass pass_tm_memopt
=
3749 "tmmemopt", /* name */
3750 OPTGROUP_NONE
, /* optinfo_flags */
3751 gate_tm_memopt
, /* gate */
3752 execute_tm_memopt
, /* execute */
3755 0, /* static_pass_number */
3756 TV_TRANS_MEM
, /* tv_id */
3757 PROP_ssa
| PROP_cfg
, /* properties_required */
3758 0, /* properties_provided */
3759 0, /* properties_destroyed */
3760 0, /* todo_flags_start */
3761 0, /* todo_flags_finish */
3766 /* Interprocedual analysis for the creation of transactional clones.
3767 The aim of this pass is to find which functions are referenced in
3768 a non-irrevocable transaction context, and for those over which
3769 we have control (or user directive), create a version of the
3770 function which uses only the transactional interface to reference
3771 protected memories. This analysis proceeds in several steps:
3773 (1) Collect the set of all possible transactional clones:
3775 (a) For all local public functions marked tm_callable, push
3776 it onto the tm_callee queue.
3778 (b) For all local functions, scan for calls in transaction blocks.
3779 Push the caller and callee onto the tm_caller and tm_callee
3780 queues. Count the number of callers for each callee.
3782 (c) For each local function on the callee list, assume we will
3783 create a transactional clone. Push *all* calls onto the
3784 callee queues; count the number of clone callers separately
3785 to the number of original callers.
3787 (2) Propagate irrevocable status up the dominator tree:
3789 (a) Any external function on the callee list that is not marked
3790 tm_callable is irrevocable. Push all callers of such onto
3793 (b) For each function on the worklist, mark each block that
3794 contains an irrevocable call. Use the AND operator to
3795 propagate that mark up the dominator tree.
3797 (c) If we reach the entry block for a possible transactional
3798 clone, then the transactional clone is irrevocable, and
3799 we should not create the clone after all. Push all
3800 callers onto the worklist.
3802 (d) Place tm_irrevocable calls at the beginning of the relevant
3803 blocks. Special case here is the entry block for the entire
3804 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
3805 the library to begin the region in serial mode. Decrement
3806 the call count for all callees in the irrevocable region.
3808 (3) Create the transactional clones:
3810 Any tm_callee that still has a non-zero call count is cloned.
3813 /* This structure is stored in the AUX field of each cgraph_node. */
3814 struct tm_ipa_cg_data
3816 /* The clone of the function that got created. */
3817 struct cgraph_node
*clone
;
3819 /* The tm regions in the normal function. */
3820 struct tm_region
*all_tm_regions
;
3822 /* The blocks of the normal/clone functions that contain irrevocable
3823 calls, or blocks that are post-dominated by irrevocable calls. */
3824 bitmap irrevocable_blocks_normal
;
3825 bitmap irrevocable_blocks_clone
;
3827 /* The blocks of the normal function that are involved in transactions. */
3828 bitmap transaction_blocks_normal
;
3830 /* The number of callers to the transactional clone of this function
3831 from normal and transactional clones respectively. */
3832 unsigned tm_callers_normal
;
3833 unsigned tm_callers_clone
;
3835 /* True if all calls to this function's transactional clone
3836 are irrevocable. Also automatically true if the function
3837 has no transactional clone. */
3838 bool is_irrevocable
;
3840 /* Flags indicating the presence of this function in various queues. */
3841 bool in_callee_queue
;
3844 /* Flags indicating the kind of scan desired while in the worklist. */
3845 bool want_irr_scan_normal
;
3848 typedef vec
<cgraph_node_ptr
> cgraph_node_queue
;
3850 /* Return the ipa data associated with NODE, allocating zeroed memory
3851 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
3852 and set *NODE accordingly. */
3854 static struct tm_ipa_cg_data
*
3855 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
3857 struct tm_ipa_cg_data
*d
;
3859 if (traverse_aliases
&& (*node
)->alias
)
3860 *node
= cgraph_get_node ((*node
)->thunk
.alias
);
3862 d
= (struct tm_ipa_cg_data
*) (*node
)->symbol
.aux
;
3866 d
= (struct tm_ipa_cg_data
*)
3867 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
3868 (*node
)->symbol
.aux
= (void *) d
;
3869 memset (d
, 0, sizeof (*d
));
3875 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
3876 it is already present. */
3879 maybe_push_queue (struct cgraph_node
*node
,
3880 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
3885 queue_p
->safe_push (node
);
3889 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
3890 code path. QUEUE are the basic blocks inside the transaction
3891 represented in REGION.
3893 Later in split_code_paths() we will add the conditional to choose
3894 between the two alternatives. */
3897 ipa_uninstrument_transaction (struct tm_region
*region
,
3898 vec
<basic_block
> queue
)
3900 gimple transaction
= region
->transaction_stmt
;
3901 basic_block transaction_bb
= gimple_bb (transaction
);
3902 int n
= queue
.length ();
3903 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
3905 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
);
3906 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
3907 add_phi_args_after_copy (new_bbs
, n
, e
);
3909 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
3910 // a) EDGE_FALLTHRU into the transaction
3911 // b) EDGE_TM_ABORT out of the transaction
3912 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
3917 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
3918 Queue all callees within block BB. */
3921 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
3922 basic_block bb
, bool for_clone
)
3924 gimple_stmt_iterator gsi
;
3926 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3928 gimple stmt
= gsi_stmt (gsi
);
3929 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3931 tree fndecl
= gimple_call_fndecl (stmt
);
3934 struct tm_ipa_cg_data
*d
;
3936 struct cgraph_node
*node
;
3938 if (is_tm_ending_fndecl (fndecl
))
3940 if (find_tm_replacement_function (fndecl
))
3943 node
= cgraph_get_node (fndecl
);
3944 gcc_assert (node
!= NULL
);
3945 d
= get_cg_data (&node
, true);
3947 pcallers
= (for_clone
? &d
->tm_callers_clone
3948 : &d
->tm_callers_normal
);
3951 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
3957 /* Scan all calls in NODE that are within a transaction region,
3958 and push the resulting nodes into the callee queue. */
3961 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
3962 cgraph_node_queue
*callees_p
)
3964 struct tm_region
*r
;
3966 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
3967 d
->all_tm_regions
= all_tm_regions
;
3969 for (r
= all_tm_regions
; r
; r
= r
->next
)
3971 vec
<basic_block
> bbs
;
3975 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
3976 d
->transaction_blocks_normal
, false);
3978 // Generate the uninstrumented code path for this transaction.
3979 ipa_uninstrument_transaction (r
, bbs
);
3981 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
3982 ipa_tm_scan_calls_block (callees_p
, bb
, false);
3987 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
3988 // copying them, rather than forcing us to do this externally.
3989 rebuild_cgraph_edges ();
3991 // ??? In ipa_uninstrument_transaction we don't try to update dominators
3992 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
3993 // Instead, just release dominators here so update_ssa recomputes them.
3994 free_dominance_info (CDI_DOMINATORS
);
3996 // When building the uninstrumented code path, copy_bbs will have invoked
3997 // create_new_def_for starting an "ssa update context". There is only one
3998 // instance of this context, so resolve ssa updates before moving on to
3999 // the next function.
4000 update_ssa (TODO_update_ssa
);
4003 /* Scan all calls in NODE as if this is the transactional clone,
4004 and push the destinations into the callee queue. */
4007 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4008 cgraph_node_queue
*callees_p
)
4010 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
4013 FOR_EACH_BB_FN (bb
, fn
)
4014 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4017 /* The function NODE has been detected to be irrevocable. Push all
4018 of its callers onto WORKLIST for the purpose of re-scanning them. */
4021 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4022 cgraph_node_queue
*worklist_p
)
4024 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4025 struct cgraph_edge
*e
;
4027 d
->is_irrevocable
= true;
4029 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4032 struct cgraph_node
*caller
;
4034 /* Don't examine recursive calls. */
4035 if (e
->caller
== node
)
4037 /* Even if we think we can go irrevocable, believe the user
4039 if (is_tm_safe_or_pure (e
->caller
->symbol
.decl
))
4043 d
= get_cg_data (&caller
, true);
4045 /* Check if the callee is in a transactional region. If so,
4046 schedule the function for normal re-scan as well. */
4047 bb
= gimple_bb (e
->call_stmt
);
4048 gcc_assert (bb
!= NULL
);
4049 if (d
->transaction_blocks_normal
4050 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4051 d
->want_irr_scan_normal
= true;
4053 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4057 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4058 within the block is irrevocable. */
4061 ipa_tm_scan_irr_block (basic_block bb
)
4063 gimple_stmt_iterator gsi
;
4066 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4068 gimple stmt
= gsi_stmt (gsi
);
4069 switch (gimple_code (stmt
))
4072 if (gimple_assign_single_p (stmt
))
4074 tree lhs
= gimple_assign_lhs (stmt
);
4075 tree rhs
= gimple_assign_rhs1 (stmt
);
4076 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4083 tree lhs
= gimple_call_lhs (stmt
);
4084 if (lhs
&& volatile_var_p (lhs
))
4087 if (is_tm_pure_call (stmt
))
4090 fn
= gimple_call_fn (stmt
);
4092 /* Functions with the attribute are by definition irrevocable. */
4093 if (is_tm_irrevocable (fn
))
4096 /* For direct function calls, go ahead and check for replacement
4097 functions, or transitive irrevocable functions. For indirect
4098 functions, we'll ask the runtime. */
4099 if (TREE_CODE (fn
) == ADDR_EXPR
)
4101 struct tm_ipa_cg_data
*d
;
4102 struct cgraph_node
*node
;
4104 fn
= TREE_OPERAND (fn
, 0);
4105 if (is_tm_ending_fndecl (fn
))
4107 if (find_tm_replacement_function (fn
))
4110 node
= cgraph_get_node(fn
);
4111 d
= get_cg_data (&node
, true);
4113 /* Return true if irrevocable, but above all, believe
4115 if (d
->is_irrevocable
4116 && !is_tm_safe_or_pure (fn
))
4123 /* ??? The Approved Method of indicating that an inline
4124 assembly statement is not relevant to the transaction
4125 is to wrap it in a __tm_waiver block. This is not
4126 yet implemented, so we can't check for it. */
4127 if (is_tm_safe (current_function_decl
))
4129 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4130 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4131 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4143 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4144 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4145 scanning past OLD_IRR or EXIT_BLOCKS. */
4148 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4149 bitmap old_irr
, bitmap exit_blocks
)
4151 bool any_new_irr
= false;
4154 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4158 basic_block bb
= pqueue
->pop ();
4160 /* Don't re-scan blocks we know already are irrevocable. */
4161 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4164 if (ipa_tm_scan_irr_block (bb
))
4166 bitmap_set_bit (new_irr
, bb
->index
);
4169 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4171 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4172 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4174 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4175 pqueue
->safe_push (e
->dest
);
4179 while (!pqueue
->is_empty ());
4181 BITMAP_FREE (visited_blocks
);
4186 /* Propagate the irrevocable property both up and down the dominator tree.
4187 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4188 TM regions; OLD_IRR are the results of a previous scan of the dominator
4189 tree which has been fully propagated; NEW_IRR is the set of new blocks
4190 which are gaining the irrevocable property during the current scan. */
4193 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4194 bitmap old_irr
, bitmap exit_blocks
)
4196 vec
<basic_block
> bbs
;
4197 bitmap all_region_blocks
;
4199 /* If this block is in the old set, no need to rescan. */
4200 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4203 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4204 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4205 all_region_blocks
, false);
4208 basic_block bb
= bbs
.pop ();
4209 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4210 bool all_son_irr
= false;
4214 /* Propagate up. If my children are, I am too, but we must have
4215 at least one child that is. */
4218 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4220 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4222 all_son_irr
= false;
4230 /* Add block to new_irr if it hasn't already been processed. */
4231 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4233 bitmap_set_bit (new_irr
, bb
->index
);
4239 /* Propagate down to everyone we immediately dominate. */
4243 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4245 son
= next_dom_son (CDI_DOMINATORS
, son
))
4247 /* Make sure block is actually in a TM region, and it
4248 isn't already in old_irr. */
4249 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4250 && bitmap_bit_p (all_region_blocks
, son
->index
))
4251 bitmap_set_bit (new_irr
, son
->index
);
4255 while (!bbs
.is_empty ());
4257 BITMAP_FREE (all_region_blocks
);
4262 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4264 gimple_stmt_iterator gsi
;
4266 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4268 gimple stmt
= gsi_stmt (gsi
);
4269 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4271 tree fndecl
= gimple_call_fndecl (stmt
);
4274 struct tm_ipa_cg_data
*d
;
4276 struct cgraph_node
*tnode
;
4278 if (is_tm_ending_fndecl (fndecl
))
4280 if (find_tm_replacement_function (fndecl
))
4283 tnode
= cgraph_get_node (fndecl
);
4284 d
= get_cg_data (&tnode
, true);
4286 pcallers
= (for_clone
? &d
->tm_callers_clone
4287 : &d
->tm_callers_normal
);
4289 gcc_assert (*pcallers
> 0);
4296 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4297 as well as other irrevocable actions such as inline assembly. Mark all
4298 such blocks as irrevocable and decrement the number of calls to
4299 transactional clones. Return true if, for the transactional clone, the
4300 entire function is irrevocable. */
4303 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4305 struct tm_ipa_cg_data
*d
;
4306 bitmap new_irr
, old_irr
;
4307 vec
<basic_block
> queue
;
4310 /* Builtin operators (operator new, and such). */
4311 if (DECL_STRUCT_FUNCTION (node
->symbol
.decl
) == NULL
4312 || DECL_STRUCT_FUNCTION (node
->symbol
.decl
)->cfg
== NULL
)
4315 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
4316 calculate_dominance_info (CDI_DOMINATORS
);
4318 d
= get_cg_data (&node
, true);
4320 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4322 /* Scan each tm region, propagating irrevocable status through the tree. */
4325 old_irr
= d
->irrevocable_blocks_clone
;
4326 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR
));
4327 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4329 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR
), new_irr
,
4331 ret
= bitmap_bit_p (new_irr
, single_succ (ENTRY_BLOCK_PTR
)->index
);
4336 struct tm_region
*region
;
4338 old_irr
= d
->irrevocable_blocks_normal
;
4339 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4341 queue
.quick_push (region
->entry_block
);
4342 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4343 region
->exit_blocks
))
4344 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4345 region
->exit_blocks
);
4349 /* If we found any new irrevocable blocks, reduce the call count for
4350 transactional clones within the irrevocable blocks. Save the new
4351 set of irrevocable blocks for next time. */
4352 if (!bitmap_empty_p (new_irr
))
4354 bitmap_iterator bmi
;
4357 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4358 ipa_tm_decrement_clone_counts (BASIC_BLOCK (i
), for_clone
);
4362 bitmap_ior_into (old_irr
, new_irr
);
4363 BITMAP_FREE (new_irr
);
4366 d
->irrevocable_blocks_clone
= new_irr
;
4368 d
->irrevocable_blocks_normal
= new_irr
;
4370 if (dump_file
&& new_irr
)
4373 bitmap_iterator bmi
;
4376 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4377 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4378 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4382 BITMAP_FREE (new_irr
);
4390 /* Return true if, for the transactional clone of NODE, any call
4391 may enter irrevocable mode. */
4394 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4396 struct tm_ipa_cg_data
*d
;
4400 d
= get_cg_data (&node
, true);
4401 decl
= node
->symbol
.decl
;
4402 flags
= flags_from_decl_or_type (decl
);
4404 /* Handle some TM builtins. Ordinarily these aren't actually generated
4405 at this point, but handling these functions when written in by the
4406 user makes it easier to build unit tests. */
4407 if (flags
& ECF_TM_BUILTIN
)
4410 /* Filter out all functions that are marked. */
4411 if (flags
& ECF_TM_PURE
)
4413 if (is_tm_safe (decl
))
4415 if (is_tm_irrevocable (decl
))
4417 if (is_tm_callable (decl
))
4419 if (find_tm_replacement_function (decl
))
4422 /* If we aren't seeing the final version of the function we don't
4423 know what it will contain at runtime. */
4424 if (cgraph_function_body_availability (node
) < AVAIL_AVAILABLE
)
4427 /* If the function must go irrevocable, then of course true. */
4428 if (d
->is_irrevocable
)
4431 /* If there are any blocks marked irrevocable, then the function
4432 as a whole may enter irrevocable. */
4433 if (d
->irrevocable_blocks_clone
)
4436 /* We may have previously marked this function as tm_may_enter_irr;
4437 see pass_diagnose_tm_blocks. */
4438 if (node
->local
.tm_may_enter_irr
)
4441 /* Recurse on the main body for aliases. In general, this will
4442 result in one of the bits above being set so that we will not
4443 have to recurse next time. */
4445 return ipa_tm_mayenterirr_function (cgraph_get_node (node
->thunk
.alias
));
4447 /* What remains is unmarked local functions without items that force
4448 the function to go irrevocable. */
4452 /* Diagnose calls from transaction_safe functions to unmarked
4453 functions that are determined to not be safe. */
4456 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4458 struct cgraph_edge
*e
;
4460 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4461 if (!is_tm_callable (e
->callee
->symbol
.decl
)
4462 && e
->callee
->local
.tm_may_enter_irr
)
4463 error_at (gimple_location (e
->call_stmt
),
4464 "unsafe function call %qD within "
4465 "%<transaction_safe%> function", e
->callee
->symbol
.decl
);
4468 /* Diagnose call from atomic transactions to unmarked functions
4469 that are determined to not be safe. */
4472 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4473 struct tm_region
*all_tm_regions
)
4475 struct tm_region
*r
;
4477 for (r
= all_tm_regions
; r
; r
= r
->next
)
4478 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4480 /* Atomic transactions can be nested inside relaxed. */
4482 ipa_tm_diagnose_transaction (node
, r
->inner
);
4486 vec
<basic_block
> bbs
;
4487 gimple_stmt_iterator gsi
;
4491 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4492 r
->irr_blocks
, NULL
, false);
4494 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4495 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4497 gimple stmt
= gsi_stmt (gsi
);
4500 if (gimple_code (stmt
) == GIMPLE_ASM
)
4502 error_at (gimple_location (stmt
),
4503 "asm not allowed in atomic transaction");
4507 if (!is_gimple_call (stmt
))
4509 fndecl
= gimple_call_fndecl (stmt
);
4511 /* Indirect function calls have been diagnosed already. */
4515 /* Stop at the end of the transaction. */
4516 if (is_tm_ending_fndecl (fndecl
))
4518 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4523 /* Marked functions have been diagnosed already. */
4524 if (is_tm_pure_call (stmt
))
4526 if (is_tm_callable (fndecl
))
4529 if (cgraph_local_info (fndecl
)->tm_may_enter_irr
)
4530 error_at (gimple_location (stmt
),
4531 "unsafe function call %qD within "
4532 "atomic transaction", fndecl
);
4539 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4540 OLD_DECL. The returned value is a freshly malloced pointer that
4541 should be freed by the caller. */
4544 tm_mangle (tree old_asm_id
)
4546 const char *old_asm_name
;
4549 struct demangle_component
*dc
;
4552 /* Determine if the symbol is already a valid C++ mangled name. Do this
4553 even for C, which might be interfacing with C++ code via appropriately
4554 ugly identifiers. */
4555 /* ??? We could probably do just as well checking for "_Z" and be done. */
4556 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4557 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4564 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4565 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4569 old_asm_name
+= 2; /* Skip _Z */
4573 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4574 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4575 /* Don't play silly games, you! */
4578 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4579 /* I'd really like to know if we can ever be passed one of
4580 these from the C++ front end. The Logical Thing would
4581 seem that hidden-alias should be outer-most, so that we
4582 get hidden-alias of a transaction-clone and not vice-versa. */
4590 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4594 new_asm_id
= get_identifier (tm_name
);
4601 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4603 cgraph_mark_force_output_node (node
);
4604 /* ??? function_and_variable_visibility will reset
4605 the needed bit, without actually checking. */
4609 /* Callback data for ipa_tm_create_version_alias. */
4610 struct create_version_alias_info
4612 struct cgraph_node
*old_node
;
4616 /* A subroutine of ipa_tm_create_version, called via
4617 cgraph_for_node_and_aliases. Create new tm clones for each of
4618 the existing aliases. */
4620 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4622 struct create_version_alias_info
*info
4623 = (struct create_version_alias_info
*)data
;
4624 tree old_decl
, new_decl
, tm_name
;
4625 struct cgraph_node
*new_node
;
4627 if (!node
->same_body_alias
)
4630 old_decl
= node
->symbol
.decl
;
4631 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4632 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4633 TREE_CODE (old_decl
), tm_name
,
4634 TREE_TYPE (old_decl
));
4636 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4637 SET_DECL_RTL (new_decl
, NULL
);
4639 /* Based loosely on C++'s make_alias_for(). */
4640 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4641 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4642 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4643 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4644 DECL_EXTERNAL (new_decl
) = 0;
4645 DECL_ARTIFICIAL (new_decl
) = 1;
4646 TREE_ADDRESSABLE (new_decl
) = 1;
4647 TREE_USED (new_decl
) = 1;
4648 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4650 /* Perform the same remapping to the comdat group. */
4651 if (DECL_ONE_ONLY (new_decl
))
4652 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4654 new_node
= cgraph_same_body_alias (NULL
, new_decl
, info
->new_decl
);
4655 new_node
->tm_clone
= true;
4656 new_node
->symbol
.externally_visible
= info
->old_node
->symbol
.externally_visible
;
4657 /* ?? Do not traverse aliases here. */
4658 get_cg_data (&node
, false)->clone
= new_node
;
4660 record_tm_clone_pair (old_decl
, new_decl
);
4662 if (info
->old_node
->symbol
.force_output
4663 || ipa_ref_list_first_referring (&info
->old_node
->symbol
.ref_list
))
4664 ipa_tm_mark_force_output_node (new_node
);
4668 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4669 appropriate for the transactional clone. */
4672 ipa_tm_create_version (struct cgraph_node
*old_node
)
4674 tree new_decl
, old_decl
, tm_name
;
4675 struct cgraph_node
*new_node
;
4677 old_decl
= old_node
->symbol
.decl
;
4678 new_decl
= copy_node (old_decl
);
4680 /* DECL_ASSEMBLER_NAME needs to be set before we call
4681 cgraph_copy_node_for_versioning below, because cgraph_node will
4682 fill the assembler_name_hash. */
4683 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4684 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4685 SET_DECL_RTL (new_decl
, NULL
);
4686 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4688 /* Perform the same remapping to the comdat group. */
4689 if (DECL_ONE_ONLY (new_decl
))
4690 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4692 new_node
= cgraph_copy_node_for_versioning (old_node
, new_decl
,
4693 vec
<cgraph_edge_p
>(),
4695 new_node
->symbol
.externally_visible
= old_node
->symbol
.externally_visible
;
4696 new_node
->lowered
= true;
4697 new_node
->tm_clone
= 1;
4698 get_cg_data (&old_node
, true)->clone
= new_node
;
4700 if (cgraph_function_body_availability (old_node
) >= AVAIL_OVERWRITABLE
)
4702 /* Remap extern inline to static inline. */
4703 /* ??? Is it worth trying to use make_decl_one_only? */
4704 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4706 DECL_EXTERNAL (new_decl
) = 0;
4707 TREE_PUBLIC (new_decl
) = 0;
4708 DECL_WEAK (new_decl
) = 0;
4711 tree_function_versioning (old_decl
, new_decl
,
4716 record_tm_clone_pair (old_decl
, new_decl
);
4718 cgraph_call_function_insertion_hooks (new_node
);
4719 if (old_node
->symbol
.force_output
4720 || ipa_ref_list_first_referring (&old_node
->symbol
.ref_list
))
4721 ipa_tm_mark_force_output_node (new_node
);
4723 /* Do the same thing, but for any aliases of the original node. */
4725 struct create_version_alias_info data
;
4726 data
.old_node
= old_node
;
4727 data
.new_decl
= new_decl
;
4728 cgraph_for_node_and_aliases (old_node
, ipa_tm_create_version_alias
,
4733 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4736 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4739 gimple_stmt_iterator gsi
;
4742 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4744 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4745 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4747 split_block_after_labels (bb
);
4748 gsi
= gsi_after_labels (bb
);
4749 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4751 cgraph_create_edge (node
,
4752 cgraph_get_create_node
4753 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4755 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4759 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4762 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4763 struct tm_region
*region
,
4764 gimple_stmt_iterator
*gsi
, gimple stmt
)
4766 tree gettm_fn
, ret
, old_fn
, callfn
;
4770 old_fn
= gimple_call_fn (stmt
);
4772 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
4774 tree fndecl
= TREE_OPERAND (old_fn
, 0);
4775 tree clone
= get_tm_clone_pair (fndecl
);
4777 /* By transforming the call into a TM_GETTMCLONE, we are
4778 technically taking the address of the original function and
4779 its clone. Explain this so inlining will know this function
4781 cgraph_mark_address_taken_node (cgraph_get_node (fndecl
));
4783 cgraph_mark_address_taken_node (cgraph_get_node (clone
));
4786 safe
= is_tm_safe (TREE_TYPE (old_fn
));
4787 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
4788 : BUILT_IN_TM_GETTMCLONE_IRR
);
4789 ret
= create_tmp_var (ptr_type_node
, NULL
);
4792 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4794 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
4795 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
4796 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
4798 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
4799 ret
= make_ssa_name (ret
, g
);
4800 gimple_call_set_lhs (g
, ret
);
4802 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
4804 cgraph_create_edge (node
, cgraph_get_create_node (gettm_fn
), g
, 0,
4805 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4808 /* Cast return value from tm_gettmclone* into appropriate function
4810 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
4811 g2
= gimple_build_assign (callfn
,
4812 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
4813 callfn
= make_ssa_name (callfn
, g2
);
4814 gimple_assign_set_lhs (g2
, callfn
);
4815 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
4817 /* ??? This is a hack to preserve the NOTHROW bit on the call,
4818 which we would have derived from the decl. Failure to save
4819 this bit means we might have to split the basic block. */
4820 if (gimple_call_nothrow_p (stmt
))
4821 gimple_call_set_nothrow (stmt
, true);
4823 gimple_call_set_fn (stmt
, callfn
);
4825 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
4826 for a call statement. Fix it. */
4828 tree lhs
= gimple_call_lhs (stmt
);
4829 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
4831 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
4835 temp
= create_tmp_reg (rettype
, 0);
4836 gimple_call_set_lhs (stmt
, temp
);
4838 g2
= gimple_build_assign (lhs
,
4839 fold_build1 (VIEW_CONVERT_EXPR
,
4840 TREE_TYPE (lhs
), temp
));
4841 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
4850 /* Helper function for ipa_tm_transform_calls*. Given a call
4851 statement in GSI which resides inside transaction REGION, redirect
4852 the call to either its wrapper function, or its clone. */
4855 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
4856 struct tm_region
*region
,
4857 gimple_stmt_iterator
*gsi
,
4858 bool *need_ssa_rename_p
)
4860 gimple stmt
= gsi_stmt (*gsi
);
4861 struct cgraph_node
*new_node
;
4862 struct cgraph_edge
*e
= cgraph_edge (node
, stmt
);
4863 tree fndecl
= gimple_call_fndecl (stmt
);
4865 /* For indirect calls, pass the address through the runtime. */
4868 *need_ssa_rename_p
|=
4869 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4873 /* Handle some TM builtins. Ordinarily these aren't actually generated
4874 at this point, but handling these functions when written in by the
4875 user makes it easier to build unit tests. */
4876 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
4879 /* Fixup recursive calls inside clones. */
4880 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
4881 for recursion but not update the call statements themselves? */
4882 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
4884 gimple_call_set_fndecl (stmt
, current_function_decl
);
4888 /* If there is a replacement, use it. */
4889 fndecl
= find_tm_replacement_function (fndecl
);
4892 new_node
= cgraph_get_create_node (fndecl
);
4894 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
4896 We can't do this earlier in record_tm_replacement because
4897 cgraph_remove_unreachable_nodes is called before we inject
4898 references to the node. Further, we can't do this in some
4899 nice central place in ipa_tm_execute because we don't have
4900 the exact list of wrapper functions that would be used.
4901 Marking more wrappers than necessary results in the creation
4902 of unnecessary cgraph_nodes, which can cause some of the
4903 other IPA passes to crash.
4905 We do need to mark these nodes so that we get the proper
4906 result in expand_call_tm. */
4907 /* ??? This seems broken. How is it that we're marking the
4908 CALLEE as may_enter_irr? Surely we should be marking the
4909 CALLER. Also note that find_tm_replacement_function also
4910 contains mappings into the TM runtime, e.g. memcpy. These
4911 we know won't go irrevocable. */
4912 new_node
->local
.tm_may_enter_irr
= 1;
4916 struct tm_ipa_cg_data
*d
;
4917 struct cgraph_node
*tnode
= e
->callee
;
4919 d
= get_cg_data (&tnode
, true);
4920 new_node
= d
->clone
;
4922 /* As we've already skipped pure calls and appropriate builtins,
4923 and we've already marked irrevocable blocks, if we can't come
4924 up with a static replacement, then ask the runtime. */
4925 if (new_node
== NULL
)
4927 *need_ssa_rename_p
|=
4928 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4932 fndecl
= new_node
->symbol
.decl
;
4935 cgraph_redirect_edge_callee (e
, new_node
);
4936 gimple_call_set_fndecl (stmt
, fndecl
);
4939 /* Helper function for ipa_tm_transform_calls. For a given BB,
4940 install calls to tm_irrevocable when IRR_BLOCKS are reached,
4941 redirect other calls to the generated transactional clone. */
4944 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
4945 basic_block bb
, bitmap irr_blocks
)
4947 gimple_stmt_iterator gsi
;
4948 bool need_ssa_rename
= false;
4950 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4952 ipa_tm_insert_irr_call (node
, region
, bb
);
4956 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4958 gimple stmt
= gsi_stmt (gsi
);
4960 if (!is_gimple_call (stmt
))
4962 if (is_tm_pure_call (stmt
))
4965 /* Redirect edges to the appropriate replacement or clone. */
4966 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
4969 return need_ssa_rename
;
4972 /* Walk the CFG for REGION, beginning at BB. Install calls to
4973 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
4974 the generated transactional clone. */
4977 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
4978 basic_block bb
, bitmap irr_blocks
)
4980 bool need_ssa_rename
= false;
4983 vec
<basic_block
> queue
= vec
<basic_block
>();
4984 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4986 queue
.safe_push (bb
);
4992 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
4994 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4997 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5000 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5001 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5003 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5004 queue
.safe_push (e
->dest
);
5007 while (!queue
.is_empty ());
5010 BITMAP_FREE (visited_blocks
);
5012 return need_ssa_rename
;
5015 /* Transform the calls within the TM regions within NODE. */
5018 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5020 struct tm_ipa_cg_data
*d
;
5021 struct tm_region
*region
;
5022 bool need_ssa_rename
= false;
5024 d
= get_cg_data (&node
, true);
5026 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
5027 calculate_dominance_info (CDI_DOMINATORS
);
5029 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5031 /* If we're sure to go irrevocable, don't transform anything. */
5032 if (d
->irrevocable_blocks_normal
5033 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5034 region
->entry_block
->index
))
5036 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
);
5037 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5042 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5043 d
->irrevocable_blocks_normal
);
5046 if (need_ssa_rename
)
5047 update_ssa (TODO_update_ssa_only_virtuals
);
5052 /* Transform the calls within the transactional clone of NODE. */
5055 ipa_tm_transform_clone (struct cgraph_node
*node
)
5057 struct tm_ipa_cg_data
*d
;
5058 bool need_ssa_rename
;
5060 d
= get_cg_data (&node
, true);
5062 /* If this function makes no calls and has no irrevocable blocks,
5063 then there's nothing to do. */
5064 /* ??? Remove non-aborting top-level transactions. */
5065 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5068 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->symbol
.decl
));
5069 calculate_dominance_info (CDI_DOMINATORS
);
5072 ipa_tm_transform_calls (d
->clone
, NULL
, single_succ (ENTRY_BLOCK_PTR
),
5073 d
->irrevocable_blocks_clone
);
5075 if (need_ssa_rename
)
5076 update_ssa (TODO_update_ssa_only_virtuals
);
5081 /* Main entry point for the transactional memory IPA pass. */
5084 ipa_tm_execute (void)
5086 cgraph_node_queue tm_callees
= cgraph_node_queue();
5087 /* List of functions that will go irrevocable. */
5088 cgraph_node_queue irr_worklist
= cgraph_node_queue();
5090 struct cgraph_node
*node
;
5091 struct tm_ipa_cg_data
*d
;
5092 enum availability a
;
5095 #ifdef ENABLE_CHECKING
5099 bitmap_obstack_initialize (&tm_obstack
);
5100 initialize_original_copy_tables ();
5102 /* For all local functions marked tm_callable, queue them. */
5103 FOR_EACH_DEFINED_FUNCTION (node
)
5104 if (is_tm_callable (node
->symbol
.decl
)
5105 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5107 d
= get_cg_data (&node
, true);
5108 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5111 /* For all local reachable functions... */
5112 FOR_EACH_DEFINED_FUNCTION (node
)
5114 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5116 /* ... marked tm_pure, record that fact for the runtime by
5117 indicating that the pure function is its own tm_callable.
5118 No need to do this if the function's address can't be taken. */
5119 if (is_tm_pure (node
->symbol
.decl
))
5121 if (!node
->local
.local
)
5122 record_tm_clone_pair (node
->symbol
.decl
, node
->symbol
.decl
);
5126 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
5127 calculate_dominance_info (CDI_DOMINATORS
);
5129 tm_region_init (NULL
);
5132 d
= get_cg_data (&node
, true);
5134 /* Scan for calls that are in each transaction, and
5135 generate the uninstrumented code path. */
5136 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5138 /* Put it in the worklist so we can scan the function
5139 later (ipa_tm_scan_irr_function) and mark the
5140 irrevocable blocks. */
5141 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5142 d
->want_irr_scan_normal
= true;
5148 /* For every local function on the callee list, scan as if we will be
5149 creating a transactional clone, queueing all new functions we find
5151 for (i
= 0; i
< tm_callees
.length (); ++i
)
5153 node
= tm_callees
[i
];
5154 a
= cgraph_function_body_availability (node
);
5155 d
= get_cg_data (&node
, true);
5157 /* Put it in the worklist so we can scan the function later
5158 (ipa_tm_scan_irr_function) and mark the irrevocable
5160 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5162 /* Some callees cannot be arbitrarily cloned. These will always be
5163 irrevocable. Mark these now, so that we need not scan them. */
5164 if (is_tm_irrevocable (node
->symbol
.decl
))
5165 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5166 else if (a
<= AVAIL_NOT_AVAILABLE
5167 && !is_tm_safe_or_pure (node
->symbol
.decl
))
5168 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5169 else if (a
>= AVAIL_OVERWRITABLE
)
5171 if (!tree_versionable_function_p (node
->symbol
.decl
))
5172 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5173 else if (!d
->is_irrevocable
)
5175 /* If this is an alias, make sure its base is queued as well.
5176 we need not scan the callees now, as the base will do. */
5179 node
= cgraph_get_node (node
->thunk
.alias
);
5180 d
= get_cg_data (&node
, true);
5181 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5185 /* Add all nodes called by this function into
5186 tm_callees as well. */
5187 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5192 /* Iterate scans until no more work to be done. Prefer not to use
5193 vec::pop because the worklist tends to follow a breadth-first
5194 search of the callgraph, which should allow convergance with a
5195 minimum number of scans. But we also don't want the worklist
5196 array to grow without bound, so we shift the array up periodically. */
5197 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5199 if (i
> 256 && i
== irr_worklist
.length () / 8)
5201 irr_worklist
.block_remove (0, i
);
5205 node
= irr_worklist
[i
];
5206 d
= get_cg_data (&node
, true);
5207 d
->in_worklist
= false;
5209 if (d
->want_irr_scan_normal
)
5211 d
->want_irr_scan_normal
= false;
5212 ipa_tm_scan_irr_function (node
, false);
5214 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5215 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5218 /* For every function on the callee list, collect the tm_may_enter_irr
5220 irr_worklist
.truncate (0);
5221 for (i
= 0; i
< tm_callees
.length (); ++i
)
5223 node
= tm_callees
[i
];
5224 if (ipa_tm_mayenterirr_function (node
))
5226 d
= get_cg_data (&node
, true);
5227 gcc_assert (d
->in_worklist
== false);
5228 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5232 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5233 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5235 struct cgraph_node
*caller
;
5236 struct cgraph_edge
*e
;
5237 struct ipa_ref
*ref
;
5240 if (i
> 256 && i
== irr_worklist
.length () / 8)
5242 irr_worklist
.block_remove (0, i
);
5246 node
= irr_worklist
[i
];
5247 d
= get_cg_data (&node
, true);
5248 d
->in_worklist
= false;
5249 node
->local
.tm_may_enter_irr
= true;
5251 /* Propagate back to normal callers. */
5252 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5255 if (!is_tm_safe_or_pure (caller
->symbol
.decl
)
5256 && !caller
->local
.tm_may_enter_irr
)
5258 d
= get_cg_data (&caller
, true);
5259 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5263 /* Propagate back to referring aliases as well. */
5264 for (j
= 0; ipa_ref_list_referring_iterate (&node
->symbol
.ref_list
, j
, ref
); j
++)
5266 caller
= cgraph (ref
->referring
);
5267 if (ref
->use
== IPA_REF_ALIAS
5268 && !caller
->local
.tm_may_enter_irr
)
5270 /* ?? Do not traverse aliases here. */
5271 d
= get_cg_data (&caller
, false);
5272 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5277 /* Now validate all tm_safe functions, and all atomic regions in
5279 FOR_EACH_DEFINED_FUNCTION (node
)
5281 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5283 d
= get_cg_data (&node
, true);
5284 if (is_tm_safe (node
->symbol
.decl
))
5285 ipa_tm_diagnose_tm_safe (node
);
5286 else if (d
->all_tm_regions
)
5287 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5290 /* Create clones. Do those that are not irrevocable and have a
5291 positive call count. Do those publicly visible functions that
5292 the user directed us to clone. */
5293 for (i
= 0; i
< tm_callees
.length (); ++i
)
5297 node
= tm_callees
[i
];
5298 if (node
->same_body_alias
)
5301 a
= cgraph_function_body_availability (node
);
5302 d
= get_cg_data (&node
, true);
5304 if (a
<= AVAIL_NOT_AVAILABLE
)
5305 doit
= is_tm_callable (node
->symbol
.decl
);
5306 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->symbol
.decl
))
5308 else if (!d
->is_irrevocable
5309 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5313 ipa_tm_create_version (node
);
5316 /* Redirect calls to the new clones, and insert irrevocable marks. */
5317 for (i
= 0; i
< tm_callees
.length (); ++i
)
5319 node
= tm_callees
[i
];
5322 d
= get_cg_data (&node
, true);
5324 ipa_tm_transform_clone (node
);
5327 FOR_EACH_DEFINED_FUNCTION (node
)
5329 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5331 d
= get_cg_data (&node
, true);
5332 if (d
->all_tm_regions
)
5333 ipa_tm_transform_transaction (node
);
5336 /* Free and clear all data structures. */
5337 tm_callees
.release ();
5338 irr_worklist
.release ();
5339 bitmap_obstack_release (&tm_obstack
);
5340 free_original_copy_tables ();
5342 FOR_EACH_FUNCTION (node
)
5343 node
->symbol
.aux
= NULL
;
5345 #ifdef ENABLE_CHECKING
5352 struct simple_ipa_opt_pass pass_ipa_tm
=
5357 OPTGROUP_NONE
, /* optinfo_flags */
5359 ipa_tm_execute
, /* execute */
5362 0, /* static_pass_number */
5363 TV_TRANS_MEM
, /* tv_id */
5364 PROP_ssa
| PROP_cfg
, /* properties_required */
5365 0, /* properties_provided */
5366 0, /* properties_destroyed */
5367 0, /* todo_flags_start */
5368 0, /* todo_flags_finish */
5372 #include "gt-trans-mem.h"