1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
23 #include "hash-table.h"
31 #include "gimple-iterator.h"
32 #include "gimplify-me.h"
33 #include "gimple-walk.h"
34 #include "gimple-ssa.h"
37 #include "stringpool.h"
38 #include "tree-ssanames.h"
39 #include "tree-into-ssa.h"
40 #include "tree-pass.h"
41 #include "tree-inline.h"
42 #include "diagnostic-core.h"
45 #include "trans-mem.h"
48 #include "langhooks.h"
49 #include "gimple-pretty-print.h"
51 #include "tree-ssa-address.h"
54 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 2000 - 1)
55 #define PROB_VERY_LIKELY (PROB_ALWAYS - PROB_VERY_UNLIKELY)
56 #define PROB_UNLIKELY (REG_BR_PROB_BASE / 5 - 1)
57 #define PROB_LIKELY (PROB_ALWAYS - PROB_VERY_LIKELY)
58 #define PROB_ALWAYS (REG_BR_PROB_BASE)
60 #define A_RUNINSTRUMENTEDCODE 0x0001
61 #define A_RUNUNINSTRUMENTEDCODE 0x0002
62 #define A_SAVELIVEVARIABLES 0x0004
63 #define A_RESTORELIVEVARIABLES 0x0008
64 #define A_ABORTTRANSACTION 0x0010
66 #define AR_USERABORT 0x0001
67 #define AR_USERRETRY 0x0002
68 #define AR_TMCONFLICT 0x0004
69 #define AR_EXCEPTIONBLOCKABORT 0x0008
70 #define AR_OUTERABORT 0x0010
72 #define MODE_SERIALIRREVOCABLE 0x0000
75 /* The representation of a transaction changes several times during the
76 lowering process. In the beginning, in the front-end we have the
77 GENERIC tree TRANSACTION_EXPR. For example,
85 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
86 trivially replaced with a GIMPLE_TRANSACTION node.
88 During pass_lower_tm, we examine the body of transactions looking
89 for aborts. Transactions that do not contain an abort may be
90 merged into an outer transaction. We also add a TRY-FINALLY node
91 to arrange for the transaction to be committed on any exit.
93 [??? Think about how this arrangement affects throw-with-commit
94 and throw-with-abort operations. In this case we want the TRY to
95 handle gotos, but not to catch any exceptions because the transaction
96 will already be closed.]
98 GIMPLE_TRANSACTION [label=NULL] {
105 __builtin___tm_abort ();
107 __builtin___tm_commit ();
111 During pass_lower_eh, we create EH regions for the transactions,
112 intermixed with the regular EH stuff. This gives us a nice persistent
113 mapping (all the way through rtl) from transactional memory operation
114 back to the transaction, which allows us to get the abnormal edges
115 correct to model transaction aborts and restarts:
117 GIMPLE_TRANSACTION [label=over]
123 __builtin___tm_abort ();
124 __builtin___tm_commit ();
127 This is the end of all_lowering_passes, and so is what is present
128 during the IPA passes, and through all of the optimization passes.
130 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
131 functions and mark functions for cloning.
133 At the end of gimple optimization, before exiting SSA form,
134 pass_tm_edges replaces statements that perform transactional
135 memory operations with the appropriate TM builtins, and swap
136 out function calls with their transactional clones. At this
137 point we introduce the abnormal transaction restart edges and
138 complete lowering of the GIMPLE_TRANSACTION node.
140 x = __builtin___tm_start (MAY_ABORT);
142 if (x & abort_transaction)
145 t0 = __builtin___tm_load (global);
147 __builtin___tm_store (&global, t1);
149 __builtin___tm_abort ();
150 __builtin___tm_commit ();
154 static void *expand_regions (struct tm_region
*,
155 void *(*callback
)(struct tm_region
*, void *),
159 /* Return the attributes we want to examine for X, or NULL if it's not
160 something we examine. We look at function types, but allow pointers
161 to function types and function decls and peek through. */
164 get_attrs_for (const_tree x
)
166 switch (TREE_CODE (x
))
169 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
176 if (TREE_CODE (x
) != POINTER_TYPE
)
182 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
188 return TYPE_ATTRIBUTES (x
);
192 /* Return true if X has been marked TM_PURE. */
195 is_tm_pure (const_tree x
)
199 switch (TREE_CODE (x
))
210 if (TREE_CODE (x
) != POINTER_TYPE
)
216 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
221 flags
= flags_from_decl_or_type (x
);
222 return (flags
& ECF_TM_PURE
) != 0;
225 /* Return true if X has been marked TM_IRREVOCABLE. */
228 is_tm_irrevocable (tree x
)
230 tree attrs
= get_attrs_for (x
);
232 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
235 /* A call to the irrevocable builtin is by definition,
237 if (TREE_CODE (x
) == ADDR_EXPR
)
238 x
= TREE_OPERAND (x
, 0);
239 if (TREE_CODE (x
) == FUNCTION_DECL
240 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
241 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
247 /* Return true if X has been marked TM_SAFE. */
250 is_tm_safe (const_tree x
)
254 tree attrs
= get_attrs_for (x
);
257 if (lookup_attribute ("transaction_safe", attrs
))
259 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
266 /* Return true if CALL is const, or tm_pure. */
269 is_tm_pure_call (gimple call
)
271 tree fn
= gimple_call_fn (call
);
273 if (TREE_CODE (fn
) == ADDR_EXPR
)
275 fn
= TREE_OPERAND (fn
, 0);
276 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
281 return is_tm_pure (fn
);
284 /* Return true if X has been marked TM_CALLABLE. */
287 is_tm_callable (tree x
)
289 tree attrs
= get_attrs_for (x
);
292 if (lookup_attribute ("transaction_callable", attrs
))
294 if (lookup_attribute ("transaction_safe", attrs
))
296 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
302 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
305 is_tm_may_cancel_outer (tree x
)
307 tree attrs
= get_attrs_for (x
);
309 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
313 /* Return true for built in functions that "end" a transaction. */
316 is_tm_ending_fndecl (tree fndecl
)
318 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
319 switch (DECL_FUNCTION_CODE (fndecl
))
321 case BUILT_IN_TM_COMMIT
:
322 case BUILT_IN_TM_COMMIT_EH
:
323 case BUILT_IN_TM_ABORT
:
324 case BUILT_IN_TM_IRREVOCABLE
:
333 /* Return true if STMT is a built in function call that "ends" a
337 is_tm_ending (gimple stmt
)
341 if (gimple_code (stmt
) != GIMPLE_CALL
)
344 fndecl
= gimple_call_fndecl (stmt
);
345 return (fndecl
!= NULL_TREE
346 && is_tm_ending_fndecl (fndecl
));
349 /* Return true if STMT is a TM load. */
352 is_tm_load (gimple stmt
)
356 if (gimple_code (stmt
) != GIMPLE_CALL
)
359 fndecl
= gimple_call_fndecl (stmt
);
360 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
361 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
364 /* Same as above, but for simple TM loads, that is, not the
365 after-write, after-read, etc optimized variants. */
368 is_tm_simple_load (gimple stmt
)
372 if (gimple_code (stmt
) != GIMPLE_CALL
)
375 fndecl
= gimple_call_fndecl (stmt
);
376 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
378 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
379 return (fcode
== BUILT_IN_TM_LOAD_1
380 || fcode
== BUILT_IN_TM_LOAD_2
381 || fcode
== BUILT_IN_TM_LOAD_4
382 || fcode
== BUILT_IN_TM_LOAD_8
383 || fcode
== BUILT_IN_TM_LOAD_FLOAT
384 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
385 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
386 || fcode
== BUILT_IN_TM_LOAD_M64
387 || fcode
== BUILT_IN_TM_LOAD_M128
388 || fcode
== BUILT_IN_TM_LOAD_M256
);
393 /* Return true if STMT is a TM store. */
396 is_tm_store (gimple stmt
)
400 if (gimple_code (stmt
) != GIMPLE_CALL
)
403 fndecl
= gimple_call_fndecl (stmt
);
404 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
405 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
408 /* Same as above, but for simple TM stores, that is, not the
409 after-write, after-read, etc optimized variants. */
412 is_tm_simple_store (gimple stmt
)
416 if (gimple_code (stmt
) != GIMPLE_CALL
)
419 fndecl
= gimple_call_fndecl (stmt
);
420 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
422 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
423 return (fcode
== BUILT_IN_TM_STORE_1
424 || fcode
== BUILT_IN_TM_STORE_2
425 || fcode
== BUILT_IN_TM_STORE_4
426 || fcode
== BUILT_IN_TM_STORE_8
427 || fcode
== BUILT_IN_TM_STORE_FLOAT
428 || fcode
== BUILT_IN_TM_STORE_DOUBLE
429 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
430 || fcode
== BUILT_IN_TM_STORE_M64
431 || fcode
== BUILT_IN_TM_STORE_M128
432 || fcode
== BUILT_IN_TM_STORE_M256
);
437 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
440 is_tm_abort (tree fndecl
)
443 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
444 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
447 /* Build a GENERIC tree for a user abort. This is called by front ends
448 while transforming the __tm_abort statement. */
451 build_tm_abort_call (location_t loc
, bool is_outer
)
453 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
454 build_int_cst (integer_type_node
,
456 | (is_outer
? AR_OUTERABORT
: 0)));
459 /* Common gateing function for several of the TM passes. */
467 /* Map for aribtrary function replacement under TM, as created
468 by the tm_wrap attribute. */
470 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
474 record_tm_replacement (tree from
, tree to
)
476 struct tree_map
**slot
, *h
;
478 /* Do not inline wrapper functions that will get replaced in the TM
481 Suppose you have foo() that will get replaced into tmfoo(). Make
482 sure the inliner doesn't try to outsmart us and inline foo()
483 before we get a chance to do the TM replacement. */
484 DECL_UNINLINABLE (from
) = 1;
486 if (tm_wrap_map
== NULL
)
487 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
489 h
= ggc_alloc_tree_map ();
490 h
->hash
= htab_hash_pointer (from
);
494 slot
= (struct tree_map
**)
495 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
499 /* Return a TM-aware replacement function for DECL. */
502 find_tm_replacement_function (tree fndecl
)
506 struct tree_map
*h
, in
;
508 in
.base
.from
= fndecl
;
509 in
.hash
= htab_hash_pointer (fndecl
);
510 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
515 /* ??? We may well want TM versions of most of the common <string.h>
516 functions. For now, we've already these two defined. */
517 /* Adjust expand_call_tm() attributes as necessary for the cases
519 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
520 switch (DECL_FUNCTION_CODE (fndecl
))
522 case BUILT_IN_MEMCPY
:
523 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
524 case BUILT_IN_MEMMOVE
:
525 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
526 case BUILT_IN_MEMSET
:
527 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
535 /* When appropriate, record TM replacement for memory allocation functions.
537 FROM is the FNDECL to wrap. */
539 tm_malloc_replacement (tree from
)
544 if (TREE_CODE (from
) != FUNCTION_DECL
)
547 /* If we have a previous replacement, the user must be explicitly
548 wrapping malloc/calloc/free. They better know what they're
550 if (find_tm_replacement_function (from
))
553 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
555 if (!strcmp (str
, "malloc"))
556 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
557 else if (!strcmp (str
, "calloc"))
558 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
559 else if (!strcmp (str
, "free"))
560 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
564 TREE_NOTHROW (to
) = 0;
566 record_tm_replacement (from
, to
);
569 /* Diagnostics for tm_safe functions/regions. Called by the front end
570 once we've lowered the function to high-gimple. */
572 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
573 Process exactly one statement. WI->INFO is set to non-null when in
574 the context of a tm_safe function, and null for a __transaction block. */
576 #define DIAG_TM_OUTER 1
577 #define DIAG_TM_SAFE 2
578 #define DIAG_TM_RELAXED 4
582 unsigned int summary_flags
: 8;
583 unsigned int block_flags
: 8;
584 unsigned int func_flags
: 8;
585 unsigned int saw_volatile
: 1;
589 /* Return true if T is a volatile variable of some kind. */
592 volatile_var_p (tree t
)
594 return (SSA_VAR_P (t
)
595 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
598 /* Tree callback function for diagnose_tm pass. */
601 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
604 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
605 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
607 if (volatile_var_p (*tp
)
608 && d
->block_flags
& DIAG_TM_SAFE
612 error_at (gimple_location (d
->stmt
),
613 "invalid volatile use of %qD inside transaction",
621 is_tm_safe_or_pure (const_tree x
)
623 return is_tm_safe (x
) || is_tm_pure (x
);
627 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
628 struct walk_stmt_info
*wi
)
630 gimple stmt
= gsi_stmt (*gsi
);
631 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
633 /* Save stmt for use in leaf analysis. */
636 switch (gimple_code (stmt
))
640 tree fn
= gimple_call_fn (stmt
);
642 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
643 && is_tm_may_cancel_outer (fn
))
644 error_at (gimple_location (stmt
),
645 "%<transaction_may_cancel_outer%> function call not within"
646 " outer transaction or %<transaction_may_cancel_outer%>");
648 if (d
->summary_flags
& DIAG_TM_SAFE
)
650 bool is_safe
, direct_call_p
;
653 if (TREE_CODE (fn
) == ADDR_EXPR
654 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
656 direct_call_p
= true;
657 replacement
= TREE_OPERAND (fn
, 0);
658 replacement
= find_tm_replacement_function (replacement
);
664 direct_call_p
= false;
665 replacement
= NULL_TREE
;
668 if (is_tm_safe_or_pure (fn
))
670 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
672 /* A function explicitly marked transaction_callable as
673 opposed to transaction_safe is being defined to be
674 unsafe as part of its ABI, regardless of its contents. */
677 else if (direct_call_p
)
679 if (flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
681 else if (replacement
)
683 /* ??? At present we've been considering replacements
684 merely transaction_callable, and therefore might
685 enter irrevocable. The tm_wrap attribute has not
686 yet made it into the new language spec. */
691 /* ??? Diagnostics for unmarked direct calls moved into
692 the IPA pass. Section 3.2 of the spec details how
693 functions not marked should be considered "implicitly
694 safe" based on having examined the function body. */
700 /* An unmarked indirect call. Consider it unsafe even
701 though optimization may yet figure out how to inline. */
707 if (TREE_CODE (fn
) == ADDR_EXPR
)
708 fn
= TREE_OPERAND (fn
, 0);
709 if (d
->block_flags
& DIAG_TM_SAFE
)
712 error_at (gimple_location (stmt
),
713 "unsafe function call %qD within "
714 "atomic transaction", fn
);
717 if (!DECL_P (fn
) || DECL_NAME (fn
))
718 error_at (gimple_location (stmt
),
719 "unsafe function call %qE within "
720 "atomic transaction", fn
);
722 error_at (gimple_location (stmt
),
723 "unsafe indirect function call within "
724 "atomic transaction");
730 error_at (gimple_location (stmt
),
731 "unsafe function call %qD within "
732 "%<transaction_safe%> function", fn
);
735 if (!DECL_P (fn
) || DECL_NAME (fn
))
736 error_at (gimple_location (stmt
),
737 "unsafe function call %qE within "
738 "%<transaction_safe%> function", fn
);
740 error_at (gimple_location (stmt
),
741 "unsafe indirect function call within "
742 "%<transaction_safe%> function");
751 /* ??? We ought to come up with a way to add attributes to
752 asm statements, and then add "transaction_safe" to it.
753 Either that or get the language spec to resurrect __tm_waiver. */
754 if (d
->block_flags
& DIAG_TM_SAFE
)
755 error_at (gimple_location (stmt
),
756 "asm not allowed in atomic transaction");
757 else if (d
->func_flags
& DIAG_TM_SAFE
)
758 error_at (gimple_location (stmt
),
759 "asm not allowed in %<transaction_safe%> function");
762 case GIMPLE_TRANSACTION
:
764 unsigned char inner_flags
= DIAG_TM_SAFE
;
766 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
768 if (d
->block_flags
& DIAG_TM_SAFE
)
769 error_at (gimple_location (stmt
),
770 "relaxed transaction in atomic transaction");
771 else if (d
->func_flags
& DIAG_TM_SAFE
)
772 error_at (gimple_location (stmt
),
773 "relaxed transaction in %<transaction_safe%> function");
774 inner_flags
= DIAG_TM_RELAXED
;
776 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
779 error_at (gimple_location (stmt
),
780 "outer transaction in transaction");
781 else if (d
->func_flags
& DIAG_TM_OUTER
)
782 error_at (gimple_location (stmt
),
783 "outer transaction in "
784 "%<transaction_may_cancel_outer%> function");
785 else if (d
->func_flags
& DIAG_TM_SAFE
)
786 error_at (gimple_location (stmt
),
787 "outer transaction in %<transaction_safe%> function");
788 inner_flags
|= DIAG_TM_OUTER
;
791 *handled_ops_p
= true;
792 if (gimple_transaction_body (stmt
))
794 struct walk_stmt_info wi_inner
;
795 struct diagnose_tm d_inner
;
797 memset (&d_inner
, 0, sizeof (d_inner
));
798 d_inner
.func_flags
= d
->func_flags
;
799 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
800 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
802 memset (&wi_inner
, 0, sizeof (wi_inner
));
803 wi_inner
.info
= &d_inner
;
805 walk_gimple_seq (gimple_transaction_body (stmt
),
806 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
819 diagnose_tm_blocks (void)
821 struct walk_stmt_info wi
;
822 struct diagnose_tm d
;
824 memset (&d
, 0, sizeof (d
));
825 if (is_tm_may_cancel_outer (current_function_decl
))
826 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
827 else if (is_tm_safe (current_function_decl
))
828 d
.func_flags
= DIAG_TM_SAFE
;
829 d
.summary_flags
= d
.func_flags
;
831 memset (&wi
, 0, sizeof (wi
));
834 walk_gimple_seq (gimple_body (current_function_decl
),
835 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
842 const pass_data pass_data_diagnose_tm_blocks
=
844 GIMPLE_PASS
, /* type */
845 "*diagnose_tm_blocks", /* name */
846 OPTGROUP_NONE
, /* optinfo_flags */
848 true, /* has_execute */
849 TV_TRANS_MEM
, /* tv_id */
850 PROP_gimple_any
, /* properties_required */
851 0, /* properties_provided */
852 0, /* properties_destroyed */
853 0, /* todo_flags_start */
854 0, /* todo_flags_finish */
857 class pass_diagnose_tm_blocks
: public gimple_opt_pass
860 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
861 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
864 /* opt_pass methods: */
865 bool gate () { return gate_tm (); }
866 unsigned int execute () { return diagnose_tm_blocks (); }
868 }; // class pass_diagnose_tm_blocks
873 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
875 return new pass_diagnose_tm_blocks (ctxt
);
878 /* Instead of instrumenting thread private memory, we save the
879 addresses in a log which we later use to save/restore the addresses
880 upon transaction start/restart.
882 The log is keyed by address, where each element contains individual
883 statements among different code paths that perform the store.
885 This log is later used to generate either plain save/restore of the
886 addresses upon transaction start/restart, or calls to the ITM_L*
889 So for something like:
891 struct large { int x[1000]; };
892 struct large lala = { 0 };
898 We can either save/restore:
901 trxn = _ITM_startTransaction ();
902 if (trxn & a_saveLiveVariables)
903 tmp_lala1 = lala.x[i];
904 else if (a & a_restoreLiveVariables)
905 lala.x[i] = tmp_lala1;
907 or use the logging functions:
910 trxn = _ITM_startTransaction ();
911 _ITM_LU4 (&lala.x[i]);
913 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
914 far up the dominator tree to shadow all of the writes to a given
915 location (thus reducing the total number of logging calls), but not
916 so high as to be called on a path that does not perform a
919 /* One individual log entry. We may have multiple statements for the
920 same location if neither dominate each other (on different
922 typedef struct tm_log_entry
924 /* Address to save. */
926 /* Entry block for the transaction this address occurs in. */
927 basic_block entry_block
;
928 /* Dominating statements the store occurs in. */
930 /* Initially, while we are building the log, we place a nonzero
931 value here to mean that this address *will* be saved with a
932 save/restore sequence. Later, when generating the save sequence
933 we place the SSA temp generated here. */
938 /* Log entry hashtable helpers. */
940 struct log_entry_hasher
942 typedef tm_log_entry value_type
;
943 typedef tm_log_entry compare_type
;
944 static inline hashval_t
hash (const value_type
*);
945 static inline bool equal (const value_type
*, const compare_type
*);
946 static inline void remove (value_type
*);
949 /* Htab support. Return hash value for a `tm_log_entry'. */
951 log_entry_hasher::hash (const value_type
*log
)
953 return iterative_hash_expr (log
->addr
, 0);
956 /* Htab support. Return true if two log entries are the same. */
958 log_entry_hasher::equal (const value_type
*log1
, const compare_type
*log2
)
962 rth: I suggest that we get rid of the component refs etc.
963 I.e. resolve the reference to base + offset.
965 We may need to actually finish a merge with mainline for this,
966 since we'd like to be presented with Richi's MEM_REF_EXPRs more
967 often than not. But in the meantime your tm_log_entry could save
968 the results of get_inner_reference.
970 See: g++.dg/tm/pr46653.C
973 /* Special case plain equality because operand_equal_p() below will
974 return FALSE if the addresses are equal but they have
975 side-effects (e.g. a volatile address). */
976 if (log1
->addr
== log2
->addr
)
979 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
982 /* Htab support. Free one tm_log_entry. */
984 log_entry_hasher::remove (value_type
*lp
)
986 lp
->stmts
.release ();
991 /* The actual log. */
992 static hash_table
<log_entry_hasher
> tm_log
;
994 /* Addresses to log with a save/restore sequence. These should be in
996 static vec
<tree
> tm_log_save_addresses
;
998 enum thread_memory_type
1002 mem_transaction_local
,
1006 typedef struct tm_new_mem_map
1008 /* SSA_NAME being dereferenced. */
1010 enum thread_memory_type local_new_memory
;
1013 /* Hashtable helpers. */
1015 struct tm_mem_map_hasher
: typed_free_remove
<tm_new_mem_map_t
>
1017 typedef tm_new_mem_map_t value_type
;
1018 typedef tm_new_mem_map_t compare_type
;
1019 static inline hashval_t
hash (const value_type
*);
1020 static inline bool equal (const value_type
*, const compare_type
*);
1024 tm_mem_map_hasher::hash (const value_type
*v
)
1026 return (intptr_t)v
->val
>> 4;
1030 tm_mem_map_hasher::equal (const value_type
*v
, const compare_type
*c
)
1032 return v
->val
== c
->val
;
1035 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1036 of memory (malloc, alloc, etc). */
1037 static hash_table
<tm_mem_map_hasher
> tm_new_mem_hash
;
1039 /* Initialize logging data structures. */
1044 tm_new_mem_hash
.create (5);
1045 tm_log_save_addresses
.create (5);
1048 /* Free logging data structures. */
1050 tm_log_delete (void)
1053 tm_new_mem_hash
.dispose ();
1054 tm_log_save_addresses
.release ();
1057 /* Return true if MEM is a transaction invariant memory for the TM
1058 region starting at REGION_ENTRY_BLOCK. */
1060 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1062 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1063 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1067 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1068 return def_bb
!= region_entry_block
1069 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1072 mem
= strip_invariant_refs (mem
);
1073 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1076 /* Given an address ADDR in STMT, find it in the memory log or add it,
1077 making sure to keep only the addresses highest in the dominator
1080 ENTRY_BLOCK is the entry_block for the transaction.
1082 If we find the address in the log, make sure it's either the same
1083 address, or an equivalent one that dominates ADDR.
1085 If we find the address, but neither ADDR dominates the found
1086 address, nor the found one dominates ADDR, we're on different
1087 execution paths. Add it.
1089 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1092 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1094 tm_log_entry
**slot
;
1095 struct tm_log_entry l
, *lp
;
1098 slot
= tm_log
.find_slot (&l
, INSERT
);
1101 tree type
= TREE_TYPE (addr
);
1103 lp
= XNEW (struct tm_log_entry
);
1107 /* Small invariant addresses can be handled as save/restores. */
1109 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1110 && TYPE_SIZE_UNIT (type
) != NULL
1111 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1112 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1113 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1114 /* We must be able to copy this type normally. I.e., no
1115 special constructors and the like. */
1116 && !TREE_ADDRESSABLE (type
))
1118 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1119 lp
->stmts
.create (0);
1120 lp
->entry_block
= entry_block
;
1121 /* Save addresses separately in dominator order so we don't
1122 get confused by overlapping addresses in the save/restore
1124 tm_log_save_addresses
.safe_push (lp
->addr
);
1128 /* Use the logging functions. */
1129 lp
->stmts
.create (5);
1130 lp
->stmts
.quick_push (stmt
);
1131 lp
->save_var
= NULL
;
1141 /* If we're generating a save/restore sequence, we don't care
1142 about statements. */
1146 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1148 if (stmt
== oldstmt
)
1150 /* We already have a store to the same address, higher up the
1151 dominator tree. Nothing to do. */
1152 if (dominated_by_p (CDI_DOMINATORS
,
1153 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1155 /* We should be processing blocks in dominator tree order. */
1156 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1157 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1159 /* Store is on a different code path. */
1160 lp
->stmts
.safe_push (stmt
);
1164 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1165 result, insert the new statements before GSI. */
1168 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1170 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1171 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1173 x
= build_fold_addr_expr (x
);
1174 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1177 /* Instrument one address with the logging functions.
1178 ADDR is the address to save.
1179 STMT is the statement before which to place it. */
1181 tm_log_emit_stmt (tree addr
, gimple stmt
)
1183 tree type
= TREE_TYPE (addr
);
1184 tree size
= TYPE_SIZE_UNIT (type
);
1185 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1187 enum built_in_function code
= BUILT_IN_TM_LOG
;
1189 if (type
== float_type_node
)
1190 code
= BUILT_IN_TM_LOG_FLOAT
;
1191 else if (type
== double_type_node
)
1192 code
= BUILT_IN_TM_LOG_DOUBLE
;
1193 else if (type
== long_double_type_node
)
1194 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1195 else if (tree_fits_uhwi_p (size
))
1197 unsigned int n
= tree_to_uhwi (size
);
1201 code
= BUILT_IN_TM_LOG_1
;
1204 code
= BUILT_IN_TM_LOG_2
;
1207 code
= BUILT_IN_TM_LOG_4
;
1210 code
= BUILT_IN_TM_LOG_8
;
1213 code
= BUILT_IN_TM_LOG
;
1214 if (TREE_CODE (type
) == VECTOR_TYPE
)
1216 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1217 code
= BUILT_IN_TM_LOG_M64
;
1218 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1219 code
= BUILT_IN_TM_LOG_M128
;
1220 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1221 code
= BUILT_IN_TM_LOG_M256
;
1227 addr
= gimplify_addr (&gsi
, addr
);
1228 if (code
== BUILT_IN_TM_LOG
)
1229 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1231 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1232 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1235 /* Go through the log and instrument address that must be instrumented
1236 with the logging functions. Leave the save/restore addresses for
1241 hash_table
<log_entry_hasher
>::iterator hi
;
1242 struct tm_log_entry
*lp
;
1244 FOR_EACH_HASH_TABLE_ELEMENT (tm_log
, lp
, tm_log_entry_t
, hi
)
1251 fprintf (dump_file
, "TM thread private mem logging: ");
1252 print_generic_expr (dump_file
, lp
->addr
, 0);
1253 fprintf (dump_file
, "\n");
1259 fprintf (dump_file
, "DUMPING to variable\n");
1265 fprintf (dump_file
, "DUMPING with logging functions\n");
1266 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1267 tm_log_emit_stmt (lp
->addr
, stmt
);
1272 /* Emit the save sequence for the corresponding addresses in the log.
1273 ENTRY_BLOCK is the entry block for the transaction.
1274 BB is the basic block to insert the code in. */
1276 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1279 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1281 struct tm_log_entry l
, *lp
;
1283 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1285 l
.addr
= tm_log_save_addresses
[i
];
1286 lp
= *(tm_log
.find_slot (&l
, NO_INSERT
));
1287 gcc_assert (lp
->save_var
!= NULL
);
1289 /* We only care about variables in the current transaction. */
1290 if (lp
->entry_block
!= entry_block
)
1293 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1295 /* Make sure we can create an SSA_NAME for this type. For
1296 instance, aggregates aren't allowed, in which case the system
1297 will create a VOP for us and everything will just work. */
1298 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1300 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1301 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1304 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1308 /* Emit the restore sequence for the corresponding addresses in the log.
1309 ENTRY_BLOCK is the entry block for the transaction.
1310 BB is the basic block to insert the code in. */
1312 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1315 struct tm_log_entry l
, *lp
;
1316 gimple_stmt_iterator gsi
;
1319 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1321 l
.addr
= tm_log_save_addresses
[i
];
1322 lp
= *(tm_log
.find_slot (&l
, NO_INSERT
));
1323 gcc_assert (lp
->save_var
!= NULL
);
1325 /* We only care about variables in the current transaction. */
1326 if (lp
->entry_block
!= entry_block
)
1329 /* Restores are in LIFO order from the saves in case we have
1331 gsi
= gsi_start_bb (bb
);
1333 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1334 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1339 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1340 struct walk_stmt_info
*);
1341 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1342 struct walk_stmt_info
*);
1344 /* Evaluate an address X being dereferenced and determine if it
1345 originally points to a non aliased new chunk of memory (malloc,
1348 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1349 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1350 Return MEM_NON_LOCAL otherwise.
1352 ENTRY_BLOCK is the entry block to the transaction containing the
1353 dereference of X. */
1354 static enum thread_memory_type
1355 thread_private_new_memory (basic_block entry_block
, tree x
)
1358 enum tree_code code
;
1359 tm_new_mem_map_t
**slot
;
1360 tm_new_mem_map_t elt
, *elt_p
;
1362 enum thread_memory_type retval
= mem_transaction_local
;
1365 || TREE_CODE (x
) != SSA_NAME
1366 /* Possible uninitialized use, or a function argument. In
1367 either case, we don't care. */
1368 || SSA_NAME_IS_DEFAULT_DEF (x
))
1369 return mem_non_local
;
1371 /* Look in cache first. */
1373 slot
= tm_new_mem_hash
.find_slot (&elt
, INSERT
);
1376 return elt_p
->local_new_memory
;
1378 /* Optimistically assume the memory is transaction local during
1379 processing. This catches recursion into this variable. */
1380 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1382 elt_p
->local_new_memory
= mem_transaction_local
;
1384 /* Search DEF chain to find the original definition of this address. */
1387 if (ptr_deref_may_alias_global_p (x
))
1389 /* Address escapes. This is not thread-private. */
1390 retval
= mem_non_local
;
1391 goto new_memory_ret
;
1394 stmt
= SSA_NAME_DEF_STMT (x
);
1396 /* If the malloc call is outside the transaction, this is
1398 if (retval
!= mem_thread_local
1399 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1400 retval
= mem_thread_local
;
1402 if (is_gimple_assign (stmt
))
1404 code
= gimple_assign_rhs_code (stmt
);
1405 /* x = foo ==> foo */
1406 if (code
== SSA_NAME
)
1407 x
= gimple_assign_rhs1 (stmt
);
1408 /* x = foo + n ==> foo */
1409 else if (code
== POINTER_PLUS_EXPR
)
1410 x
= gimple_assign_rhs1 (stmt
);
1411 /* x = (cast*) foo ==> foo */
1412 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1413 x
= gimple_assign_rhs1 (stmt
);
1414 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1415 else if (code
== COND_EXPR
)
1417 tree op1
= gimple_assign_rhs2 (stmt
);
1418 tree op2
= gimple_assign_rhs3 (stmt
);
1419 enum thread_memory_type mem
;
1420 retval
= thread_private_new_memory (entry_block
, op1
);
1421 if (retval
== mem_non_local
)
1422 goto new_memory_ret
;
1423 mem
= thread_private_new_memory (entry_block
, op2
);
1424 retval
= MIN (retval
, mem
);
1425 goto new_memory_ret
;
1429 retval
= mem_non_local
;
1430 goto new_memory_ret
;
1435 if (gimple_code (stmt
) == GIMPLE_PHI
)
1438 enum thread_memory_type mem
;
1439 tree phi_result
= gimple_phi_result (stmt
);
1441 /* If any of the ancestors are non-local, we are sure to
1442 be non-local. Otherwise we can avoid doing anything
1443 and inherit what has already been generated. */
1445 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1447 tree op
= PHI_ARG_DEF (stmt
, i
);
1449 /* Exclude self-assignment. */
1450 if (phi_result
== op
)
1453 mem
= thread_private_new_memory (entry_block
, op
);
1454 if (mem
== mem_non_local
)
1457 goto new_memory_ret
;
1459 retval
= MIN (retval
, mem
);
1461 goto new_memory_ret
;
1466 while (TREE_CODE (x
) == SSA_NAME
);
1468 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1469 /* Thread-local or transaction-local. */
1472 retval
= mem_non_local
;
1475 elt_p
->local_new_memory
= retval
;
1479 /* Determine whether X has to be instrumented using a read
1482 ENTRY_BLOCK is the entry block for the region where stmt resides
1483 in. NULL if unknown.
1485 STMT is the statement in which X occurs in. It is used for thread
1486 private memory instrumentation. If no TPM instrumentation is
1487 desired, STMT should be null. */
1489 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1492 while (handled_component_p (x
))
1493 x
= TREE_OPERAND (x
, 0);
1495 switch (TREE_CODE (x
))
1500 enum thread_memory_type ret
;
1502 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1503 if (ret
== mem_non_local
)
1505 if (stmt
&& ret
== mem_thread_local
)
1506 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1507 tm_log_add (entry_block
, orig
, stmt
);
1509 /* Transaction-locals require nothing at all. For malloc, a
1510 transaction restart frees the memory and we reallocate.
1511 For alloca, the stack pointer gets reset by the retry and
1516 case TARGET_MEM_REF
:
1517 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1519 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1520 if (TREE_CODE (x
) == PARM_DECL
)
1522 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1528 if (DECL_BY_REFERENCE (x
))
1530 /* ??? This value is a pointer, but aggregate_value_p has been
1531 jigged to return true which confuses needs_to_live_in_memory.
1532 This ought to be cleaned up generically.
1534 FIXME: Verify this still happens after the next mainline
1535 merge. Testcase ie g++.dg/tm/pr47554.C.
1540 if (is_global_var (x
))
1541 return !TREE_READONLY (x
);
1542 if (/* FIXME: This condition should actually go below in the
1543 tm_log_add() call, however is_call_clobbered() depends on
1544 aliasing info which is not available during
1545 gimplification. Since requires_barrier() gets called
1546 during lower_sequence_tm/gimplification, leave the call
1547 to needs_to_live_in_memory until we eliminate
1548 lower_sequence_tm altogether. */
1549 needs_to_live_in_memory (x
))
1553 /* For local memory that doesn't escape (aka thread private
1554 memory), we can either save the value at the beginning of
1555 the transaction and restore on restart, or call a tm
1556 function to dynamically save and restore on restart
1559 tm_log_add (entry_block
, orig
, stmt
);
1568 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1569 a transaction region. */
1572 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1574 gimple stmt
= gsi_stmt (*gsi
);
1576 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1577 *state
|= GTMA_HAVE_LOAD
;
1578 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1579 *state
|= GTMA_HAVE_STORE
;
1582 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1585 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1587 gimple stmt
= gsi_stmt (*gsi
);
1590 if (is_tm_pure_call (stmt
))
1593 /* Check if this call is a transaction abort. */
1594 fn
= gimple_call_fndecl (stmt
);
1595 if (is_tm_abort (fn
))
1596 *state
|= GTMA_HAVE_ABORT
;
1598 /* Note that something may happen. */
1599 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1602 /* Lower a GIMPLE_TRANSACTION statement. */
1605 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1607 gimple g
, stmt
= gsi_stmt (*gsi
);
1608 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1609 unsigned int this_state
= 0;
1610 struct walk_stmt_info this_wi
;
1612 /* First, lower the body. The scanning that we do inside gives
1613 us some idea of what we're dealing with. */
1614 memset (&this_wi
, 0, sizeof (this_wi
));
1615 this_wi
.info
= (void *) &this_state
;
1616 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1617 lower_sequence_tm
, NULL
, &this_wi
);
1619 /* If there was absolutely nothing transaction related inside the
1620 transaction, we may elide it. Likewise if this is a nested
1621 transaction and does not contain an abort. */
1623 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1626 *outer_state
|= this_state
;
1628 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1630 gimple_transaction_set_body (stmt
, NULL
);
1632 gsi_remove (gsi
, true);
1633 wi
->removed_stmt
= true;
1637 /* Wrap the body of the transaction in a try-finally node so that
1638 the commit call is always properly called. */
1639 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1640 if (flag_exceptions
)
1643 gimple_seq n_seq
, e_seq
;
1645 n_seq
= gimple_seq_alloc_with_stmt (g
);
1648 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1649 1, integer_zero_node
);
1650 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1651 gimple_call_set_lhs (g
, ptr
);
1652 gimple_seq_add_stmt (&e_seq
, g
);
1654 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1656 gimple_seq_add_stmt (&e_seq
, g
);
1658 g
= gimple_build_eh_else (n_seq
, e_seq
);
1661 g
= gimple_build_try (gimple_transaction_body (stmt
),
1662 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1663 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1665 gimple_transaction_set_body (stmt
, NULL
);
1667 /* If the transaction calls abort or if this is an outer transaction,
1668 add an "over" label afterwards. */
1669 if ((this_state
& (GTMA_HAVE_ABORT
))
1670 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1672 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1673 gimple_transaction_set_label (stmt
, label
);
1674 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1677 /* Record the set of operations found for use later. */
1678 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1679 gimple_transaction_set_subcode (stmt
, this_state
);
1682 /* Iterate through the statements in the sequence, lowering them all
1683 as appropriate for being in a transaction. */
1686 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1687 struct walk_stmt_info
*wi
)
1689 unsigned int *state
= (unsigned int *) wi
->info
;
1690 gimple stmt
= gsi_stmt (*gsi
);
1692 *handled_ops_p
= true;
1693 switch (gimple_code (stmt
))
1696 /* Only memory reads/writes need to be instrumented. */
1697 if (gimple_assign_single_p (stmt
))
1698 examine_assign_tm (state
, gsi
);
1702 examine_call_tm (state
, gsi
);
1706 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1709 case GIMPLE_TRANSACTION
:
1710 lower_transaction (gsi
, wi
);
1714 *handled_ops_p
= !gimple_has_substatements (stmt
);
1721 /* Iterate through the statements in the sequence, lowering them all
1722 as appropriate for being outside of a transaction. */
1725 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1726 struct walk_stmt_info
* wi
)
1728 gimple stmt
= gsi_stmt (*gsi
);
1730 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1732 *handled_ops_p
= true;
1733 lower_transaction (gsi
, wi
);
1736 *handled_ops_p
= !gimple_has_substatements (stmt
);
1741 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1742 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1743 been moved out, and all the data required for constructing a proper
1744 CFG has been recorded. */
1747 execute_lower_tm (void)
1749 struct walk_stmt_info wi
;
1752 /* Transactional clones aren't created until a later pass. */
1753 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1755 body
= gimple_body (current_function_decl
);
1756 memset (&wi
, 0, sizeof (wi
));
1757 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1758 gimple_set_body (current_function_decl
, body
);
1765 const pass_data pass_data_lower_tm
=
1767 GIMPLE_PASS
, /* type */
1768 "tmlower", /* name */
1769 OPTGROUP_NONE
, /* optinfo_flags */
1770 true, /* has_gate */
1771 true, /* has_execute */
1772 TV_TRANS_MEM
, /* tv_id */
1773 PROP_gimple_lcf
, /* properties_required */
1774 0, /* properties_provided */
1775 0, /* properties_destroyed */
1776 0, /* todo_flags_start */
1777 0, /* todo_flags_finish */
1780 class pass_lower_tm
: public gimple_opt_pass
1783 pass_lower_tm (gcc::context
*ctxt
)
1784 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1787 /* opt_pass methods: */
1788 bool gate () { return gate_tm (); }
1789 unsigned int execute () { return execute_lower_tm (); }
1791 }; // class pass_lower_tm
1796 make_pass_lower_tm (gcc::context
*ctxt
)
1798 return new pass_lower_tm (ctxt
);
1801 /* Collect region information for each transaction. */
1805 /* Link to the next unnested transaction. */
1806 struct tm_region
*next
;
1808 /* Link to the next inner transaction. */
1809 struct tm_region
*inner
;
1811 /* Link to the next outer transaction. */
1812 struct tm_region
*outer
;
1814 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1815 After TM_MARK, this gets replaced by a call to
1816 BUILT_IN_TM_START. */
1817 gimple transaction_stmt
;
1819 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1820 BUILT_IN_TM_START, this field is true if the transaction is an
1821 outer transaction. */
1822 bool original_transaction_was_outer
;
1824 /* Return value from BUILT_IN_TM_START. */
1827 /* The entry block to this region. This will always be the first
1828 block of the body of the transaction. */
1829 basic_block entry_block
;
1831 /* The first block after an expanded call to _ITM_beginTransaction. */
1832 basic_block restart_block
;
1834 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1835 These blocks are still a part of the region (i.e., the border is
1836 inclusive). Note that this set is only complete for paths in the CFG
1837 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1838 the edge to the "over" label. */
1841 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1845 typedef struct tm_region
*tm_region_p
;
1847 /* True if there are pending edge statements to be committed for the
1848 current function being scanned in the tmmark pass. */
1849 bool pending_edge_inserts_p
;
1851 static struct tm_region
*all_tm_regions
;
1852 static bitmap_obstack tm_obstack
;
1855 /* A subroutine of tm_region_init. Record the existence of the
1856 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1858 static struct tm_region
*
1859 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1861 struct tm_region
*region
;
1863 region
= (struct tm_region
*)
1864 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1868 region
->next
= outer
->inner
;
1869 outer
->inner
= region
;
1873 region
->next
= all_tm_regions
;
1874 all_tm_regions
= region
;
1876 region
->inner
= NULL
;
1877 region
->outer
= outer
;
1879 region
->transaction_stmt
= stmt
;
1880 region
->original_transaction_was_outer
= false;
1881 region
->tm_state
= NULL
;
1883 /* There are either one or two edges out of the block containing
1884 the GIMPLE_TRANSACTION, one to the actual region and one to the
1885 "over" label if the region contains an abort. The former will
1886 always be the one marked FALLTHRU. */
1887 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1889 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1890 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1895 /* A subroutine of tm_region_init. Record all the exit and
1896 irrevocable blocks in BB into the region's exit_blocks and
1897 irr_blocks bitmaps. Returns the new region being scanned. */
1899 static struct tm_region
*
1900 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1902 gimple_stmt_iterator gsi
;
1906 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1909 /* Check to see if this is the end of a region by seeing if it
1910 contains a call to __builtin_tm_commit{,_eh}. Note that the
1911 outermost region for DECL_IS_TM_CLONE need not collect this. */
1912 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1915 if (gimple_code (g
) == GIMPLE_CALL
)
1917 tree fn
= gimple_call_fndecl (g
);
1918 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1920 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1921 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1922 && region
->exit_blocks
)
1924 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1925 region
= region
->outer
;
1928 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1929 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1936 /* Collect all of the transaction regions within the current function
1937 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1938 an "outermost" region for use by tm clones. */
1941 tm_region_init (struct tm_region
*region
)
1947 vec
<basic_block
> queue
= vNULL
;
1948 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1949 struct tm_region
*old_region
;
1950 vec
<tm_region_p
> bb_regions
= vNULL
;
1952 all_tm_regions
= region
;
1953 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1955 /* We could store this information in bb->aux, but we may get called
1956 through get_all_tm_blocks() from another pass that may be already
1958 bb_regions
.safe_grow_cleared (last_basic_block
);
1960 queue
.safe_push (bb
);
1961 bb_regions
[bb
->index
] = region
;
1965 region
= bb_regions
[bb
->index
];
1966 bb_regions
[bb
->index
] = NULL
;
1968 /* Record exit and irrevocable blocks. */
1969 region
= tm_region_init_1 (region
, bb
);
1971 /* Check for the last statement in the block beginning a new region. */
1973 old_region
= region
;
1974 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1975 region
= tm_region_init_0 (region
, bb
, g
);
1977 /* Process subsequent blocks. */
1978 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1979 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1981 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1982 queue
.safe_push (e
->dest
);
1984 /* If the current block started a new region, make sure that only
1985 the entry block of the new region is associated with this region.
1986 Other successors are still part of the old region. */
1987 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1988 bb_regions
[e
->dest
->index
] = old_region
;
1990 bb_regions
[e
->dest
->index
] = region
;
1993 while (!queue
.is_empty ());
1995 BITMAP_FREE (visited_blocks
);
1996 bb_regions
.release ();
1999 /* The "gate" function for all transactional memory expansion and optimization
2000 passes. We collect region information for each top-level transaction, and
2001 if we don't find any, we skip all of the TM passes. Each region will have
2002 all of the exit blocks recorded, and the originating statement. */
2010 calculate_dominance_info (CDI_DOMINATORS
);
2011 bitmap_obstack_initialize (&tm_obstack
);
2013 /* If the function is a TM_CLONE, then the entire function is the region. */
2014 if (decl_is_tm_clone (current_function_decl
))
2016 struct tm_region
*region
= (struct tm_region
*)
2017 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2018 memset (region
, 0, sizeof (*region
));
2019 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2020 /* For a clone, the entire function is the region. But even if
2021 we don't need to record any exit blocks, we may need to
2022 record irrevocable blocks. */
2023 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2025 tm_region_init (region
);
2029 tm_region_init (NULL
);
2031 /* If we didn't find any regions, cleanup and skip the whole tree
2032 of tm-related optimizations. */
2033 if (all_tm_regions
== NULL
)
2035 bitmap_obstack_release (&tm_obstack
);
2045 const pass_data pass_data_tm_init
=
2047 GIMPLE_PASS
, /* type */
2048 "*tminit", /* name */
2049 OPTGROUP_NONE
, /* optinfo_flags */
2050 true, /* has_gate */
2051 false, /* has_execute */
2052 TV_TRANS_MEM
, /* tv_id */
2053 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2054 0, /* properties_provided */
2055 0, /* properties_destroyed */
2056 0, /* todo_flags_start */
2057 0, /* todo_flags_finish */
2060 class pass_tm_init
: public gimple_opt_pass
2063 pass_tm_init (gcc::context
*ctxt
)
2064 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2067 /* opt_pass methods: */
2068 bool gate () { return gate_tm_init (); }
2070 }; // class pass_tm_init
2075 make_pass_tm_init (gcc::context
*ctxt
)
2077 return new pass_tm_init (ctxt
);
2080 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2081 represented by STATE. */
2084 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2086 if (region
&& region
->transaction_stmt
)
2088 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
2089 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
2093 /* Construct a memory load in a transactional context. Return the
2094 gimple statement performing the load, or NULL if there is no
2095 TM_LOAD builtin of the appropriate size to do the load.
2097 LOC is the location to use for the new statement(s). */
2100 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2102 enum built_in_function code
= END_BUILTINS
;
2103 tree t
, type
= TREE_TYPE (rhs
), decl
;
2106 if (type
== float_type_node
)
2107 code
= BUILT_IN_TM_LOAD_FLOAT
;
2108 else if (type
== double_type_node
)
2109 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2110 else if (type
== long_double_type_node
)
2111 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2112 else if (TYPE_SIZE_UNIT (type
) != NULL
2113 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2115 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2118 code
= BUILT_IN_TM_LOAD_1
;
2121 code
= BUILT_IN_TM_LOAD_2
;
2124 code
= BUILT_IN_TM_LOAD_4
;
2127 code
= BUILT_IN_TM_LOAD_8
;
2132 if (code
== END_BUILTINS
)
2134 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2139 decl
= builtin_decl_explicit (code
);
2141 t
= gimplify_addr (gsi
, rhs
);
2142 gcall
= gimple_build_call (decl
, 1, t
);
2143 gimple_set_location (gcall
, loc
);
2145 t
= TREE_TYPE (TREE_TYPE (decl
));
2146 if (useless_type_conversion_p (type
, t
))
2148 gimple_call_set_lhs (gcall
, lhs
);
2149 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2156 temp
= create_tmp_reg (t
, NULL
);
2157 gimple_call_set_lhs (gcall
, temp
);
2158 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2160 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2161 g
= gimple_build_assign (lhs
, t
);
2162 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2169 /* Similarly for storing TYPE in a transactional context. */
2172 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2174 enum built_in_function code
= END_BUILTINS
;
2175 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2178 if (type
== float_type_node
)
2179 code
= BUILT_IN_TM_STORE_FLOAT
;
2180 else if (type
== double_type_node
)
2181 code
= BUILT_IN_TM_STORE_DOUBLE
;
2182 else if (type
== long_double_type_node
)
2183 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2184 else if (TYPE_SIZE_UNIT (type
) != NULL
2185 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2187 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2190 code
= BUILT_IN_TM_STORE_1
;
2193 code
= BUILT_IN_TM_STORE_2
;
2196 code
= BUILT_IN_TM_STORE_4
;
2199 code
= BUILT_IN_TM_STORE_8
;
2204 if (code
== END_BUILTINS
)
2206 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2211 fn
= builtin_decl_explicit (code
);
2213 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2215 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2217 /* Handle the easy initialization to zero. */
2218 if (!CONSTRUCTOR_ELTS (rhs
))
2219 rhs
= build_int_cst (simple_type
, 0);
2222 /* ...otherwise punt to the caller and probably use
2223 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2224 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2229 else if (!useless_type_conversion_p (simple_type
, type
))
2234 temp
= create_tmp_reg (simple_type
, NULL
);
2235 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2236 g
= gimple_build_assign (temp
, t
);
2237 gimple_set_location (g
, loc
);
2238 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2243 t
= gimplify_addr (gsi
, lhs
);
2244 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2245 gimple_set_location (gcall
, loc
);
2246 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2252 /* Expand an assignment statement into transactional builtins. */
2255 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2257 gimple stmt
= gsi_stmt (*gsi
);
2258 location_t loc
= gimple_location (stmt
);
2259 tree lhs
= gimple_assign_lhs (stmt
);
2260 tree rhs
= gimple_assign_rhs1 (stmt
);
2261 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2262 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2263 gimple gcall
= NULL
;
2265 if (!load_p
&& !store_p
)
2267 /* Add thread private addresses to log if applicable. */
2268 requires_barrier (region
->entry_block
, lhs
, stmt
);
2273 // Remove original load/store statement.
2274 gsi_remove (gsi
, true);
2276 if (load_p
&& !store_p
)
2278 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2279 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2281 else if (store_p
&& !load_p
)
2283 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2284 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2288 tree lhs_addr
, rhs_addr
, tmp
;
2291 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2293 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2295 /* ??? Figure out if there's any possible overlap between the LHS
2296 and the RHS and if not, use MEMCPY. */
2298 if (load_p
&& is_gimple_reg (lhs
))
2300 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2301 lhs_addr
= build_fold_addr_expr (tmp
);
2306 lhs_addr
= gimplify_addr (gsi
, lhs
);
2308 rhs_addr
= gimplify_addr (gsi
, rhs
);
2309 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2310 3, lhs_addr
, rhs_addr
,
2311 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2312 gimple_set_location (gcall
, loc
);
2313 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2317 gcall
= gimple_build_assign (lhs
, tmp
);
2318 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2322 /* Now that we have the load/store in its instrumented form, add
2323 thread private addresses to the log if applicable. */
2325 requires_barrier (region
->entry_block
, lhs
, gcall
);
2327 // The calls to build_tm_{store,load} above inserted the instrumented
2328 // call into the stream.
2329 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2333 /* Expand a call statement as appropriate for a transaction. That is,
2334 either verify that the call does not affect the transaction, or
2335 redirect the call to a clone that handles transactions, or change
2336 the transaction state to IRREVOCABLE. Return true if the call is
2337 one of the builtins that end a transaction. */
2340 expand_call_tm (struct tm_region
*region
,
2341 gimple_stmt_iterator
*gsi
)
2343 gimple stmt
= gsi_stmt (*gsi
);
2344 tree lhs
= gimple_call_lhs (stmt
);
2346 struct cgraph_node
*node
;
2347 bool retval
= false;
2349 fn_decl
= gimple_call_fndecl (stmt
);
2351 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2352 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2353 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2354 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2355 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2357 if (is_tm_pure_call (stmt
))
2361 retval
= is_tm_ending_fndecl (fn_decl
);
2364 /* Assume all non-const/pure calls write to memory, except
2365 transaction ending builtins. */
2366 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2369 /* For indirect calls, we already generated a call into the runtime. */
2372 tree fn
= gimple_call_fn (stmt
);
2374 /* We are guaranteed never to go irrevocable on a safe or pure
2375 call, and the pure call was handled above. */
2376 if (is_tm_safe (fn
))
2379 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2384 node
= cgraph_get_node (fn_decl
);
2385 /* All calls should have cgraph here. */
2388 /* We can have a nodeless call here if some pass after IPA-tm
2389 added uninstrumented calls. For example, loop distribution
2390 can transform certain loop constructs into __builtin_mem*
2391 calls. In this case, see if we have a suitable TM
2392 replacement and fill in the gaps. */
2393 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2394 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2395 gcc_assert (code
== BUILT_IN_MEMCPY
2396 || code
== BUILT_IN_MEMMOVE
2397 || code
== BUILT_IN_MEMSET
);
2399 tree repl
= find_tm_replacement_function (fn_decl
);
2402 gimple_call_set_fndecl (stmt
, repl
);
2404 node
= cgraph_create_node (repl
);
2405 node
->local
.tm_may_enter_irr
= false;
2406 return expand_call_tm (region
, gsi
);
2410 if (node
->local
.tm_may_enter_irr
)
2411 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2413 if (is_tm_abort (fn_decl
))
2415 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2419 /* Instrument the store if needed.
2421 If the assignment happens inside the function call (return slot
2422 optimization), there is no instrumentation to be done, since
2423 the callee should have done the right thing. */
2424 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2425 && !gimple_call_return_slot_opt_p (stmt
))
2427 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2428 location_t loc
= gimple_location (stmt
);
2429 edge fallthru_edge
= NULL
;
2431 /* Remember if the call was going to throw. */
2432 if (stmt_can_throw_internal (stmt
))
2436 basic_block bb
= gimple_bb (stmt
);
2438 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2439 if (e
->flags
& EDGE_FALLTHRU
)
2446 gimple_call_set_lhs (stmt
, tmp
);
2448 stmt
= gimple_build_assign (lhs
, tmp
);
2449 gimple_set_location (stmt
, loc
);
2451 /* We cannot throw in the middle of a BB. If the call was going
2452 to throw, place the instrumentation on the fallthru edge, so
2453 the call remains the last statement in the block. */
2456 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2457 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2458 expand_assign_tm (region
, &fallthru_gsi
);
2459 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2460 pending_edge_inserts_p
= true;
2464 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2465 expand_assign_tm (region
, gsi
);
2468 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2475 /* Expand all statements in BB as appropriate for being inside
2479 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2481 gimple_stmt_iterator gsi
;
2483 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2485 gimple stmt
= gsi_stmt (gsi
);
2486 switch (gimple_code (stmt
))
2489 /* Only memory reads/writes need to be instrumented. */
2490 if (gimple_assign_single_p (stmt
)
2491 && !gimple_clobber_p (stmt
))
2493 expand_assign_tm (region
, &gsi
);
2499 if (expand_call_tm (region
, &gsi
))
2509 if (!gsi_end_p (gsi
))
2514 /* Return the list of basic-blocks in REGION.
2516 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2517 following a TM_IRREVOCABLE call.
2519 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2520 uninstrumented code path blocks in the list of basic blocks
2521 returned, false otherwise. */
2523 static vec
<basic_block
>
2524 get_tm_region_blocks (basic_block entry_block
,
2527 bitmap all_region_blocks
,
2528 bool stop_at_irrevocable_p
,
2529 bool include_uninstrumented_p
= true)
2531 vec
<basic_block
> bbs
= vNULL
;
2535 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2538 bbs
.safe_push (entry_block
);
2539 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2543 basic_block bb
= bbs
[i
++];
2546 bitmap_bit_p (exit_blocks
, bb
->index
))
2549 if (stop_at_irrevocable_p
2551 && bitmap_bit_p (irr_blocks
, bb
->index
))
2554 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2555 if ((include_uninstrumented_p
2556 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2557 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2559 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2560 bbs
.safe_push (e
->dest
);
2563 while (i
< bbs
.length ());
2565 if (all_region_blocks
)
2566 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2568 BITMAP_FREE (visited_blocks
);
2572 // Callback data for collect_bb2reg.
2575 vec
<tm_region_p
> *bb2reg
;
2576 bool include_uninstrumented_p
;
2579 // Callback for expand_regions, collect innermost region data for each bb.
2581 collect_bb2reg (struct tm_region
*region
, void *data
)
2583 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2584 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2585 vec
<basic_block
> queue
;
2589 queue
= get_tm_region_blocks (region
->entry_block
,
2590 region
->exit_blocks
,
2593 /*stop_at_irr_p=*/true,
2594 stuff
->include_uninstrumented_p
);
2596 // We expect expand_region to perform a post-order traversal of the region
2597 // tree. Therefore the last region seen for any bb is the innermost.
2598 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2599 (*bb2reg
)[bb
->index
] = region
;
2605 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2606 // which a basic block belongs. Note that we only consider the instrumented
2607 // code paths for the region; the uninstrumented code paths are ignored if
2608 // INCLUDE_UNINSTRUMENTED_P is false.
2610 // ??? This data is very similar to the bb_regions array that is collected
2611 // during tm_region_init. Or, rather, this data is similar to what could
2612 // be used within tm_region_init. The actual computation in tm_region_init
2613 // begins and ends with bb_regions entirely full of NULL pointers, due to
2614 // the way in which pointers are swapped in and out of the array.
2616 // ??? Our callers expect that blocks are not shared between transactions.
2617 // When the optimizers get too smart, and blocks are shared, then during
2618 // the tm_mark phase we'll add log entries to only one of the two transactions,
2619 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2620 // cycles. The symptom being SSA defs that do not dominate their uses.
2621 // Note that the optimizers were locally correct with their transformation,
2622 // as we have no info within the program that suggests that the blocks cannot
2625 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2626 // only known instance of this block sharing.
2628 static vec
<tm_region_p
>
2629 get_bb_regions_instrumented (bool traverse_clones
,
2630 bool include_uninstrumented_p
)
2632 unsigned n
= last_basic_block
;
2633 struct bb2reg_stuff stuff
;
2634 vec
<tm_region_p
> ret
;
2637 ret
.safe_grow_cleared (n
);
2638 stuff
.bb2reg
= &ret
;
2639 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2640 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2645 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2649 compute_transaction_bits (void)
2651 struct tm_region
*region
;
2652 vec
<basic_block
> queue
;
2656 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2657 certainly don't need it to calculate CDI_DOMINATOR info. */
2661 bb
->flags
&= ~BB_IN_TRANSACTION
;
2663 for (region
= all_tm_regions
; region
; region
= region
->next
)
2665 queue
= get_tm_region_blocks (region
->entry_block
,
2666 region
->exit_blocks
,
2669 /*stop_at_irr_p=*/true);
2670 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2671 bb
->flags
|= BB_IN_TRANSACTION
;
2676 bitmap_obstack_release (&tm_obstack
);
2679 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2680 call to BUILT_IN_TM_START. */
2683 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2685 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2686 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2687 tree tm_state
= region
->tm_state
;
2688 tree tm_state_type
= TREE_TYPE (tm_state
);
2689 edge abort_edge
= NULL
;
2690 edge inst_edge
= NULL
;
2691 edge uninst_edge
= NULL
;
2692 edge fallthru_edge
= NULL
;
2694 // Identify the various successors of the transaction start.
2698 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2700 if (e
->flags
& EDGE_TM_ABORT
)
2702 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2706 if (e
->flags
& EDGE_FALLTHRU
)
2711 /* ??? There are plenty of bits here we're not computing. */
2713 int subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2715 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2716 flags
|= PR_DOESGOIRREVOCABLE
;
2717 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2718 flags
|= PR_HASNOIRREVOCABLE
;
2719 /* If the transaction does not have an abort in lexical scope and is not
2720 marked as an outer transaction, then it will never abort. */
2721 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2722 flags
|= PR_HASNOABORT
;
2723 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2724 flags
|= PR_READONLY
;
2725 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2726 flags
|= PR_INSTRUMENTEDCODE
;
2728 flags
|= PR_UNINSTRUMENTEDCODE
;
2729 if (subcode
& GTMA_IS_OUTER
)
2730 region
->original_transaction_was_outer
= true;
2731 tree t
= build_int_cst (tm_state_type
, flags
);
2732 gimple call
= gimple_build_call (tm_start
, 1, t
);
2733 gimple_call_set_lhs (call
, tm_state
);
2734 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2736 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2737 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2738 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2739 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2740 gsi_remove (&gsi
, true);
2741 region
->transaction_stmt
= call
;
2744 // Generate log saves.
2745 if (!tm_log_save_addresses
.is_empty ())
2746 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2748 // In the beginning, we've no tests to perform on transaction restart.
2749 // Note that after this point, transaction_bb becomes the "most recent
2750 // block containing tests for the transaction".
2751 region
->restart_block
= region
->entry_block
;
2753 // Generate log restores.
2754 if (!tm_log_save_addresses
.is_empty ())
2756 basic_block test_bb
= create_empty_bb (transaction_bb
);
2757 basic_block code_bb
= create_empty_bb (test_bb
);
2758 basic_block join_bb
= create_empty_bb (code_bb
);
2759 if (current_loops
&& transaction_bb
->loop_father
)
2761 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2762 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2763 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2765 if (region
->restart_block
== region
->entry_block
)
2766 region
->restart_block
= test_bb
;
2768 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2769 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2770 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2772 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2773 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2775 t2
= build_int_cst (tm_state_type
, 0);
2776 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2777 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2779 tm_log_emit_restores (region
->entry_block
, code_bb
);
2781 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2782 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2783 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2784 redirect_edge_pred (fallthru_edge
, join_bb
);
2786 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2787 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2789 ei
->probability
= PROB_ALWAYS
;
2790 et
->probability
= PROB_LIKELY
;
2791 ef
->probability
= PROB_UNLIKELY
;
2792 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2793 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2795 code_bb
->count
= et
->count
;
2796 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2798 transaction_bb
= join_bb
;
2801 // If we have an ABORT edge, create a test to perform the abort.
2804 basic_block test_bb
= create_empty_bb (transaction_bb
);
2805 if (current_loops
&& transaction_bb
->loop_father
)
2806 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2807 if (region
->restart_block
== region
->entry_block
)
2808 region
->restart_block
= test_bb
;
2810 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2811 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2812 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2814 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2815 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2817 t2
= build_int_cst (tm_state_type
, 0);
2818 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2819 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2821 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2822 test_bb
->frequency
= transaction_bb
->frequency
;
2823 test_bb
->count
= transaction_bb
->count
;
2824 ei
->probability
= PROB_ALWAYS
;
2826 // Not abort edge. If both are live, chose one at random as we'll
2827 // we'll be fixing that up below.
2828 redirect_edge_pred (fallthru_edge
, test_bb
);
2829 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2830 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2831 fallthru_edge
->count
2832 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2835 redirect_edge_pred (abort_edge
, test_bb
);
2836 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2837 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2839 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2841 transaction_bb
= test_bb
;
2844 // If we have both instrumented and uninstrumented code paths, select one.
2845 if (inst_edge
&& uninst_edge
)
2847 basic_block test_bb
= create_empty_bb (transaction_bb
);
2848 if (current_loops
&& transaction_bb
->loop_father
)
2849 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2850 if (region
->restart_block
== region
->entry_block
)
2851 region
->restart_block
= test_bb
;
2853 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2854 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2856 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2858 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2859 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2861 t2
= build_int_cst (tm_state_type
, 0);
2862 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2863 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2865 // Create the edge into test_bb first, as we want to copy values
2866 // out of the fallthru edge.
2867 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2868 e
->probability
= fallthru_edge
->probability
;
2869 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2870 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2872 // Now update the edges to the inst/uninist implementations.
2873 // For now assume that the paths are equally likely. When using HTM,
2874 // we'll try the uninst path first and fallback to inst path if htm
2875 // buffers are exceeded. Without HTM we start with the inst path and
2876 // use the uninst path when falling back to serial mode.
2877 redirect_edge_pred (inst_edge
, test_bb
);
2878 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2879 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2881 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2883 redirect_edge_pred (uninst_edge
, test_bb
);
2884 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2885 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2887 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2890 // If we have no previous special cases, and we have PHIs at the beginning
2891 // of the atomic region, this means we have a loop at the beginning of the
2892 // atomic region that shares the first block. This can cause problems with
2893 // the transaction restart abnormal edges to be added in the tm_edges pass.
2894 // Solve this by adding a new empty block to receive the abnormal edges.
2895 if (region
->restart_block
== region
->entry_block
2896 && phi_nodes (region
->entry_block
))
2898 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2899 region
->restart_block
= empty_bb
;
2900 if (current_loops
&& transaction_bb
->loop_father
)
2901 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2903 redirect_edge_pred (fallthru_edge
, empty_bb
);
2904 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2910 /* Generate the temporary to be used for the return value of
2911 BUILT_IN_TM_START. */
2914 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2916 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2918 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2920 // Reset the subcode, post optimizations. We'll fill this in
2921 // again as we process blocks.
2922 if (region
->exit_blocks
)
2924 unsigned int subcode
2925 = gimple_transaction_subcode (region
->transaction_stmt
);
2927 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2928 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2929 | GTMA_MAY_ENTER_IRREVOCABLE
2930 | GTMA_HAS_NO_INSTRUMENTATION
);
2932 subcode
&= GTMA_DECLARATION_MASK
;
2933 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2939 // Propagate flags from inner transactions outwards.
2941 propagate_tm_flags_out (struct tm_region
*region
)
2945 propagate_tm_flags_out (region
->inner
);
2947 if (region
->outer
&& region
->outer
->transaction_stmt
)
2949 unsigned s
= gimple_transaction_subcode (region
->transaction_stmt
);
2950 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2951 | GTMA_MAY_ENTER_IRREVOCABLE
);
2952 s
|= gimple_transaction_subcode (region
->outer
->transaction_stmt
);
2953 gimple_transaction_set_subcode (region
->outer
->transaction_stmt
, s
);
2956 propagate_tm_flags_out (region
->next
);
2959 /* Entry point to the MARK phase of TM expansion. Here we replace
2960 transactional memory statements with calls to builtins, and function
2961 calls with their transactional clones (if available). But we don't
2962 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2965 execute_tm_mark (void)
2967 pending_edge_inserts_p
= false;
2969 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2970 /*traverse_clones=*/true);
2974 vec
<tm_region_p
> bb_regions
2975 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2976 /*include_uninstrumented_p=*/false);
2977 struct tm_region
*r
;
2980 // Expand memory operations into calls into the runtime.
2981 // This collects log entries as well.
2982 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
2986 if (r
->transaction_stmt
)
2988 unsigned sub
= gimple_transaction_subcode (r
->transaction_stmt
);
2990 /* If we're sure to go irrevocable, there won't be
2991 anything to expand, since the run-time will go
2992 irrevocable right away. */
2993 if (sub
& GTMA_DOES_GO_IRREVOCABLE
2994 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
2997 expand_block_tm (r
, BASIC_BLOCK (i
));
3001 bb_regions
.release ();
3003 // Propagate flags from inner transactions outwards.
3004 propagate_tm_flags_out (all_tm_regions
);
3006 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3007 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3008 /*traverse_clones=*/false);
3013 if (pending_edge_inserts_p
)
3014 gsi_commit_edge_inserts ();
3015 free_dominance_info (CDI_DOMINATORS
);
3021 const pass_data pass_data_tm_mark
=
3023 GIMPLE_PASS
, /* type */
3024 "tmmark", /* name */
3025 OPTGROUP_NONE
, /* optinfo_flags */
3026 false, /* has_gate */
3027 true, /* has_execute */
3028 TV_TRANS_MEM
, /* tv_id */
3029 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3030 0, /* properties_provided */
3031 0, /* properties_destroyed */
3032 0, /* todo_flags_start */
3033 ( TODO_update_ssa
| TODO_verify_ssa
), /* todo_flags_finish */
3036 class pass_tm_mark
: public gimple_opt_pass
3039 pass_tm_mark (gcc::context
*ctxt
)
3040 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3043 /* opt_pass methods: */
3044 unsigned int execute () { return execute_tm_mark (); }
3046 }; // class pass_tm_mark
3051 make_pass_tm_mark (gcc::context
*ctxt
)
3053 return new pass_tm_mark (ctxt
);
3057 /* Create an abnormal edge from STMT at iter, splitting the block
3058 as necessary. Adjust *PNEXT as needed for the split block. */
3061 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3062 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3064 basic_block bb
= gimple_bb (stmt
);
3065 if (!gsi_one_before_end_p (iter
))
3067 edge e
= split_block (bb
, stmt
);
3068 *pnext
= gsi_start_bb (e
->dest
);
3070 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3072 // Record the need for the edge for the benefit of the rtl passes.
3073 if (cfun
->gimple_df
->tm_restart
== NULL
)
3074 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
3075 struct_ptr_eq
, ggc_free
);
3077 struct tm_restart_node dummy
;
3079 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3081 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
3082 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
3085 n
= ggc_alloc_tm_restart_node ();
3090 tree old
= n
->label_or_list
;
3091 if (TREE_CODE (old
) == LABEL_DECL
)
3092 old
= tree_cons (NULL
, old
, NULL
);
3093 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3097 /* Split block BB as necessary for every builtin function we added, and
3098 wire up the abnormal back edges implied by the transaction restart. */
3101 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3103 gimple_stmt_iterator gsi
, next_gsi
;
3105 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3107 gimple stmt
= gsi_stmt (gsi
);
3110 gsi_next (&next_gsi
);
3112 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3113 if (gimple_code (stmt
) != GIMPLE_CALL
3114 || (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) == 0)
3117 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
)) == BUILT_IN_TM_ABORT
)
3119 // If we have a ``_transaction_cancel [[outer]]'', there is only
3120 // one abnormal edge: to the transaction marked OUTER.
3121 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3122 // constant argument, which we can examine here. Users invoking
3123 // TM_ABORT directly get what they deserve.
3124 tree arg
= gimple_call_arg (stmt
, 0);
3125 if (TREE_CODE (arg
) == INTEGER_CST
3126 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3127 && !decl_is_tm_clone (current_function_decl
))
3129 // Find the GTMA_IS_OUTER transaction.
3130 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3131 if (o
->original_transaction_was_outer
)
3133 split_bb_make_tm_edge (stmt
, o
->restart_block
,
3138 // Otherwise, the front-end should have semantically checked
3139 // outer aborts, but in either case the target region is not
3140 // within this function.
3144 // Non-outer, TM aborts have an abnormal edge to the inner-most
3145 // transaction, the one being aborted;
3146 split_bb_make_tm_edge (stmt
, region
->restart_block
, gsi
, &next_gsi
);
3149 // All TM builtins have an abnormal edge to the outer-most transaction.
3150 // We never restart inner transactions. For tm clones, we know a-priori
3151 // that the outer-most transaction is outside the function.
3152 if (decl_is_tm_clone (current_function_decl
))
3155 if (cfun
->gimple_df
->tm_restart
== NULL
)
3156 cfun
->gimple_df
->tm_restart
3157 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
3159 // All TM builtins have an abnormal edge to the outer-most transaction.
3160 // We never restart inner transactions.
3161 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3164 split_bb_make_tm_edge (stmt
, o
->restart_block
, gsi
, &next_gsi
);
3168 // Delete any tail-call annotation that may have been added.
3169 // The tail-call pass may have mis-identified the commit as being
3170 // a candidate because we had not yet added this restart edge.
3171 gimple_call_set_tail (stmt
, false);
3175 /* Entry point to the final expansion of transactional nodes. */
3178 execute_tm_edges (void)
3180 vec
<tm_region_p
> bb_regions
3181 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3182 /*include_uninstrumented_p=*/true);
3183 struct tm_region
*r
;
3186 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3188 expand_block_edges (r
, BASIC_BLOCK (i
));
3190 bb_regions
.release ();
3192 /* We've got to release the dominance info now, to indicate that it
3193 must be rebuilt completely. Otherwise we'll crash trying to update
3194 the SSA web in the TODO section following this pass. */
3195 free_dominance_info (CDI_DOMINATORS
);
3196 bitmap_obstack_release (&tm_obstack
);
3197 all_tm_regions
= NULL
;
3204 const pass_data pass_data_tm_edges
=
3206 GIMPLE_PASS
, /* type */
3207 "tmedge", /* name */
3208 OPTGROUP_NONE
, /* optinfo_flags */
3209 false, /* has_gate */
3210 true, /* has_execute */
3211 TV_TRANS_MEM
, /* tv_id */
3212 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3213 0, /* properties_provided */
3214 0, /* properties_destroyed */
3215 0, /* todo_flags_start */
3216 ( TODO_update_ssa
| TODO_verify_ssa
), /* todo_flags_finish */
3219 class pass_tm_edges
: public gimple_opt_pass
3222 pass_tm_edges (gcc::context
*ctxt
)
3223 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3226 /* opt_pass methods: */
3227 unsigned int execute () { return execute_tm_edges (); }
3229 }; // class pass_tm_edges
3234 make_pass_tm_edges (gcc::context
*ctxt
)
3236 return new pass_tm_edges (ctxt
);
3239 /* Helper function for expand_regions. Expand REGION and recurse to
3240 the inner region. Call CALLBACK on each region. CALLBACK returns
3241 NULL to continue the traversal, otherwise a non-null value which
3242 this function will return as well. TRAVERSE_CLONES is true if we
3243 should traverse transactional clones. */
3246 expand_regions_1 (struct tm_region
*region
,
3247 void *(*callback
)(struct tm_region
*, void *),
3249 bool traverse_clones
)
3251 void *retval
= NULL
;
3252 if (region
->exit_blocks
3253 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3255 retval
= callback (region
, data
);
3261 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3268 /* Traverse the regions enclosed and including REGION. Execute
3269 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3270 continue the traversal, otherwise a non-null value which this
3271 function will return as well. TRAVERSE_CLONES is true if we should
3272 traverse transactional clones. */
3275 expand_regions (struct tm_region
*region
,
3276 void *(*callback
)(struct tm_region
*, void *),
3278 bool traverse_clones
)
3280 void *retval
= NULL
;
3283 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3286 region
= region
->next
;
3292 /* A unique TM memory operation. */
3293 typedef struct tm_memop
3295 /* Unique ID that all memory operations to the same location have. */
3296 unsigned int value_id
;
3297 /* Address of load/store. */
3301 /* TM memory operation hashtable helpers. */
3303 struct tm_memop_hasher
: typed_free_remove
<tm_memop
>
3305 typedef tm_memop value_type
;
3306 typedef tm_memop compare_type
;
3307 static inline hashval_t
hash (const value_type
*);
3308 static inline bool equal (const value_type
*, const compare_type
*);
3311 /* Htab support. Return a hash value for a `tm_memop'. */
3313 tm_memop_hasher::hash (const value_type
*mem
)
3315 tree addr
= mem
->addr
;
3316 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3317 actually done with operand_equal_p (see tm_memop_eq). */
3318 if (TREE_CODE (addr
) == ADDR_EXPR
)
3319 addr
= TREE_OPERAND (addr
, 0);
3320 return iterative_hash_expr (addr
, 0);
3323 /* Htab support. Return true if two tm_memop's are the same. */
3325 tm_memop_hasher::equal (const value_type
*mem1
, const compare_type
*mem2
)
3327 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3330 /* Sets for solving data flow equations in the memory optimization pass. */
3331 struct tm_memopt_bitmaps
3333 /* Stores available to this BB upon entry. Basically, stores that
3334 dominate this BB. */
3335 bitmap store_avail_in
;
3336 /* Stores available at the end of this BB. */
3337 bitmap store_avail_out
;
3338 bitmap store_antic_in
;
3339 bitmap store_antic_out
;
3340 /* Reads available to this BB upon entry. Basically, reads that
3341 dominate this BB. */
3342 bitmap read_avail_in
;
3343 /* Reads available at the end of this BB. */
3344 bitmap read_avail_out
;
3345 /* Reads performed in this BB. */
3347 /* Writes performed in this BB. */
3350 /* Temporary storage for pass. */
3351 /* Is the current BB in the worklist? */
3352 bool avail_in_worklist_p
;
3353 /* Have we visited this BB? */
3357 static bitmap_obstack tm_memopt_obstack
;
3359 /* Unique counter for TM loads and stores. Loads and stores of the
3360 same address get the same ID. */
3361 static unsigned int tm_memopt_value_id
;
3362 static hash_table
<tm_memop_hasher
> tm_memopt_value_numbers
;
3364 #define STORE_AVAIL_IN(BB) \
3365 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3366 #define STORE_AVAIL_OUT(BB) \
3367 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3368 #define STORE_ANTIC_IN(BB) \
3369 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3370 #define STORE_ANTIC_OUT(BB) \
3371 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3372 #define READ_AVAIL_IN(BB) \
3373 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3374 #define READ_AVAIL_OUT(BB) \
3375 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3376 #define READ_LOCAL(BB) \
3377 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3378 #define STORE_LOCAL(BB) \
3379 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3380 #define AVAIL_IN_WORKLIST_P(BB) \
3381 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3382 #define BB_VISITED_P(BB) \
3383 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3385 /* Given a TM load/store in STMT, return the value number for the address
3389 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3391 struct tm_memop tmpmem
, *mem
;
3394 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3395 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3396 slot
= tm_memopt_value_numbers
.find_slot (&tmpmem
, op
);
3399 else if (op
== INSERT
)
3401 mem
= XNEW (struct tm_memop
);
3403 mem
->value_id
= tm_memopt_value_id
++;
3404 mem
->addr
= tmpmem
.addr
;
3408 return mem
->value_id
;
3411 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3414 tm_memopt_accumulate_memops (basic_block bb
)
3416 gimple_stmt_iterator gsi
;
3418 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3420 gimple stmt
= gsi_stmt (gsi
);
3424 if (is_tm_store (stmt
))
3425 bits
= STORE_LOCAL (bb
);
3426 else if (is_tm_load (stmt
))
3427 bits
= READ_LOCAL (bb
);
3431 loc
= tm_memopt_value_number (stmt
, INSERT
);
3432 bitmap_set_bit (bits
, loc
);
3435 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3436 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3437 gimple_bb (stmt
)->index
);
3438 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3439 fprintf (dump_file
, "\n");
3444 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3447 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3451 const char *comma
= "";
3453 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3454 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3456 hash_table
<tm_memop_hasher
>::iterator hi
;
3457 struct tm_memop
*mem
= NULL
;
3459 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3460 FOR_EACH_HASH_TABLE_ELEMENT (tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3461 if (mem
->value_id
== i
)
3463 gcc_assert (mem
->value_id
== i
);
3464 fprintf (dump_file
, "%s", comma
);
3466 print_generic_expr (dump_file
, mem
->addr
, 0);
3468 fprintf (dump_file
, "]\n");
3471 /* Prettily dump all of the memopt sets in BLOCKS. */
3474 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3479 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3481 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3482 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3483 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3484 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3485 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3486 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3487 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3491 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3494 tm_memopt_compute_avin (basic_block bb
)
3499 /* Seed with the AVOUT of any predecessor. */
3500 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3502 e
= EDGE_PRED (bb
, ix
);
3503 /* Make sure we have already visited this BB, and is thus
3506 If e->src->aux is NULL, this predecessor is actually on an
3507 enclosing transaction. We only care about the current
3508 transaction, so ignore it. */
3509 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3511 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3512 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3517 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3519 e
= EDGE_PRED (bb
, ix
);
3520 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3522 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3523 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3527 BB_VISITED_P (bb
) = true;
3530 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3533 tm_memopt_compute_antin (basic_block bb
)
3538 /* Seed with the ANTIC_OUT of any successor. */
3539 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3541 e
= EDGE_SUCC (bb
, ix
);
3542 /* Make sure we have already visited this BB, and is thus
3544 if (BB_VISITED_P (e
->dest
))
3546 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3551 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3553 e
= EDGE_SUCC (bb
, ix
);
3554 if (BB_VISITED_P (e
->dest
))
3555 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3558 BB_VISITED_P (bb
) = true;
3561 /* Compute the AVAIL sets for every basic block in BLOCKS.
3563 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3565 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3566 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3568 This is basically what we do in lcm's compute_available(), but here
3569 we calculate two sets of sets (one for STOREs and one for READs),
3570 and we work on a region instead of the entire CFG.
3572 REGION is the TM region.
3573 BLOCKS are the basic blocks in the region. */
3576 tm_memopt_compute_available (struct tm_region
*region
,
3577 vec
<basic_block
> blocks
)
3580 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3581 unsigned int qlen
, i
;
3585 /* Allocate a worklist array/queue. Entries are only added to the
3586 list if they were not already on the list. So the size is
3587 bounded by the number of basic blocks in the region. */
3588 qlen
= blocks
.length () - 1;
3589 qin
= qout
= worklist
=
3590 XNEWVEC (basic_block
, qlen
);
3592 /* Put every block in the region on the worklist. */
3593 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3595 /* Seed AVAIL_OUT with the LOCAL set. */
3596 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3597 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3599 AVAIL_IN_WORKLIST_P (bb
) = true;
3600 /* No need to insert the entry block, since it has an AVIN of
3601 null, and an AVOUT that has already been seeded in. */
3602 if (bb
!= region
->entry_block
)
3606 /* The entry block has been initialized with the local sets. */
3607 BB_VISITED_P (region
->entry_block
) = true;
3610 qend
= &worklist
[qlen
];
3612 /* Iterate until the worklist is empty. */
3615 /* Take the first entry off the worklist. */
3622 /* This block can be added to the worklist again if necessary. */
3623 AVAIL_IN_WORKLIST_P (bb
) = false;
3624 tm_memopt_compute_avin (bb
);
3626 /* Note: We do not add the LOCAL sets here because we already
3627 seeded the AVAIL_OUT sets with them. */
3628 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3629 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3631 && (region
->exit_blocks
== NULL
3632 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3633 /* If the out state of this block changed, then we need to add
3634 its successors to the worklist if they are not already in. */
3635 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3636 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3637 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3640 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3651 dump_tm_memopt_sets (blocks
);
3654 /* Compute ANTIC sets for every basic block in BLOCKS.
3656 We compute STORE_ANTIC_OUT as follows:
3658 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3659 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3661 REGION is the TM region.
3662 BLOCKS are the basic blocks in the region. */
3665 tm_memopt_compute_antic (struct tm_region
*region
,
3666 vec
<basic_block
> blocks
)
3669 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3674 /* Allocate a worklist array/queue. Entries are only added to the
3675 list if they were not already on the list. So the size is
3676 bounded by the number of basic blocks in the region. */
3677 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3679 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3683 /* Seed ANTIC_OUT with the LOCAL set. */
3684 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3686 /* Put every block in the region on the worklist. */
3687 AVAIL_IN_WORKLIST_P (bb
) = true;
3688 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3689 and their ANTIC_OUT has already been seeded in. */
3690 if (region
->exit_blocks
3691 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3698 /* The exit blocks have been initialized with the local sets. */
3699 if (region
->exit_blocks
)
3703 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3704 BB_VISITED_P (BASIC_BLOCK (i
)) = true;
3708 qend
= &worklist
[qlen
];
3710 /* Iterate until the worklist is empty. */
3713 /* Take the first entry off the worklist. */
3720 /* This block can be added to the worklist again if necessary. */
3721 AVAIL_IN_WORKLIST_P (bb
) = false;
3722 tm_memopt_compute_antin (bb
);
3724 /* Note: We do not add the LOCAL sets here because we already
3725 seeded the ANTIC_OUT sets with them. */
3726 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3727 && bb
!= region
->entry_block
)
3728 /* If the out state of this block changed, then we need to add
3729 its predecessors to the worklist if they are not already in. */
3730 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3731 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3734 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3745 dump_tm_memopt_sets (blocks
);
3748 /* Offsets of load variants from TM_LOAD. For example,
3749 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3750 See gtm-builtins.def. */
3751 #define TRANSFORM_RAR 1
3752 #define TRANSFORM_RAW 2
3753 #define TRANSFORM_RFW 3
3754 /* Offsets of store variants from TM_STORE. */
3755 #define TRANSFORM_WAR 1
3756 #define TRANSFORM_WAW 2
3758 /* Inform about a load/store optimization. */
3761 dump_tm_memopt_transform (gimple stmt
)
3765 fprintf (dump_file
, "TM memopt: transforming: ");
3766 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3767 fprintf (dump_file
, "\n");
3771 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3772 by a builtin that is OFFSET entries down in the builtins table in
3773 gtm-builtins.def. */
3776 tm_memopt_transform_stmt (unsigned int offset
,
3778 gimple_stmt_iterator
*gsi
)
3780 tree fn
= gimple_call_fn (stmt
);
3781 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3782 TREE_OPERAND (fn
, 0)
3783 = builtin_decl_explicit ((enum built_in_function
)
3784 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3786 gimple_call_set_fn (stmt
, fn
);
3787 gsi_replace (gsi
, stmt
, true);
3788 dump_tm_memopt_transform (stmt
);
3791 /* Perform the actual TM memory optimization transformations in the
3792 basic blocks in BLOCKS. */
3795 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3799 gimple_stmt_iterator gsi
;
3801 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3803 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3805 gimple stmt
= gsi_stmt (gsi
);
3806 bitmap read_avail
= READ_AVAIL_IN (bb
);
3807 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3808 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3811 if (is_tm_simple_load (stmt
))
3813 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3814 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3815 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3816 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3818 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3819 bitmap_set_bit (store_avail
, loc
);
3821 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3822 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3824 bitmap_set_bit (read_avail
, loc
);
3826 else if (is_tm_simple_store (stmt
))
3828 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3829 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3830 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3833 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3834 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3835 bitmap_set_bit (store_avail
, loc
);
3842 /* Return a new set of bitmaps for a BB. */
3844 static struct tm_memopt_bitmaps
*
3845 tm_memopt_init_sets (void)
3847 struct tm_memopt_bitmaps
*b
3848 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3849 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3850 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3851 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3852 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3853 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3854 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3855 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3856 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3857 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3861 /* Free sets computed for each BB. */
3864 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3869 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3873 /* Clear the visited bit for every basic block in BLOCKS. */
3876 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3881 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3882 BB_VISITED_P (bb
) = false;
3885 /* Replace TM load/stores with hints for the runtime. We handle
3886 things like read-after-write, write-after-read, read-after-read,
3887 read-for-write, etc. */
3890 execute_tm_memopt (void)
3892 struct tm_region
*region
;
3893 vec
<basic_block
> bbs
;
3895 tm_memopt_value_id
= 0;
3896 tm_memopt_value_numbers
.create (10);
3898 for (region
= all_tm_regions
; region
; region
= region
->next
)
3900 /* All the TM stores/loads in the current region. */
3904 bitmap_obstack_initialize (&tm_memopt_obstack
);
3906 /* Save all BBs for the current region. */
3907 bbs
= get_tm_region_blocks (region
->entry_block
,
3908 region
->exit_blocks
,
3913 /* Collect all the memory operations. */
3914 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3916 bb
->aux
= tm_memopt_init_sets ();
3917 tm_memopt_accumulate_memops (bb
);
3920 /* Solve data flow equations and transform each block accordingly. */
3921 tm_memopt_clear_visited (bbs
);
3922 tm_memopt_compute_available (region
, bbs
);
3923 tm_memopt_clear_visited (bbs
);
3924 tm_memopt_compute_antic (region
, bbs
);
3925 tm_memopt_transform_blocks (bbs
);
3927 tm_memopt_free_sets (bbs
);
3929 bitmap_obstack_release (&tm_memopt_obstack
);
3930 tm_memopt_value_numbers
.empty ();
3933 tm_memopt_value_numbers
.dispose ();
3938 gate_tm_memopt (void)
3940 return flag_tm
&& optimize
> 0;
3945 const pass_data pass_data_tm_memopt
=
3947 GIMPLE_PASS
, /* type */
3948 "tmmemopt", /* name */
3949 OPTGROUP_NONE
, /* optinfo_flags */
3950 true, /* has_gate */
3951 true, /* has_execute */
3952 TV_TRANS_MEM
, /* tv_id */
3953 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3954 0, /* properties_provided */
3955 0, /* properties_destroyed */
3956 0, /* todo_flags_start */
3957 0, /* todo_flags_finish */
3960 class pass_tm_memopt
: public gimple_opt_pass
3963 pass_tm_memopt (gcc::context
*ctxt
)
3964 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3967 /* opt_pass methods: */
3968 bool gate () { return gate_tm_memopt (); }
3969 unsigned int execute () { return execute_tm_memopt (); }
3971 }; // class pass_tm_memopt
3976 make_pass_tm_memopt (gcc::context
*ctxt
)
3978 return new pass_tm_memopt (ctxt
);
3982 /* Interprocedual analysis for the creation of transactional clones.
3983 The aim of this pass is to find which functions are referenced in
3984 a non-irrevocable transaction context, and for those over which
3985 we have control (or user directive), create a version of the
3986 function which uses only the transactional interface to reference
3987 protected memories. This analysis proceeds in several steps:
3989 (1) Collect the set of all possible transactional clones:
3991 (a) For all local public functions marked tm_callable, push
3992 it onto the tm_callee queue.
3994 (b) For all local functions, scan for calls in transaction blocks.
3995 Push the caller and callee onto the tm_caller and tm_callee
3996 queues. Count the number of callers for each callee.
3998 (c) For each local function on the callee list, assume we will
3999 create a transactional clone. Push *all* calls onto the
4000 callee queues; count the number of clone callers separately
4001 to the number of original callers.
4003 (2) Propagate irrevocable status up the dominator tree:
4005 (a) Any external function on the callee list that is not marked
4006 tm_callable is irrevocable. Push all callers of such onto
4009 (b) For each function on the worklist, mark each block that
4010 contains an irrevocable call. Use the AND operator to
4011 propagate that mark up the dominator tree.
4013 (c) If we reach the entry block for a possible transactional
4014 clone, then the transactional clone is irrevocable, and
4015 we should not create the clone after all. Push all
4016 callers onto the worklist.
4018 (d) Place tm_irrevocable calls at the beginning of the relevant
4019 blocks. Special case here is the entry block for the entire
4020 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4021 the library to begin the region in serial mode. Decrement
4022 the call count for all callees in the irrevocable region.
4024 (3) Create the transactional clones:
4026 Any tm_callee that still has a non-zero call count is cloned.
4029 /* This structure is stored in the AUX field of each cgraph_node. */
4030 struct tm_ipa_cg_data
4032 /* The clone of the function that got created. */
4033 struct cgraph_node
*clone
;
4035 /* The tm regions in the normal function. */
4036 struct tm_region
*all_tm_regions
;
4038 /* The blocks of the normal/clone functions that contain irrevocable
4039 calls, or blocks that are post-dominated by irrevocable calls. */
4040 bitmap irrevocable_blocks_normal
;
4041 bitmap irrevocable_blocks_clone
;
4043 /* The blocks of the normal function that are involved in transactions. */
4044 bitmap transaction_blocks_normal
;
4046 /* The number of callers to the transactional clone of this function
4047 from normal and transactional clones respectively. */
4048 unsigned tm_callers_normal
;
4049 unsigned tm_callers_clone
;
4051 /* True if all calls to this function's transactional clone
4052 are irrevocable. Also automatically true if the function
4053 has no transactional clone. */
4054 bool is_irrevocable
;
4056 /* Flags indicating the presence of this function in various queues. */
4057 bool in_callee_queue
;
4060 /* Flags indicating the kind of scan desired while in the worklist. */
4061 bool want_irr_scan_normal
;
4064 typedef vec
<cgraph_node_ptr
> cgraph_node_queue
;
4066 /* Return the ipa data associated with NODE, allocating zeroed memory
4067 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4068 and set *NODE accordingly. */
4070 static struct tm_ipa_cg_data
*
4071 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4073 struct tm_ipa_cg_data
*d
;
4075 if (traverse_aliases
&& (*node
)->alias
)
4076 *node
= cgraph_alias_target (*node
);
4078 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4082 d
= (struct tm_ipa_cg_data
*)
4083 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4084 (*node
)->aux
= (void *) d
;
4085 memset (d
, 0, sizeof (*d
));
4091 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4092 it is already present. */
4095 maybe_push_queue (struct cgraph_node
*node
,
4096 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4101 queue_p
->safe_push (node
);
4105 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4106 code path. QUEUE are the basic blocks inside the transaction
4107 represented in REGION.
4109 Later in split_code_paths() we will add the conditional to choose
4110 between the two alternatives. */
4113 ipa_uninstrument_transaction (struct tm_region
*region
,
4114 vec
<basic_block
> queue
)
4116 gimple transaction
= region
->transaction_stmt
;
4117 basic_block transaction_bb
= gimple_bb (transaction
);
4118 int n
= queue
.length ();
4119 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4121 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4123 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4124 add_phi_args_after_copy (new_bbs
, n
, e
);
4126 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4127 // a) EDGE_FALLTHRU into the transaction
4128 // b) EDGE_TM_ABORT out of the transaction
4129 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4134 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4135 Queue all callees within block BB. */
4138 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4139 basic_block bb
, bool for_clone
)
4141 gimple_stmt_iterator gsi
;
4143 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4145 gimple stmt
= gsi_stmt (gsi
);
4146 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4148 tree fndecl
= gimple_call_fndecl (stmt
);
4151 struct tm_ipa_cg_data
*d
;
4153 struct cgraph_node
*node
;
4155 if (is_tm_ending_fndecl (fndecl
))
4157 if (find_tm_replacement_function (fndecl
))
4160 node
= cgraph_get_node (fndecl
);
4161 gcc_assert (node
!= NULL
);
4162 d
= get_cg_data (&node
, true);
4164 pcallers
= (for_clone
? &d
->tm_callers_clone
4165 : &d
->tm_callers_normal
);
4168 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4174 /* Scan all calls in NODE that are within a transaction region,
4175 and push the resulting nodes into the callee queue. */
4178 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4179 cgraph_node_queue
*callees_p
)
4181 struct tm_region
*r
;
4183 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4184 d
->all_tm_regions
= all_tm_regions
;
4186 for (r
= all_tm_regions
; r
; r
= r
->next
)
4188 vec
<basic_block
> bbs
;
4192 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4193 d
->transaction_blocks_normal
, false);
4195 // Generate the uninstrumented code path for this transaction.
4196 ipa_uninstrument_transaction (r
, bbs
);
4198 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4199 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4204 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4205 // copying them, rather than forcing us to do this externally.
4206 rebuild_cgraph_edges ();
4208 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4209 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4210 // Instead, just release dominators here so update_ssa recomputes them.
4211 free_dominance_info (CDI_DOMINATORS
);
4213 // When building the uninstrumented code path, copy_bbs will have invoked
4214 // create_new_def_for starting an "ssa update context". There is only one
4215 // instance of this context, so resolve ssa updates before moving on to
4216 // the next function.
4217 update_ssa (TODO_update_ssa
);
4220 /* Scan all calls in NODE as if this is the transactional clone,
4221 and push the destinations into the callee queue. */
4224 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4225 cgraph_node_queue
*callees_p
)
4227 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4230 FOR_EACH_BB_FN (bb
, fn
)
4231 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4234 /* The function NODE has been detected to be irrevocable. Push all
4235 of its callers onto WORKLIST for the purpose of re-scanning them. */
4238 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4239 cgraph_node_queue
*worklist_p
)
4241 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4242 struct cgraph_edge
*e
;
4244 d
->is_irrevocable
= true;
4246 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4249 struct cgraph_node
*caller
;
4251 /* Don't examine recursive calls. */
4252 if (e
->caller
== node
)
4254 /* Even if we think we can go irrevocable, believe the user
4256 if (is_tm_safe_or_pure (e
->caller
->decl
))
4260 d
= get_cg_data (&caller
, true);
4262 /* Check if the callee is in a transactional region. If so,
4263 schedule the function for normal re-scan as well. */
4264 bb
= gimple_bb (e
->call_stmt
);
4265 gcc_assert (bb
!= NULL
);
4266 if (d
->transaction_blocks_normal
4267 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4268 d
->want_irr_scan_normal
= true;
4270 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4274 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4275 within the block is irrevocable. */
4278 ipa_tm_scan_irr_block (basic_block bb
)
4280 gimple_stmt_iterator gsi
;
4283 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4285 gimple stmt
= gsi_stmt (gsi
);
4286 switch (gimple_code (stmt
))
4289 if (gimple_assign_single_p (stmt
))
4291 tree lhs
= gimple_assign_lhs (stmt
);
4292 tree rhs
= gimple_assign_rhs1 (stmt
);
4293 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4300 tree lhs
= gimple_call_lhs (stmt
);
4301 if (lhs
&& volatile_var_p (lhs
))
4304 if (is_tm_pure_call (stmt
))
4307 fn
= gimple_call_fn (stmt
);
4309 /* Functions with the attribute are by definition irrevocable. */
4310 if (is_tm_irrevocable (fn
))
4313 /* For direct function calls, go ahead and check for replacement
4314 functions, or transitive irrevocable functions. For indirect
4315 functions, we'll ask the runtime. */
4316 if (TREE_CODE (fn
) == ADDR_EXPR
)
4318 struct tm_ipa_cg_data
*d
;
4319 struct cgraph_node
*node
;
4321 fn
= TREE_OPERAND (fn
, 0);
4322 if (is_tm_ending_fndecl (fn
))
4324 if (find_tm_replacement_function (fn
))
4327 node
= cgraph_get_node (fn
);
4328 d
= get_cg_data (&node
, true);
4330 /* Return true if irrevocable, but above all, believe
4332 if (d
->is_irrevocable
4333 && !is_tm_safe_or_pure (fn
))
4340 /* ??? The Approved Method of indicating that an inline
4341 assembly statement is not relevant to the transaction
4342 is to wrap it in a __tm_waiver block. This is not
4343 yet implemented, so we can't check for it. */
4344 if (is_tm_safe (current_function_decl
))
4346 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4347 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4348 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4360 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4361 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4362 scanning past OLD_IRR or EXIT_BLOCKS. */
4365 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4366 bitmap old_irr
, bitmap exit_blocks
)
4368 bool any_new_irr
= false;
4371 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4375 basic_block bb
= pqueue
->pop ();
4377 /* Don't re-scan blocks we know already are irrevocable. */
4378 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4381 if (ipa_tm_scan_irr_block (bb
))
4383 bitmap_set_bit (new_irr
, bb
->index
);
4386 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4388 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4389 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4391 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4392 pqueue
->safe_push (e
->dest
);
4396 while (!pqueue
->is_empty ());
4398 BITMAP_FREE (visited_blocks
);
4403 /* Propagate the irrevocable property both up and down the dominator tree.
4404 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4405 TM regions; OLD_IRR are the results of a previous scan of the dominator
4406 tree which has been fully propagated; NEW_IRR is the set of new blocks
4407 which are gaining the irrevocable property during the current scan. */
4410 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4411 bitmap old_irr
, bitmap exit_blocks
)
4413 vec
<basic_block
> bbs
;
4414 bitmap all_region_blocks
;
4416 /* If this block is in the old set, no need to rescan. */
4417 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4420 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4421 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4422 all_region_blocks
, false);
4425 basic_block bb
= bbs
.pop ();
4426 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4427 bool all_son_irr
= false;
4431 /* Propagate up. If my children are, I am too, but we must have
4432 at least one child that is. */
4435 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4437 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4439 all_son_irr
= false;
4447 /* Add block to new_irr if it hasn't already been processed. */
4448 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4450 bitmap_set_bit (new_irr
, bb
->index
);
4456 /* Propagate down to everyone we immediately dominate. */
4460 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4462 son
= next_dom_son (CDI_DOMINATORS
, son
))
4464 /* Make sure block is actually in a TM region, and it
4465 isn't already in old_irr. */
4466 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4467 && bitmap_bit_p (all_region_blocks
, son
->index
))
4468 bitmap_set_bit (new_irr
, son
->index
);
4472 while (!bbs
.is_empty ());
4474 BITMAP_FREE (all_region_blocks
);
4479 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4481 gimple_stmt_iterator gsi
;
4483 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4485 gimple stmt
= gsi_stmt (gsi
);
4486 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4488 tree fndecl
= gimple_call_fndecl (stmt
);
4491 struct tm_ipa_cg_data
*d
;
4493 struct cgraph_node
*tnode
;
4495 if (is_tm_ending_fndecl (fndecl
))
4497 if (find_tm_replacement_function (fndecl
))
4500 tnode
= cgraph_get_node (fndecl
);
4501 d
= get_cg_data (&tnode
, true);
4503 pcallers
= (for_clone
? &d
->tm_callers_clone
4504 : &d
->tm_callers_normal
);
4506 gcc_assert (*pcallers
> 0);
4513 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4514 as well as other irrevocable actions such as inline assembly. Mark all
4515 such blocks as irrevocable and decrement the number of calls to
4516 transactional clones. Return true if, for the transactional clone, the
4517 entire function is irrevocable. */
4520 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4522 struct tm_ipa_cg_data
*d
;
4523 bitmap new_irr
, old_irr
;
4524 vec
<basic_block
> queue
;
4527 /* Builtin operators (operator new, and such). */
4528 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4529 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4532 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4533 calculate_dominance_info (CDI_DOMINATORS
);
4535 d
= get_cg_data (&node
, true);
4537 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4539 /* Scan each tm region, propagating irrevocable status through the tree. */
4542 old_irr
= d
->irrevocable_blocks_clone
;
4543 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4544 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4546 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4549 ret
= bitmap_bit_p (new_irr
,
4550 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4555 struct tm_region
*region
;
4557 old_irr
= d
->irrevocable_blocks_normal
;
4558 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4560 queue
.quick_push (region
->entry_block
);
4561 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4562 region
->exit_blocks
))
4563 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4564 region
->exit_blocks
);
4568 /* If we found any new irrevocable blocks, reduce the call count for
4569 transactional clones within the irrevocable blocks. Save the new
4570 set of irrevocable blocks for next time. */
4571 if (!bitmap_empty_p (new_irr
))
4573 bitmap_iterator bmi
;
4576 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4577 ipa_tm_decrement_clone_counts (BASIC_BLOCK (i
), for_clone
);
4581 bitmap_ior_into (old_irr
, new_irr
);
4582 BITMAP_FREE (new_irr
);
4585 d
->irrevocable_blocks_clone
= new_irr
;
4587 d
->irrevocable_blocks_normal
= new_irr
;
4589 if (dump_file
&& new_irr
)
4592 bitmap_iterator bmi
;
4595 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4596 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4597 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4601 BITMAP_FREE (new_irr
);
4609 /* Return true if, for the transactional clone of NODE, any call
4610 may enter irrevocable mode. */
4613 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4615 struct tm_ipa_cg_data
*d
;
4619 d
= get_cg_data (&node
, true);
4621 flags
= flags_from_decl_or_type (decl
);
4623 /* Handle some TM builtins. Ordinarily these aren't actually generated
4624 at this point, but handling these functions when written in by the
4625 user makes it easier to build unit tests. */
4626 if (flags
& ECF_TM_BUILTIN
)
4629 /* Filter out all functions that are marked. */
4630 if (flags
& ECF_TM_PURE
)
4632 if (is_tm_safe (decl
))
4634 if (is_tm_irrevocable (decl
))
4636 if (is_tm_callable (decl
))
4638 if (find_tm_replacement_function (decl
))
4641 /* If we aren't seeing the final version of the function we don't
4642 know what it will contain at runtime. */
4643 if (cgraph_function_body_availability (node
) < AVAIL_AVAILABLE
)
4646 /* If the function must go irrevocable, then of course true. */
4647 if (d
->is_irrevocable
)
4650 /* If there are any blocks marked irrevocable, then the function
4651 as a whole may enter irrevocable. */
4652 if (d
->irrevocable_blocks_clone
)
4655 /* We may have previously marked this function as tm_may_enter_irr;
4656 see pass_diagnose_tm_blocks. */
4657 if (node
->local
.tm_may_enter_irr
)
4660 /* Recurse on the main body for aliases. In general, this will
4661 result in one of the bits above being set so that we will not
4662 have to recurse next time. */
4664 return ipa_tm_mayenterirr_function (cgraph_get_node (node
->thunk
.alias
));
4666 /* What remains is unmarked local functions without items that force
4667 the function to go irrevocable. */
4671 /* Diagnose calls from transaction_safe functions to unmarked
4672 functions that are determined to not be safe. */
4675 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4677 struct cgraph_edge
*e
;
4679 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4680 if (!is_tm_callable (e
->callee
->decl
)
4681 && e
->callee
->local
.tm_may_enter_irr
)
4682 error_at (gimple_location (e
->call_stmt
),
4683 "unsafe function call %qD within "
4684 "%<transaction_safe%> function", e
->callee
->decl
);
4687 /* Diagnose call from atomic transactions to unmarked functions
4688 that are determined to not be safe. */
4691 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4692 struct tm_region
*all_tm_regions
)
4694 struct tm_region
*r
;
4696 for (r
= all_tm_regions
; r
; r
= r
->next
)
4697 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4699 /* Atomic transactions can be nested inside relaxed. */
4701 ipa_tm_diagnose_transaction (node
, r
->inner
);
4705 vec
<basic_block
> bbs
;
4706 gimple_stmt_iterator gsi
;
4710 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4711 r
->irr_blocks
, NULL
, false);
4713 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4714 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4716 gimple stmt
= gsi_stmt (gsi
);
4719 if (gimple_code (stmt
) == GIMPLE_ASM
)
4721 error_at (gimple_location (stmt
),
4722 "asm not allowed in atomic transaction");
4726 if (!is_gimple_call (stmt
))
4728 fndecl
= gimple_call_fndecl (stmt
);
4730 /* Indirect function calls have been diagnosed already. */
4734 /* Stop at the end of the transaction. */
4735 if (is_tm_ending_fndecl (fndecl
))
4737 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4742 /* Marked functions have been diagnosed already. */
4743 if (is_tm_pure_call (stmt
))
4745 if (is_tm_callable (fndecl
))
4748 if (cgraph_local_info (fndecl
)->tm_may_enter_irr
)
4749 error_at (gimple_location (stmt
),
4750 "unsafe function call %qD within "
4751 "atomic transaction", fndecl
);
4758 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4759 OLD_DECL. The returned value is a freshly malloced pointer that
4760 should be freed by the caller. */
4763 tm_mangle (tree old_asm_id
)
4765 const char *old_asm_name
;
4768 struct demangle_component
*dc
;
4771 /* Determine if the symbol is already a valid C++ mangled name. Do this
4772 even for C, which might be interfacing with C++ code via appropriately
4773 ugly identifiers. */
4774 /* ??? We could probably do just as well checking for "_Z" and be done. */
4775 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4776 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4783 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4784 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4788 old_asm_name
+= 2; /* Skip _Z */
4792 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4793 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4794 /* Don't play silly games, you! */
4797 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4798 /* I'd really like to know if we can ever be passed one of
4799 these from the C++ front end. The Logical Thing would
4800 seem that hidden-alias should be outer-most, so that we
4801 get hidden-alias of a transaction-clone and not vice-versa. */
4809 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4813 new_asm_id
= get_identifier (tm_name
);
4820 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4822 cgraph_mark_force_output_node (node
);
4823 node
->analyzed
= true;
4827 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4829 node
->forced_by_abi
= true;
4830 node
->analyzed
= true;
4833 /* Callback data for ipa_tm_create_version_alias. */
4834 struct create_version_alias_info
4836 struct cgraph_node
*old_node
;
4840 /* A subroutine of ipa_tm_create_version, called via
4841 cgraph_for_node_and_aliases. Create new tm clones for each of
4842 the existing aliases. */
4844 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4846 struct create_version_alias_info
*info
4847 = (struct create_version_alias_info
*)data
;
4848 tree old_decl
, new_decl
, tm_name
;
4849 struct cgraph_node
*new_node
;
4851 if (!node
->cpp_implicit_alias
)
4854 old_decl
= node
->decl
;
4855 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4856 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4857 TREE_CODE (old_decl
), tm_name
,
4858 TREE_TYPE (old_decl
));
4860 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4861 SET_DECL_RTL (new_decl
, NULL
);
4863 /* Based loosely on C++'s make_alias_for(). */
4864 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4865 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4866 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4867 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4868 DECL_EXTERNAL (new_decl
) = 0;
4869 DECL_ARTIFICIAL (new_decl
) = 1;
4870 TREE_ADDRESSABLE (new_decl
) = 1;
4871 TREE_USED (new_decl
) = 1;
4872 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4874 /* Perform the same remapping to the comdat group. */
4875 if (DECL_ONE_ONLY (new_decl
))
4876 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4878 new_node
= cgraph_same_body_alias (NULL
, new_decl
, info
->new_decl
);
4879 new_node
->tm_clone
= true;
4880 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4881 /* ?? Do not traverse aliases here. */
4882 get_cg_data (&node
, false)->clone
= new_node
;
4884 record_tm_clone_pair (old_decl
, new_decl
);
4886 if (info
->old_node
->force_output
4887 || ipa_ref_list_first_referring (&info
->old_node
->ref_list
))
4888 ipa_tm_mark_force_output_node (new_node
);
4889 if (info
->old_node
->forced_by_abi
)
4890 ipa_tm_mark_forced_by_abi_node (new_node
);
4894 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4895 appropriate for the transactional clone. */
4898 ipa_tm_create_version (struct cgraph_node
*old_node
)
4900 tree new_decl
, old_decl
, tm_name
;
4901 struct cgraph_node
*new_node
;
4903 old_decl
= old_node
->decl
;
4904 new_decl
= copy_node (old_decl
);
4906 /* DECL_ASSEMBLER_NAME needs to be set before we call
4907 cgraph_copy_node_for_versioning below, because cgraph_node will
4908 fill the assembler_name_hash. */
4909 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4910 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4911 SET_DECL_RTL (new_decl
, NULL
);
4912 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4914 /* Perform the same remapping to the comdat group. */
4915 if (DECL_ONE_ONLY (new_decl
))
4916 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4918 new_node
= cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
4919 new_node
->local
.local
= false;
4920 new_node
->externally_visible
= old_node
->externally_visible
;
4921 new_node
->lowered
= true;
4922 new_node
->tm_clone
= 1;
4923 get_cg_data (&old_node
, true)->clone
= new_node
;
4925 if (cgraph_function_body_availability (old_node
) >= AVAIL_OVERWRITABLE
)
4927 /* Remap extern inline to static inline. */
4928 /* ??? Is it worth trying to use make_decl_one_only? */
4929 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4931 DECL_EXTERNAL (new_decl
) = 0;
4932 TREE_PUBLIC (new_decl
) = 0;
4933 DECL_WEAK (new_decl
) = 0;
4936 tree_function_versioning (old_decl
, new_decl
,
4941 record_tm_clone_pair (old_decl
, new_decl
);
4943 cgraph_call_function_insertion_hooks (new_node
);
4944 if (old_node
->force_output
4945 || ipa_ref_list_first_referring (&old_node
->ref_list
))
4946 ipa_tm_mark_force_output_node (new_node
);
4947 if (old_node
->forced_by_abi
)
4948 ipa_tm_mark_forced_by_abi_node (new_node
);
4950 /* Do the same thing, but for any aliases of the original node. */
4952 struct create_version_alias_info data
;
4953 data
.old_node
= old_node
;
4954 data
.new_decl
= new_decl
;
4955 cgraph_for_node_and_aliases (old_node
, ipa_tm_create_version_alias
,
4960 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4963 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4966 gimple_stmt_iterator gsi
;
4969 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4971 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4972 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4974 split_block_after_labels (bb
);
4975 gsi
= gsi_after_labels (bb
);
4976 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4978 cgraph_create_edge (node
,
4979 cgraph_get_create_node
4980 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4982 compute_call_stmt_bb_frequency (node
->decl
,
4986 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4989 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4990 struct tm_region
*region
,
4991 gimple_stmt_iterator
*gsi
, gimple stmt
)
4993 tree gettm_fn
, ret
, old_fn
, callfn
;
4997 old_fn
= gimple_call_fn (stmt
);
4999 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5001 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5002 tree clone
= get_tm_clone_pair (fndecl
);
5004 /* By transforming the call into a TM_GETTMCLONE, we are
5005 technically taking the address of the original function and
5006 its clone. Explain this so inlining will know this function
5008 cgraph_mark_address_taken_node (cgraph_get_node (fndecl
));
5010 cgraph_mark_address_taken_node (cgraph_get_node (clone
));
5013 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5014 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5015 : BUILT_IN_TM_GETTMCLONE_IRR
);
5016 ret
= create_tmp_var (ptr_type_node
, NULL
);
5019 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5021 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5022 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5023 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5025 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5026 ret
= make_ssa_name (ret
, g
);
5027 gimple_call_set_lhs (g
, ret
);
5029 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5031 cgraph_create_edge (node
, cgraph_get_create_node (gettm_fn
), g
, 0,
5032 compute_call_stmt_bb_frequency (node
->decl
,
5035 /* Cast return value from tm_gettmclone* into appropriate function
5037 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
5038 g2
= gimple_build_assign (callfn
,
5039 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5040 callfn
= make_ssa_name (callfn
, g2
);
5041 gimple_assign_set_lhs (g2
, callfn
);
5042 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5044 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5045 which we would have derived from the decl. Failure to save
5046 this bit means we might have to split the basic block. */
5047 if (gimple_call_nothrow_p (stmt
))
5048 gimple_call_set_nothrow (stmt
, true);
5050 gimple_call_set_fn (stmt
, callfn
);
5052 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5053 for a call statement. Fix it. */
5055 tree lhs
= gimple_call_lhs (stmt
);
5056 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5058 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5062 temp
= create_tmp_reg (rettype
, 0);
5063 gimple_call_set_lhs (stmt
, temp
);
5065 g2
= gimple_build_assign (lhs
,
5066 fold_build1 (VIEW_CONVERT_EXPR
,
5067 TREE_TYPE (lhs
), temp
));
5068 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5077 /* Helper function for ipa_tm_transform_calls*. Given a call
5078 statement in GSI which resides inside transaction REGION, redirect
5079 the call to either its wrapper function, or its clone. */
5082 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5083 struct tm_region
*region
,
5084 gimple_stmt_iterator
*gsi
,
5085 bool *need_ssa_rename_p
)
5087 gimple stmt
= gsi_stmt (*gsi
);
5088 struct cgraph_node
*new_node
;
5089 struct cgraph_edge
*e
= cgraph_edge (node
, stmt
);
5090 tree fndecl
= gimple_call_fndecl (stmt
);
5092 /* For indirect calls, pass the address through the runtime. */
5095 *need_ssa_rename_p
|=
5096 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5100 /* Handle some TM builtins. Ordinarily these aren't actually generated
5101 at this point, but handling these functions when written in by the
5102 user makes it easier to build unit tests. */
5103 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5106 /* Fixup recursive calls inside clones. */
5107 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5108 for recursion but not update the call statements themselves? */
5109 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5111 gimple_call_set_fndecl (stmt
, current_function_decl
);
5115 /* If there is a replacement, use it. */
5116 fndecl
= find_tm_replacement_function (fndecl
);
5119 new_node
= cgraph_get_create_node (fndecl
);
5121 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5123 We can't do this earlier in record_tm_replacement because
5124 cgraph_remove_unreachable_nodes is called before we inject
5125 references to the node. Further, we can't do this in some
5126 nice central place in ipa_tm_execute because we don't have
5127 the exact list of wrapper functions that would be used.
5128 Marking more wrappers than necessary results in the creation
5129 of unnecessary cgraph_nodes, which can cause some of the
5130 other IPA passes to crash.
5132 We do need to mark these nodes so that we get the proper
5133 result in expand_call_tm. */
5134 /* ??? This seems broken. How is it that we're marking the
5135 CALLEE as may_enter_irr? Surely we should be marking the
5136 CALLER. Also note that find_tm_replacement_function also
5137 contains mappings into the TM runtime, e.g. memcpy. These
5138 we know won't go irrevocable. */
5139 new_node
->local
.tm_may_enter_irr
= 1;
5143 struct tm_ipa_cg_data
*d
;
5144 struct cgraph_node
*tnode
= e
->callee
;
5146 d
= get_cg_data (&tnode
, true);
5147 new_node
= d
->clone
;
5149 /* As we've already skipped pure calls and appropriate builtins,
5150 and we've already marked irrevocable blocks, if we can't come
5151 up with a static replacement, then ask the runtime. */
5152 if (new_node
== NULL
)
5154 *need_ssa_rename_p
|=
5155 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5159 fndecl
= new_node
->decl
;
5162 cgraph_redirect_edge_callee (e
, new_node
);
5163 gimple_call_set_fndecl (stmt
, fndecl
);
5166 /* Helper function for ipa_tm_transform_calls. For a given BB,
5167 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5168 redirect other calls to the generated transactional clone. */
5171 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5172 basic_block bb
, bitmap irr_blocks
)
5174 gimple_stmt_iterator gsi
;
5175 bool need_ssa_rename
= false;
5177 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5179 ipa_tm_insert_irr_call (node
, region
, bb
);
5183 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5185 gimple stmt
= gsi_stmt (gsi
);
5187 if (!is_gimple_call (stmt
))
5189 if (is_tm_pure_call (stmt
))
5192 /* Redirect edges to the appropriate replacement or clone. */
5193 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5196 return need_ssa_rename
;
5199 /* Walk the CFG for REGION, beginning at BB. Install calls to
5200 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5201 the generated transactional clone. */
5204 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5205 basic_block bb
, bitmap irr_blocks
)
5207 bool need_ssa_rename
= false;
5210 vec
<basic_block
> queue
= vNULL
;
5211 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5213 queue
.safe_push (bb
);
5219 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5221 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5224 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5227 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5228 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5230 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5231 queue
.safe_push (e
->dest
);
5234 while (!queue
.is_empty ());
5237 BITMAP_FREE (visited_blocks
);
5239 return need_ssa_rename
;
5242 /* Transform the calls within the TM regions within NODE. */
5245 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5247 struct tm_ipa_cg_data
*d
;
5248 struct tm_region
*region
;
5249 bool need_ssa_rename
= false;
5251 d
= get_cg_data (&node
, true);
5253 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5254 calculate_dominance_info (CDI_DOMINATORS
);
5256 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5258 /* If we're sure to go irrevocable, don't transform anything. */
5259 if (d
->irrevocable_blocks_normal
5260 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5261 region
->entry_block
->index
))
5263 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5264 | GTMA_MAY_ENTER_IRREVOCABLE
5265 | GTMA_HAS_NO_INSTRUMENTATION
);
5270 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5271 d
->irrevocable_blocks_normal
);
5274 if (need_ssa_rename
)
5275 update_ssa (TODO_update_ssa_only_virtuals
);
5280 /* Transform the calls within the transactional clone of NODE. */
5283 ipa_tm_transform_clone (struct cgraph_node
*node
)
5285 struct tm_ipa_cg_data
*d
;
5286 bool need_ssa_rename
;
5288 d
= get_cg_data (&node
, true);
5290 /* If this function makes no calls and has no irrevocable blocks,
5291 then there's nothing to do. */
5292 /* ??? Remove non-aborting top-level transactions. */
5293 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5296 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5297 calculate_dominance_info (CDI_DOMINATORS
);
5300 ipa_tm_transform_calls (d
->clone
, NULL
,
5301 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5302 d
->irrevocable_blocks_clone
);
5304 if (need_ssa_rename
)
5305 update_ssa (TODO_update_ssa_only_virtuals
);
5310 /* Main entry point for the transactional memory IPA pass. */
5313 ipa_tm_execute (void)
5315 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5316 /* List of functions that will go irrevocable. */
5317 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5319 struct cgraph_node
*node
;
5320 struct tm_ipa_cg_data
*d
;
5321 enum availability a
;
5324 #ifdef ENABLE_CHECKING
5328 bitmap_obstack_initialize (&tm_obstack
);
5329 initialize_original_copy_tables ();
5331 /* For all local functions marked tm_callable, queue them. */
5332 FOR_EACH_DEFINED_FUNCTION (node
)
5333 if (is_tm_callable (node
->decl
)
5334 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5336 d
= get_cg_data (&node
, true);
5337 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5340 /* For all local reachable functions... */
5341 FOR_EACH_DEFINED_FUNCTION (node
)
5343 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5345 /* ... marked tm_pure, record that fact for the runtime by
5346 indicating that the pure function is its own tm_callable.
5347 No need to do this if the function's address can't be taken. */
5348 if (is_tm_pure (node
->decl
))
5350 if (!node
->local
.local
)
5351 record_tm_clone_pair (node
->decl
, node
->decl
);
5355 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5356 calculate_dominance_info (CDI_DOMINATORS
);
5358 tm_region_init (NULL
);
5361 d
= get_cg_data (&node
, true);
5363 /* Scan for calls that are in each transaction, and
5364 generate the uninstrumented code path. */
5365 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5367 /* Put it in the worklist so we can scan the function
5368 later (ipa_tm_scan_irr_function) and mark the
5369 irrevocable blocks. */
5370 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5371 d
->want_irr_scan_normal
= true;
5377 /* For every local function on the callee list, scan as if we will be
5378 creating a transactional clone, queueing all new functions we find
5380 for (i
= 0; i
< tm_callees
.length (); ++i
)
5382 node
= tm_callees
[i
];
5383 a
= cgraph_function_body_availability (node
);
5384 d
= get_cg_data (&node
, true);
5386 /* Put it in the worklist so we can scan the function later
5387 (ipa_tm_scan_irr_function) and mark the irrevocable
5389 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5391 /* Some callees cannot be arbitrarily cloned. These will always be
5392 irrevocable. Mark these now, so that we need not scan them. */
5393 if (is_tm_irrevocable (node
->decl
))
5394 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5395 else if (a
<= AVAIL_NOT_AVAILABLE
5396 && !is_tm_safe_or_pure (node
->decl
))
5397 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5398 else if (a
>= AVAIL_OVERWRITABLE
)
5400 if (!tree_versionable_function_p (node
->decl
))
5401 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5402 else if (!d
->is_irrevocable
)
5404 /* If this is an alias, make sure its base is queued as well.
5405 we need not scan the callees now, as the base will do. */
5408 node
= cgraph_get_node (node
->thunk
.alias
);
5409 d
= get_cg_data (&node
, true);
5410 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5414 /* Add all nodes called by this function into
5415 tm_callees as well. */
5416 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5421 /* Iterate scans until no more work to be done. Prefer not to use
5422 vec::pop because the worklist tends to follow a breadth-first
5423 search of the callgraph, which should allow convergance with a
5424 minimum number of scans. But we also don't want the worklist
5425 array to grow without bound, so we shift the array up periodically. */
5426 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5428 if (i
> 256 && i
== irr_worklist
.length () / 8)
5430 irr_worklist
.block_remove (0, i
);
5434 node
= irr_worklist
[i
];
5435 d
= get_cg_data (&node
, true);
5436 d
->in_worklist
= false;
5438 if (d
->want_irr_scan_normal
)
5440 d
->want_irr_scan_normal
= false;
5441 ipa_tm_scan_irr_function (node
, false);
5443 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5444 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5447 /* For every function on the callee list, collect the tm_may_enter_irr
5449 irr_worklist
.truncate (0);
5450 for (i
= 0; i
< tm_callees
.length (); ++i
)
5452 node
= tm_callees
[i
];
5453 if (ipa_tm_mayenterirr_function (node
))
5455 d
= get_cg_data (&node
, true);
5456 gcc_assert (d
->in_worklist
== false);
5457 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5461 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5462 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5464 struct cgraph_node
*caller
;
5465 struct cgraph_edge
*e
;
5466 struct ipa_ref
*ref
;
5469 if (i
> 256 && i
== irr_worklist
.length () / 8)
5471 irr_worklist
.block_remove (0, i
);
5475 node
= irr_worklist
[i
];
5476 d
= get_cg_data (&node
, true);
5477 d
->in_worklist
= false;
5478 node
->local
.tm_may_enter_irr
= true;
5480 /* Propagate back to normal callers. */
5481 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5484 if (!is_tm_safe_or_pure (caller
->decl
)
5485 && !caller
->local
.tm_may_enter_irr
)
5487 d
= get_cg_data (&caller
, true);
5488 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5492 /* Propagate back to referring aliases as well. */
5493 for (j
= 0; ipa_ref_list_referring_iterate (&node
->ref_list
, j
, ref
); j
++)
5495 caller
= cgraph (ref
->referring
);
5496 if (ref
->use
== IPA_REF_ALIAS
5497 && !caller
->local
.tm_may_enter_irr
)
5499 /* ?? Do not traverse aliases here. */
5500 d
= get_cg_data (&caller
, false);
5501 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5506 /* Now validate all tm_safe functions, and all atomic regions in
5508 FOR_EACH_DEFINED_FUNCTION (node
)
5510 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5512 d
= get_cg_data (&node
, true);
5513 if (is_tm_safe (node
->decl
))
5514 ipa_tm_diagnose_tm_safe (node
);
5515 else if (d
->all_tm_regions
)
5516 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5519 /* Create clones. Do those that are not irrevocable and have a
5520 positive call count. Do those publicly visible functions that
5521 the user directed us to clone. */
5522 for (i
= 0; i
< tm_callees
.length (); ++i
)
5526 node
= tm_callees
[i
];
5527 if (node
->cpp_implicit_alias
)
5530 a
= cgraph_function_body_availability (node
);
5531 d
= get_cg_data (&node
, true);
5533 if (a
<= AVAIL_NOT_AVAILABLE
)
5534 doit
= is_tm_callable (node
->decl
);
5535 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5537 else if (!d
->is_irrevocable
5538 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5542 ipa_tm_create_version (node
);
5545 /* Redirect calls to the new clones, and insert irrevocable marks. */
5546 for (i
= 0; i
< tm_callees
.length (); ++i
)
5548 node
= tm_callees
[i
];
5551 d
= get_cg_data (&node
, true);
5553 ipa_tm_transform_clone (node
);
5556 FOR_EACH_DEFINED_FUNCTION (node
)
5558 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5560 d
= get_cg_data (&node
, true);
5561 if (d
->all_tm_regions
)
5562 ipa_tm_transform_transaction (node
);
5565 /* Free and clear all data structures. */
5566 tm_callees
.release ();
5567 irr_worklist
.release ();
5568 bitmap_obstack_release (&tm_obstack
);
5569 free_original_copy_tables ();
5571 FOR_EACH_FUNCTION (node
)
5574 #ifdef ENABLE_CHECKING
5583 const pass_data pass_data_ipa_tm
=
5585 SIMPLE_IPA_PASS
, /* type */
5587 OPTGROUP_NONE
, /* optinfo_flags */
5588 true, /* has_gate */
5589 true, /* has_execute */
5590 TV_TRANS_MEM
, /* tv_id */
5591 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5592 0, /* properties_provided */
5593 0, /* properties_destroyed */
5594 0, /* todo_flags_start */
5595 0, /* todo_flags_finish */
5598 class pass_ipa_tm
: public simple_ipa_opt_pass
5601 pass_ipa_tm (gcc::context
*ctxt
)
5602 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5605 /* opt_pass methods: */
5606 bool gate () { return gate_tm (); }
5607 unsigned int execute () { return ipa_tm_execute (); }
5609 }; // class pass_ipa_tm
5613 simple_ipa_opt_pass
*
5614 make_pass_ipa_tm (gcc::context
*ctxt
)
5616 return new pass_ipa_tm (ctxt
);
5619 #include "gt-trans-mem.h"