1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2014 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
23 #include "hash-table.h"
25 #include "basic-block.h"
26 #include "tree-ssa-alias.h"
27 #include "internal-fn.h"
29 #include "gimple-expr.h"
37 #include "gimple-iterator.h"
38 #include "gimplify-me.h"
39 #include "gimple-walk.h"
40 #include "gimple-ssa.h"
43 #include "stringpool.h"
44 #include "tree-ssanames.h"
45 #include "tree-into-ssa.h"
46 #include "tree-pass.h"
47 #include "tree-inline.h"
48 #include "diagnostic-core.h"
51 #include "trans-mem.h"
54 #include "langhooks.h"
55 #include "gimple-pretty-print.h"
57 #include "tree-ssa-address.h"
61 #define A_RUNINSTRUMENTEDCODE 0x0001
62 #define A_RUNUNINSTRUMENTEDCODE 0x0002
63 #define A_SAVELIVEVARIABLES 0x0004
64 #define A_RESTORELIVEVARIABLES 0x0008
65 #define A_ABORTTRANSACTION 0x0010
67 #define AR_USERABORT 0x0001
68 #define AR_USERRETRY 0x0002
69 #define AR_TMCONFLICT 0x0004
70 #define AR_EXCEPTIONBLOCKABORT 0x0008
71 #define AR_OUTERABORT 0x0010
73 #define MODE_SERIALIRREVOCABLE 0x0000
76 /* The representation of a transaction changes several times during the
77 lowering process. In the beginning, in the front-end we have the
78 GENERIC tree TRANSACTION_EXPR. For example,
86 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
87 trivially replaced with a GIMPLE_TRANSACTION node.
89 During pass_lower_tm, we examine the body of transactions looking
90 for aborts. Transactions that do not contain an abort may be
91 merged into an outer transaction. We also add a TRY-FINALLY node
92 to arrange for the transaction to be committed on any exit.
94 [??? Think about how this arrangement affects throw-with-commit
95 and throw-with-abort operations. In this case we want the TRY to
96 handle gotos, but not to catch any exceptions because the transaction
97 will already be closed.]
99 GIMPLE_TRANSACTION [label=NULL] {
106 __builtin___tm_abort ();
108 __builtin___tm_commit ();
112 During pass_lower_eh, we create EH regions for the transactions,
113 intermixed with the regular EH stuff. This gives us a nice persistent
114 mapping (all the way through rtl) from transactional memory operation
115 back to the transaction, which allows us to get the abnormal edges
116 correct to model transaction aborts and restarts:
118 GIMPLE_TRANSACTION [label=over]
124 __builtin___tm_abort ();
125 __builtin___tm_commit ();
128 This is the end of all_lowering_passes, and so is what is present
129 during the IPA passes, and through all of the optimization passes.
131 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
132 functions and mark functions for cloning.
134 At the end of gimple optimization, before exiting SSA form,
135 pass_tm_edges replaces statements that perform transactional
136 memory operations with the appropriate TM builtins, and swap
137 out function calls with their transactional clones. At this
138 point we introduce the abnormal transaction restart edges and
139 complete lowering of the GIMPLE_TRANSACTION node.
141 x = __builtin___tm_start (MAY_ABORT);
143 if (x & abort_transaction)
146 t0 = __builtin___tm_load (global);
148 __builtin___tm_store (&global, t1);
150 __builtin___tm_abort ();
151 __builtin___tm_commit ();
155 static void *expand_regions (struct tm_region
*,
156 void *(*callback
)(struct tm_region
*, void *),
160 /* Return the attributes we want to examine for X, or NULL if it's not
161 something we examine. We look at function types, but allow pointers
162 to function types and function decls and peek through. */
165 get_attrs_for (const_tree x
)
167 switch (TREE_CODE (x
))
170 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
177 if (TREE_CODE (x
) != POINTER_TYPE
)
183 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
189 return TYPE_ATTRIBUTES (x
);
193 /* Return true if X has been marked TM_PURE. */
196 is_tm_pure (const_tree x
)
200 switch (TREE_CODE (x
))
211 if (TREE_CODE (x
) != POINTER_TYPE
)
217 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
222 flags
= flags_from_decl_or_type (x
);
223 return (flags
& ECF_TM_PURE
) != 0;
226 /* Return true if X has been marked TM_IRREVOCABLE. */
229 is_tm_irrevocable (tree x
)
231 tree attrs
= get_attrs_for (x
);
233 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
236 /* A call to the irrevocable builtin is by definition,
238 if (TREE_CODE (x
) == ADDR_EXPR
)
239 x
= TREE_OPERAND (x
, 0);
240 if (TREE_CODE (x
) == FUNCTION_DECL
241 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
242 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
248 /* Return true if X has been marked TM_SAFE. */
251 is_tm_safe (const_tree x
)
255 tree attrs
= get_attrs_for (x
);
258 if (lookup_attribute ("transaction_safe", attrs
))
260 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
267 /* Return true if CALL is const, or tm_pure. */
270 is_tm_pure_call (gimple call
)
272 tree fn
= gimple_call_fn (call
);
274 if (TREE_CODE (fn
) == ADDR_EXPR
)
276 fn
= TREE_OPERAND (fn
, 0);
277 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
282 return is_tm_pure (fn
);
285 /* Return true if X has been marked TM_CALLABLE. */
288 is_tm_callable (tree x
)
290 tree attrs
= get_attrs_for (x
);
293 if (lookup_attribute ("transaction_callable", attrs
))
295 if (lookup_attribute ("transaction_safe", attrs
))
297 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
303 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
306 is_tm_may_cancel_outer (tree x
)
308 tree attrs
= get_attrs_for (x
);
310 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
314 /* Return true for built in functions that "end" a transaction. */
317 is_tm_ending_fndecl (tree fndecl
)
319 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
320 switch (DECL_FUNCTION_CODE (fndecl
))
322 case BUILT_IN_TM_COMMIT
:
323 case BUILT_IN_TM_COMMIT_EH
:
324 case BUILT_IN_TM_ABORT
:
325 case BUILT_IN_TM_IRREVOCABLE
:
334 /* Return true if STMT is a built in function call that "ends" a
338 is_tm_ending (gimple stmt
)
342 if (gimple_code (stmt
) != GIMPLE_CALL
)
345 fndecl
= gimple_call_fndecl (stmt
);
346 return (fndecl
!= NULL_TREE
347 && is_tm_ending_fndecl (fndecl
));
350 /* Return true if STMT is a TM load. */
353 is_tm_load (gimple stmt
)
357 if (gimple_code (stmt
) != GIMPLE_CALL
)
360 fndecl
= gimple_call_fndecl (stmt
);
361 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
362 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
365 /* Same as above, but for simple TM loads, that is, not the
366 after-write, after-read, etc optimized variants. */
369 is_tm_simple_load (gimple stmt
)
373 if (gimple_code (stmt
) != GIMPLE_CALL
)
376 fndecl
= gimple_call_fndecl (stmt
);
377 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
379 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
380 return (fcode
== BUILT_IN_TM_LOAD_1
381 || fcode
== BUILT_IN_TM_LOAD_2
382 || fcode
== BUILT_IN_TM_LOAD_4
383 || fcode
== BUILT_IN_TM_LOAD_8
384 || fcode
== BUILT_IN_TM_LOAD_FLOAT
385 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
386 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
387 || fcode
== BUILT_IN_TM_LOAD_M64
388 || fcode
== BUILT_IN_TM_LOAD_M128
389 || fcode
== BUILT_IN_TM_LOAD_M256
);
394 /* Return true if STMT is a TM store. */
397 is_tm_store (gimple stmt
)
401 if (gimple_code (stmt
) != GIMPLE_CALL
)
404 fndecl
= gimple_call_fndecl (stmt
);
405 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
406 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
409 /* Same as above, but for simple TM stores, that is, not the
410 after-write, after-read, etc optimized variants. */
413 is_tm_simple_store (gimple stmt
)
417 if (gimple_code (stmt
) != GIMPLE_CALL
)
420 fndecl
= gimple_call_fndecl (stmt
);
421 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
423 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
424 return (fcode
== BUILT_IN_TM_STORE_1
425 || fcode
== BUILT_IN_TM_STORE_2
426 || fcode
== BUILT_IN_TM_STORE_4
427 || fcode
== BUILT_IN_TM_STORE_8
428 || fcode
== BUILT_IN_TM_STORE_FLOAT
429 || fcode
== BUILT_IN_TM_STORE_DOUBLE
430 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
431 || fcode
== BUILT_IN_TM_STORE_M64
432 || fcode
== BUILT_IN_TM_STORE_M128
433 || fcode
== BUILT_IN_TM_STORE_M256
);
438 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
441 is_tm_abort (tree fndecl
)
444 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
445 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
448 /* Build a GENERIC tree for a user abort. This is called by front ends
449 while transforming the __tm_abort statement. */
452 build_tm_abort_call (location_t loc
, bool is_outer
)
454 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
455 build_int_cst (integer_type_node
,
457 | (is_outer
? AR_OUTERABORT
: 0)));
460 /* Map for aribtrary function replacement under TM, as created
461 by the tm_wrap attribute. */
463 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
467 record_tm_replacement (tree from
, tree to
)
469 struct tree_map
**slot
, *h
;
471 /* Do not inline wrapper functions that will get replaced in the TM
474 Suppose you have foo() that will get replaced into tmfoo(). Make
475 sure the inliner doesn't try to outsmart us and inline foo()
476 before we get a chance to do the TM replacement. */
477 DECL_UNINLINABLE (from
) = 1;
479 if (tm_wrap_map
== NULL
)
480 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
482 h
= ggc_alloc
<tree_map
> ();
483 h
->hash
= htab_hash_pointer (from
);
487 slot
= (struct tree_map
**)
488 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
492 /* Return a TM-aware replacement function for DECL. */
495 find_tm_replacement_function (tree fndecl
)
499 struct tree_map
*h
, in
;
501 in
.base
.from
= fndecl
;
502 in
.hash
= htab_hash_pointer (fndecl
);
503 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
508 /* ??? We may well want TM versions of most of the common <string.h>
509 functions. For now, we've already these two defined. */
510 /* Adjust expand_call_tm() attributes as necessary for the cases
512 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
513 switch (DECL_FUNCTION_CODE (fndecl
))
515 case BUILT_IN_MEMCPY
:
516 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
517 case BUILT_IN_MEMMOVE
:
518 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
519 case BUILT_IN_MEMSET
:
520 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
528 /* When appropriate, record TM replacement for memory allocation functions.
530 FROM is the FNDECL to wrap. */
532 tm_malloc_replacement (tree from
)
537 if (TREE_CODE (from
) != FUNCTION_DECL
)
540 /* If we have a previous replacement, the user must be explicitly
541 wrapping malloc/calloc/free. They better know what they're
543 if (find_tm_replacement_function (from
))
546 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
548 if (!strcmp (str
, "malloc"))
549 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
550 else if (!strcmp (str
, "calloc"))
551 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
552 else if (!strcmp (str
, "free"))
553 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
557 TREE_NOTHROW (to
) = 0;
559 record_tm_replacement (from
, to
);
562 /* Diagnostics for tm_safe functions/regions. Called by the front end
563 once we've lowered the function to high-gimple. */
565 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
566 Process exactly one statement. WI->INFO is set to non-null when in
567 the context of a tm_safe function, and null for a __transaction block. */
569 #define DIAG_TM_OUTER 1
570 #define DIAG_TM_SAFE 2
571 #define DIAG_TM_RELAXED 4
575 unsigned int summary_flags
: 8;
576 unsigned int block_flags
: 8;
577 unsigned int func_flags
: 8;
578 unsigned int saw_volatile
: 1;
582 /* Return true if T is a volatile variable of some kind. */
585 volatile_var_p (tree t
)
587 return (SSA_VAR_P (t
)
588 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
591 /* Tree callback function for diagnose_tm pass. */
594 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
597 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
598 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
600 if (volatile_var_p (*tp
)
601 && d
->block_flags
& DIAG_TM_SAFE
605 error_at (gimple_location (d
->stmt
),
606 "invalid volatile use of %qD inside transaction",
614 is_tm_safe_or_pure (const_tree x
)
616 return is_tm_safe (x
) || is_tm_pure (x
);
620 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
621 struct walk_stmt_info
*wi
)
623 gimple stmt
= gsi_stmt (*gsi
);
624 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
626 /* Save stmt for use in leaf analysis. */
629 switch (gimple_code (stmt
))
633 tree fn
= gimple_call_fn (stmt
);
635 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
636 && is_tm_may_cancel_outer (fn
))
637 error_at (gimple_location (stmt
),
638 "%<transaction_may_cancel_outer%> function call not within"
639 " outer transaction or %<transaction_may_cancel_outer%>");
641 if (d
->summary_flags
& DIAG_TM_SAFE
)
643 bool is_safe
, direct_call_p
;
646 if (TREE_CODE (fn
) == ADDR_EXPR
647 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
649 direct_call_p
= true;
650 replacement
= TREE_OPERAND (fn
, 0);
651 replacement
= find_tm_replacement_function (replacement
);
657 direct_call_p
= false;
658 replacement
= NULL_TREE
;
661 if (is_tm_safe_or_pure (fn
))
663 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
665 /* A function explicitly marked transaction_callable as
666 opposed to transaction_safe is being defined to be
667 unsafe as part of its ABI, regardless of its contents. */
670 else if (direct_call_p
)
672 if (IS_TYPE_OR_DECL_P (fn
)
673 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
675 else if (replacement
)
677 /* ??? At present we've been considering replacements
678 merely transaction_callable, and therefore might
679 enter irrevocable. The tm_wrap attribute has not
680 yet made it into the new language spec. */
685 /* ??? Diagnostics for unmarked direct calls moved into
686 the IPA pass. Section 3.2 of the spec details how
687 functions not marked should be considered "implicitly
688 safe" based on having examined the function body. */
694 /* An unmarked indirect call. Consider it unsafe even
695 though optimization may yet figure out how to inline. */
701 if (TREE_CODE (fn
) == ADDR_EXPR
)
702 fn
= TREE_OPERAND (fn
, 0);
703 if (d
->block_flags
& DIAG_TM_SAFE
)
706 error_at (gimple_location (stmt
),
707 "unsafe function call %qD within "
708 "atomic transaction", fn
);
711 if (!DECL_P (fn
) || DECL_NAME (fn
))
712 error_at (gimple_location (stmt
),
713 "unsafe function call %qE within "
714 "atomic transaction", fn
);
716 error_at (gimple_location (stmt
),
717 "unsafe indirect function call within "
718 "atomic transaction");
724 error_at (gimple_location (stmt
),
725 "unsafe function call %qD within "
726 "%<transaction_safe%> function", fn
);
729 if (!DECL_P (fn
) || DECL_NAME (fn
))
730 error_at (gimple_location (stmt
),
731 "unsafe function call %qE within "
732 "%<transaction_safe%> function", fn
);
734 error_at (gimple_location (stmt
),
735 "unsafe indirect function call within "
736 "%<transaction_safe%> function");
745 /* ??? We ought to come up with a way to add attributes to
746 asm statements, and then add "transaction_safe" to it.
747 Either that or get the language spec to resurrect __tm_waiver. */
748 if (d
->block_flags
& DIAG_TM_SAFE
)
749 error_at (gimple_location (stmt
),
750 "asm not allowed in atomic transaction");
751 else if (d
->func_flags
& DIAG_TM_SAFE
)
752 error_at (gimple_location (stmt
),
753 "asm not allowed in %<transaction_safe%> function");
756 case GIMPLE_TRANSACTION
:
758 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
759 unsigned char inner_flags
= DIAG_TM_SAFE
;
761 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
763 if (d
->block_flags
& DIAG_TM_SAFE
)
764 error_at (gimple_location (stmt
),
765 "relaxed transaction in atomic transaction");
766 else if (d
->func_flags
& DIAG_TM_SAFE
)
767 error_at (gimple_location (stmt
),
768 "relaxed transaction in %<transaction_safe%> function");
769 inner_flags
= DIAG_TM_RELAXED
;
771 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
774 error_at (gimple_location (stmt
),
775 "outer transaction in transaction");
776 else if (d
->func_flags
& DIAG_TM_OUTER
)
777 error_at (gimple_location (stmt
),
778 "outer transaction in "
779 "%<transaction_may_cancel_outer%> function");
780 else if (d
->func_flags
& DIAG_TM_SAFE
)
781 error_at (gimple_location (stmt
),
782 "outer transaction in %<transaction_safe%> function");
783 inner_flags
|= DIAG_TM_OUTER
;
786 *handled_ops_p
= true;
787 if (gimple_transaction_body (trans_stmt
))
789 struct walk_stmt_info wi_inner
;
790 struct diagnose_tm d_inner
;
792 memset (&d_inner
, 0, sizeof (d_inner
));
793 d_inner
.func_flags
= d
->func_flags
;
794 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
795 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
797 memset (&wi_inner
, 0, sizeof (wi_inner
));
798 wi_inner
.info
= &d_inner
;
800 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
801 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
814 diagnose_tm_blocks (void)
816 struct walk_stmt_info wi
;
817 struct diagnose_tm d
;
819 memset (&d
, 0, sizeof (d
));
820 if (is_tm_may_cancel_outer (current_function_decl
))
821 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
822 else if (is_tm_safe (current_function_decl
))
823 d
.func_flags
= DIAG_TM_SAFE
;
824 d
.summary_flags
= d
.func_flags
;
826 memset (&wi
, 0, sizeof (wi
));
829 walk_gimple_seq (gimple_body (current_function_decl
),
830 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
837 const pass_data pass_data_diagnose_tm_blocks
=
839 GIMPLE_PASS
, /* type */
840 "*diagnose_tm_blocks", /* name */
841 OPTGROUP_NONE
, /* optinfo_flags */
842 TV_TRANS_MEM
, /* tv_id */
843 PROP_gimple_any
, /* properties_required */
844 0, /* properties_provided */
845 0, /* properties_destroyed */
846 0, /* todo_flags_start */
847 0, /* todo_flags_finish */
850 class pass_diagnose_tm_blocks
: public gimple_opt_pass
853 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
854 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
857 /* opt_pass methods: */
858 virtual bool gate (function
*) { return flag_tm
; }
859 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
861 }; // class pass_diagnose_tm_blocks
866 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
868 return new pass_diagnose_tm_blocks (ctxt
);
871 /* Instead of instrumenting thread private memory, we save the
872 addresses in a log which we later use to save/restore the addresses
873 upon transaction start/restart.
875 The log is keyed by address, where each element contains individual
876 statements among different code paths that perform the store.
878 This log is later used to generate either plain save/restore of the
879 addresses upon transaction start/restart, or calls to the ITM_L*
882 So for something like:
884 struct large { int x[1000]; };
885 struct large lala = { 0 };
891 We can either save/restore:
894 trxn = _ITM_startTransaction ();
895 if (trxn & a_saveLiveVariables)
896 tmp_lala1 = lala.x[i];
897 else if (a & a_restoreLiveVariables)
898 lala.x[i] = tmp_lala1;
900 or use the logging functions:
903 trxn = _ITM_startTransaction ();
904 _ITM_LU4 (&lala.x[i]);
906 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
907 far up the dominator tree to shadow all of the writes to a given
908 location (thus reducing the total number of logging calls), but not
909 so high as to be called on a path that does not perform a
912 /* One individual log entry. We may have multiple statements for the
913 same location if neither dominate each other (on different
915 typedef struct tm_log_entry
917 /* Address to save. */
919 /* Entry block for the transaction this address occurs in. */
920 basic_block entry_block
;
921 /* Dominating statements the store occurs in. */
923 /* Initially, while we are building the log, we place a nonzero
924 value here to mean that this address *will* be saved with a
925 save/restore sequence. Later, when generating the save sequence
926 we place the SSA temp generated here. */
931 /* Log entry hashtable helpers. */
933 struct log_entry_hasher
935 typedef tm_log_entry value_type
;
936 typedef tm_log_entry compare_type
;
937 static inline hashval_t
hash (const value_type
*);
938 static inline bool equal (const value_type
*, const compare_type
*);
939 static inline void remove (value_type
*);
942 /* Htab support. Return hash value for a `tm_log_entry'. */
944 log_entry_hasher::hash (const value_type
*log
)
946 return iterative_hash_expr (log
->addr
, 0);
949 /* Htab support. Return true if two log entries are the same. */
951 log_entry_hasher::equal (const value_type
*log1
, const compare_type
*log2
)
955 rth: I suggest that we get rid of the component refs etc.
956 I.e. resolve the reference to base + offset.
958 We may need to actually finish a merge with mainline for this,
959 since we'd like to be presented with Richi's MEM_REF_EXPRs more
960 often than not. But in the meantime your tm_log_entry could save
961 the results of get_inner_reference.
963 See: g++.dg/tm/pr46653.C
966 /* Special case plain equality because operand_equal_p() below will
967 return FALSE if the addresses are equal but they have
968 side-effects (e.g. a volatile address). */
969 if (log1
->addr
== log2
->addr
)
972 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
975 /* Htab support. Free one tm_log_entry. */
977 log_entry_hasher::remove (value_type
*lp
)
979 lp
->stmts
.release ();
984 /* The actual log. */
985 static hash_table
<log_entry_hasher
> *tm_log
;
987 /* Addresses to log with a save/restore sequence. These should be in
989 static vec
<tree
> tm_log_save_addresses
;
991 enum thread_memory_type
995 mem_transaction_local
,
999 typedef struct tm_new_mem_map
1001 /* SSA_NAME being dereferenced. */
1003 enum thread_memory_type local_new_memory
;
1006 /* Hashtable helpers. */
1008 struct tm_mem_map_hasher
: typed_free_remove
<tm_new_mem_map_t
>
1010 typedef tm_new_mem_map_t value_type
;
1011 typedef tm_new_mem_map_t compare_type
;
1012 static inline hashval_t
hash (const value_type
*);
1013 static inline bool equal (const value_type
*, const compare_type
*);
1017 tm_mem_map_hasher::hash (const value_type
*v
)
1019 return (intptr_t)v
->val
>> 4;
1023 tm_mem_map_hasher::equal (const value_type
*v
, const compare_type
*c
)
1025 return v
->val
== c
->val
;
1028 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1029 of memory (malloc, alloc, etc). */
1030 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1032 /* Initialize logging data structures. */
1036 tm_log
= new hash_table
<log_entry_hasher
> (10);
1037 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1038 tm_log_save_addresses
.create (5);
1041 /* Free logging data structures. */
1043 tm_log_delete (void)
1047 delete tm_new_mem_hash
;
1048 tm_new_mem_hash
= NULL
;
1049 tm_log_save_addresses
.release ();
1052 /* Return true if MEM is a transaction invariant memory for the TM
1053 region starting at REGION_ENTRY_BLOCK. */
1055 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1057 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1058 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1062 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1063 return def_bb
!= region_entry_block
1064 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1067 mem
= strip_invariant_refs (mem
);
1068 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1071 /* Given an address ADDR in STMT, find it in the memory log or add it,
1072 making sure to keep only the addresses highest in the dominator
1075 ENTRY_BLOCK is the entry_block for the transaction.
1077 If we find the address in the log, make sure it's either the same
1078 address, or an equivalent one that dominates ADDR.
1080 If we find the address, but neither ADDR dominates the found
1081 address, nor the found one dominates ADDR, we're on different
1082 execution paths. Add it.
1084 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1087 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1089 tm_log_entry
**slot
;
1090 struct tm_log_entry l
, *lp
;
1093 slot
= tm_log
->find_slot (&l
, INSERT
);
1096 tree type
= TREE_TYPE (addr
);
1098 lp
= XNEW (struct tm_log_entry
);
1102 /* Small invariant addresses can be handled as save/restores. */
1104 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1105 && TYPE_SIZE_UNIT (type
) != NULL
1106 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1107 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1108 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1109 /* We must be able to copy this type normally. I.e., no
1110 special constructors and the like. */
1111 && !TREE_ADDRESSABLE (type
))
1113 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1114 lp
->stmts
.create (0);
1115 lp
->entry_block
= entry_block
;
1116 /* Save addresses separately in dominator order so we don't
1117 get confused by overlapping addresses in the save/restore
1119 tm_log_save_addresses
.safe_push (lp
->addr
);
1123 /* Use the logging functions. */
1124 lp
->stmts
.create (5);
1125 lp
->stmts
.quick_push (stmt
);
1126 lp
->save_var
= NULL
;
1136 /* If we're generating a save/restore sequence, we don't care
1137 about statements. */
1141 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1143 if (stmt
== oldstmt
)
1145 /* We already have a store to the same address, higher up the
1146 dominator tree. Nothing to do. */
1147 if (dominated_by_p (CDI_DOMINATORS
,
1148 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1150 /* We should be processing blocks in dominator tree order. */
1151 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1152 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1154 /* Store is on a different code path. */
1155 lp
->stmts
.safe_push (stmt
);
1159 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1160 result, insert the new statements before GSI. */
1163 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1165 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1166 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1168 x
= build_fold_addr_expr (x
);
1169 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1172 /* Instrument one address with the logging functions.
1173 ADDR is the address to save.
1174 STMT is the statement before which to place it. */
1176 tm_log_emit_stmt (tree addr
, gimple stmt
)
1178 tree type
= TREE_TYPE (addr
);
1179 tree size
= TYPE_SIZE_UNIT (type
);
1180 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1182 enum built_in_function code
= BUILT_IN_TM_LOG
;
1184 if (type
== float_type_node
)
1185 code
= BUILT_IN_TM_LOG_FLOAT
;
1186 else if (type
== double_type_node
)
1187 code
= BUILT_IN_TM_LOG_DOUBLE
;
1188 else if (type
== long_double_type_node
)
1189 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1190 else if (tree_fits_uhwi_p (size
))
1192 unsigned int n
= tree_to_uhwi (size
);
1196 code
= BUILT_IN_TM_LOG_1
;
1199 code
= BUILT_IN_TM_LOG_2
;
1202 code
= BUILT_IN_TM_LOG_4
;
1205 code
= BUILT_IN_TM_LOG_8
;
1208 code
= BUILT_IN_TM_LOG
;
1209 if (TREE_CODE (type
) == VECTOR_TYPE
)
1211 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1212 code
= BUILT_IN_TM_LOG_M64
;
1213 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1214 code
= BUILT_IN_TM_LOG_M128
;
1215 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1216 code
= BUILT_IN_TM_LOG_M256
;
1222 addr
= gimplify_addr (&gsi
, addr
);
1223 if (code
== BUILT_IN_TM_LOG
)
1224 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1226 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1227 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1230 /* Go through the log and instrument address that must be instrumented
1231 with the logging functions. Leave the save/restore addresses for
1236 hash_table
<log_entry_hasher
>::iterator hi
;
1237 struct tm_log_entry
*lp
;
1239 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1246 fprintf (dump_file
, "TM thread private mem logging: ");
1247 print_generic_expr (dump_file
, lp
->addr
, 0);
1248 fprintf (dump_file
, "\n");
1254 fprintf (dump_file
, "DUMPING to variable\n");
1260 fprintf (dump_file
, "DUMPING with logging functions\n");
1261 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1262 tm_log_emit_stmt (lp
->addr
, stmt
);
1267 /* Emit the save sequence for the corresponding addresses in the log.
1268 ENTRY_BLOCK is the entry block for the transaction.
1269 BB is the basic block to insert the code in. */
1271 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1274 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1276 struct tm_log_entry l
, *lp
;
1278 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1280 l
.addr
= tm_log_save_addresses
[i
];
1281 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1282 gcc_assert (lp
->save_var
!= NULL
);
1284 /* We only care about variables in the current transaction. */
1285 if (lp
->entry_block
!= entry_block
)
1288 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1290 /* Make sure we can create an SSA_NAME for this type. For
1291 instance, aggregates aren't allowed, in which case the system
1292 will create a VOP for us and everything will just work. */
1293 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1295 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1296 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1299 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1303 /* Emit the restore sequence for the corresponding addresses in the log.
1304 ENTRY_BLOCK is the entry block for the transaction.
1305 BB is the basic block to insert the code in. */
1307 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1310 struct tm_log_entry l
, *lp
;
1311 gimple_stmt_iterator gsi
;
1314 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1316 l
.addr
= tm_log_save_addresses
[i
];
1317 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1318 gcc_assert (lp
->save_var
!= NULL
);
1320 /* We only care about variables in the current transaction. */
1321 if (lp
->entry_block
!= entry_block
)
1324 /* Restores are in LIFO order from the saves in case we have
1326 gsi
= gsi_start_bb (bb
);
1328 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1329 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1334 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1335 struct walk_stmt_info
*);
1336 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1337 struct walk_stmt_info
*);
1339 /* Evaluate an address X being dereferenced and determine if it
1340 originally points to a non aliased new chunk of memory (malloc,
1343 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1344 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1345 Return MEM_NON_LOCAL otherwise.
1347 ENTRY_BLOCK is the entry block to the transaction containing the
1348 dereference of X. */
1349 static enum thread_memory_type
1350 thread_private_new_memory (basic_block entry_block
, tree x
)
1353 enum tree_code code
;
1354 tm_new_mem_map_t
**slot
;
1355 tm_new_mem_map_t elt
, *elt_p
;
1357 enum thread_memory_type retval
= mem_transaction_local
;
1360 || TREE_CODE (x
) != SSA_NAME
1361 /* Possible uninitialized use, or a function argument. In
1362 either case, we don't care. */
1363 || SSA_NAME_IS_DEFAULT_DEF (x
))
1364 return mem_non_local
;
1366 /* Look in cache first. */
1368 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1371 return elt_p
->local_new_memory
;
1373 /* Optimistically assume the memory is transaction local during
1374 processing. This catches recursion into this variable. */
1375 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1377 elt_p
->local_new_memory
= mem_transaction_local
;
1379 /* Search DEF chain to find the original definition of this address. */
1382 if (ptr_deref_may_alias_global_p (x
))
1384 /* Address escapes. This is not thread-private. */
1385 retval
= mem_non_local
;
1386 goto new_memory_ret
;
1389 stmt
= SSA_NAME_DEF_STMT (x
);
1391 /* If the malloc call is outside the transaction, this is
1393 if (retval
!= mem_thread_local
1394 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1395 retval
= mem_thread_local
;
1397 if (is_gimple_assign (stmt
))
1399 code
= gimple_assign_rhs_code (stmt
);
1400 /* x = foo ==> foo */
1401 if (code
== SSA_NAME
)
1402 x
= gimple_assign_rhs1 (stmt
);
1403 /* x = foo + n ==> foo */
1404 else if (code
== POINTER_PLUS_EXPR
)
1405 x
= gimple_assign_rhs1 (stmt
);
1406 /* x = (cast*) foo ==> foo */
1407 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1408 x
= gimple_assign_rhs1 (stmt
);
1409 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1410 else if (code
== COND_EXPR
)
1412 tree op1
= gimple_assign_rhs2 (stmt
);
1413 tree op2
= gimple_assign_rhs3 (stmt
);
1414 enum thread_memory_type mem
;
1415 retval
= thread_private_new_memory (entry_block
, op1
);
1416 if (retval
== mem_non_local
)
1417 goto new_memory_ret
;
1418 mem
= thread_private_new_memory (entry_block
, op2
);
1419 retval
= MIN (retval
, mem
);
1420 goto new_memory_ret
;
1424 retval
= mem_non_local
;
1425 goto new_memory_ret
;
1430 if (gimple_code (stmt
) == GIMPLE_PHI
)
1433 enum thread_memory_type mem
;
1434 tree phi_result
= gimple_phi_result (stmt
);
1436 /* If any of the ancestors are non-local, we are sure to
1437 be non-local. Otherwise we can avoid doing anything
1438 and inherit what has already been generated. */
1440 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1442 tree op
= PHI_ARG_DEF (stmt
, i
);
1444 /* Exclude self-assignment. */
1445 if (phi_result
== op
)
1448 mem
= thread_private_new_memory (entry_block
, op
);
1449 if (mem
== mem_non_local
)
1452 goto new_memory_ret
;
1454 retval
= MIN (retval
, mem
);
1456 goto new_memory_ret
;
1461 while (TREE_CODE (x
) == SSA_NAME
);
1463 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1464 /* Thread-local or transaction-local. */
1467 retval
= mem_non_local
;
1470 elt_p
->local_new_memory
= retval
;
1474 /* Determine whether X has to be instrumented using a read
1477 ENTRY_BLOCK is the entry block for the region where stmt resides
1478 in. NULL if unknown.
1480 STMT is the statement in which X occurs in. It is used for thread
1481 private memory instrumentation. If no TPM instrumentation is
1482 desired, STMT should be null. */
1484 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1487 while (handled_component_p (x
))
1488 x
= TREE_OPERAND (x
, 0);
1490 switch (TREE_CODE (x
))
1495 enum thread_memory_type ret
;
1497 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1498 if (ret
== mem_non_local
)
1500 if (stmt
&& ret
== mem_thread_local
)
1501 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1502 tm_log_add (entry_block
, orig
, stmt
);
1504 /* Transaction-locals require nothing at all. For malloc, a
1505 transaction restart frees the memory and we reallocate.
1506 For alloca, the stack pointer gets reset by the retry and
1511 case TARGET_MEM_REF
:
1512 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1514 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1515 if (TREE_CODE (x
) == PARM_DECL
)
1517 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1523 if (DECL_BY_REFERENCE (x
))
1525 /* ??? This value is a pointer, but aggregate_value_p has been
1526 jigged to return true which confuses needs_to_live_in_memory.
1527 This ought to be cleaned up generically.
1529 FIXME: Verify this still happens after the next mainline
1530 merge. Testcase ie g++.dg/tm/pr47554.C.
1535 if (is_global_var (x
))
1536 return !TREE_READONLY (x
);
1537 if (/* FIXME: This condition should actually go below in the
1538 tm_log_add() call, however is_call_clobbered() depends on
1539 aliasing info which is not available during
1540 gimplification. Since requires_barrier() gets called
1541 during lower_sequence_tm/gimplification, leave the call
1542 to needs_to_live_in_memory until we eliminate
1543 lower_sequence_tm altogether. */
1544 needs_to_live_in_memory (x
))
1548 /* For local memory that doesn't escape (aka thread private
1549 memory), we can either save the value at the beginning of
1550 the transaction and restore on restart, or call a tm
1551 function to dynamically save and restore on restart
1554 tm_log_add (entry_block
, orig
, stmt
);
1563 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1564 a transaction region. */
1567 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1569 gimple stmt
= gsi_stmt (*gsi
);
1571 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1572 *state
|= GTMA_HAVE_LOAD
;
1573 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1574 *state
|= GTMA_HAVE_STORE
;
1577 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1580 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1582 gimple stmt
= gsi_stmt (*gsi
);
1585 if (is_tm_pure_call (stmt
))
1588 /* Check if this call is a transaction abort. */
1589 fn
= gimple_call_fndecl (stmt
);
1590 if (is_tm_abort (fn
))
1591 *state
|= GTMA_HAVE_ABORT
;
1593 /* Note that something may happen. */
1594 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1597 /* Lower a GIMPLE_TRANSACTION statement. */
1600 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1603 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1604 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1605 unsigned int this_state
= 0;
1606 struct walk_stmt_info this_wi
;
1608 /* First, lower the body. The scanning that we do inside gives
1609 us some idea of what we're dealing with. */
1610 memset (&this_wi
, 0, sizeof (this_wi
));
1611 this_wi
.info
= (void *) &this_state
;
1612 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1613 lower_sequence_tm
, NULL
, &this_wi
);
1615 /* If there was absolutely nothing transaction related inside the
1616 transaction, we may elide it. Likewise if this is a nested
1617 transaction and does not contain an abort. */
1619 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1622 *outer_state
|= this_state
;
1624 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1626 gimple_transaction_set_body (stmt
, NULL
);
1628 gsi_remove (gsi
, true);
1629 wi
->removed_stmt
= true;
1633 /* Wrap the body of the transaction in a try-finally node so that
1634 the commit call is always properly called. */
1635 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1636 if (flag_exceptions
)
1639 gimple_seq n_seq
, e_seq
;
1641 n_seq
= gimple_seq_alloc_with_stmt (g
);
1644 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1645 1, integer_zero_node
);
1646 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1647 gimple_call_set_lhs (g
, ptr
);
1648 gimple_seq_add_stmt (&e_seq
, g
);
1650 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1652 gimple_seq_add_stmt (&e_seq
, g
);
1654 g
= gimple_build_eh_else (n_seq
, e_seq
);
1657 g
= gimple_build_try (gimple_transaction_body (stmt
),
1658 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1659 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1661 gimple_transaction_set_body (stmt
, NULL
);
1663 /* If the transaction calls abort or if this is an outer transaction,
1664 add an "over" label afterwards. */
1665 if ((this_state
& (GTMA_HAVE_ABORT
))
1666 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1668 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1669 gimple_transaction_set_label (stmt
, label
);
1670 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1673 /* Record the set of operations found for use later. */
1674 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1675 gimple_transaction_set_subcode (stmt
, this_state
);
1678 /* Iterate through the statements in the sequence, lowering them all
1679 as appropriate for being in a transaction. */
1682 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1683 struct walk_stmt_info
*wi
)
1685 unsigned int *state
= (unsigned int *) wi
->info
;
1686 gimple stmt
= gsi_stmt (*gsi
);
1688 *handled_ops_p
= true;
1689 switch (gimple_code (stmt
))
1692 /* Only memory reads/writes need to be instrumented. */
1693 if (gimple_assign_single_p (stmt
))
1694 examine_assign_tm (state
, gsi
);
1698 examine_call_tm (state
, gsi
);
1702 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1705 case GIMPLE_TRANSACTION
:
1706 lower_transaction (gsi
, wi
);
1710 *handled_ops_p
= !gimple_has_substatements (stmt
);
1717 /* Iterate through the statements in the sequence, lowering them all
1718 as appropriate for being outside of a transaction. */
1721 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1722 struct walk_stmt_info
* wi
)
1724 gimple stmt
= gsi_stmt (*gsi
);
1726 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1728 *handled_ops_p
= true;
1729 lower_transaction (gsi
, wi
);
1732 *handled_ops_p
= !gimple_has_substatements (stmt
);
1737 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1738 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1739 been moved out, and all the data required for constructing a proper
1740 CFG has been recorded. */
1743 execute_lower_tm (void)
1745 struct walk_stmt_info wi
;
1748 /* Transactional clones aren't created until a later pass. */
1749 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1751 body
= gimple_body (current_function_decl
);
1752 memset (&wi
, 0, sizeof (wi
));
1753 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1754 gimple_set_body (current_function_decl
, body
);
1761 const pass_data pass_data_lower_tm
=
1763 GIMPLE_PASS
, /* type */
1764 "tmlower", /* name */
1765 OPTGROUP_NONE
, /* optinfo_flags */
1766 TV_TRANS_MEM
, /* tv_id */
1767 PROP_gimple_lcf
, /* properties_required */
1768 0, /* properties_provided */
1769 0, /* properties_destroyed */
1770 0, /* todo_flags_start */
1771 0, /* todo_flags_finish */
1774 class pass_lower_tm
: public gimple_opt_pass
1777 pass_lower_tm (gcc::context
*ctxt
)
1778 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1781 /* opt_pass methods: */
1782 virtual bool gate (function
*) { return flag_tm
; }
1783 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1785 }; // class pass_lower_tm
1790 make_pass_lower_tm (gcc::context
*ctxt
)
1792 return new pass_lower_tm (ctxt
);
1795 /* Collect region information for each transaction. */
1801 /* The field "transaction_stmt" is initially a gtransaction *,
1802 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1804 Helper method to get it as a gtransaction *, with code-checking
1805 in a checked-build. */
1808 get_transaction_stmt () const
1810 return as_a
<gtransaction
*> (transaction_stmt
);
1815 /* Link to the next unnested transaction. */
1816 struct tm_region
*next
;
1818 /* Link to the next inner transaction. */
1819 struct tm_region
*inner
;
1821 /* Link to the next outer transaction. */
1822 struct tm_region
*outer
;
1824 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1825 After TM_MARK, this gets replaced by a call to
1827 Hence this will be either a gtransaction *or a gcall *. */
1828 gimple transaction_stmt
;
1830 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1831 BUILT_IN_TM_START, this field is true if the transaction is an
1832 outer transaction. */
1833 bool original_transaction_was_outer
;
1835 /* Return value from BUILT_IN_TM_START. */
1838 /* The entry block to this region. This will always be the first
1839 block of the body of the transaction. */
1840 basic_block entry_block
;
1842 /* The first block after an expanded call to _ITM_beginTransaction. */
1843 basic_block restart_block
;
1845 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1846 These blocks are still a part of the region (i.e., the border is
1847 inclusive). Note that this set is only complete for paths in the CFG
1848 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1849 the edge to the "over" label. */
1852 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1856 typedef struct tm_region
*tm_region_p
;
1858 /* True if there are pending edge statements to be committed for the
1859 current function being scanned in the tmmark pass. */
1860 bool pending_edge_inserts_p
;
1862 static struct tm_region
*all_tm_regions
;
1863 static bitmap_obstack tm_obstack
;
1866 /* A subroutine of tm_region_init. Record the existence of the
1867 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1869 static struct tm_region
*
1870 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1873 struct tm_region
*region
;
1875 region
= (struct tm_region
*)
1876 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1880 region
->next
= outer
->inner
;
1881 outer
->inner
= region
;
1885 region
->next
= all_tm_regions
;
1886 all_tm_regions
= region
;
1888 region
->inner
= NULL
;
1889 region
->outer
= outer
;
1891 region
->transaction_stmt
= stmt
;
1892 region
->original_transaction_was_outer
= false;
1893 region
->tm_state
= NULL
;
1895 /* There are either one or two edges out of the block containing
1896 the GIMPLE_TRANSACTION, one to the actual region and one to the
1897 "over" label if the region contains an abort. The former will
1898 always be the one marked FALLTHRU. */
1899 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1901 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1902 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1907 /* A subroutine of tm_region_init. Record all the exit and
1908 irrevocable blocks in BB into the region's exit_blocks and
1909 irr_blocks bitmaps. Returns the new region being scanned. */
1911 static struct tm_region
*
1912 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1914 gimple_stmt_iterator gsi
;
1918 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1921 /* Check to see if this is the end of a region by seeing if it
1922 contains a call to __builtin_tm_commit{,_eh}. Note that the
1923 outermost region for DECL_IS_TM_CLONE need not collect this. */
1924 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1927 if (gimple_code (g
) == GIMPLE_CALL
)
1929 tree fn
= gimple_call_fndecl (g
);
1930 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1932 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1933 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1934 && region
->exit_blocks
)
1936 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1937 region
= region
->outer
;
1940 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1941 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1948 /* Collect all of the transaction regions within the current function
1949 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1950 an "outermost" region for use by tm clones. */
1953 tm_region_init (struct tm_region
*region
)
1959 auto_vec
<basic_block
> queue
;
1960 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1961 struct tm_region
*old_region
;
1962 auto_vec
<tm_region_p
> bb_regions
;
1964 all_tm_regions
= region
;
1965 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1967 /* We could store this information in bb->aux, but we may get called
1968 through get_all_tm_blocks() from another pass that may be already
1970 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1972 queue
.safe_push (bb
);
1973 bb_regions
[bb
->index
] = region
;
1977 region
= bb_regions
[bb
->index
];
1978 bb_regions
[bb
->index
] = NULL
;
1980 /* Record exit and irrevocable blocks. */
1981 region
= tm_region_init_1 (region
, bb
);
1983 /* Check for the last statement in the block beginning a new region. */
1985 old_region
= region
;
1987 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
1988 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
1990 /* Process subsequent blocks. */
1991 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1992 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1994 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1995 queue
.safe_push (e
->dest
);
1997 /* If the current block started a new region, make sure that only
1998 the entry block of the new region is associated with this region.
1999 Other successors are still part of the old region. */
2000 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2001 bb_regions
[e
->dest
->index
] = old_region
;
2003 bb_regions
[e
->dest
->index
] = region
;
2006 while (!queue
.is_empty ());
2007 BITMAP_FREE (visited_blocks
);
2010 /* The "gate" function for all transactional memory expansion and optimization
2011 passes. We collect region information for each top-level transaction, and
2012 if we don't find any, we skip all of the TM passes. Each region will have
2013 all of the exit blocks recorded, and the originating statement. */
2021 calculate_dominance_info (CDI_DOMINATORS
);
2022 bitmap_obstack_initialize (&tm_obstack
);
2024 /* If the function is a TM_CLONE, then the entire function is the region. */
2025 if (decl_is_tm_clone (current_function_decl
))
2027 struct tm_region
*region
= (struct tm_region
*)
2028 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2029 memset (region
, 0, sizeof (*region
));
2030 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2031 /* For a clone, the entire function is the region. But even if
2032 we don't need to record any exit blocks, we may need to
2033 record irrevocable blocks. */
2034 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2036 tm_region_init (region
);
2040 tm_region_init (NULL
);
2042 /* If we didn't find any regions, cleanup and skip the whole tree
2043 of tm-related optimizations. */
2044 if (all_tm_regions
== NULL
)
2046 bitmap_obstack_release (&tm_obstack
);
2056 const pass_data pass_data_tm_init
=
2058 GIMPLE_PASS
, /* type */
2059 "*tminit", /* name */
2060 OPTGROUP_NONE
, /* optinfo_flags */
2061 TV_TRANS_MEM
, /* tv_id */
2062 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2063 0, /* properties_provided */
2064 0, /* properties_destroyed */
2065 0, /* todo_flags_start */
2066 0, /* todo_flags_finish */
2069 class pass_tm_init
: public gimple_opt_pass
2072 pass_tm_init (gcc::context
*ctxt
)
2073 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2076 /* opt_pass methods: */
2077 virtual bool gate (function
*) { return gate_tm_init (); }
2079 }; // class pass_tm_init
2084 make_pass_tm_init (gcc::context
*ctxt
)
2086 return new pass_tm_init (ctxt
);
2089 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2090 represented by STATE. */
2093 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2095 if (region
&& region
->transaction_stmt
)
2097 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2098 flags
|= gimple_transaction_subcode (transaction_stmt
);
2099 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2103 /* Construct a memory load in a transactional context. Return the
2104 gimple statement performing the load, or NULL if there is no
2105 TM_LOAD builtin of the appropriate size to do the load.
2107 LOC is the location to use for the new statement(s). */
2110 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2112 enum built_in_function code
= END_BUILTINS
;
2113 tree t
, type
= TREE_TYPE (rhs
), decl
;
2116 if (type
== float_type_node
)
2117 code
= BUILT_IN_TM_LOAD_FLOAT
;
2118 else if (type
== double_type_node
)
2119 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2120 else if (type
== long_double_type_node
)
2121 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2122 else if (TYPE_SIZE_UNIT (type
) != NULL
2123 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2125 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2128 code
= BUILT_IN_TM_LOAD_1
;
2131 code
= BUILT_IN_TM_LOAD_2
;
2134 code
= BUILT_IN_TM_LOAD_4
;
2137 code
= BUILT_IN_TM_LOAD_8
;
2142 if (code
== END_BUILTINS
)
2144 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2149 decl
= builtin_decl_explicit (code
);
2151 t
= gimplify_addr (gsi
, rhs
);
2152 gcall
= gimple_build_call (decl
, 1, t
);
2153 gimple_set_location (gcall
, loc
);
2155 t
= TREE_TYPE (TREE_TYPE (decl
));
2156 if (useless_type_conversion_p (type
, t
))
2158 gimple_call_set_lhs (gcall
, lhs
);
2159 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2166 temp
= create_tmp_reg (t
, NULL
);
2167 gimple_call_set_lhs (gcall
, temp
);
2168 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2170 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2171 g
= gimple_build_assign (lhs
, t
);
2172 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2179 /* Similarly for storing TYPE in a transactional context. */
2182 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2184 enum built_in_function code
= END_BUILTINS
;
2185 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2188 if (type
== float_type_node
)
2189 code
= BUILT_IN_TM_STORE_FLOAT
;
2190 else if (type
== double_type_node
)
2191 code
= BUILT_IN_TM_STORE_DOUBLE
;
2192 else if (type
== long_double_type_node
)
2193 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2194 else if (TYPE_SIZE_UNIT (type
) != NULL
2195 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2197 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2200 code
= BUILT_IN_TM_STORE_1
;
2203 code
= BUILT_IN_TM_STORE_2
;
2206 code
= BUILT_IN_TM_STORE_4
;
2209 code
= BUILT_IN_TM_STORE_8
;
2214 if (code
== END_BUILTINS
)
2216 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2221 fn
= builtin_decl_explicit (code
);
2223 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2225 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2227 /* Handle the easy initialization to zero. */
2228 if (!CONSTRUCTOR_ELTS (rhs
))
2229 rhs
= build_int_cst (simple_type
, 0);
2232 /* ...otherwise punt to the caller and probably use
2233 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2234 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2239 else if (!useless_type_conversion_p (simple_type
, type
))
2244 temp
= create_tmp_reg (simple_type
, NULL
);
2245 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2246 g
= gimple_build_assign (temp
, t
);
2247 gimple_set_location (g
, loc
);
2248 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2253 t
= gimplify_addr (gsi
, lhs
);
2254 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2255 gimple_set_location (gcall
, loc
);
2256 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2262 /* Expand an assignment statement into transactional builtins. */
2265 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2267 gimple stmt
= gsi_stmt (*gsi
);
2268 location_t loc
= gimple_location (stmt
);
2269 tree lhs
= gimple_assign_lhs (stmt
);
2270 tree rhs
= gimple_assign_rhs1 (stmt
);
2271 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2272 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2273 gimple gcall
= NULL
;
2275 if (!load_p
&& !store_p
)
2277 /* Add thread private addresses to log if applicable. */
2278 requires_barrier (region
->entry_block
, lhs
, stmt
);
2283 // Remove original load/store statement.
2284 gsi_remove (gsi
, true);
2286 if (load_p
&& !store_p
)
2288 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2289 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2291 else if (store_p
&& !load_p
)
2293 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2294 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2298 tree lhs_addr
, rhs_addr
, tmp
;
2301 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2303 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2305 /* ??? Figure out if there's any possible overlap between the LHS
2306 and the RHS and if not, use MEMCPY. */
2308 if (load_p
&& is_gimple_reg (lhs
))
2310 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2311 lhs_addr
= build_fold_addr_expr (tmp
);
2316 lhs_addr
= gimplify_addr (gsi
, lhs
);
2318 rhs_addr
= gimplify_addr (gsi
, rhs
);
2319 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2320 3, lhs_addr
, rhs_addr
,
2321 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2322 gimple_set_location (gcall
, loc
);
2323 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2327 gcall
= gimple_build_assign (lhs
, tmp
);
2328 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2332 /* Now that we have the load/store in its instrumented form, add
2333 thread private addresses to the log if applicable. */
2335 requires_barrier (region
->entry_block
, lhs
, gcall
);
2337 // The calls to build_tm_{store,load} above inserted the instrumented
2338 // call into the stream.
2339 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2343 /* Expand a call statement as appropriate for a transaction. That is,
2344 either verify that the call does not affect the transaction, or
2345 redirect the call to a clone that handles transactions, or change
2346 the transaction state to IRREVOCABLE. Return true if the call is
2347 one of the builtins that end a transaction. */
2350 expand_call_tm (struct tm_region
*region
,
2351 gimple_stmt_iterator
*gsi
)
2353 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2354 tree lhs
= gimple_call_lhs (stmt
);
2356 struct cgraph_node
*node
;
2357 bool retval
= false;
2359 fn_decl
= gimple_call_fndecl (stmt
);
2361 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2362 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2363 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2364 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2365 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2367 if (is_tm_pure_call (stmt
))
2371 retval
= is_tm_ending_fndecl (fn_decl
);
2374 /* Assume all non-const/pure calls write to memory, except
2375 transaction ending builtins. */
2376 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2379 /* For indirect calls, we already generated a call into the runtime. */
2382 tree fn
= gimple_call_fn (stmt
);
2384 /* We are guaranteed never to go irrevocable on a safe or pure
2385 call, and the pure call was handled above. */
2386 if (is_tm_safe (fn
))
2389 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2394 node
= cgraph_node::get (fn_decl
);
2395 /* All calls should have cgraph here. */
2398 /* We can have a nodeless call here if some pass after IPA-tm
2399 added uninstrumented calls. For example, loop distribution
2400 can transform certain loop constructs into __builtin_mem*
2401 calls. In this case, see if we have a suitable TM
2402 replacement and fill in the gaps. */
2403 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2404 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2405 gcc_assert (code
== BUILT_IN_MEMCPY
2406 || code
== BUILT_IN_MEMMOVE
2407 || code
== BUILT_IN_MEMSET
);
2409 tree repl
= find_tm_replacement_function (fn_decl
);
2412 gimple_call_set_fndecl (stmt
, repl
);
2414 node
= cgraph_node::create (repl
);
2415 node
->local
.tm_may_enter_irr
= false;
2416 return expand_call_tm (region
, gsi
);
2420 if (node
->local
.tm_may_enter_irr
)
2421 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2423 if (is_tm_abort (fn_decl
))
2425 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2429 /* Instrument the store if needed.
2431 If the assignment happens inside the function call (return slot
2432 optimization), there is no instrumentation to be done, since
2433 the callee should have done the right thing. */
2434 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2435 && !gimple_call_return_slot_opt_p (stmt
))
2437 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2438 location_t loc
= gimple_location (stmt
);
2439 edge fallthru_edge
= NULL
;
2440 gassign
*assign_stmt
;
2442 /* Remember if the call was going to throw. */
2443 if (stmt_can_throw_internal (stmt
))
2447 basic_block bb
= gimple_bb (stmt
);
2449 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2450 if (e
->flags
& EDGE_FALLTHRU
)
2457 gimple_call_set_lhs (stmt
, tmp
);
2459 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2460 gimple_set_location (assign_stmt
, loc
);
2462 /* We cannot throw in the middle of a BB. If the call was going
2463 to throw, place the instrumentation on the fallthru edge, so
2464 the call remains the last statement in the block. */
2467 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2468 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2469 expand_assign_tm (region
, &fallthru_gsi
);
2470 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2471 pending_edge_inserts_p
= true;
2475 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2476 expand_assign_tm (region
, gsi
);
2479 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2486 /* Expand all statements in BB as appropriate for being inside
2490 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2492 gimple_stmt_iterator gsi
;
2494 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2496 gimple stmt
= gsi_stmt (gsi
);
2497 switch (gimple_code (stmt
))
2500 /* Only memory reads/writes need to be instrumented. */
2501 if (gimple_assign_single_p (stmt
)
2502 && !gimple_clobber_p (stmt
))
2504 expand_assign_tm (region
, &gsi
);
2510 if (expand_call_tm (region
, &gsi
))
2520 if (!gsi_end_p (gsi
))
2525 /* Return the list of basic-blocks in REGION.
2527 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2528 following a TM_IRREVOCABLE call.
2530 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2531 uninstrumented code path blocks in the list of basic blocks
2532 returned, false otherwise. */
2534 static vec
<basic_block
>
2535 get_tm_region_blocks (basic_block entry_block
,
2538 bitmap all_region_blocks
,
2539 bool stop_at_irrevocable_p
,
2540 bool include_uninstrumented_p
= true)
2542 vec
<basic_block
> bbs
= vNULL
;
2546 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2549 bbs
.safe_push (entry_block
);
2550 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2554 basic_block bb
= bbs
[i
++];
2557 bitmap_bit_p (exit_blocks
, bb
->index
))
2560 if (stop_at_irrevocable_p
2562 && bitmap_bit_p (irr_blocks
, bb
->index
))
2565 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2566 if ((include_uninstrumented_p
2567 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2568 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2570 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2571 bbs
.safe_push (e
->dest
);
2574 while (i
< bbs
.length ());
2576 if (all_region_blocks
)
2577 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2579 BITMAP_FREE (visited_blocks
);
2583 // Callback data for collect_bb2reg.
2586 vec
<tm_region_p
> *bb2reg
;
2587 bool include_uninstrumented_p
;
2590 // Callback for expand_regions, collect innermost region data for each bb.
2592 collect_bb2reg (struct tm_region
*region
, void *data
)
2594 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2595 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2596 vec
<basic_block
> queue
;
2600 queue
= get_tm_region_blocks (region
->entry_block
,
2601 region
->exit_blocks
,
2604 /*stop_at_irr_p=*/true,
2605 stuff
->include_uninstrumented_p
);
2607 // We expect expand_region to perform a post-order traversal of the region
2608 // tree. Therefore the last region seen for any bb is the innermost.
2609 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2610 (*bb2reg
)[bb
->index
] = region
;
2616 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2617 // which a basic block belongs. Note that we only consider the instrumented
2618 // code paths for the region; the uninstrumented code paths are ignored if
2619 // INCLUDE_UNINSTRUMENTED_P is false.
2621 // ??? This data is very similar to the bb_regions array that is collected
2622 // during tm_region_init. Or, rather, this data is similar to what could
2623 // be used within tm_region_init. The actual computation in tm_region_init
2624 // begins and ends with bb_regions entirely full of NULL pointers, due to
2625 // the way in which pointers are swapped in and out of the array.
2627 // ??? Our callers expect that blocks are not shared between transactions.
2628 // When the optimizers get too smart, and blocks are shared, then during
2629 // the tm_mark phase we'll add log entries to only one of the two transactions,
2630 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2631 // cycles. The symptom being SSA defs that do not dominate their uses.
2632 // Note that the optimizers were locally correct with their transformation,
2633 // as we have no info within the program that suggests that the blocks cannot
2636 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2637 // only known instance of this block sharing.
2639 static vec
<tm_region_p
>
2640 get_bb_regions_instrumented (bool traverse_clones
,
2641 bool include_uninstrumented_p
)
2643 unsigned n
= last_basic_block_for_fn (cfun
);
2644 struct bb2reg_stuff stuff
;
2645 vec
<tm_region_p
> ret
;
2648 ret
.safe_grow_cleared (n
);
2649 stuff
.bb2reg
= &ret
;
2650 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2651 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2656 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2660 compute_transaction_bits (void)
2662 struct tm_region
*region
;
2663 vec
<basic_block
> queue
;
2667 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2668 certainly don't need it to calculate CDI_DOMINATOR info. */
2671 FOR_EACH_BB_FN (bb
, cfun
)
2672 bb
->flags
&= ~BB_IN_TRANSACTION
;
2674 for (region
= all_tm_regions
; region
; region
= region
->next
)
2676 queue
= get_tm_region_blocks (region
->entry_block
,
2677 region
->exit_blocks
,
2680 /*stop_at_irr_p=*/true);
2681 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2682 bb
->flags
|= BB_IN_TRANSACTION
;
2687 bitmap_obstack_release (&tm_obstack
);
2690 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2691 call to BUILT_IN_TM_START. */
2694 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2696 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2697 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2698 tree tm_state
= region
->tm_state
;
2699 tree tm_state_type
= TREE_TYPE (tm_state
);
2700 edge abort_edge
= NULL
;
2701 edge inst_edge
= NULL
;
2702 edge uninst_edge
= NULL
;
2703 edge fallthru_edge
= NULL
;
2705 // Identify the various successors of the transaction start.
2709 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2711 if (e
->flags
& EDGE_TM_ABORT
)
2713 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2717 if (e
->flags
& EDGE_FALLTHRU
)
2722 /* ??? There are plenty of bits here we're not computing. */
2724 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2726 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2727 flags
|= PR_DOESGOIRREVOCABLE
;
2728 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2729 flags
|= PR_HASNOIRREVOCABLE
;
2730 /* If the transaction does not have an abort in lexical scope and is not
2731 marked as an outer transaction, then it will never abort. */
2732 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2733 flags
|= PR_HASNOABORT
;
2734 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2735 flags
|= PR_READONLY
;
2736 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2737 flags
|= PR_INSTRUMENTEDCODE
;
2739 flags
|= PR_UNINSTRUMENTEDCODE
;
2740 if (subcode
& GTMA_IS_OUTER
)
2741 region
->original_transaction_was_outer
= true;
2742 tree t
= build_int_cst (tm_state_type
, flags
);
2743 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2744 gimple_call_set_lhs (call
, tm_state
);
2745 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2747 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2748 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2749 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2750 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2751 gsi_remove (&gsi
, true);
2752 region
->transaction_stmt
= call
;
2755 // Generate log saves.
2756 if (!tm_log_save_addresses
.is_empty ())
2757 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2759 // In the beginning, we've no tests to perform on transaction restart.
2760 // Note that after this point, transaction_bb becomes the "most recent
2761 // block containing tests for the transaction".
2762 region
->restart_block
= region
->entry_block
;
2764 // Generate log restores.
2765 if (!tm_log_save_addresses
.is_empty ())
2767 basic_block test_bb
= create_empty_bb (transaction_bb
);
2768 basic_block code_bb
= create_empty_bb (test_bb
);
2769 basic_block join_bb
= create_empty_bb (code_bb
);
2770 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2771 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2772 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2773 if (region
->restart_block
== region
->entry_block
)
2774 region
->restart_block
= test_bb
;
2776 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2777 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2778 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2780 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2781 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2783 t2
= build_int_cst (tm_state_type
, 0);
2784 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2785 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2787 tm_log_emit_restores (region
->entry_block
, code_bb
);
2789 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2790 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2791 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2792 redirect_edge_pred (fallthru_edge
, join_bb
);
2794 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2795 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2797 ei
->probability
= PROB_ALWAYS
;
2798 et
->probability
= PROB_LIKELY
;
2799 ef
->probability
= PROB_UNLIKELY
;
2800 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2801 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2803 code_bb
->count
= et
->count
;
2804 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2806 transaction_bb
= join_bb
;
2809 // If we have an ABORT edge, create a test to perform the abort.
2812 basic_block test_bb
= create_empty_bb (transaction_bb
);
2813 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2814 if (region
->restart_block
== region
->entry_block
)
2815 region
->restart_block
= test_bb
;
2817 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2818 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2819 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2821 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2822 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2824 t2
= build_int_cst (tm_state_type
, 0);
2825 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2826 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2828 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2829 test_bb
->frequency
= transaction_bb
->frequency
;
2830 test_bb
->count
= transaction_bb
->count
;
2831 ei
->probability
= PROB_ALWAYS
;
2833 // Not abort edge. If both are live, chose one at random as we'll
2834 // we'll be fixing that up below.
2835 redirect_edge_pred (fallthru_edge
, test_bb
);
2836 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2837 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2838 fallthru_edge
->count
2839 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2842 redirect_edge_pred (abort_edge
, test_bb
);
2843 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2844 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2846 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2848 transaction_bb
= test_bb
;
2851 // If we have both instrumented and uninstrumented code paths, select one.
2852 if (inst_edge
&& uninst_edge
)
2854 basic_block test_bb
= create_empty_bb (transaction_bb
);
2855 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2856 if (region
->restart_block
== region
->entry_block
)
2857 region
->restart_block
= test_bb
;
2859 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2860 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2862 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2864 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2865 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2867 t2
= build_int_cst (tm_state_type
, 0);
2868 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2869 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2871 // Create the edge into test_bb first, as we want to copy values
2872 // out of the fallthru edge.
2873 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2874 e
->probability
= fallthru_edge
->probability
;
2875 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2876 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2878 // Now update the edges to the inst/uninist implementations.
2879 // For now assume that the paths are equally likely. When using HTM,
2880 // we'll try the uninst path first and fallback to inst path if htm
2881 // buffers are exceeded. Without HTM we start with the inst path and
2882 // use the uninst path when falling back to serial mode.
2883 redirect_edge_pred (inst_edge
, test_bb
);
2884 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2885 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2887 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2889 redirect_edge_pred (uninst_edge
, test_bb
);
2890 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2891 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2893 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2896 // If we have no previous special cases, and we have PHIs at the beginning
2897 // of the atomic region, this means we have a loop at the beginning of the
2898 // atomic region that shares the first block. This can cause problems with
2899 // the transaction restart abnormal edges to be added in the tm_edges pass.
2900 // Solve this by adding a new empty block to receive the abnormal edges.
2901 if (region
->restart_block
== region
->entry_block
2902 && phi_nodes (region
->entry_block
))
2904 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2905 region
->restart_block
= empty_bb
;
2906 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2908 redirect_edge_pred (fallthru_edge
, empty_bb
);
2909 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2915 /* Generate the temporary to be used for the return value of
2916 BUILT_IN_TM_START. */
2919 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2921 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2923 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2925 // Reset the subcode, post optimizations. We'll fill this in
2926 // again as we process blocks.
2927 if (region
->exit_blocks
)
2929 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2930 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2932 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2933 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2934 | GTMA_MAY_ENTER_IRREVOCABLE
2935 | GTMA_HAS_NO_INSTRUMENTATION
);
2937 subcode
&= GTMA_DECLARATION_MASK
;
2938 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2944 // Propagate flags from inner transactions outwards.
2946 propagate_tm_flags_out (struct tm_region
*region
)
2950 propagate_tm_flags_out (region
->inner
);
2952 if (region
->outer
&& region
->outer
->transaction_stmt
)
2955 gimple_transaction_subcode (region
->get_transaction_stmt ());
2956 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2957 | GTMA_MAY_ENTER_IRREVOCABLE
);
2958 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2959 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2963 propagate_tm_flags_out (region
->next
);
2966 /* Entry point to the MARK phase of TM expansion. Here we replace
2967 transactional memory statements with calls to builtins, and function
2968 calls with their transactional clones (if available). But we don't
2969 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2972 execute_tm_mark (void)
2974 pending_edge_inserts_p
= false;
2976 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2977 /*traverse_clones=*/true);
2981 vec
<tm_region_p
> bb_regions
2982 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2983 /*include_uninstrumented_p=*/false);
2984 struct tm_region
*r
;
2987 // Expand memory operations into calls into the runtime.
2988 // This collects log entries as well.
2989 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
2993 if (r
->transaction_stmt
)
2996 gimple_transaction_subcode (r
->get_transaction_stmt ());
2998 /* If we're sure to go irrevocable, there won't be
2999 anything to expand, since the run-time will go
3000 irrevocable right away. */
3001 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3002 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3005 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3009 bb_regions
.release ();
3011 // Propagate flags from inner transactions outwards.
3012 propagate_tm_flags_out (all_tm_regions
);
3014 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3015 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3016 /*traverse_clones=*/false);
3021 if (pending_edge_inserts_p
)
3022 gsi_commit_edge_inserts ();
3023 free_dominance_info (CDI_DOMINATORS
);
3029 const pass_data pass_data_tm_mark
=
3031 GIMPLE_PASS
, /* type */
3032 "tmmark", /* name */
3033 OPTGROUP_NONE
, /* optinfo_flags */
3034 TV_TRANS_MEM
, /* tv_id */
3035 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3036 0, /* properties_provided */
3037 0, /* properties_destroyed */
3038 0, /* todo_flags_start */
3039 TODO_update_ssa
, /* todo_flags_finish */
3042 class pass_tm_mark
: public gimple_opt_pass
3045 pass_tm_mark (gcc::context
*ctxt
)
3046 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3049 /* opt_pass methods: */
3050 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3052 }; // class pass_tm_mark
3057 make_pass_tm_mark (gcc::context
*ctxt
)
3059 return new pass_tm_mark (ctxt
);
3063 /* Create an abnormal edge from STMT at iter, splitting the block
3064 as necessary. Adjust *PNEXT as needed for the split block. */
3067 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3068 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3070 basic_block bb
= gimple_bb (stmt
);
3071 if (!gsi_one_before_end_p (iter
))
3073 edge e
= split_block (bb
, stmt
);
3074 *pnext
= gsi_start_bb (e
->dest
);
3076 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3078 // Record the need for the edge for the benefit of the rtl passes.
3079 if (cfun
->gimple_df
->tm_restart
== NULL
)
3080 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
3081 struct_ptr_eq
, ggc_free
);
3083 struct tm_restart_node dummy
;
3085 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3087 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
3088 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
3091 n
= ggc_alloc
<tm_restart_node
> ();
3096 tree old
= n
->label_or_list
;
3097 if (TREE_CODE (old
) == LABEL_DECL
)
3098 old
= tree_cons (NULL
, old
, NULL
);
3099 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3103 /* Split block BB as necessary for every builtin function we added, and
3104 wire up the abnormal back edges implied by the transaction restart. */
3107 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3109 gimple_stmt_iterator gsi
, next_gsi
;
3111 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3113 gimple stmt
= gsi_stmt (gsi
);
3117 gsi_next (&next_gsi
);
3119 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3120 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3122 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3125 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3126 == BUILT_IN_TM_ABORT
)
3128 // If we have a ``_transaction_cancel [[outer]]'', there is only
3129 // one abnormal edge: to the transaction marked OUTER.
3130 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3131 // constant argument, which we can examine here. Users invoking
3132 // TM_ABORT directly get what they deserve.
3133 tree arg
= gimple_call_arg (call_stmt
, 0);
3134 if (TREE_CODE (arg
) == INTEGER_CST
3135 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3136 && !decl_is_tm_clone (current_function_decl
))
3138 // Find the GTMA_IS_OUTER transaction.
3139 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3140 if (o
->original_transaction_was_outer
)
3142 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3147 // Otherwise, the front-end should have semantically checked
3148 // outer aborts, but in either case the target region is not
3149 // within this function.
3153 // Non-outer, TM aborts have an abnormal edge to the inner-most
3154 // transaction, the one being aborted;
3155 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3159 // All TM builtins have an abnormal edge to the outer-most transaction.
3160 // We never restart inner transactions. For tm clones, we know a-priori
3161 // that the outer-most transaction is outside the function.
3162 if (decl_is_tm_clone (current_function_decl
))
3165 if (cfun
->gimple_df
->tm_restart
== NULL
)
3166 cfun
->gimple_df
->tm_restart
3167 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
3169 // All TM builtins have an abnormal edge to the outer-most transaction.
3170 // We never restart inner transactions.
3171 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3174 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3178 // Delete any tail-call annotation that may have been added.
3179 // The tail-call pass may have mis-identified the commit as being
3180 // a candidate because we had not yet added this restart edge.
3181 gimple_call_set_tail (call_stmt
, false);
3185 /* Entry point to the final expansion of transactional nodes. */
3189 const pass_data pass_data_tm_edges
=
3191 GIMPLE_PASS
, /* type */
3192 "tmedge", /* name */
3193 OPTGROUP_NONE
, /* optinfo_flags */
3194 TV_TRANS_MEM
, /* tv_id */
3195 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3196 0, /* properties_provided */
3197 0, /* properties_destroyed */
3198 0, /* todo_flags_start */
3199 TODO_update_ssa
, /* todo_flags_finish */
3202 class pass_tm_edges
: public gimple_opt_pass
3205 pass_tm_edges (gcc::context
*ctxt
)
3206 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3209 /* opt_pass methods: */
3210 virtual unsigned int execute (function
*);
3212 }; // class pass_tm_edges
3215 pass_tm_edges::execute (function
*fun
)
3217 vec
<tm_region_p
> bb_regions
3218 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3219 /*include_uninstrumented_p=*/true);
3220 struct tm_region
*r
;
3223 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3225 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3227 bb_regions
.release ();
3229 /* We've got to release the dominance info now, to indicate that it
3230 must be rebuilt completely. Otherwise we'll crash trying to update
3231 the SSA web in the TODO section following this pass. */
3232 free_dominance_info (CDI_DOMINATORS
);
3233 bitmap_obstack_release (&tm_obstack
);
3234 all_tm_regions
= NULL
;
3242 make_pass_tm_edges (gcc::context
*ctxt
)
3244 return new pass_tm_edges (ctxt
);
3247 /* Helper function for expand_regions. Expand REGION and recurse to
3248 the inner region. Call CALLBACK on each region. CALLBACK returns
3249 NULL to continue the traversal, otherwise a non-null value which
3250 this function will return as well. TRAVERSE_CLONES is true if we
3251 should traverse transactional clones. */
3254 expand_regions_1 (struct tm_region
*region
,
3255 void *(*callback
)(struct tm_region
*, void *),
3257 bool traverse_clones
)
3259 void *retval
= NULL
;
3260 if (region
->exit_blocks
3261 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3263 retval
= callback (region
, data
);
3269 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3276 /* Traverse the regions enclosed and including REGION. Execute
3277 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3278 continue the traversal, otherwise a non-null value which this
3279 function will return as well. TRAVERSE_CLONES is true if we should
3280 traverse transactional clones. */
3283 expand_regions (struct tm_region
*region
,
3284 void *(*callback
)(struct tm_region
*, void *),
3286 bool traverse_clones
)
3288 void *retval
= NULL
;
3291 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3294 region
= region
->next
;
3300 /* A unique TM memory operation. */
3301 typedef struct tm_memop
3303 /* Unique ID that all memory operations to the same location have. */
3304 unsigned int value_id
;
3305 /* Address of load/store. */
3309 /* TM memory operation hashtable helpers. */
3311 struct tm_memop_hasher
: typed_free_remove
<tm_memop
>
3313 typedef tm_memop value_type
;
3314 typedef tm_memop compare_type
;
3315 static inline hashval_t
hash (const value_type
*);
3316 static inline bool equal (const value_type
*, const compare_type
*);
3319 /* Htab support. Return a hash value for a `tm_memop'. */
3321 tm_memop_hasher::hash (const value_type
*mem
)
3323 tree addr
= mem
->addr
;
3324 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3325 actually done with operand_equal_p (see tm_memop_eq). */
3326 if (TREE_CODE (addr
) == ADDR_EXPR
)
3327 addr
= TREE_OPERAND (addr
, 0);
3328 return iterative_hash_expr (addr
, 0);
3331 /* Htab support. Return true if two tm_memop's are the same. */
3333 tm_memop_hasher::equal (const value_type
*mem1
, const compare_type
*mem2
)
3335 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3338 /* Sets for solving data flow equations in the memory optimization pass. */
3339 struct tm_memopt_bitmaps
3341 /* Stores available to this BB upon entry. Basically, stores that
3342 dominate this BB. */
3343 bitmap store_avail_in
;
3344 /* Stores available at the end of this BB. */
3345 bitmap store_avail_out
;
3346 bitmap store_antic_in
;
3347 bitmap store_antic_out
;
3348 /* Reads available to this BB upon entry. Basically, reads that
3349 dominate this BB. */
3350 bitmap read_avail_in
;
3351 /* Reads available at the end of this BB. */
3352 bitmap read_avail_out
;
3353 /* Reads performed in this BB. */
3355 /* Writes performed in this BB. */
3358 /* Temporary storage for pass. */
3359 /* Is the current BB in the worklist? */
3360 bool avail_in_worklist_p
;
3361 /* Have we visited this BB? */
3365 static bitmap_obstack tm_memopt_obstack
;
3367 /* Unique counter for TM loads and stores. Loads and stores of the
3368 same address get the same ID. */
3369 static unsigned int tm_memopt_value_id
;
3370 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3372 #define STORE_AVAIL_IN(BB) \
3373 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3374 #define STORE_AVAIL_OUT(BB) \
3375 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3376 #define STORE_ANTIC_IN(BB) \
3377 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3378 #define STORE_ANTIC_OUT(BB) \
3379 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3380 #define READ_AVAIL_IN(BB) \
3381 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3382 #define READ_AVAIL_OUT(BB) \
3383 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3384 #define READ_LOCAL(BB) \
3385 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3386 #define STORE_LOCAL(BB) \
3387 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3388 #define AVAIL_IN_WORKLIST_P(BB) \
3389 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3390 #define BB_VISITED_P(BB) \
3391 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3393 /* Given a TM load/store in STMT, return the value number for the address
3397 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3399 struct tm_memop tmpmem
, *mem
;
3402 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3403 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3404 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3407 else if (op
== INSERT
)
3409 mem
= XNEW (struct tm_memop
);
3411 mem
->value_id
= tm_memopt_value_id
++;
3412 mem
->addr
= tmpmem
.addr
;
3416 return mem
->value_id
;
3419 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3422 tm_memopt_accumulate_memops (basic_block bb
)
3424 gimple_stmt_iterator gsi
;
3426 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3428 gimple stmt
= gsi_stmt (gsi
);
3432 if (is_tm_store (stmt
))
3433 bits
= STORE_LOCAL (bb
);
3434 else if (is_tm_load (stmt
))
3435 bits
= READ_LOCAL (bb
);
3439 loc
= tm_memopt_value_number (stmt
, INSERT
);
3440 bitmap_set_bit (bits
, loc
);
3443 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3444 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3445 gimple_bb (stmt
)->index
);
3446 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3447 fprintf (dump_file
, "\n");
3452 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3455 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3459 const char *comma
= "";
3461 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3462 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3464 hash_table
<tm_memop_hasher
>::iterator hi
;
3465 struct tm_memop
*mem
= NULL
;
3467 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3468 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3469 if (mem
->value_id
== i
)
3471 gcc_assert (mem
->value_id
== i
);
3472 fprintf (dump_file
, "%s", comma
);
3474 print_generic_expr (dump_file
, mem
->addr
, 0);
3476 fprintf (dump_file
, "]\n");
3479 /* Prettily dump all of the memopt sets in BLOCKS. */
3482 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3487 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3489 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3490 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3491 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3492 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3493 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3494 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3495 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3499 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3502 tm_memopt_compute_avin (basic_block bb
)
3507 /* Seed with the AVOUT of any predecessor. */
3508 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3510 e
= EDGE_PRED (bb
, ix
);
3511 /* Make sure we have already visited this BB, and is thus
3514 If e->src->aux is NULL, this predecessor is actually on an
3515 enclosing transaction. We only care about the current
3516 transaction, so ignore it. */
3517 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3519 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3520 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3525 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3527 e
= EDGE_PRED (bb
, ix
);
3528 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3530 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3531 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3535 BB_VISITED_P (bb
) = true;
3538 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3541 tm_memopt_compute_antin (basic_block bb
)
3546 /* Seed with the ANTIC_OUT of any successor. */
3547 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3549 e
= EDGE_SUCC (bb
, ix
);
3550 /* Make sure we have already visited this BB, and is thus
3552 if (BB_VISITED_P (e
->dest
))
3554 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3559 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3561 e
= EDGE_SUCC (bb
, ix
);
3562 if (BB_VISITED_P (e
->dest
))
3563 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3566 BB_VISITED_P (bb
) = true;
3569 /* Compute the AVAIL sets for every basic block in BLOCKS.
3571 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3573 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3574 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3576 This is basically what we do in lcm's compute_available(), but here
3577 we calculate two sets of sets (one for STOREs and one for READs),
3578 and we work on a region instead of the entire CFG.
3580 REGION is the TM region.
3581 BLOCKS are the basic blocks in the region. */
3584 tm_memopt_compute_available (struct tm_region
*region
,
3585 vec
<basic_block
> blocks
)
3588 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3589 unsigned int qlen
, i
;
3593 /* Allocate a worklist array/queue. Entries are only added to the
3594 list if they were not already on the list. So the size is
3595 bounded by the number of basic blocks in the region. */
3596 qlen
= blocks
.length () - 1;
3597 qin
= qout
= worklist
=
3598 XNEWVEC (basic_block
, qlen
);
3600 /* Put every block in the region on the worklist. */
3601 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3603 /* Seed AVAIL_OUT with the LOCAL set. */
3604 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3605 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3607 AVAIL_IN_WORKLIST_P (bb
) = true;
3608 /* No need to insert the entry block, since it has an AVIN of
3609 null, and an AVOUT that has already been seeded in. */
3610 if (bb
!= region
->entry_block
)
3614 /* The entry block has been initialized with the local sets. */
3615 BB_VISITED_P (region
->entry_block
) = true;
3618 qend
= &worklist
[qlen
];
3620 /* Iterate until the worklist is empty. */
3623 /* Take the first entry off the worklist. */
3630 /* This block can be added to the worklist again if necessary. */
3631 AVAIL_IN_WORKLIST_P (bb
) = false;
3632 tm_memopt_compute_avin (bb
);
3634 /* Note: We do not add the LOCAL sets here because we already
3635 seeded the AVAIL_OUT sets with them. */
3636 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3637 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3639 && (region
->exit_blocks
== NULL
3640 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3641 /* If the out state of this block changed, then we need to add
3642 its successors to the worklist if they are not already in. */
3643 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3644 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3645 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3648 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3659 dump_tm_memopt_sets (blocks
);
3662 /* Compute ANTIC sets for every basic block in BLOCKS.
3664 We compute STORE_ANTIC_OUT as follows:
3666 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3667 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3669 REGION is the TM region.
3670 BLOCKS are the basic blocks in the region. */
3673 tm_memopt_compute_antic (struct tm_region
*region
,
3674 vec
<basic_block
> blocks
)
3677 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3682 /* Allocate a worklist array/queue. Entries are only added to the
3683 list if they were not already on the list. So the size is
3684 bounded by the number of basic blocks in the region. */
3685 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3687 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3691 /* Seed ANTIC_OUT with the LOCAL set. */
3692 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3694 /* Put every block in the region on the worklist. */
3695 AVAIL_IN_WORKLIST_P (bb
) = true;
3696 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3697 and their ANTIC_OUT has already been seeded in. */
3698 if (region
->exit_blocks
3699 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3706 /* The exit blocks have been initialized with the local sets. */
3707 if (region
->exit_blocks
)
3711 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3712 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3716 qend
= &worklist
[qlen
];
3718 /* Iterate until the worklist is empty. */
3721 /* Take the first entry off the worklist. */
3728 /* This block can be added to the worklist again if necessary. */
3729 AVAIL_IN_WORKLIST_P (bb
) = false;
3730 tm_memopt_compute_antin (bb
);
3732 /* Note: We do not add the LOCAL sets here because we already
3733 seeded the ANTIC_OUT sets with them. */
3734 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3735 && bb
!= region
->entry_block
)
3736 /* If the out state of this block changed, then we need to add
3737 its predecessors to the worklist if they are not already in. */
3738 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3739 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3742 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3753 dump_tm_memopt_sets (blocks
);
3756 /* Offsets of load variants from TM_LOAD. For example,
3757 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3758 See gtm-builtins.def. */
3759 #define TRANSFORM_RAR 1
3760 #define TRANSFORM_RAW 2
3761 #define TRANSFORM_RFW 3
3762 /* Offsets of store variants from TM_STORE. */
3763 #define TRANSFORM_WAR 1
3764 #define TRANSFORM_WAW 2
3766 /* Inform about a load/store optimization. */
3769 dump_tm_memopt_transform (gimple stmt
)
3773 fprintf (dump_file
, "TM memopt: transforming: ");
3774 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3775 fprintf (dump_file
, "\n");
3779 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3780 by a builtin that is OFFSET entries down in the builtins table in
3781 gtm-builtins.def. */
3784 tm_memopt_transform_stmt (unsigned int offset
,
3786 gimple_stmt_iterator
*gsi
)
3788 tree fn
= gimple_call_fn (stmt
);
3789 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3790 TREE_OPERAND (fn
, 0)
3791 = builtin_decl_explicit ((enum built_in_function
)
3792 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3794 gimple_call_set_fn (stmt
, fn
);
3795 gsi_replace (gsi
, stmt
, true);
3796 dump_tm_memopt_transform (stmt
);
3799 /* Perform the actual TM memory optimization transformations in the
3800 basic blocks in BLOCKS. */
3803 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3807 gimple_stmt_iterator gsi
;
3809 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3811 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3813 gimple stmt
= gsi_stmt (gsi
);
3814 bitmap read_avail
= READ_AVAIL_IN (bb
);
3815 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3816 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3819 if (is_tm_simple_load (stmt
))
3821 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3822 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3823 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3824 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3825 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3827 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3828 bitmap_set_bit (store_avail
, loc
);
3830 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3831 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3833 bitmap_set_bit (read_avail
, loc
);
3835 else if (is_tm_simple_store (stmt
))
3837 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3838 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3839 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3840 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3843 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3844 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3845 bitmap_set_bit (store_avail
, loc
);
3852 /* Return a new set of bitmaps for a BB. */
3854 static struct tm_memopt_bitmaps
*
3855 tm_memopt_init_sets (void)
3857 struct tm_memopt_bitmaps
*b
3858 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3859 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3860 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3861 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3862 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3863 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3864 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3865 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3866 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3867 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3871 /* Free sets computed for each BB. */
3874 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3879 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3883 /* Clear the visited bit for every basic block in BLOCKS. */
3886 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3891 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3892 BB_VISITED_P (bb
) = false;
3895 /* Replace TM load/stores with hints for the runtime. We handle
3896 things like read-after-write, write-after-read, read-after-read,
3897 read-for-write, etc. */
3900 execute_tm_memopt (void)
3902 struct tm_region
*region
;
3903 vec
<basic_block
> bbs
;
3905 tm_memopt_value_id
= 0;
3906 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3908 for (region
= all_tm_regions
; region
; region
= region
->next
)
3910 /* All the TM stores/loads in the current region. */
3914 bitmap_obstack_initialize (&tm_memopt_obstack
);
3916 /* Save all BBs for the current region. */
3917 bbs
= get_tm_region_blocks (region
->entry_block
,
3918 region
->exit_blocks
,
3923 /* Collect all the memory operations. */
3924 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3926 bb
->aux
= tm_memopt_init_sets ();
3927 tm_memopt_accumulate_memops (bb
);
3930 /* Solve data flow equations and transform each block accordingly. */
3931 tm_memopt_clear_visited (bbs
);
3932 tm_memopt_compute_available (region
, bbs
);
3933 tm_memopt_clear_visited (bbs
);
3934 tm_memopt_compute_antic (region
, bbs
);
3935 tm_memopt_transform_blocks (bbs
);
3937 tm_memopt_free_sets (bbs
);
3939 bitmap_obstack_release (&tm_memopt_obstack
);
3940 tm_memopt_value_numbers
->empty ();
3943 delete tm_memopt_value_numbers
;
3944 tm_memopt_value_numbers
= NULL
;
3950 const pass_data pass_data_tm_memopt
=
3952 GIMPLE_PASS
, /* type */
3953 "tmmemopt", /* name */
3954 OPTGROUP_NONE
, /* optinfo_flags */
3955 TV_TRANS_MEM
, /* tv_id */
3956 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3957 0, /* properties_provided */
3958 0, /* properties_destroyed */
3959 0, /* todo_flags_start */
3960 0, /* todo_flags_finish */
3963 class pass_tm_memopt
: public gimple_opt_pass
3966 pass_tm_memopt (gcc::context
*ctxt
)
3967 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3970 /* opt_pass methods: */
3971 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3972 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3974 }; // class pass_tm_memopt
3979 make_pass_tm_memopt (gcc::context
*ctxt
)
3981 return new pass_tm_memopt (ctxt
);
3985 /* Interprocedual analysis for the creation of transactional clones.
3986 The aim of this pass is to find which functions are referenced in
3987 a non-irrevocable transaction context, and for those over which
3988 we have control (or user directive), create a version of the
3989 function which uses only the transactional interface to reference
3990 protected memories. This analysis proceeds in several steps:
3992 (1) Collect the set of all possible transactional clones:
3994 (a) For all local public functions marked tm_callable, push
3995 it onto the tm_callee queue.
3997 (b) For all local functions, scan for calls in transaction blocks.
3998 Push the caller and callee onto the tm_caller and tm_callee
3999 queues. Count the number of callers for each callee.
4001 (c) For each local function on the callee list, assume we will
4002 create a transactional clone. Push *all* calls onto the
4003 callee queues; count the number of clone callers separately
4004 to the number of original callers.
4006 (2) Propagate irrevocable status up the dominator tree:
4008 (a) Any external function on the callee list that is not marked
4009 tm_callable is irrevocable. Push all callers of such onto
4012 (b) For each function on the worklist, mark each block that
4013 contains an irrevocable call. Use the AND operator to
4014 propagate that mark up the dominator tree.
4016 (c) If we reach the entry block for a possible transactional
4017 clone, then the transactional clone is irrevocable, and
4018 we should not create the clone after all. Push all
4019 callers onto the worklist.
4021 (d) Place tm_irrevocable calls at the beginning of the relevant
4022 blocks. Special case here is the entry block for the entire
4023 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4024 the library to begin the region in serial mode. Decrement
4025 the call count for all callees in the irrevocable region.
4027 (3) Create the transactional clones:
4029 Any tm_callee that still has a non-zero call count is cloned.
4032 /* This structure is stored in the AUX field of each cgraph_node. */
4033 struct tm_ipa_cg_data
4035 /* The clone of the function that got created. */
4036 struct cgraph_node
*clone
;
4038 /* The tm regions in the normal function. */
4039 struct tm_region
*all_tm_regions
;
4041 /* The blocks of the normal/clone functions that contain irrevocable
4042 calls, or blocks that are post-dominated by irrevocable calls. */
4043 bitmap irrevocable_blocks_normal
;
4044 bitmap irrevocable_blocks_clone
;
4046 /* The blocks of the normal function that are involved in transactions. */
4047 bitmap transaction_blocks_normal
;
4049 /* The number of callers to the transactional clone of this function
4050 from normal and transactional clones respectively. */
4051 unsigned tm_callers_normal
;
4052 unsigned tm_callers_clone
;
4054 /* True if all calls to this function's transactional clone
4055 are irrevocable. Also automatically true if the function
4056 has no transactional clone. */
4057 bool is_irrevocable
;
4059 /* Flags indicating the presence of this function in various queues. */
4060 bool in_callee_queue
;
4063 /* Flags indicating the kind of scan desired while in the worklist. */
4064 bool want_irr_scan_normal
;
4067 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4069 /* Return the ipa data associated with NODE, allocating zeroed memory
4070 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4071 and set *NODE accordingly. */
4073 static struct tm_ipa_cg_data
*
4074 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4076 struct tm_ipa_cg_data
*d
;
4078 if (traverse_aliases
&& (*node
)->alias
)
4079 *node
= (*node
)->get_alias_target ();
4081 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4085 d
= (struct tm_ipa_cg_data
*)
4086 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4087 (*node
)->aux
= (void *) d
;
4088 memset (d
, 0, sizeof (*d
));
4094 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4095 it is already present. */
4098 maybe_push_queue (struct cgraph_node
*node
,
4099 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4104 queue_p
->safe_push (node
);
4108 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4109 code path. QUEUE are the basic blocks inside the transaction
4110 represented in REGION.
4112 Later in split_code_paths() we will add the conditional to choose
4113 between the two alternatives. */
4116 ipa_uninstrument_transaction (struct tm_region
*region
,
4117 vec
<basic_block
> queue
)
4119 gimple transaction
= region
->transaction_stmt
;
4120 basic_block transaction_bb
= gimple_bb (transaction
);
4121 int n
= queue
.length ();
4122 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4124 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4126 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4127 add_phi_args_after_copy (new_bbs
, n
, e
);
4129 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4130 // a) EDGE_FALLTHRU into the transaction
4131 // b) EDGE_TM_ABORT out of the transaction
4132 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4137 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4138 Queue all callees within block BB. */
4141 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4142 basic_block bb
, bool for_clone
)
4144 gimple_stmt_iterator gsi
;
4146 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4148 gimple stmt
= gsi_stmt (gsi
);
4149 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4151 tree fndecl
= gimple_call_fndecl (stmt
);
4154 struct tm_ipa_cg_data
*d
;
4156 struct cgraph_node
*node
;
4158 if (is_tm_ending_fndecl (fndecl
))
4160 if (find_tm_replacement_function (fndecl
))
4163 node
= cgraph_node::get (fndecl
);
4164 gcc_assert (node
!= NULL
);
4165 d
= get_cg_data (&node
, true);
4167 pcallers
= (for_clone
? &d
->tm_callers_clone
4168 : &d
->tm_callers_normal
);
4171 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4177 /* Scan all calls in NODE that are within a transaction region,
4178 and push the resulting nodes into the callee queue. */
4181 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4182 cgraph_node_queue
*callees_p
)
4184 struct tm_region
*r
;
4186 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4187 d
->all_tm_regions
= all_tm_regions
;
4189 for (r
= all_tm_regions
; r
; r
= r
->next
)
4191 vec
<basic_block
> bbs
;
4195 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4196 d
->transaction_blocks_normal
, false);
4198 // Generate the uninstrumented code path for this transaction.
4199 ipa_uninstrument_transaction (r
, bbs
);
4201 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4202 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4207 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4208 // copying them, rather than forcing us to do this externally.
4209 cgraph_edge::rebuild_edges ();
4211 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4212 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4213 // Instead, just release dominators here so update_ssa recomputes them.
4214 free_dominance_info (CDI_DOMINATORS
);
4216 // When building the uninstrumented code path, copy_bbs will have invoked
4217 // create_new_def_for starting an "ssa update context". There is only one
4218 // instance of this context, so resolve ssa updates before moving on to
4219 // the next function.
4220 update_ssa (TODO_update_ssa
);
4223 /* Scan all calls in NODE as if this is the transactional clone,
4224 and push the destinations into the callee queue. */
4227 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4228 cgraph_node_queue
*callees_p
)
4230 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4233 FOR_EACH_BB_FN (bb
, fn
)
4234 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4237 /* The function NODE has been detected to be irrevocable. Push all
4238 of its callers onto WORKLIST for the purpose of re-scanning them. */
4241 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4242 cgraph_node_queue
*worklist_p
)
4244 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4245 struct cgraph_edge
*e
;
4247 d
->is_irrevocable
= true;
4249 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4252 struct cgraph_node
*caller
;
4254 /* Don't examine recursive calls. */
4255 if (e
->caller
== node
)
4257 /* Even if we think we can go irrevocable, believe the user
4259 if (is_tm_safe_or_pure (e
->caller
->decl
))
4263 d
= get_cg_data (&caller
, true);
4265 /* Check if the callee is in a transactional region. If so,
4266 schedule the function for normal re-scan as well. */
4267 bb
= gimple_bb (e
->call_stmt
);
4268 gcc_assert (bb
!= NULL
);
4269 if (d
->transaction_blocks_normal
4270 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4271 d
->want_irr_scan_normal
= true;
4273 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4277 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4278 within the block is irrevocable. */
4281 ipa_tm_scan_irr_block (basic_block bb
)
4283 gimple_stmt_iterator gsi
;
4286 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4288 gimple stmt
= gsi_stmt (gsi
);
4289 switch (gimple_code (stmt
))
4292 if (gimple_assign_single_p (stmt
))
4294 tree lhs
= gimple_assign_lhs (stmt
);
4295 tree rhs
= gimple_assign_rhs1 (stmt
);
4296 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4303 tree lhs
= gimple_call_lhs (stmt
);
4304 if (lhs
&& volatile_var_p (lhs
))
4307 if (is_tm_pure_call (stmt
))
4310 fn
= gimple_call_fn (stmt
);
4312 /* Functions with the attribute are by definition irrevocable. */
4313 if (is_tm_irrevocable (fn
))
4316 /* For direct function calls, go ahead and check for replacement
4317 functions, or transitive irrevocable functions. For indirect
4318 functions, we'll ask the runtime. */
4319 if (TREE_CODE (fn
) == ADDR_EXPR
)
4321 struct tm_ipa_cg_data
*d
;
4322 struct cgraph_node
*node
;
4324 fn
= TREE_OPERAND (fn
, 0);
4325 if (is_tm_ending_fndecl (fn
))
4327 if (find_tm_replacement_function (fn
))
4330 node
= cgraph_node::get (fn
);
4331 d
= get_cg_data (&node
, true);
4333 /* Return true if irrevocable, but above all, believe
4335 if (d
->is_irrevocable
4336 && !is_tm_safe_or_pure (fn
))
4343 /* ??? The Approved Method of indicating that an inline
4344 assembly statement is not relevant to the transaction
4345 is to wrap it in a __tm_waiver block. This is not
4346 yet implemented, so we can't check for it. */
4347 if (is_tm_safe (current_function_decl
))
4349 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4350 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4351 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4363 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4364 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4365 scanning past OLD_IRR or EXIT_BLOCKS. */
4368 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4369 bitmap old_irr
, bitmap exit_blocks
)
4371 bool any_new_irr
= false;
4374 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4378 basic_block bb
= pqueue
->pop ();
4380 /* Don't re-scan blocks we know already are irrevocable. */
4381 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4384 if (ipa_tm_scan_irr_block (bb
))
4386 bitmap_set_bit (new_irr
, bb
->index
);
4389 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4391 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4392 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4394 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4395 pqueue
->safe_push (e
->dest
);
4399 while (!pqueue
->is_empty ());
4401 BITMAP_FREE (visited_blocks
);
4406 /* Propagate the irrevocable property both up and down the dominator tree.
4407 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4408 TM regions; OLD_IRR are the results of a previous scan of the dominator
4409 tree which has been fully propagated; NEW_IRR is the set of new blocks
4410 which are gaining the irrevocable property during the current scan. */
4413 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4414 bitmap old_irr
, bitmap exit_blocks
)
4416 vec
<basic_block
> bbs
;
4417 bitmap all_region_blocks
;
4419 /* If this block is in the old set, no need to rescan. */
4420 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4423 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4424 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4425 all_region_blocks
, false);
4428 basic_block bb
= bbs
.pop ();
4429 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4430 bool all_son_irr
= false;
4434 /* Propagate up. If my children are, I am too, but we must have
4435 at least one child that is. */
4438 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4440 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4442 all_son_irr
= false;
4450 /* Add block to new_irr if it hasn't already been processed. */
4451 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4453 bitmap_set_bit (new_irr
, bb
->index
);
4459 /* Propagate down to everyone we immediately dominate. */
4463 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4465 son
= next_dom_son (CDI_DOMINATORS
, son
))
4467 /* Make sure block is actually in a TM region, and it
4468 isn't already in old_irr. */
4469 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4470 && bitmap_bit_p (all_region_blocks
, son
->index
))
4471 bitmap_set_bit (new_irr
, son
->index
);
4475 while (!bbs
.is_empty ());
4477 BITMAP_FREE (all_region_blocks
);
4482 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4484 gimple_stmt_iterator gsi
;
4486 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4488 gimple stmt
= gsi_stmt (gsi
);
4489 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4491 tree fndecl
= gimple_call_fndecl (stmt
);
4494 struct tm_ipa_cg_data
*d
;
4496 struct cgraph_node
*tnode
;
4498 if (is_tm_ending_fndecl (fndecl
))
4500 if (find_tm_replacement_function (fndecl
))
4503 tnode
= cgraph_node::get (fndecl
);
4504 d
= get_cg_data (&tnode
, true);
4506 pcallers
= (for_clone
? &d
->tm_callers_clone
4507 : &d
->tm_callers_normal
);
4509 gcc_assert (*pcallers
> 0);
4516 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4517 as well as other irrevocable actions such as inline assembly. Mark all
4518 such blocks as irrevocable and decrement the number of calls to
4519 transactional clones. Return true if, for the transactional clone, the
4520 entire function is irrevocable. */
4523 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4525 struct tm_ipa_cg_data
*d
;
4526 bitmap new_irr
, old_irr
;
4529 /* Builtin operators (operator new, and such). */
4530 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4531 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4534 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4535 calculate_dominance_info (CDI_DOMINATORS
);
4537 d
= get_cg_data (&node
, true);
4538 auto_vec
<basic_block
, 10> queue
;
4539 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4541 /* Scan each tm region, propagating irrevocable status through the tree. */
4544 old_irr
= d
->irrevocable_blocks_clone
;
4545 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4546 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4548 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4551 ret
= bitmap_bit_p (new_irr
,
4552 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4557 struct tm_region
*region
;
4559 old_irr
= d
->irrevocable_blocks_normal
;
4560 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4562 queue
.quick_push (region
->entry_block
);
4563 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4564 region
->exit_blocks
))
4565 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4566 region
->exit_blocks
);
4570 /* If we found any new irrevocable blocks, reduce the call count for
4571 transactional clones within the irrevocable blocks. Save the new
4572 set of irrevocable blocks for next time. */
4573 if (!bitmap_empty_p (new_irr
))
4575 bitmap_iterator bmi
;
4578 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4579 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4584 bitmap_ior_into (old_irr
, new_irr
);
4585 BITMAP_FREE (new_irr
);
4588 d
->irrevocable_blocks_clone
= new_irr
;
4590 d
->irrevocable_blocks_normal
= new_irr
;
4592 if (dump_file
&& new_irr
)
4595 bitmap_iterator bmi
;
4598 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4599 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4600 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4604 BITMAP_FREE (new_irr
);
4611 /* Return true if, for the transactional clone of NODE, any call
4612 may enter irrevocable mode. */
4615 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4617 struct tm_ipa_cg_data
*d
;
4621 d
= get_cg_data (&node
, true);
4623 flags
= flags_from_decl_or_type (decl
);
4625 /* Handle some TM builtins. Ordinarily these aren't actually generated
4626 at this point, but handling these functions when written in by the
4627 user makes it easier to build unit tests. */
4628 if (flags
& ECF_TM_BUILTIN
)
4631 /* Filter out all functions that are marked. */
4632 if (flags
& ECF_TM_PURE
)
4634 if (is_tm_safe (decl
))
4636 if (is_tm_irrevocable (decl
))
4638 if (is_tm_callable (decl
))
4640 if (find_tm_replacement_function (decl
))
4643 /* If we aren't seeing the final version of the function we don't
4644 know what it will contain at runtime. */
4645 if (node
->get_availability () < AVAIL_AVAILABLE
)
4648 /* If the function must go irrevocable, then of course true. */
4649 if (d
->is_irrevocable
)
4652 /* If there are any blocks marked irrevocable, then the function
4653 as a whole may enter irrevocable. */
4654 if (d
->irrevocable_blocks_clone
)
4657 /* We may have previously marked this function as tm_may_enter_irr;
4658 see pass_diagnose_tm_blocks. */
4659 if (node
->local
.tm_may_enter_irr
)
4662 /* Recurse on the main body for aliases. In general, this will
4663 result in one of the bits above being set so that we will not
4664 have to recurse next time. */
4666 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4668 /* What remains is unmarked local functions without items that force
4669 the function to go irrevocable. */
4673 /* Diagnose calls from transaction_safe functions to unmarked
4674 functions that are determined to not be safe. */
4677 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4679 struct cgraph_edge
*e
;
4681 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4682 if (!is_tm_callable (e
->callee
->decl
)
4683 && e
->callee
->local
.tm_may_enter_irr
)
4684 error_at (gimple_location (e
->call_stmt
),
4685 "unsafe function call %qD within "
4686 "%<transaction_safe%> function", e
->callee
->decl
);
4689 /* Diagnose call from atomic transactions to unmarked functions
4690 that are determined to not be safe. */
4693 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4694 struct tm_region
*all_tm_regions
)
4696 struct tm_region
*r
;
4698 for (r
= all_tm_regions
; r
; r
= r
->next
)
4699 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4702 /* Atomic transactions can be nested inside relaxed. */
4704 ipa_tm_diagnose_transaction (node
, r
->inner
);
4708 vec
<basic_block
> bbs
;
4709 gimple_stmt_iterator gsi
;
4713 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4714 r
->irr_blocks
, NULL
, false);
4716 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4717 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4719 gimple stmt
= gsi_stmt (gsi
);
4722 if (gimple_code (stmt
) == GIMPLE_ASM
)
4724 error_at (gimple_location (stmt
),
4725 "asm not allowed in atomic transaction");
4729 if (!is_gimple_call (stmt
))
4731 fndecl
= gimple_call_fndecl (stmt
);
4733 /* Indirect function calls have been diagnosed already. */
4737 /* Stop at the end of the transaction. */
4738 if (is_tm_ending_fndecl (fndecl
))
4740 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4745 /* Marked functions have been diagnosed already. */
4746 if (is_tm_pure_call (stmt
))
4748 if (is_tm_callable (fndecl
))
4751 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4752 error_at (gimple_location (stmt
),
4753 "unsafe function call %qD within "
4754 "atomic transaction", fndecl
);
4761 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4762 OLD_DECL. The returned value is a freshly malloced pointer that
4763 should be freed by the caller. */
4766 tm_mangle (tree old_asm_id
)
4768 const char *old_asm_name
;
4771 struct demangle_component
*dc
;
4774 /* Determine if the symbol is already a valid C++ mangled name. Do this
4775 even for C, which might be interfacing with C++ code via appropriately
4776 ugly identifiers. */
4777 /* ??? We could probably do just as well checking for "_Z" and be done. */
4778 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4779 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4786 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4787 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4791 old_asm_name
+= 2; /* Skip _Z */
4795 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4796 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4797 /* Don't play silly games, you! */
4800 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4801 /* I'd really like to know if we can ever be passed one of
4802 these from the C++ front end. The Logical Thing would
4803 seem that hidden-alias should be outer-most, so that we
4804 get hidden-alias of a transaction-clone and not vice-versa. */
4812 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4816 new_asm_id
= get_identifier (tm_name
);
4823 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4825 node
->mark_force_output ();
4826 node
->analyzed
= true;
4830 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4832 node
->forced_by_abi
= true;
4833 node
->analyzed
= true;
4836 /* Callback data for ipa_tm_create_version_alias. */
4837 struct create_version_alias_info
4839 struct cgraph_node
*old_node
;
4843 /* A subroutine of ipa_tm_create_version, called via
4844 cgraph_for_node_and_aliases. Create new tm clones for each of
4845 the existing aliases. */
4847 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4849 struct create_version_alias_info
*info
4850 = (struct create_version_alias_info
*)data
;
4851 tree old_decl
, new_decl
, tm_name
;
4852 struct cgraph_node
*new_node
;
4854 if (!node
->cpp_implicit_alias
)
4857 old_decl
= node
->decl
;
4858 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4859 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4860 TREE_CODE (old_decl
), tm_name
,
4861 TREE_TYPE (old_decl
));
4863 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4864 SET_DECL_RTL (new_decl
, NULL
);
4866 /* Based loosely on C++'s make_alias_for(). */
4867 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4868 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4869 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4870 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4871 DECL_EXTERNAL (new_decl
) = 0;
4872 DECL_ARTIFICIAL (new_decl
) = 1;
4873 TREE_ADDRESSABLE (new_decl
) = 1;
4874 TREE_USED (new_decl
) = 1;
4875 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4877 /* Perform the same remapping to the comdat group. */
4878 if (DECL_ONE_ONLY (new_decl
))
4879 varpool_node::get (new_decl
)->set_comdat_group
4880 (tm_mangle (decl_comdat_group_id (old_decl
)));
4882 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4883 new_node
->tm_clone
= true;
4884 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4885 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4886 /* ?? Do not traverse aliases here. */
4887 get_cg_data (&node
, false)->clone
= new_node
;
4889 record_tm_clone_pair (old_decl
, new_decl
);
4891 if (info
->old_node
->force_output
4892 || info
->old_node
->ref_list
.first_referring ())
4893 ipa_tm_mark_force_output_node (new_node
);
4894 if (info
->old_node
->forced_by_abi
)
4895 ipa_tm_mark_forced_by_abi_node (new_node
);
4899 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4900 appropriate for the transactional clone. */
4903 ipa_tm_create_version (struct cgraph_node
*old_node
)
4905 tree new_decl
, old_decl
, tm_name
;
4906 struct cgraph_node
*new_node
;
4908 old_decl
= old_node
->decl
;
4909 new_decl
= copy_node (old_decl
);
4911 /* DECL_ASSEMBLER_NAME needs to be set before we call
4912 cgraph_copy_node_for_versioning below, because cgraph_node will
4913 fill the assembler_name_hash. */
4914 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4915 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4916 SET_DECL_RTL (new_decl
, NULL
);
4917 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4919 /* Perform the same remapping to the comdat group. */
4920 if (DECL_ONE_ONLY (new_decl
))
4921 varpool_node::get (new_decl
)->set_comdat_group
4922 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4924 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4925 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4926 new_node
->local
.local
= false;
4927 new_node
->externally_visible
= old_node
->externally_visible
;
4928 new_node
->lowered
= true;
4929 new_node
->tm_clone
= 1;
4930 get_cg_data (&old_node
, true)->clone
= new_node
;
4932 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4934 /* Remap extern inline to static inline. */
4935 /* ??? Is it worth trying to use make_decl_one_only? */
4936 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4938 DECL_EXTERNAL (new_decl
) = 0;
4939 TREE_PUBLIC (new_decl
) = 0;
4940 DECL_WEAK (new_decl
) = 0;
4943 tree_function_versioning (old_decl
, new_decl
,
4948 record_tm_clone_pair (old_decl
, new_decl
);
4950 symtab
->call_cgraph_insertion_hooks (new_node
);
4951 if (old_node
->force_output
4952 || old_node
->ref_list
.first_referring ())
4953 ipa_tm_mark_force_output_node (new_node
);
4954 if (old_node
->forced_by_abi
)
4955 ipa_tm_mark_forced_by_abi_node (new_node
);
4957 /* Do the same thing, but for any aliases of the original node. */
4959 struct create_version_alias_info data
;
4960 data
.old_node
= old_node
;
4961 data
.new_decl
= new_decl
;
4962 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4967 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4970 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4973 gimple_stmt_iterator gsi
;
4976 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4978 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4979 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4981 split_block_after_labels (bb
);
4982 gsi
= gsi_after_labels (bb
);
4983 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4985 node
->create_edge (cgraph_node::get_create
4986 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4988 compute_call_stmt_bb_frequency (node
->decl
,
4992 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4995 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4996 struct tm_region
*region
,
4997 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
4999 tree gettm_fn
, ret
, old_fn
, callfn
;
5004 old_fn
= gimple_call_fn (stmt
);
5006 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5008 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5009 tree clone
= get_tm_clone_pair (fndecl
);
5011 /* By transforming the call into a TM_GETTMCLONE, we are
5012 technically taking the address of the original function and
5013 its clone. Explain this so inlining will know this function
5015 cgraph_node::get (fndecl
)->mark_address_taken () ;
5017 cgraph_node::get (clone
)->mark_address_taken ();
5020 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5021 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5022 : BUILT_IN_TM_GETTMCLONE_IRR
);
5023 ret
= create_tmp_var (ptr_type_node
, NULL
);
5026 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5028 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5029 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5030 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5032 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5033 ret
= make_ssa_name (ret
, g
);
5034 gimple_call_set_lhs (g
, ret
);
5036 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5038 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5039 compute_call_stmt_bb_frequency (node
->decl
,
5042 /* Cast return value from tm_gettmclone* into appropriate function
5044 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
5045 g2
= gimple_build_assign (callfn
,
5046 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5047 callfn
= make_ssa_name (callfn
, g2
);
5048 gimple_assign_set_lhs (g2
, callfn
);
5049 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5051 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5052 which we would have derived from the decl. Failure to save
5053 this bit means we might have to split the basic block. */
5054 if (gimple_call_nothrow_p (stmt
))
5055 gimple_call_set_nothrow (stmt
, true);
5057 gimple_call_set_fn (stmt
, callfn
);
5059 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5060 for a call statement. Fix it. */
5062 tree lhs
= gimple_call_lhs (stmt
);
5063 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5065 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5069 temp
= create_tmp_reg (rettype
, 0);
5070 gimple_call_set_lhs (stmt
, temp
);
5072 g2
= gimple_build_assign (lhs
,
5073 fold_build1 (VIEW_CONVERT_EXPR
,
5074 TREE_TYPE (lhs
), temp
));
5075 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5080 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5081 if (e
&& e
->indirect_info
)
5082 e
->indirect_info
->polymorphic
= false;
5087 /* Helper function for ipa_tm_transform_calls*. Given a call
5088 statement in GSI which resides inside transaction REGION, redirect
5089 the call to either its wrapper function, or its clone. */
5092 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5093 struct tm_region
*region
,
5094 gimple_stmt_iterator
*gsi
,
5095 bool *need_ssa_rename_p
)
5097 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5098 struct cgraph_node
*new_node
;
5099 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5100 tree fndecl
= gimple_call_fndecl (stmt
);
5102 /* For indirect calls, pass the address through the runtime. */
5105 *need_ssa_rename_p
|=
5106 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5110 /* Handle some TM builtins. Ordinarily these aren't actually generated
5111 at this point, but handling these functions when written in by the
5112 user makes it easier to build unit tests. */
5113 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5116 /* Fixup recursive calls inside clones. */
5117 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5118 for recursion but not update the call statements themselves? */
5119 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5121 gimple_call_set_fndecl (stmt
, current_function_decl
);
5125 /* If there is a replacement, use it. */
5126 fndecl
= find_tm_replacement_function (fndecl
);
5129 new_node
= cgraph_node::get_create (fndecl
);
5131 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5133 We can't do this earlier in record_tm_replacement because
5134 cgraph_remove_unreachable_nodes is called before we inject
5135 references to the node. Further, we can't do this in some
5136 nice central place in ipa_tm_execute because we don't have
5137 the exact list of wrapper functions that would be used.
5138 Marking more wrappers than necessary results in the creation
5139 of unnecessary cgraph_nodes, which can cause some of the
5140 other IPA passes to crash.
5142 We do need to mark these nodes so that we get the proper
5143 result in expand_call_tm. */
5144 /* ??? This seems broken. How is it that we're marking the
5145 CALLEE as may_enter_irr? Surely we should be marking the
5146 CALLER. Also note that find_tm_replacement_function also
5147 contains mappings into the TM runtime, e.g. memcpy. These
5148 we know won't go irrevocable. */
5149 new_node
->local
.tm_may_enter_irr
= 1;
5153 struct tm_ipa_cg_data
*d
;
5154 struct cgraph_node
*tnode
= e
->callee
;
5156 d
= get_cg_data (&tnode
, true);
5157 new_node
= d
->clone
;
5159 /* As we've already skipped pure calls and appropriate builtins,
5160 and we've already marked irrevocable blocks, if we can't come
5161 up with a static replacement, then ask the runtime. */
5162 if (new_node
== NULL
)
5164 *need_ssa_rename_p
|=
5165 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5169 fndecl
= new_node
->decl
;
5172 e
->redirect_callee (new_node
);
5173 gimple_call_set_fndecl (stmt
, fndecl
);
5176 /* Helper function for ipa_tm_transform_calls. For a given BB,
5177 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5178 redirect other calls to the generated transactional clone. */
5181 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5182 basic_block bb
, bitmap irr_blocks
)
5184 gimple_stmt_iterator gsi
;
5185 bool need_ssa_rename
= false;
5187 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5189 ipa_tm_insert_irr_call (node
, region
, bb
);
5193 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5195 gimple stmt
= gsi_stmt (gsi
);
5197 if (!is_gimple_call (stmt
))
5199 if (is_tm_pure_call (stmt
))
5202 /* Redirect edges to the appropriate replacement or clone. */
5203 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5206 return need_ssa_rename
;
5209 /* Walk the CFG for REGION, beginning at BB. Install calls to
5210 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5211 the generated transactional clone. */
5214 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5215 basic_block bb
, bitmap irr_blocks
)
5217 bool need_ssa_rename
= false;
5220 auto_vec
<basic_block
> queue
;
5221 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5223 queue
.safe_push (bb
);
5229 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5231 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5234 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5237 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5238 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5240 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5241 queue
.safe_push (e
->dest
);
5244 while (!queue
.is_empty ());
5246 BITMAP_FREE (visited_blocks
);
5248 return need_ssa_rename
;
5251 /* Transform the calls within the TM regions within NODE. */
5254 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5256 struct tm_ipa_cg_data
*d
;
5257 struct tm_region
*region
;
5258 bool need_ssa_rename
= false;
5260 d
= get_cg_data (&node
, true);
5262 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5263 calculate_dominance_info (CDI_DOMINATORS
);
5265 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5267 /* If we're sure to go irrevocable, don't transform anything. */
5268 if (d
->irrevocable_blocks_normal
5269 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5270 region
->entry_block
->index
))
5272 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5273 | GTMA_MAY_ENTER_IRREVOCABLE
5274 | GTMA_HAS_NO_INSTRUMENTATION
);
5279 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5280 d
->irrevocable_blocks_normal
);
5283 if (need_ssa_rename
)
5284 update_ssa (TODO_update_ssa_only_virtuals
);
5289 /* Transform the calls within the transactional clone of NODE. */
5292 ipa_tm_transform_clone (struct cgraph_node
*node
)
5294 struct tm_ipa_cg_data
*d
;
5295 bool need_ssa_rename
;
5297 d
= get_cg_data (&node
, true);
5299 /* If this function makes no calls and has no irrevocable blocks,
5300 then there's nothing to do. */
5301 /* ??? Remove non-aborting top-level transactions. */
5302 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5305 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5306 calculate_dominance_info (CDI_DOMINATORS
);
5309 ipa_tm_transform_calls (d
->clone
, NULL
,
5310 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5311 d
->irrevocable_blocks_clone
);
5313 if (need_ssa_rename
)
5314 update_ssa (TODO_update_ssa_only_virtuals
);
5319 /* Main entry point for the transactional memory IPA pass. */
5322 ipa_tm_execute (void)
5324 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5325 /* List of functions that will go irrevocable. */
5326 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5328 struct cgraph_node
*node
;
5329 struct tm_ipa_cg_data
*d
;
5330 enum availability a
;
5333 #ifdef ENABLE_CHECKING
5334 cgraph_node::verify_cgraph_nodes ();
5337 bitmap_obstack_initialize (&tm_obstack
);
5338 initialize_original_copy_tables ();
5340 /* For all local functions marked tm_callable, queue them. */
5341 FOR_EACH_DEFINED_FUNCTION (node
)
5342 if (is_tm_callable (node
->decl
)
5343 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5345 d
= get_cg_data (&node
, true);
5346 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5349 /* For all local reachable functions... */
5350 FOR_EACH_DEFINED_FUNCTION (node
)
5352 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5354 /* ... marked tm_pure, record that fact for the runtime by
5355 indicating that the pure function is its own tm_callable.
5356 No need to do this if the function's address can't be taken. */
5357 if (is_tm_pure (node
->decl
))
5359 if (!node
->local
.local
)
5360 record_tm_clone_pair (node
->decl
, node
->decl
);
5364 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5365 calculate_dominance_info (CDI_DOMINATORS
);
5367 tm_region_init (NULL
);
5370 d
= get_cg_data (&node
, true);
5372 /* Scan for calls that are in each transaction, and
5373 generate the uninstrumented code path. */
5374 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5376 /* Put it in the worklist so we can scan the function
5377 later (ipa_tm_scan_irr_function) and mark the
5378 irrevocable blocks. */
5379 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5380 d
->want_irr_scan_normal
= true;
5386 /* For every local function on the callee list, scan as if we will be
5387 creating a transactional clone, queueing all new functions we find
5389 for (i
= 0; i
< tm_callees
.length (); ++i
)
5391 node
= tm_callees
[i
];
5392 a
= node
->get_availability ();
5393 d
= get_cg_data (&node
, true);
5395 /* Put it in the worklist so we can scan the function later
5396 (ipa_tm_scan_irr_function) and mark the irrevocable
5398 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5400 /* Some callees cannot be arbitrarily cloned. These will always be
5401 irrevocable. Mark these now, so that we need not scan them. */
5402 if (is_tm_irrevocable (node
->decl
))
5403 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5404 else if (a
<= AVAIL_NOT_AVAILABLE
5405 && !is_tm_safe_or_pure (node
->decl
))
5406 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5407 else if (a
>= AVAIL_INTERPOSABLE
)
5409 if (!tree_versionable_function_p (node
->decl
))
5410 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5411 else if (!d
->is_irrevocable
)
5413 /* If this is an alias, make sure its base is queued as well.
5414 we need not scan the callees now, as the base will do. */
5417 node
= cgraph_node::get (node
->thunk
.alias
);
5418 d
= get_cg_data (&node
, true);
5419 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5423 /* Add all nodes called by this function into
5424 tm_callees as well. */
5425 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5430 /* Iterate scans until no more work to be done. Prefer not to use
5431 vec::pop because the worklist tends to follow a breadth-first
5432 search of the callgraph, which should allow convergance with a
5433 minimum number of scans. But we also don't want the worklist
5434 array to grow without bound, so we shift the array up periodically. */
5435 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5437 if (i
> 256 && i
== irr_worklist
.length () / 8)
5439 irr_worklist
.block_remove (0, i
);
5443 node
= irr_worklist
[i
];
5444 d
= get_cg_data (&node
, true);
5445 d
->in_worklist
= false;
5447 if (d
->want_irr_scan_normal
)
5449 d
->want_irr_scan_normal
= false;
5450 ipa_tm_scan_irr_function (node
, false);
5452 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5453 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5456 /* For every function on the callee list, collect the tm_may_enter_irr
5458 irr_worklist
.truncate (0);
5459 for (i
= 0; i
< tm_callees
.length (); ++i
)
5461 node
= tm_callees
[i
];
5462 if (ipa_tm_mayenterirr_function (node
))
5464 d
= get_cg_data (&node
, true);
5465 gcc_assert (d
->in_worklist
== false);
5466 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5470 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5471 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5473 struct cgraph_node
*caller
;
5474 struct cgraph_edge
*e
;
5475 struct ipa_ref
*ref
;
5477 if (i
> 256 && i
== irr_worklist
.length () / 8)
5479 irr_worklist
.block_remove (0, i
);
5483 node
= irr_worklist
[i
];
5484 d
= get_cg_data (&node
, true);
5485 d
->in_worklist
= false;
5486 node
->local
.tm_may_enter_irr
= true;
5488 /* Propagate back to normal callers. */
5489 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5492 if (!is_tm_safe_or_pure (caller
->decl
)
5493 && !caller
->local
.tm_may_enter_irr
)
5495 d
= get_cg_data (&caller
, true);
5496 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5500 /* Propagate back to referring aliases as well. */
5501 FOR_EACH_ALIAS (node
, ref
)
5503 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5504 if (!caller
->local
.tm_may_enter_irr
)
5506 /* ?? Do not traverse aliases here. */
5507 d
= get_cg_data (&caller
, false);
5508 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5513 /* Now validate all tm_safe functions, and all atomic regions in
5515 FOR_EACH_DEFINED_FUNCTION (node
)
5517 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5519 d
= get_cg_data (&node
, true);
5520 if (is_tm_safe (node
->decl
))
5521 ipa_tm_diagnose_tm_safe (node
);
5522 else if (d
->all_tm_regions
)
5523 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5526 /* Create clones. Do those that are not irrevocable and have a
5527 positive call count. Do those publicly visible functions that
5528 the user directed us to clone. */
5529 for (i
= 0; i
< tm_callees
.length (); ++i
)
5533 node
= tm_callees
[i
];
5534 if (node
->cpp_implicit_alias
)
5537 a
= node
->get_availability ();
5538 d
= get_cg_data (&node
, true);
5540 if (a
<= AVAIL_NOT_AVAILABLE
)
5541 doit
= is_tm_callable (node
->decl
);
5542 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5544 else if (!d
->is_irrevocable
5545 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5549 ipa_tm_create_version (node
);
5552 /* Redirect calls to the new clones, and insert irrevocable marks. */
5553 for (i
= 0; i
< tm_callees
.length (); ++i
)
5555 node
= tm_callees
[i
];
5558 d
= get_cg_data (&node
, true);
5560 ipa_tm_transform_clone (node
);
5563 FOR_EACH_DEFINED_FUNCTION (node
)
5565 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5567 d
= get_cg_data (&node
, true);
5568 if (d
->all_tm_regions
)
5569 ipa_tm_transform_transaction (node
);
5572 /* Free and clear all data structures. */
5573 tm_callees
.release ();
5574 irr_worklist
.release ();
5575 bitmap_obstack_release (&tm_obstack
);
5576 free_original_copy_tables ();
5578 FOR_EACH_FUNCTION (node
)
5581 #ifdef ENABLE_CHECKING
5582 cgraph_node::verify_cgraph_nodes ();
5590 const pass_data pass_data_ipa_tm
=
5592 SIMPLE_IPA_PASS
, /* type */
5594 OPTGROUP_NONE
, /* optinfo_flags */
5595 TV_TRANS_MEM
, /* tv_id */
5596 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5597 0, /* properties_provided */
5598 0, /* properties_destroyed */
5599 0, /* todo_flags_start */
5600 0, /* todo_flags_finish */
5603 class pass_ipa_tm
: public simple_ipa_opt_pass
5606 pass_ipa_tm (gcc::context
*ctxt
)
5607 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5610 /* opt_pass methods: */
5611 virtual bool gate (function
*) { return flag_tm
; }
5612 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5614 }; // class pass_ipa_tm
5618 simple_ipa_opt_pass
*
5619 make_pass_ipa_tm (gcc::context
*ctxt
)
5621 return new pass_ipa_tm (ctxt
);
5624 #include "gt-trans-mem.h"