1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2014 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
23 #include "hash-table.h"
25 #include "basic-block.h"
26 #include "tree-ssa-alias.h"
27 #include "internal-fn.h"
29 #include "gimple-expr.h"
38 #include "hard-reg-set.h"
44 #include "gimple-iterator.h"
45 #include "gimplify-me.h"
46 #include "gimple-walk.h"
47 #include "gimple-ssa.h"
50 #include "stringpool.h"
51 #include "tree-ssanames.h"
52 #include "tree-into-ssa.h"
53 #include "tree-pass.h"
54 #include "tree-inline.h"
55 #include "diagnostic-core.h"
58 #include "trans-mem.h"
61 #include "langhooks.h"
62 #include "gimple-pretty-print.h"
64 #include "tree-ssa-address.h"
68 #define A_RUNINSTRUMENTEDCODE 0x0001
69 #define A_RUNUNINSTRUMENTEDCODE 0x0002
70 #define A_SAVELIVEVARIABLES 0x0004
71 #define A_RESTORELIVEVARIABLES 0x0008
72 #define A_ABORTTRANSACTION 0x0010
74 #define AR_USERABORT 0x0001
75 #define AR_USERRETRY 0x0002
76 #define AR_TMCONFLICT 0x0004
77 #define AR_EXCEPTIONBLOCKABORT 0x0008
78 #define AR_OUTERABORT 0x0010
80 #define MODE_SERIALIRREVOCABLE 0x0000
83 /* The representation of a transaction changes several times during the
84 lowering process. In the beginning, in the front-end we have the
85 GENERIC tree TRANSACTION_EXPR. For example,
93 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
94 trivially replaced with a GIMPLE_TRANSACTION node.
96 During pass_lower_tm, we examine the body of transactions looking
97 for aborts. Transactions that do not contain an abort may be
98 merged into an outer transaction. We also add a TRY-FINALLY node
99 to arrange for the transaction to be committed on any exit.
101 [??? Think about how this arrangement affects throw-with-commit
102 and throw-with-abort operations. In this case we want the TRY to
103 handle gotos, but not to catch any exceptions because the transaction
104 will already be closed.]
106 GIMPLE_TRANSACTION [label=NULL] {
113 __builtin___tm_abort ();
115 __builtin___tm_commit ();
119 During pass_lower_eh, we create EH regions for the transactions,
120 intermixed with the regular EH stuff. This gives us a nice persistent
121 mapping (all the way through rtl) from transactional memory operation
122 back to the transaction, which allows us to get the abnormal edges
123 correct to model transaction aborts and restarts:
125 GIMPLE_TRANSACTION [label=over]
131 __builtin___tm_abort ();
132 __builtin___tm_commit ();
135 This is the end of all_lowering_passes, and so is what is present
136 during the IPA passes, and through all of the optimization passes.
138 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
139 functions and mark functions for cloning.
141 At the end of gimple optimization, before exiting SSA form,
142 pass_tm_edges replaces statements that perform transactional
143 memory operations with the appropriate TM builtins, and swap
144 out function calls with their transactional clones. At this
145 point we introduce the abnormal transaction restart edges and
146 complete lowering of the GIMPLE_TRANSACTION node.
148 x = __builtin___tm_start (MAY_ABORT);
150 if (x & abort_transaction)
153 t0 = __builtin___tm_load (global);
155 __builtin___tm_store (&global, t1);
157 __builtin___tm_abort ();
158 __builtin___tm_commit ();
162 static void *expand_regions (struct tm_region
*,
163 void *(*callback
)(struct tm_region
*, void *),
167 /* Return the attributes we want to examine for X, or NULL if it's not
168 something we examine. We look at function types, but allow pointers
169 to function types and function decls and peek through. */
172 get_attrs_for (const_tree x
)
174 switch (TREE_CODE (x
))
177 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
184 if (TREE_CODE (x
) != POINTER_TYPE
)
190 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
196 return TYPE_ATTRIBUTES (x
);
200 /* Return true if X has been marked TM_PURE. */
203 is_tm_pure (const_tree x
)
207 switch (TREE_CODE (x
))
218 if (TREE_CODE (x
) != POINTER_TYPE
)
224 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
229 flags
= flags_from_decl_or_type (x
);
230 return (flags
& ECF_TM_PURE
) != 0;
233 /* Return true if X has been marked TM_IRREVOCABLE. */
236 is_tm_irrevocable (tree x
)
238 tree attrs
= get_attrs_for (x
);
240 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
243 /* A call to the irrevocable builtin is by definition,
245 if (TREE_CODE (x
) == ADDR_EXPR
)
246 x
= TREE_OPERAND (x
, 0);
247 if (TREE_CODE (x
) == FUNCTION_DECL
248 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
249 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
255 /* Return true if X has been marked TM_SAFE. */
258 is_tm_safe (const_tree x
)
262 tree attrs
= get_attrs_for (x
);
265 if (lookup_attribute ("transaction_safe", attrs
))
267 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
274 /* Return true if CALL is const, or tm_pure. */
277 is_tm_pure_call (gimple call
)
279 tree fn
= gimple_call_fn (call
);
281 if (TREE_CODE (fn
) == ADDR_EXPR
)
283 fn
= TREE_OPERAND (fn
, 0);
284 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
289 return is_tm_pure (fn
);
292 /* Return true if X has been marked TM_CALLABLE. */
295 is_tm_callable (tree x
)
297 tree attrs
= get_attrs_for (x
);
300 if (lookup_attribute ("transaction_callable", attrs
))
302 if (lookup_attribute ("transaction_safe", attrs
))
304 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
310 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
313 is_tm_may_cancel_outer (tree x
)
315 tree attrs
= get_attrs_for (x
);
317 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
321 /* Return true for built in functions that "end" a transaction. */
324 is_tm_ending_fndecl (tree fndecl
)
326 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
327 switch (DECL_FUNCTION_CODE (fndecl
))
329 case BUILT_IN_TM_COMMIT
:
330 case BUILT_IN_TM_COMMIT_EH
:
331 case BUILT_IN_TM_ABORT
:
332 case BUILT_IN_TM_IRREVOCABLE
:
341 /* Return true if STMT is a built in function call that "ends" a
345 is_tm_ending (gimple stmt
)
349 if (gimple_code (stmt
) != GIMPLE_CALL
)
352 fndecl
= gimple_call_fndecl (stmt
);
353 return (fndecl
!= NULL_TREE
354 && is_tm_ending_fndecl (fndecl
));
357 /* Return true if STMT is a TM load. */
360 is_tm_load (gimple stmt
)
364 if (gimple_code (stmt
) != GIMPLE_CALL
)
367 fndecl
= gimple_call_fndecl (stmt
);
368 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
369 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
372 /* Same as above, but for simple TM loads, that is, not the
373 after-write, after-read, etc optimized variants. */
376 is_tm_simple_load (gimple stmt
)
380 if (gimple_code (stmt
) != GIMPLE_CALL
)
383 fndecl
= gimple_call_fndecl (stmt
);
384 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
386 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
387 return (fcode
== BUILT_IN_TM_LOAD_1
388 || fcode
== BUILT_IN_TM_LOAD_2
389 || fcode
== BUILT_IN_TM_LOAD_4
390 || fcode
== BUILT_IN_TM_LOAD_8
391 || fcode
== BUILT_IN_TM_LOAD_FLOAT
392 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
393 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
394 || fcode
== BUILT_IN_TM_LOAD_M64
395 || fcode
== BUILT_IN_TM_LOAD_M128
396 || fcode
== BUILT_IN_TM_LOAD_M256
);
401 /* Return true if STMT is a TM store. */
404 is_tm_store (gimple stmt
)
408 if (gimple_code (stmt
) != GIMPLE_CALL
)
411 fndecl
= gimple_call_fndecl (stmt
);
412 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
413 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
416 /* Same as above, but for simple TM stores, that is, not the
417 after-write, after-read, etc optimized variants. */
420 is_tm_simple_store (gimple stmt
)
424 if (gimple_code (stmt
) != GIMPLE_CALL
)
427 fndecl
= gimple_call_fndecl (stmt
);
428 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
430 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
431 return (fcode
== BUILT_IN_TM_STORE_1
432 || fcode
== BUILT_IN_TM_STORE_2
433 || fcode
== BUILT_IN_TM_STORE_4
434 || fcode
== BUILT_IN_TM_STORE_8
435 || fcode
== BUILT_IN_TM_STORE_FLOAT
436 || fcode
== BUILT_IN_TM_STORE_DOUBLE
437 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
438 || fcode
== BUILT_IN_TM_STORE_M64
439 || fcode
== BUILT_IN_TM_STORE_M128
440 || fcode
== BUILT_IN_TM_STORE_M256
);
445 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
448 is_tm_abort (tree fndecl
)
451 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
452 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
455 /* Build a GENERIC tree for a user abort. This is called by front ends
456 while transforming the __tm_abort statement. */
459 build_tm_abort_call (location_t loc
, bool is_outer
)
461 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
462 build_int_cst (integer_type_node
,
464 | (is_outer
? AR_OUTERABORT
: 0)));
467 /* Map for aribtrary function replacement under TM, as created
468 by the tm_wrap attribute. */
470 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
474 record_tm_replacement (tree from
, tree to
)
476 struct tree_map
**slot
, *h
;
478 /* Do not inline wrapper functions that will get replaced in the TM
481 Suppose you have foo() that will get replaced into tmfoo(). Make
482 sure the inliner doesn't try to outsmart us and inline foo()
483 before we get a chance to do the TM replacement. */
484 DECL_UNINLINABLE (from
) = 1;
486 if (tm_wrap_map
== NULL
)
487 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
489 h
= ggc_alloc
<tree_map
> ();
490 h
->hash
= htab_hash_pointer (from
);
494 slot
= (struct tree_map
**)
495 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
499 /* Return a TM-aware replacement function for DECL. */
502 find_tm_replacement_function (tree fndecl
)
506 struct tree_map
*h
, in
;
508 in
.base
.from
= fndecl
;
509 in
.hash
= htab_hash_pointer (fndecl
);
510 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
515 /* ??? We may well want TM versions of most of the common <string.h>
516 functions. For now, we've already these two defined. */
517 /* Adjust expand_call_tm() attributes as necessary for the cases
519 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
520 switch (DECL_FUNCTION_CODE (fndecl
))
522 case BUILT_IN_MEMCPY
:
523 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
524 case BUILT_IN_MEMMOVE
:
525 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
526 case BUILT_IN_MEMSET
:
527 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
535 /* When appropriate, record TM replacement for memory allocation functions.
537 FROM is the FNDECL to wrap. */
539 tm_malloc_replacement (tree from
)
544 if (TREE_CODE (from
) != FUNCTION_DECL
)
547 /* If we have a previous replacement, the user must be explicitly
548 wrapping malloc/calloc/free. They better know what they're
550 if (find_tm_replacement_function (from
))
553 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
555 if (!strcmp (str
, "malloc"))
556 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
557 else if (!strcmp (str
, "calloc"))
558 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
559 else if (!strcmp (str
, "free"))
560 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
564 TREE_NOTHROW (to
) = 0;
566 record_tm_replacement (from
, to
);
569 /* Diagnostics for tm_safe functions/regions. Called by the front end
570 once we've lowered the function to high-gimple. */
572 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
573 Process exactly one statement. WI->INFO is set to non-null when in
574 the context of a tm_safe function, and null for a __transaction block. */
576 #define DIAG_TM_OUTER 1
577 #define DIAG_TM_SAFE 2
578 #define DIAG_TM_RELAXED 4
582 unsigned int summary_flags
: 8;
583 unsigned int block_flags
: 8;
584 unsigned int func_flags
: 8;
585 unsigned int saw_volatile
: 1;
589 /* Return true if T is a volatile variable of some kind. */
592 volatile_var_p (tree t
)
594 return (SSA_VAR_P (t
)
595 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
598 /* Tree callback function for diagnose_tm pass. */
601 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
604 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
605 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
607 if (volatile_var_p (*tp
)
608 && d
->block_flags
& DIAG_TM_SAFE
612 error_at (gimple_location (d
->stmt
),
613 "invalid volatile use of %qD inside transaction",
621 is_tm_safe_or_pure (const_tree x
)
623 return is_tm_safe (x
) || is_tm_pure (x
);
627 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
628 struct walk_stmt_info
*wi
)
630 gimple stmt
= gsi_stmt (*gsi
);
631 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
633 /* Save stmt for use in leaf analysis. */
636 switch (gimple_code (stmt
))
640 tree fn
= gimple_call_fn (stmt
);
642 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
643 && is_tm_may_cancel_outer (fn
))
644 error_at (gimple_location (stmt
),
645 "%<transaction_may_cancel_outer%> function call not within"
646 " outer transaction or %<transaction_may_cancel_outer%>");
648 if (d
->summary_flags
& DIAG_TM_SAFE
)
650 bool is_safe
, direct_call_p
;
653 if (TREE_CODE (fn
) == ADDR_EXPR
654 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
656 direct_call_p
= true;
657 replacement
= TREE_OPERAND (fn
, 0);
658 replacement
= find_tm_replacement_function (replacement
);
664 direct_call_p
= false;
665 replacement
= NULL_TREE
;
668 if (is_tm_safe_or_pure (fn
))
670 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
672 /* A function explicitly marked transaction_callable as
673 opposed to transaction_safe is being defined to be
674 unsafe as part of its ABI, regardless of its contents. */
677 else if (direct_call_p
)
679 if (IS_TYPE_OR_DECL_P (fn
)
680 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
682 else if (replacement
)
684 /* ??? At present we've been considering replacements
685 merely transaction_callable, and therefore might
686 enter irrevocable. The tm_wrap attribute has not
687 yet made it into the new language spec. */
692 /* ??? Diagnostics for unmarked direct calls moved into
693 the IPA pass. Section 3.2 of the spec details how
694 functions not marked should be considered "implicitly
695 safe" based on having examined the function body. */
701 /* An unmarked indirect call. Consider it unsafe even
702 though optimization may yet figure out how to inline. */
708 if (TREE_CODE (fn
) == ADDR_EXPR
)
709 fn
= TREE_OPERAND (fn
, 0);
710 if (d
->block_flags
& DIAG_TM_SAFE
)
713 error_at (gimple_location (stmt
),
714 "unsafe function call %qD within "
715 "atomic transaction", fn
);
718 if (!DECL_P (fn
) || DECL_NAME (fn
))
719 error_at (gimple_location (stmt
),
720 "unsafe function call %qE within "
721 "atomic transaction", fn
);
723 error_at (gimple_location (stmt
),
724 "unsafe indirect function call within "
725 "atomic transaction");
731 error_at (gimple_location (stmt
),
732 "unsafe function call %qD within "
733 "%<transaction_safe%> function", fn
);
736 if (!DECL_P (fn
) || DECL_NAME (fn
))
737 error_at (gimple_location (stmt
),
738 "unsafe function call %qE within "
739 "%<transaction_safe%> function", fn
);
741 error_at (gimple_location (stmt
),
742 "unsafe indirect function call within "
743 "%<transaction_safe%> function");
752 /* ??? We ought to come up with a way to add attributes to
753 asm statements, and then add "transaction_safe" to it.
754 Either that or get the language spec to resurrect __tm_waiver. */
755 if (d
->block_flags
& DIAG_TM_SAFE
)
756 error_at (gimple_location (stmt
),
757 "asm not allowed in atomic transaction");
758 else if (d
->func_flags
& DIAG_TM_SAFE
)
759 error_at (gimple_location (stmt
),
760 "asm not allowed in %<transaction_safe%> function");
763 case GIMPLE_TRANSACTION
:
765 unsigned char inner_flags
= DIAG_TM_SAFE
;
767 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
769 if (d
->block_flags
& DIAG_TM_SAFE
)
770 error_at (gimple_location (stmt
),
771 "relaxed transaction in atomic transaction");
772 else if (d
->func_flags
& DIAG_TM_SAFE
)
773 error_at (gimple_location (stmt
),
774 "relaxed transaction in %<transaction_safe%> function");
775 inner_flags
= DIAG_TM_RELAXED
;
777 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
780 error_at (gimple_location (stmt
),
781 "outer transaction in transaction");
782 else if (d
->func_flags
& DIAG_TM_OUTER
)
783 error_at (gimple_location (stmt
),
784 "outer transaction in "
785 "%<transaction_may_cancel_outer%> function");
786 else if (d
->func_flags
& DIAG_TM_SAFE
)
787 error_at (gimple_location (stmt
),
788 "outer transaction in %<transaction_safe%> function");
789 inner_flags
|= DIAG_TM_OUTER
;
792 *handled_ops_p
= true;
793 if (gimple_transaction_body (stmt
))
795 struct walk_stmt_info wi_inner
;
796 struct diagnose_tm d_inner
;
798 memset (&d_inner
, 0, sizeof (d_inner
));
799 d_inner
.func_flags
= d
->func_flags
;
800 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
801 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
803 memset (&wi_inner
, 0, sizeof (wi_inner
));
804 wi_inner
.info
= &d_inner
;
806 walk_gimple_seq (gimple_transaction_body (stmt
),
807 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
820 diagnose_tm_blocks (void)
822 struct walk_stmt_info wi
;
823 struct diagnose_tm d
;
825 memset (&d
, 0, sizeof (d
));
826 if (is_tm_may_cancel_outer (current_function_decl
))
827 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
828 else if (is_tm_safe (current_function_decl
))
829 d
.func_flags
= DIAG_TM_SAFE
;
830 d
.summary_flags
= d
.func_flags
;
832 memset (&wi
, 0, sizeof (wi
));
835 walk_gimple_seq (gimple_body (current_function_decl
),
836 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
843 const pass_data pass_data_diagnose_tm_blocks
=
845 GIMPLE_PASS
, /* type */
846 "*diagnose_tm_blocks", /* name */
847 OPTGROUP_NONE
, /* optinfo_flags */
848 TV_TRANS_MEM
, /* tv_id */
849 PROP_gimple_any
, /* properties_required */
850 0, /* properties_provided */
851 0, /* properties_destroyed */
852 0, /* todo_flags_start */
853 0, /* todo_flags_finish */
856 class pass_diagnose_tm_blocks
: public gimple_opt_pass
859 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
860 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
863 /* opt_pass methods: */
864 virtual bool gate (function
*) { return flag_tm
; }
865 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
867 }; // class pass_diagnose_tm_blocks
872 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
874 return new pass_diagnose_tm_blocks (ctxt
);
877 /* Instead of instrumenting thread private memory, we save the
878 addresses in a log which we later use to save/restore the addresses
879 upon transaction start/restart.
881 The log is keyed by address, where each element contains individual
882 statements among different code paths that perform the store.
884 This log is later used to generate either plain save/restore of the
885 addresses upon transaction start/restart, or calls to the ITM_L*
888 So for something like:
890 struct large { int x[1000]; };
891 struct large lala = { 0 };
897 We can either save/restore:
900 trxn = _ITM_startTransaction ();
901 if (trxn & a_saveLiveVariables)
902 tmp_lala1 = lala.x[i];
903 else if (a & a_restoreLiveVariables)
904 lala.x[i] = tmp_lala1;
906 or use the logging functions:
909 trxn = _ITM_startTransaction ();
910 _ITM_LU4 (&lala.x[i]);
912 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
913 far up the dominator tree to shadow all of the writes to a given
914 location (thus reducing the total number of logging calls), but not
915 so high as to be called on a path that does not perform a
918 /* One individual log entry. We may have multiple statements for the
919 same location if neither dominate each other (on different
921 typedef struct tm_log_entry
923 /* Address to save. */
925 /* Entry block for the transaction this address occurs in. */
926 basic_block entry_block
;
927 /* Dominating statements the store occurs in. */
929 /* Initially, while we are building the log, we place a nonzero
930 value here to mean that this address *will* be saved with a
931 save/restore sequence. Later, when generating the save sequence
932 we place the SSA temp generated here. */
937 /* Log entry hashtable helpers. */
939 struct log_entry_hasher
941 typedef tm_log_entry value_type
;
942 typedef tm_log_entry compare_type
;
943 static inline hashval_t
hash (const value_type
*);
944 static inline bool equal (const value_type
*, const compare_type
*);
945 static inline void remove (value_type
*);
948 /* Htab support. Return hash value for a `tm_log_entry'. */
950 log_entry_hasher::hash (const value_type
*log
)
952 return iterative_hash_expr (log
->addr
, 0);
955 /* Htab support. Return true if two log entries are the same. */
957 log_entry_hasher::equal (const value_type
*log1
, const compare_type
*log2
)
961 rth: I suggest that we get rid of the component refs etc.
962 I.e. resolve the reference to base + offset.
964 We may need to actually finish a merge with mainline for this,
965 since we'd like to be presented with Richi's MEM_REF_EXPRs more
966 often than not. But in the meantime your tm_log_entry could save
967 the results of get_inner_reference.
969 See: g++.dg/tm/pr46653.C
972 /* Special case plain equality because operand_equal_p() below will
973 return FALSE if the addresses are equal but they have
974 side-effects (e.g. a volatile address). */
975 if (log1
->addr
== log2
->addr
)
978 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
981 /* Htab support. Free one tm_log_entry. */
983 log_entry_hasher::remove (value_type
*lp
)
985 lp
->stmts
.release ();
990 /* The actual log. */
991 static hash_table
<log_entry_hasher
> *tm_log
;
993 /* Addresses to log with a save/restore sequence. These should be in
995 static vec
<tree
> tm_log_save_addresses
;
997 enum thread_memory_type
1001 mem_transaction_local
,
1005 typedef struct tm_new_mem_map
1007 /* SSA_NAME being dereferenced. */
1009 enum thread_memory_type local_new_memory
;
1012 /* Hashtable helpers. */
1014 struct tm_mem_map_hasher
: typed_free_remove
<tm_new_mem_map_t
>
1016 typedef tm_new_mem_map_t value_type
;
1017 typedef tm_new_mem_map_t compare_type
;
1018 static inline hashval_t
hash (const value_type
*);
1019 static inline bool equal (const value_type
*, const compare_type
*);
1023 tm_mem_map_hasher::hash (const value_type
*v
)
1025 return (intptr_t)v
->val
>> 4;
1029 tm_mem_map_hasher::equal (const value_type
*v
, const compare_type
*c
)
1031 return v
->val
== c
->val
;
1034 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1035 of memory (malloc, alloc, etc). */
1036 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1038 /* Initialize logging data structures. */
1042 tm_log
= new hash_table
<log_entry_hasher
> (10);
1043 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1044 tm_log_save_addresses
.create (5);
1047 /* Free logging data structures. */
1049 tm_log_delete (void)
1053 delete tm_new_mem_hash
;
1054 tm_new_mem_hash
= NULL
;
1055 tm_log_save_addresses
.release ();
1058 /* Return true if MEM is a transaction invariant memory for the TM
1059 region starting at REGION_ENTRY_BLOCK. */
1061 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1063 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1064 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1068 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1069 return def_bb
!= region_entry_block
1070 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1073 mem
= strip_invariant_refs (mem
);
1074 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1077 /* Given an address ADDR in STMT, find it in the memory log or add it,
1078 making sure to keep only the addresses highest in the dominator
1081 ENTRY_BLOCK is the entry_block for the transaction.
1083 If we find the address in the log, make sure it's either the same
1084 address, or an equivalent one that dominates ADDR.
1086 If we find the address, but neither ADDR dominates the found
1087 address, nor the found one dominates ADDR, we're on different
1088 execution paths. Add it.
1090 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1093 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1095 tm_log_entry
**slot
;
1096 struct tm_log_entry l
, *lp
;
1099 slot
= tm_log
->find_slot (&l
, INSERT
);
1102 tree type
= TREE_TYPE (addr
);
1104 lp
= XNEW (struct tm_log_entry
);
1108 /* Small invariant addresses can be handled as save/restores. */
1110 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1111 && TYPE_SIZE_UNIT (type
) != NULL
1112 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1113 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1114 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1115 /* We must be able to copy this type normally. I.e., no
1116 special constructors and the like. */
1117 && !TREE_ADDRESSABLE (type
))
1119 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1120 lp
->stmts
.create (0);
1121 lp
->entry_block
= entry_block
;
1122 /* Save addresses separately in dominator order so we don't
1123 get confused by overlapping addresses in the save/restore
1125 tm_log_save_addresses
.safe_push (lp
->addr
);
1129 /* Use the logging functions. */
1130 lp
->stmts
.create (5);
1131 lp
->stmts
.quick_push (stmt
);
1132 lp
->save_var
= NULL
;
1142 /* If we're generating a save/restore sequence, we don't care
1143 about statements. */
1147 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1149 if (stmt
== oldstmt
)
1151 /* We already have a store to the same address, higher up the
1152 dominator tree. Nothing to do. */
1153 if (dominated_by_p (CDI_DOMINATORS
,
1154 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1156 /* We should be processing blocks in dominator tree order. */
1157 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1158 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1160 /* Store is on a different code path. */
1161 lp
->stmts
.safe_push (stmt
);
1165 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1166 result, insert the new statements before GSI. */
1169 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1171 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1172 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1174 x
= build_fold_addr_expr (x
);
1175 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1178 /* Instrument one address with the logging functions.
1179 ADDR is the address to save.
1180 STMT is the statement before which to place it. */
1182 tm_log_emit_stmt (tree addr
, gimple stmt
)
1184 tree type
= TREE_TYPE (addr
);
1185 tree size
= TYPE_SIZE_UNIT (type
);
1186 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1188 enum built_in_function code
= BUILT_IN_TM_LOG
;
1190 if (type
== float_type_node
)
1191 code
= BUILT_IN_TM_LOG_FLOAT
;
1192 else if (type
== double_type_node
)
1193 code
= BUILT_IN_TM_LOG_DOUBLE
;
1194 else if (type
== long_double_type_node
)
1195 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1196 else if (tree_fits_uhwi_p (size
))
1198 unsigned int n
= tree_to_uhwi (size
);
1202 code
= BUILT_IN_TM_LOG_1
;
1205 code
= BUILT_IN_TM_LOG_2
;
1208 code
= BUILT_IN_TM_LOG_4
;
1211 code
= BUILT_IN_TM_LOG_8
;
1214 code
= BUILT_IN_TM_LOG
;
1215 if (TREE_CODE (type
) == VECTOR_TYPE
)
1217 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1218 code
= BUILT_IN_TM_LOG_M64
;
1219 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1220 code
= BUILT_IN_TM_LOG_M128
;
1221 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1222 code
= BUILT_IN_TM_LOG_M256
;
1228 addr
= gimplify_addr (&gsi
, addr
);
1229 if (code
== BUILT_IN_TM_LOG
)
1230 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1232 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1233 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1236 /* Go through the log and instrument address that must be instrumented
1237 with the logging functions. Leave the save/restore addresses for
1242 hash_table
<log_entry_hasher
>::iterator hi
;
1243 struct tm_log_entry
*lp
;
1245 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1252 fprintf (dump_file
, "TM thread private mem logging: ");
1253 print_generic_expr (dump_file
, lp
->addr
, 0);
1254 fprintf (dump_file
, "\n");
1260 fprintf (dump_file
, "DUMPING to variable\n");
1266 fprintf (dump_file
, "DUMPING with logging functions\n");
1267 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1268 tm_log_emit_stmt (lp
->addr
, stmt
);
1273 /* Emit the save sequence for the corresponding addresses in the log.
1274 ENTRY_BLOCK is the entry block for the transaction.
1275 BB is the basic block to insert the code in. */
1277 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1280 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1282 struct tm_log_entry l
, *lp
;
1284 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1286 l
.addr
= tm_log_save_addresses
[i
];
1287 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1288 gcc_assert (lp
->save_var
!= NULL
);
1290 /* We only care about variables in the current transaction. */
1291 if (lp
->entry_block
!= entry_block
)
1294 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1296 /* Make sure we can create an SSA_NAME for this type. For
1297 instance, aggregates aren't allowed, in which case the system
1298 will create a VOP for us and everything will just work. */
1299 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1301 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1302 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1305 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1309 /* Emit the restore sequence for the corresponding addresses in the log.
1310 ENTRY_BLOCK is the entry block for the transaction.
1311 BB is the basic block to insert the code in. */
1313 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1316 struct tm_log_entry l
, *lp
;
1317 gimple_stmt_iterator gsi
;
1320 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1322 l
.addr
= tm_log_save_addresses
[i
];
1323 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1324 gcc_assert (lp
->save_var
!= NULL
);
1326 /* We only care about variables in the current transaction. */
1327 if (lp
->entry_block
!= entry_block
)
1330 /* Restores are in LIFO order from the saves in case we have
1332 gsi
= gsi_start_bb (bb
);
1334 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1335 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1340 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1341 struct walk_stmt_info
*);
1342 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1343 struct walk_stmt_info
*);
1345 /* Evaluate an address X being dereferenced and determine if it
1346 originally points to a non aliased new chunk of memory (malloc,
1349 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1350 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1351 Return MEM_NON_LOCAL otherwise.
1353 ENTRY_BLOCK is the entry block to the transaction containing the
1354 dereference of X. */
1355 static enum thread_memory_type
1356 thread_private_new_memory (basic_block entry_block
, tree x
)
1359 enum tree_code code
;
1360 tm_new_mem_map_t
**slot
;
1361 tm_new_mem_map_t elt
, *elt_p
;
1363 enum thread_memory_type retval
= mem_transaction_local
;
1366 || TREE_CODE (x
) != SSA_NAME
1367 /* Possible uninitialized use, or a function argument. In
1368 either case, we don't care. */
1369 || SSA_NAME_IS_DEFAULT_DEF (x
))
1370 return mem_non_local
;
1372 /* Look in cache first. */
1374 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1377 return elt_p
->local_new_memory
;
1379 /* Optimistically assume the memory is transaction local during
1380 processing. This catches recursion into this variable. */
1381 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1383 elt_p
->local_new_memory
= mem_transaction_local
;
1385 /* Search DEF chain to find the original definition of this address. */
1388 if (ptr_deref_may_alias_global_p (x
))
1390 /* Address escapes. This is not thread-private. */
1391 retval
= mem_non_local
;
1392 goto new_memory_ret
;
1395 stmt
= SSA_NAME_DEF_STMT (x
);
1397 /* If the malloc call is outside the transaction, this is
1399 if (retval
!= mem_thread_local
1400 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1401 retval
= mem_thread_local
;
1403 if (is_gimple_assign (stmt
))
1405 code
= gimple_assign_rhs_code (stmt
);
1406 /* x = foo ==> foo */
1407 if (code
== SSA_NAME
)
1408 x
= gimple_assign_rhs1 (stmt
);
1409 /* x = foo + n ==> foo */
1410 else if (code
== POINTER_PLUS_EXPR
)
1411 x
= gimple_assign_rhs1 (stmt
);
1412 /* x = (cast*) foo ==> foo */
1413 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1414 x
= gimple_assign_rhs1 (stmt
);
1415 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1416 else if (code
== COND_EXPR
)
1418 tree op1
= gimple_assign_rhs2 (stmt
);
1419 tree op2
= gimple_assign_rhs3 (stmt
);
1420 enum thread_memory_type mem
;
1421 retval
= thread_private_new_memory (entry_block
, op1
);
1422 if (retval
== mem_non_local
)
1423 goto new_memory_ret
;
1424 mem
= thread_private_new_memory (entry_block
, op2
);
1425 retval
= MIN (retval
, mem
);
1426 goto new_memory_ret
;
1430 retval
= mem_non_local
;
1431 goto new_memory_ret
;
1436 if (gimple_code (stmt
) == GIMPLE_PHI
)
1439 enum thread_memory_type mem
;
1440 tree phi_result
= gimple_phi_result (stmt
);
1442 /* If any of the ancestors are non-local, we are sure to
1443 be non-local. Otherwise we can avoid doing anything
1444 and inherit what has already been generated. */
1446 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1448 tree op
= PHI_ARG_DEF (stmt
, i
);
1450 /* Exclude self-assignment. */
1451 if (phi_result
== op
)
1454 mem
= thread_private_new_memory (entry_block
, op
);
1455 if (mem
== mem_non_local
)
1458 goto new_memory_ret
;
1460 retval
= MIN (retval
, mem
);
1462 goto new_memory_ret
;
1467 while (TREE_CODE (x
) == SSA_NAME
);
1469 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1470 /* Thread-local or transaction-local. */
1473 retval
= mem_non_local
;
1476 elt_p
->local_new_memory
= retval
;
1480 /* Determine whether X has to be instrumented using a read
1483 ENTRY_BLOCK is the entry block for the region where stmt resides
1484 in. NULL if unknown.
1486 STMT is the statement in which X occurs in. It is used for thread
1487 private memory instrumentation. If no TPM instrumentation is
1488 desired, STMT should be null. */
1490 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1493 while (handled_component_p (x
))
1494 x
= TREE_OPERAND (x
, 0);
1496 switch (TREE_CODE (x
))
1501 enum thread_memory_type ret
;
1503 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1504 if (ret
== mem_non_local
)
1506 if (stmt
&& ret
== mem_thread_local
)
1507 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1508 tm_log_add (entry_block
, orig
, stmt
);
1510 /* Transaction-locals require nothing at all. For malloc, a
1511 transaction restart frees the memory and we reallocate.
1512 For alloca, the stack pointer gets reset by the retry and
1517 case TARGET_MEM_REF
:
1518 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1520 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1521 if (TREE_CODE (x
) == PARM_DECL
)
1523 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1529 if (DECL_BY_REFERENCE (x
))
1531 /* ??? This value is a pointer, but aggregate_value_p has been
1532 jigged to return true which confuses needs_to_live_in_memory.
1533 This ought to be cleaned up generically.
1535 FIXME: Verify this still happens after the next mainline
1536 merge. Testcase ie g++.dg/tm/pr47554.C.
1541 if (is_global_var (x
))
1542 return !TREE_READONLY (x
);
1543 if (/* FIXME: This condition should actually go below in the
1544 tm_log_add() call, however is_call_clobbered() depends on
1545 aliasing info which is not available during
1546 gimplification. Since requires_barrier() gets called
1547 during lower_sequence_tm/gimplification, leave the call
1548 to needs_to_live_in_memory until we eliminate
1549 lower_sequence_tm altogether. */
1550 needs_to_live_in_memory (x
))
1554 /* For local memory that doesn't escape (aka thread private
1555 memory), we can either save the value at the beginning of
1556 the transaction and restore on restart, or call a tm
1557 function to dynamically save and restore on restart
1560 tm_log_add (entry_block
, orig
, stmt
);
1569 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1570 a transaction region. */
1573 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1575 gimple stmt
= gsi_stmt (*gsi
);
1577 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1578 *state
|= GTMA_HAVE_LOAD
;
1579 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1580 *state
|= GTMA_HAVE_STORE
;
1583 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1586 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1588 gimple stmt
= gsi_stmt (*gsi
);
1591 if (is_tm_pure_call (stmt
))
1594 /* Check if this call is a transaction abort. */
1595 fn
= gimple_call_fndecl (stmt
);
1596 if (is_tm_abort (fn
))
1597 *state
|= GTMA_HAVE_ABORT
;
1599 /* Note that something may happen. */
1600 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1603 /* Lower a GIMPLE_TRANSACTION statement. */
1606 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1608 gimple g
, stmt
= gsi_stmt (*gsi
);
1609 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1610 unsigned int this_state
= 0;
1611 struct walk_stmt_info this_wi
;
1613 /* First, lower the body. The scanning that we do inside gives
1614 us some idea of what we're dealing with. */
1615 memset (&this_wi
, 0, sizeof (this_wi
));
1616 this_wi
.info
= (void *) &this_state
;
1617 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1618 lower_sequence_tm
, NULL
, &this_wi
);
1620 /* If there was absolutely nothing transaction related inside the
1621 transaction, we may elide it. Likewise if this is a nested
1622 transaction and does not contain an abort. */
1624 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1627 *outer_state
|= this_state
;
1629 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1631 gimple_transaction_set_body (stmt
, NULL
);
1633 gsi_remove (gsi
, true);
1634 wi
->removed_stmt
= true;
1638 /* Wrap the body of the transaction in a try-finally node so that
1639 the commit call is always properly called. */
1640 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1641 if (flag_exceptions
)
1644 gimple_seq n_seq
, e_seq
;
1646 n_seq
= gimple_seq_alloc_with_stmt (g
);
1649 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1650 1, integer_zero_node
);
1651 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1652 gimple_call_set_lhs (g
, ptr
);
1653 gimple_seq_add_stmt (&e_seq
, g
);
1655 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1657 gimple_seq_add_stmt (&e_seq
, g
);
1659 g
= gimple_build_eh_else (n_seq
, e_seq
);
1662 g
= gimple_build_try (gimple_transaction_body (stmt
),
1663 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1664 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1666 gimple_transaction_set_body (stmt
, NULL
);
1668 /* If the transaction calls abort or if this is an outer transaction,
1669 add an "over" label afterwards. */
1670 if ((this_state
& (GTMA_HAVE_ABORT
))
1671 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1673 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1674 gimple_transaction_set_label (stmt
, label
);
1675 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1678 /* Record the set of operations found for use later. */
1679 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1680 gimple_transaction_set_subcode (stmt
, this_state
);
1683 /* Iterate through the statements in the sequence, lowering them all
1684 as appropriate for being in a transaction. */
1687 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1688 struct walk_stmt_info
*wi
)
1690 unsigned int *state
= (unsigned int *) wi
->info
;
1691 gimple stmt
= gsi_stmt (*gsi
);
1693 *handled_ops_p
= true;
1694 switch (gimple_code (stmt
))
1697 /* Only memory reads/writes need to be instrumented. */
1698 if (gimple_assign_single_p (stmt
))
1699 examine_assign_tm (state
, gsi
);
1703 examine_call_tm (state
, gsi
);
1707 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1710 case GIMPLE_TRANSACTION
:
1711 lower_transaction (gsi
, wi
);
1715 *handled_ops_p
= !gimple_has_substatements (stmt
);
1722 /* Iterate through the statements in the sequence, lowering them all
1723 as appropriate for being outside of a transaction. */
1726 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1727 struct walk_stmt_info
* wi
)
1729 gimple stmt
= gsi_stmt (*gsi
);
1731 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1733 *handled_ops_p
= true;
1734 lower_transaction (gsi
, wi
);
1737 *handled_ops_p
= !gimple_has_substatements (stmt
);
1742 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1743 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1744 been moved out, and all the data required for constructing a proper
1745 CFG has been recorded. */
1748 execute_lower_tm (void)
1750 struct walk_stmt_info wi
;
1753 /* Transactional clones aren't created until a later pass. */
1754 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1756 body
= gimple_body (current_function_decl
);
1757 memset (&wi
, 0, sizeof (wi
));
1758 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1759 gimple_set_body (current_function_decl
, body
);
1766 const pass_data pass_data_lower_tm
=
1768 GIMPLE_PASS
, /* type */
1769 "tmlower", /* name */
1770 OPTGROUP_NONE
, /* optinfo_flags */
1771 TV_TRANS_MEM
, /* tv_id */
1772 PROP_gimple_lcf
, /* properties_required */
1773 0, /* properties_provided */
1774 0, /* properties_destroyed */
1775 0, /* todo_flags_start */
1776 0, /* todo_flags_finish */
1779 class pass_lower_tm
: public gimple_opt_pass
1782 pass_lower_tm (gcc::context
*ctxt
)
1783 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1786 /* opt_pass methods: */
1787 virtual bool gate (function
*) { return flag_tm
; }
1788 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1790 }; // class pass_lower_tm
1795 make_pass_lower_tm (gcc::context
*ctxt
)
1797 return new pass_lower_tm (ctxt
);
1800 /* Collect region information for each transaction. */
1804 /* Link to the next unnested transaction. */
1805 struct tm_region
*next
;
1807 /* Link to the next inner transaction. */
1808 struct tm_region
*inner
;
1810 /* Link to the next outer transaction. */
1811 struct tm_region
*outer
;
1813 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1814 After TM_MARK, this gets replaced by a call to
1815 BUILT_IN_TM_START. */
1816 gimple transaction_stmt
;
1818 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1819 BUILT_IN_TM_START, this field is true if the transaction is an
1820 outer transaction. */
1821 bool original_transaction_was_outer
;
1823 /* Return value from BUILT_IN_TM_START. */
1826 /* The entry block to this region. This will always be the first
1827 block of the body of the transaction. */
1828 basic_block entry_block
;
1830 /* The first block after an expanded call to _ITM_beginTransaction. */
1831 basic_block restart_block
;
1833 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1834 These blocks are still a part of the region (i.e., the border is
1835 inclusive). Note that this set is only complete for paths in the CFG
1836 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1837 the edge to the "over" label. */
1840 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1844 typedef struct tm_region
*tm_region_p
;
1846 /* True if there are pending edge statements to be committed for the
1847 current function being scanned in the tmmark pass. */
1848 bool pending_edge_inserts_p
;
1850 static struct tm_region
*all_tm_regions
;
1851 static bitmap_obstack tm_obstack
;
1854 /* A subroutine of tm_region_init. Record the existence of the
1855 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1857 static struct tm_region
*
1858 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1860 struct tm_region
*region
;
1862 region
= (struct tm_region
*)
1863 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1867 region
->next
= outer
->inner
;
1868 outer
->inner
= region
;
1872 region
->next
= all_tm_regions
;
1873 all_tm_regions
= region
;
1875 region
->inner
= NULL
;
1876 region
->outer
= outer
;
1878 region
->transaction_stmt
= stmt
;
1879 region
->original_transaction_was_outer
= false;
1880 region
->tm_state
= NULL
;
1882 /* There are either one or two edges out of the block containing
1883 the GIMPLE_TRANSACTION, one to the actual region and one to the
1884 "over" label if the region contains an abort. The former will
1885 always be the one marked FALLTHRU. */
1886 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1888 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1889 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1894 /* A subroutine of tm_region_init. Record all the exit and
1895 irrevocable blocks in BB into the region's exit_blocks and
1896 irr_blocks bitmaps. Returns the new region being scanned. */
1898 static struct tm_region
*
1899 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1901 gimple_stmt_iterator gsi
;
1905 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1908 /* Check to see if this is the end of a region by seeing if it
1909 contains a call to __builtin_tm_commit{,_eh}. Note that the
1910 outermost region for DECL_IS_TM_CLONE need not collect this. */
1911 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1914 if (gimple_code (g
) == GIMPLE_CALL
)
1916 tree fn
= gimple_call_fndecl (g
);
1917 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1919 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1920 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1921 && region
->exit_blocks
)
1923 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1924 region
= region
->outer
;
1927 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1928 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1935 /* Collect all of the transaction regions within the current function
1936 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1937 an "outermost" region for use by tm clones. */
1940 tm_region_init (struct tm_region
*region
)
1946 auto_vec
<basic_block
> queue
;
1947 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1948 struct tm_region
*old_region
;
1949 auto_vec
<tm_region_p
> bb_regions
;
1951 all_tm_regions
= region
;
1952 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1954 /* We could store this information in bb->aux, but we may get called
1955 through get_all_tm_blocks() from another pass that may be already
1957 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1959 queue
.safe_push (bb
);
1960 bb_regions
[bb
->index
] = region
;
1964 region
= bb_regions
[bb
->index
];
1965 bb_regions
[bb
->index
] = NULL
;
1967 /* Record exit and irrevocable blocks. */
1968 region
= tm_region_init_1 (region
, bb
);
1970 /* Check for the last statement in the block beginning a new region. */
1972 old_region
= region
;
1973 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1974 region
= tm_region_init_0 (region
, bb
, g
);
1976 /* Process subsequent blocks. */
1977 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1978 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1980 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1981 queue
.safe_push (e
->dest
);
1983 /* If the current block started a new region, make sure that only
1984 the entry block of the new region is associated with this region.
1985 Other successors are still part of the old region. */
1986 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1987 bb_regions
[e
->dest
->index
] = old_region
;
1989 bb_regions
[e
->dest
->index
] = region
;
1992 while (!queue
.is_empty ());
1993 BITMAP_FREE (visited_blocks
);
1996 /* The "gate" function for all transactional memory expansion and optimization
1997 passes. We collect region information for each top-level transaction, and
1998 if we don't find any, we skip all of the TM passes. Each region will have
1999 all of the exit blocks recorded, and the originating statement. */
2007 calculate_dominance_info (CDI_DOMINATORS
);
2008 bitmap_obstack_initialize (&tm_obstack
);
2010 /* If the function is a TM_CLONE, then the entire function is the region. */
2011 if (decl_is_tm_clone (current_function_decl
))
2013 struct tm_region
*region
= (struct tm_region
*)
2014 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2015 memset (region
, 0, sizeof (*region
));
2016 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2017 /* For a clone, the entire function is the region. But even if
2018 we don't need to record any exit blocks, we may need to
2019 record irrevocable blocks. */
2020 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2022 tm_region_init (region
);
2026 tm_region_init (NULL
);
2028 /* If we didn't find any regions, cleanup and skip the whole tree
2029 of tm-related optimizations. */
2030 if (all_tm_regions
== NULL
)
2032 bitmap_obstack_release (&tm_obstack
);
2042 const pass_data pass_data_tm_init
=
2044 GIMPLE_PASS
, /* type */
2045 "*tminit", /* name */
2046 OPTGROUP_NONE
, /* optinfo_flags */
2047 TV_TRANS_MEM
, /* tv_id */
2048 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2049 0, /* properties_provided */
2050 0, /* properties_destroyed */
2051 0, /* todo_flags_start */
2052 0, /* todo_flags_finish */
2055 class pass_tm_init
: public gimple_opt_pass
2058 pass_tm_init (gcc::context
*ctxt
)
2059 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2062 /* opt_pass methods: */
2063 virtual bool gate (function
*) { return gate_tm_init (); }
2065 }; // class pass_tm_init
2070 make_pass_tm_init (gcc::context
*ctxt
)
2072 return new pass_tm_init (ctxt
);
2075 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2076 represented by STATE. */
2079 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2081 if (region
&& region
->transaction_stmt
)
2083 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
2084 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
2088 /* Construct a memory load in a transactional context. Return the
2089 gimple statement performing the load, or NULL if there is no
2090 TM_LOAD builtin of the appropriate size to do the load.
2092 LOC is the location to use for the new statement(s). */
2095 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2097 enum built_in_function code
= END_BUILTINS
;
2098 tree t
, type
= TREE_TYPE (rhs
), decl
;
2101 if (type
== float_type_node
)
2102 code
= BUILT_IN_TM_LOAD_FLOAT
;
2103 else if (type
== double_type_node
)
2104 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2105 else if (type
== long_double_type_node
)
2106 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2107 else if (TYPE_SIZE_UNIT (type
) != NULL
2108 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2110 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2113 code
= BUILT_IN_TM_LOAD_1
;
2116 code
= BUILT_IN_TM_LOAD_2
;
2119 code
= BUILT_IN_TM_LOAD_4
;
2122 code
= BUILT_IN_TM_LOAD_8
;
2127 if (code
== END_BUILTINS
)
2129 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2134 decl
= builtin_decl_explicit (code
);
2136 t
= gimplify_addr (gsi
, rhs
);
2137 gcall
= gimple_build_call (decl
, 1, t
);
2138 gimple_set_location (gcall
, loc
);
2140 t
= TREE_TYPE (TREE_TYPE (decl
));
2141 if (useless_type_conversion_p (type
, t
))
2143 gimple_call_set_lhs (gcall
, lhs
);
2144 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2151 temp
= create_tmp_reg (t
, NULL
);
2152 gimple_call_set_lhs (gcall
, temp
);
2153 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2155 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2156 g
= gimple_build_assign (lhs
, t
);
2157 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2164 /* Similarly for storing TYPE in a transactional context. */
2167 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2169 enum built_in_function code
= END_BUILTINS
;
2170 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2173 if (type
== float_type_node
)
2174 code
= BUILT_IN_TM_STORE_FLOAT
;
2175 else if (type
== double_type_node
)
2176 code
= BUILT_IN_TM_STORE_DOUBLE
;
2177 else if (type
== long_double_type_node
)
2178 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2179 else if (TYPE_SIZE_UNIT (type
) != NULL
2180 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2182 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2185 code
= BUILT_IN_TM_STORE_1
;
2188 code
= BUILT_IN_TM_STORE_2
;
2191 code
= BUILT_IN_TM_STORE_4
;
2194 code
= BUILT_IN_TM_STORE_8
;
2199 if (code
== END_BUILTINS
)
2201 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2206 fn
= builtin_decl_explicit (code
);
2208 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2210 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2212 /* Handle the easy initialization to zero. */
2213 if (!CONSTRUCTOR_ELTS (rhs
))
2214 rhs
= build_int_cst (simple_type
, 0);
2217 /* ...otherwise punt to the caller and probably use
2218 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2219 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2224 else if (!useless_type_conversion_p (simple_type
, type
))
2229 temp
= create_tmp_reg (simple_type
, NULL
);
2230 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2231 g
= gimple_build_assign (temp
, t
);
2232 gimple_set_location (g
, loc
);
2233 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2238 t
= gimplify_addr (gsi
, lhs
);
2239 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2240 gimple_set_location (gcall
, loc
);
2241 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2247 /* Expand an assignment statement into transactional builtins. */
2250 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2252 gimple stmt
= gsi_stmt (*gsi
);
2253 location_t loc
= gimple_location (stmt
);
2254 tree lhs
= gimple_assign_lhs (stmt
);
2255 tree rhs
= gimple_assign_rhs1 (stmt
);
2256 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2257 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2258 gimple gcall
= NULL
;
2260 if (!load_p
&& !store_p
)
2262 /* Add thread private addresses to log if applicable. */
2263 requires_barrier (region
->entry_block
, lhs
, stmt
);
2268 // Remove original load/store statement.
2269 gsi_remove (gsi
, true);
2271 if (load_p
&& !store_p
)
2273 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2274 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2276 else if (store_p
&& !load_p
)
2278 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2279 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2283 tree lhs_addr
, rhs_addr
, tmp
;
2286 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2288 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2290 /* ??? Figure out if there's any possible overlap between the LHS
2291 and the RHS and if not, use MEMCPY. */
2293 if (load_p
&& is_gimple_reg (lhs
))
2295 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2296 lhs_addr
= build_fold_addr_expr (tmp
);
2301 lhs_addr
= gimplify_addr (gsi
, lhs
);
2303 rhs_addr
= gimplify_addr (gsi
, rhs
);
2304 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2305 3, lhs_addr
, rhs_addr
,
2306 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2307 gimple_set_location (gcall
, loc
);
2308 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2312 gcall
= gimple_build_assign (lhs
, tmp
);
2313 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2317 /* Now that we have the load/store in its instrumented form, add
2318 thread private addresses to the log if applicable. */
2320 requires_barrier (region
->entry_block
, lhs
, gcall
);
2322 // The calls to build_tm_{store,load} above inserted the instrumented
2323 // call into the stream.
2324 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2328 /* Expand a call statement as appropriate for a transaction. That is,
2329 either verify that the call does not affect the transaction, or
2330 redirect the call to a clone that handles transactions, or change
2331 the transaction state to IRREVOCABLE. Return true if the call is
2332 one of the builtins that end a transaction. */
2335 expand_call_tm (struct tm_region
*region
,
2336 gimple_stmt_iterator
*gsi
)
2338 gimple stmt
= gsi_stmt (*gsi
);
2339 tree lhs
= gimple_call_lhs (stmt
);
2341 struct cgraph_node
*node
;
2342 bool retval
= false;
2344 fn_decl
= gimple_call_fndecl (stmt
);
2346 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2347 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2348 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2349 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2350 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2352 if (is_tm_pure_call (stmt
))
2356 retval
= is_tm_ending_fndecl (fn_decl
);
2359 /* Assume all non-const/pure calls write to memory, except
2360 transaction ending builtins. */
2361 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2364 /* For indirect calls, we already generated a call into the runtime. */
2367 tree fn
= gimple_call_fn (stmt
);
2369 /* We are guaranteed never to go irrevocable on a safe or pure
2370 call, and the pure call was handled above. */
2371 if (is_tm_safe (fn
))
2374 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2379 node
= cgraph_node::get (fn_decl
);
2380 /* All calls should have cgraph here. */
2383 /* We can have a nodeless call here if some pass after IPA-tm
2384 added uninstrumented calls. For example, loop distribution
2385 can transform certain loop constructs into __builtin_mem*
2386 calls. In this case, see if we have a suitable TM
2387 replacement and fill in the gaps. */
2388 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2389 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2390 gcc_assert (code
== BUILT_IN_MEMCPY
2391 || code
== BUILT_IN_MEMMOVE
2392 || code
== BUILT_IN_MEMSET
);
2394 tree repl
= find_tm_replacement_function (fn_decl
);
2397 gimple_call_set_fndecl (stmt
, repl
);
2399 node
= cgraph_node::create (repl
);
2400 node
->local
.tm_may_enter_irr
= false;
2401 return expand_call_tm (region
, gsi
);
2405 if (node
->local
.tm_may_enter_irr
)
2406 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2408 if (is_tm_abort (fn_decl
))
2410 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2414 /* Instrument the store if needed.
2416 If the assignment happens inside the function call (return slot
2417 optimization), there is no instrumentation to be done, since
2418 the callee should have done the right thing. */
2419 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2420 && !gimple_call_return_slot_opt_p (stmt
))
2422 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2423 location_t loc
= gimple_location (stmt
);
2424 edge fallthru_edge
= NULL
;
2426 /* Remember if the call was going to throw. */
2427 if (stmt_can_throw_internal (stmt
))
2431 basic_block bb
= gimple_bb (stmt
);
2433 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2434 if (e
->flags
& EDGE_FALLTHRU
)
2441 gimple_call_set_lhs (stmt
, tmp
);
2443 stmt
= gimple_build_assign (lhs
, tmp
);
2444 gimple_set_location (stmt
, loc
);
2446 /* We cannot throw in the middle of a BB. If the call was going
2447 to throw, place the instrumentation on the fallthru edge, so
2448 the call remains the last statement in the block. */
2451 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2452 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2453 expand_assign_tm (region
, &fallthru_gsi
);
2454 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2455 pending_edge_inserts_p
= true;
2459 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2460 expand_assign_tm (region
, gsi
);
2463 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2470 /* Expand all statements in BB as appropriate for being inside
2474 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2476 gimple_stmt_iterator gsi
;
2478 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2480 gimple stmt
= gsi_stmt (gsi
);
2481 switch (gimple_code (stmt
))
2484 /* Only memory reads/writes need to be instrumented. */
2485 if (gimple_assign_single_p (stmt
)
2486 && !gimple_clobber_p (stmt
))
2488 expand_assign_tm (region
, &gsi
);
2494 if (expand_call_tm (region
, &gsi
))
2504 if (!gsi_end_p (gsi
))
2509 /* Return the list of basic-blocks in REGION.
2511 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2512 following a TM_IRREVOCABLE call.
2514 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2515 uninstrumented code path blocks in the list of basic blocks
2516 returned, false otherwise. */
2518 static vec
<basic_block
>
2519 get_tm_region_blocks (basic_block entry_block
,
2522 bitmap all_region_blocks
,
2523 bool stop_at_irrevocable_p
,
2524 bool include_uninstrumented_p
= true)
2526 vec
<basic_block
> bbs
= vNULL
;
2530 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2533 bbs
.safe_push (entry_block
);
2534 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2538 basic_block bb
= bbs
[i
++];
2541 bitmap_bit_p (exit_blocks
, bb
->index
))
2544 if (stop_at_irrevocable_p
2546 && bitmap_bit_p (irr_blocks
, bb
->index
))
2549 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2550 if ((include_uninstrumented_p
2551 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2552 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2554 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2555 bbs
.safe_push (e
->dest
);
2558 while (i
< bbs
.length ());
2560 if (all_region_blocks
)
2561 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2563 BITMAP_FREE (visited_blocks
);
2567 // Callback data for collect_bb2reg.
2570 vec
<tm_region_p
> *bb2reg
;
2571 bool include_uninstrumented_p
;
2574 // Callback for expand_regions, collect innermost region data for each bb.
2576 collect_bb2reg (struct tm_region
*region
, void *data
)
2578 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2579 vec
<tm_region_p
> *bb2reg
= stuff
->bb2reg
;
2580 vec
<basic_block
> queue
;
2584 queue
= get_tm_region_blocks (region
->entry_block
,
2585 region
->exit_blocks
,
2588 /*stop_at_irr_p=*/true,
2589 stuff
->include_uninstrumented_p
);
2591 // We expect expand_region to perform a post-order traversal of the region
2592 // tree. Therefore the last region seen for any bb is the innermost.
2593 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2594 (*bb2reg
)[bb
->index
] = region
;
2600 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2601 // which a basic block belongs. Note that we only consider the instrumented
2602 // code paths for the region; the uninstrumented code paths are ignored if
2603 // INCLUDE_UNINSTRUMENTED_P is false.
2605 // ??? This data is very similar to the bb_regions array that is collected
2606 // during tm_region_init. Or, rather, this data is similar to what could
2607 // be used within tm_region_init. The actual computation in tm_region_init
2608 // begins and ends with bb_regions entirely full of NULL pointers, due to
2609 // the way in which pointers are swapped in and out of the array.
2611 // ??? Our callers expect that blocks are not shared between transactions.
2612 // When the optimizers get too smart, and blocks are shared, then during
2613 // the tm_mark phase we'll add log entries to only one of the two transactions,
2614 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2615 // cycles. The symptom being SSA defs that do not dominate their uses.
2616 // Note that the optimizers were locally correct with their transformation,
2617 // as we have no info within the program that suggests that the blocks cannot
2620 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2621 // only known instance of this block sharing.
2623 static vec
<tm_region_p
>
2624 get_bb_regions_instrumented (bool traverse_clones
,
2625 bool include_uninstrumented_p
)
2627 unsigned n
= last_basic_block_for_fn (cfun
);
2628 struct bb2reg_stuff stuff
;
2629 vec
<tm_region_p
> ret
;
2632 ret
.safe_grow_cleared (n
);
2633 stuff
.bb2reg
= &ret
;
2634 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2635 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2640 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2644 compute_transaction_bits (void)
2646 struct tm_region
*region
;
2647 vec
<basic_block
> queue
;
2651 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2652 certainly don't need it to calculate CDI_DOMINATOR info. */
2655 FOR_EACH_BB_FN (bb
, cfun
)
2656 bb
->flags
&= ~BB_IN_TRANSACTION
;
2658 for (region
= all_tm_regions
; region
; region
= region
->next
)
2660 queue
= get_tm_region_blocks (region
->entry_block
,
2661 region
->exit_blocks
,
2664 /*stop_at_irr_p=*/true);
2665 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2666 bb
->flags
|= BB_IN_TRANSACTION
;
2671 bitmap_obstack_release (&tm_obstack
);
2674 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2675 call to BUILT_IN_TM_START. */
2678 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2680 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2681 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2682 tree tm_state
= region
->tm_state
;
2683 tree tm_state_type
= TREE_TYPE (tm_state
);
2684 edge abort_edge
= NULL
;
2685 edge inst_edge
= NULL
;
2686 edge uninst_edge
= NULL
;
2687 edge fallthru_edge
= NULL
;
2689 // Identify the various successors of the transaction start.
2693 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2695 if (e
->flags
& EDGE_TM_ABORT
)
2697 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2701 if (e
->flags
& EDGE_FALLTHRU
)
2706 /* ??? There are plenty of bits here we're not computing. */
2708 int subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2710 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2711 flags
|= PR_DOESGOIRREVOCABLE
;
2712 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2713 flags
|= PR_HASNOIRREVOCABLE
;
2714 /* If the transaction does not have an abort in lexical scope and is not
2715 marked as an outer transaction, then it will never abort. */
2716 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2717 flags
|= PR_HASNOABORT
;
2718 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2719 flags
|= PR_READONLY
;
2720 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2721 flags
|= PR_INSTRUMENTEDCODE
;
2723 flags
|= PR_UNINSTRUMENTEDCODE
;
2724 if (subcode
& GTMA_IS_OUTER
)
2725 region
->original_transaction_was_outer
= true;
2726 tree t
= build_int_cst (tm_state_type
, flags
);
2727 gimple call
= gimple_build_call (tm_start
, 1, t
);
2728 gimple_call_set_lhs (call
, tm_state
);
2729 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2731 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2732 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2733 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2734 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2735 gsi_remove (&gsi
, true);
2736 region
->transaction_stmt
= call
;
2739 // Generate log saves.
2740 if (!tm_log_save_addresses
.is_empty ())
2741 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2743 // In the beginning, we've no tests to perform on transaction restart.
2744 // Note that after this point, transaction_bb becomes the "most recent
2745 // block containing tests for the transaction".
2746 region
->restart_block
= region
->entry_block
;
2748 // Generate log restores.
2749 if (!tm_log_save_addresses
.is_empty ())
2751 basic_block test_bb
= create_empty_bb (transaction_bb
);
2752 basic_block code_bb
= create_empty_bb (test_bb
);
2753 basic_block join_bb
= create_empty_bb (code_bb
);
2754 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2755 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2756 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2757 if (region
->restart_block
== region
->entry_block
)
2758 region
->restart_block
= test_bb
;
2760 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2761 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2762 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2764 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2765 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2767 t2
= build_int_cst (tm_state_type
, 0);
2768 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2769 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2771 tm_log_emit_restores (region
->entry_block
, code_bb
);
2773 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2774 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2775 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2776 redirect_edge_pred (fallthru_edge
, join_bb
);
2778 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2779 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2781 ei
->probability
= PROB_ALWAYS
;
2782 et
->probability
= PROB_LIKELY
;
2783 ef
->probability
= PROB_UNLIKELY
;
2784 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2785 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2787 code_bb
->count
= et
->count
;
2788 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2790 transaction_bb
= join_bb
;
2793 // If we have an ABORT edge, create a test to perform the abort.
2796 basic_block test_bb
= create_empty_bb (transaction_bb
);
2797 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2798 if (region
->restart_block
== region
->entry_block
)
2799 region
->restart_block
= test_bb
;
2801 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2802 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2803 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2805 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2806 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2808 t2
= build_int_cst (tm_state_type
, 0);
2809 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2810 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2812 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2813 test_bb
->frequency
= transaction_bb
->frequency
;
2814 test_bb
->count
= transaction_bb
->count
;
2815 ei
->probability
= PROB_ALWAYS
;
2817 // Not abort edge. If both are live, chose one at random as we'll
2818 // we'll be fixing that up below.
2819 redirect_edge_pred (fallthru_edge
, test_bb
);
2820 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2821 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2822 fallthru_edge
->count
2823 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2826 redirect_edge_pred (abort_edge
, test_bb
);
2827 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2828 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2830 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2832 transaction_bb
= test_bb
;
2835 // If we have both instrumented and uninstrumented code paths, select one.
2836 if (inst_edge
&& uninst_edge
)
2838 basic_block test_bb
= create_empty_bb (transaction_bb
);
2839 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2840 if (region
->restart_block
== region
->entry_block
)
2841 region
->restart_block
= test_bb
;
2843 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2844 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2846 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2848 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2849 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2851 t2
= build_int_cst (tm_state_type
, 0);
2852 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2853 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2855 // Create the edge into test_bb first, as we want to copy values
2856 // out of the fallthru edge.
2857 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2858 e
->probability
= fallthru_edge
->probability
;
2859 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2860 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2862 // Now update the edges to the inst/uninist implementations.
2863 // For now assume that the paths are equally likely. When using HTM,
2864 // we'll try the uninst path first and fallback to inst path if htm
2865 // buffers are exceeded. Without HTM we start with the inst path and
2866 // use the uninst path when falling back to serial mode.
2867 redirect_edge_pred (inst_edge
, test_bb
);
2868 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2869 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2871 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2873 redirect_edge_pred (uninst_edge
, test_bb
);
2874 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2875 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2877 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2880 // If we have no previous special cases, and we have PHIs at the beginning
2881 // of the atomic region, this means we have a loop at the beginning of the
2882 // atomic region that shares the first block. This can cause problems with
2883 // the transaction restart abnormal edges to be added in the tm_edges pass.
2884 // Solve this by adding a new empty block to receive the abnormal edges.
2885 if (region
->restart_block
== region
->entry_block
2886 && phi_nodes (region
->entry_block
))
2888 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2889 region
->restart_block
= empty_bb
;
2890 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2892 redirect_edge_pred (fallthru_edge
, empty_bb
);
2893 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2899 /* Generate the temporary to be used for the return value of
2900 BUILT_IN_TM_START. */
2903 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2905 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2907 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2909 // Reset the subcode, post optimizations. We'll fill this in
2910 // again as we process blocks.
2911 if (region
->exit_blocks
)
2913 unsigned int subcode
2914 = gimple_transaction_subcode (region
->transaction_stmt
);
2916 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2917 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2918 | GTMA_MAY_ENTER_IRREVOCABLE
2919 | GTMA_HAS_NO_INSTRUMENTATION
);
2921 subcode
&= GTMA_DECLARATION_MASK
;
2922 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2928 // Propagate flags from inner transactions outwards.
2930 propagate_tm_flags_out (struct tm_region
*region
)
2934 propagate_tm_flags_out (region
->inner
);
2936 if (region
->outer
&& region
->outer
->transaction_stmt
)
2938 unsigned s
= gimple_transaction_subcode (region
->transaction_stmt
);
2939 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2940 | GTMA_MAY_ENTER_IRREVOCABLE
);
2941 s
|= gimple_transaction_subcode (region
->outer
->transaction_stmt
);
2942 gimple_transaction_set_subcode (region
->outer
->transaction_stmt
, s
);
2945 propagate_tm_flags_out (region
->next
);
2948 /* Entry point to the MARK phase of TM expansion. Here we replace
2949 transactional memory statements with calls to builtins, and function
2950 calls with their transactional clones (if available). But we don't
2951 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2954 execute_tm_mark (void)
2956 pending_edge_inserts_p
= false;
2958 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2959 /*traverse_clones=*/true);
2963 vec
<tm_region_p
> bb_regions
2964 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2965 /*include_uninstrumented_p=*/false);
2966 struct tm_region
*r
;
2969 // Expand memory operations into calls into the runtime.
2970 // This collects log entries as well.
2971 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
2975 if (r
->transaction_stmt
)
2977 unsigned sub
= gimple_transaction_subcode (r
->transaction_stmt
);
2979 /* If we're sure to go irrevocable, there won't be
2980 anything to expand, since the run-time will go
2981 irrevocable right away. */
2982 if (sub
& GTMA_DOES_GO_IRREVOCABLE
2983 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
2986 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
2990 bb_regions
.release ();
2992 // Propagate flags from inner transactions outwards.
2993 propagate_tm_flags_out (all_tm_regions
);
2995 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
2996 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
2997 /*traverse_clones=*/false);
3002 if (pending_edge_inserts_p
)
3003 gsi_commit_edge_inserts ();
3004 free_dominance_info (CDI_DOMINATORS
);
3010 const pass_data pass_data_tm_mark
=
3012 GIMPLE_PASS
, /* type */
3013 "tmmark", /* name */
3014 OPTGROUP_NONE
, /* optinfo_flags */
3015 TV_TRANS_MEM
, /* tv_id */
3016 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3017 0, /* properties_provided */
3018 0, /* properties_destroyed */
3019 0, /* todo_flags_start */
3020 TODO_update_ssa
, /* todo_flags_finish */
3023 class pass_tm_mark
: public gimple_opt_pass
3026 pass_tm_mark (gcc::context
*ctxt
)
3027 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3030 /* opt_pass methods: */
3031 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3033 }; // class pass_tm_mark
3038 make_pass_tm_mark (gcc::context
*ctxt
)
3040 return new pass_tm_mark (ctxt
);
3044 /* Create an abnormal edge from STMT at iter, splitting the block
3045 as necessary. Adjust *PNEXT as needed for the split block. */
3048 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
3049 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3051 basic_block bb
= gimple_bb (stmt
);
3052 if (!gsi_one_before_end_p (iter
))
3054 edge e
= split_block (bb
, stmt
);
3055 *pnext
= gsi_start_bb (e
->dest
);
3057 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3059 // Record the need for the edge for the benefit of the rtl passes.
3060 if (cfun
->gimple_df
->tm_restart
== NULL
)
3061 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
3062 struct_ptr_eq
, ggc_free
);
3064 struct tm_restart_node dummy
;
3066 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3068 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
3069 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
3072 n
= ggc_alloc
<tm_restart_node
> ();
3077 tree old
= n
->label_or_list
;
3078 if (TREE_CODE (old
) == LABEL_DECL
)
3079 old
= tree_cons (NULL
, old
, NULL
);
3080 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3084 /* Split block BB as necessary for every builtin function we added, and
3085 wire up the abnormal back edges implied by the transaction restart. */
3088 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3090 gimple_stmt_iterator gsi
, next_gsi
;
3092 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3094 gimple stmt
= gsi_stmt (gsi
);
3097 gsi_next (&next_gsi
);
3099 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3100 if (gimple_code (stmt
) != GIMPLE_CALL
3101 || (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) == 0)
3104 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
)) == BUILT_IN_TM_ABORT
)
3106 // If we have a ``_transaction_cancel [[outer]]'', there is only
3107 // one abnormal edge: to the transaction marked OUTER.
3108 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3109 // constant argument, which we can examine here. Users invoking
3110 // TM_ABORT directly get what they deserve.
3111 tree arg
= gimple_call_arg (stmt
, 0);
3112 if (TREE_CODE (arg
) == INTEGER_CST
3113 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3114 && !decl_is_tm_clone (current_function_decl
))
3116 // Find the GTMA_IS_OUTER transaction.
3117 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3118 if (o
->original_transaction_was_outer
)
3120 split_bb_make_tm_edge (stmt
, o
->restart_block
,
3125 // Otherwise, the front-end should have semantically checked
3126 // outer aborts, but in either case the target region is not
3127 // within this function.
3131 // Non-outer, TM aborts have an abnormal edge to the inner-most
3132 // transaction, the one being aborted;
3133 split_bb_make_tm_edge (stmt
, region
->restart_block
, gsi
, &next_gsi
);
3136 // All TM builtins have an abnormal edge to the outer-most transaction.
3137 // We never restart inner transactions. For tm clones, we know a-priori
3138 // that the outer-most transaction is outside the function.
3139 if (decl_is_tm_clone (current_function_decl
))
3142 if (cfun
->gimple_df
->tm_restart
== NULL
)
3143 cfun
->gimple_df
->tm_restart
3144 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
3146 // All TM builtins have an abnormal edge to the outer-most transaction.
3147 // We never restart inner transactions.
3148 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3151 split_bb_make_tm_edge (stmt
, o
->restart_block
, gsi
, &next_gsi
);
3155 // Delete any tail-call annotation that may have been added.
3156 // The tail-call pass may have mis-identified the commit as being
3157 // a candidate because we had not yet added this restart edge.
3158 gimple_call_set_tail (stmt
, false);
3162 /* Entry point to the final expansion of transactional nodes. */
3166 const pass_data pass_data_tm_edges
=
3168 GIMPLE_PASS
, /* type */
3169 "tmedge", /* name */
3170 OPTGROUP_NONE
, /* optinfo_flags */
3171 TV_TRANS_MEM
, /* tv_id */
3172 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3173 0, /* properties_provided */
3174 0, /* properties_destroyed */
3175 0, /* todo_flags_start */
3176 TODO_update_ssa
, /* todo_flags_finish */
3179 class pass_tm_edges
: public gimple_opt_pass
3182 pass_tm_edges (gcc::context
*ctxt
)
3183 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3186 /* opt_pass methods: */
3187 virtual unsigned int execute (function
*);
3189 }; // class pass_tm_edges
3192 pass_tm_edges::execute (function
*fun
)
3194 vec
<tm_region_p
> bb_regions
3195 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3196 /*include_uninstrumented_p=*/true);
3197 struct tm_region
*r
;
3200 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3202 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3204 bb_regions
.release ();
3206 /* We've got to release the dominance info now, to indicate that it
3207 must be rebuilt completely. Otherwise we'll crash trying to update
3208 the SSA web in the TODO section following this pass. */
3209 free_dominance_info (CDI_DOMINATORS
);
3210 bitmap_obstack_release (&tm_obstack
);
3211 all_tm_regions
= NULL
;
3219 make_pass_tm_edges (gcc::context
*ctxt
)
3221 return new pass_tm_edges (ctxt
);
3224 /* Helper function for expand_regions. Expand REGION and recurse to
3225 the inner region. Call CALLBACK on each region. CALLBACK returns
3226 NULL to continue the traversal, otherwise a non-null value which
3227 this function will return as well. TRAVERSE_CLONES is true if we
3228 should traverse transactional clones. */
3231 expand_regions_1 (struct tm_region
*region
,
3232 void *(*callback
)(struct tm_region
*, void *),
3234 bool traverse_clones
)
3236 void *retval
= NULL
;
3237 if (region
->exit_blocks
3238 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3240 retval
= callback (region
, data
);
3246 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3253 /* Traverse the regions enclosed and including REGION. Execute
3254 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3255 continue the traversal, otherwise a non-null value which this
3256 function will return as well. TRAVERSE_CLONES is true if we should
3257 traverse transactional clones. */
3260 expand_regions (struct tm_region
*region
,
3261 void *(*callback
)(struct tm_region
*, void *),
3263 bool traverse_clones
)
3265 void *retval
= NULL
;
3268 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3271 region
= region
->next
;
3277 /* A unique TM memory operation. */
3278 typedef struct tm_memop
3280 /* Unique ID that all memory operations to the same location have. */
3281 unsigned int value_id
;
3282 /* Address of load/store. */
3286 /* TM memory operation hashtable helpers. */
3288 struct tm_memop_hasher
: typed_free_remove
<tm_memop
>
3290 typedef tm_memop value_type
;
3291 typedef tm_memop compare_type
;
3292 static inline hashval_t
hash (const value_type
*);
3293 static inline bool equal (const value_type
*, const compare_type
*);
3296 /* Htab support. Return a hash value for a `tm_memop'. */
3298 tm_memop_hasher::hash (const value_type
*mem
)
3300 tree addr
= mem
->addr
;
3301 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3302 actually done with operand_equal_p (see tm_memop_eq). */
3303 if (TREE_CODE (addr
) == ADDR_EXPR
)
3304 addr
= TREE_OPERAND (addr
, 0);
3305 return iterative_hash_expr (addr
, 0);
3308 /* Htab support. Return true if two tm_memop's are the same. */
3310 tm_memop_hasher::equal (const value_type
*mem1
, const compare_type
*mem2
)
3312 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3315 /* Sets for solving data flow equations in the memory optimization pass. */
3316 struct tm_memopt_bitmaps
3318 /* Stores available to this BB upon entry. Basically, stores that
3319 dominate this BB. */
3320 bitmap store_avail_in
;
3321 /* Stores available at the end of this BB. */
3322 bitmap store_avail_out
;
3323 bitmap store_antic_in
;
3324 bitmap store_antic_out
;
3325 /* Reads available to this BB upon entry. Basically, reads that
3326 dominate this BB. */
3327 bitmap read_avail_in
;
3328 /* Reads available at the end of this BB. */
3329 bitmap read_avail_out
;
3330 /* Reads performed in this BB. */
3332 /* Writes performed in this BB. */
3335 /* Temporary storage for pass. */
3336 /* Is the current BB in the worklist? */
3337 bool avail_in_worklist_p
;
3338 /* Have we visited this BB? */
3342 static bitmap_obstack tm_memopt_obstack
;
3344 /* Unique counter for TM loads and stores. Loads and stores of the
3345 same address get the same ID. */
3346 static unsigned int tm_memopt_value_id
;
3347 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3349 #define STORE_AVAIL_IN(BB) \
3350 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3351 #define STORE_AVAIL_OUT(BB) \
3352 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3353 #define STORE_ANTIC_IN(BB) \
3354 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3355 #define STORE_ANTIC_OUT(BB) \
3356 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3357 #define READ_AVAIL_IN(BB) \
3358 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3359 #define READ_AVAIL_OUT(BB) \
3360 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3361 #define READ_LOCAL(BB) \
3362 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3363 #define STORE_LOCAL(BB) \
3364 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3365 #define AVAIL_IN_WORKLIST_P(BB) \
3366 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3367 #define BB_VISITED_P(BB) \
3368 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3370 /* Given a TM load/store in STMT, return the value number for the address
3374 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3376 struct tm_memop tmpmem
, *mem
;
3379 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3380 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3381 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3384 else if (op
== INSERT
)
3386 mem
= XNEW (struct tm_memop
);
3388 mem
->value_id
= tm_memopt_value_id
++;
3389 mem
->addr
= tmpmem
.addr
;
3393 return mem
->value_id
;
3396 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3399 tm_memopt_accumulate_memops (basic_block bb
)
3401 gimple_stmt_iterator gsi
;
3403 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3405 gimple stmt
= gsi_stmt (gsi
);
3409 if (is_tm_store (stmt
))
3410 bits
= STORE_LOCAL (bb
);
3411 else if (is_tm_load (stmt
))
3412 bits
= READ_LOCAL (bb
);
3416 loc
= tm_memopt_value_number (stmt
, INSERT
);
3417 bitmap_set_bit (bits
, loc
);
3420 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3421 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3422 gimple_bb (stmt
)->index
);
3423 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3424 fprintf (dump_file
, "\n");
3429 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3432 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3436 const char *comma
= "";
3438 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3439 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3441 hash_table
<tm_memop_hasher
>::iterator hi
;
3442 struct tm_memop
*mem
= NULL
;
3444 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3445 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3446 if (mem
->value_id
== i
)
3448 gcc_assert (mem
->value_id
== i
);
3449 fprintf (dump_file
, "%s", comma
);
3451 print_generic_expr (dump_file
, mem
->addr
, 0);
3453 fprintf (dump_file
, "]\n");
3456 /* Prettily dump all of the memopt sets in BLOCKS. */
3459 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3464 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3466 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3467 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3468 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3469 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3470 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3471 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3472 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3476 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3479 tm_memopt_compute_avin (basic_block bb
)
3484 /* Seed with the AVOUT of any predecessor. */
3485 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3487 e
= EDGE_PRED (bb
, ix
);
3488 /* Make sure we have already visited this BB, and is thus
3491 If e->src->aux is NULL, this predecessor is actually on an
3492 enclosing transaction. We only care about the current
3493 transaction, so ignore it. */
3494 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3496 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3497 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3502 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3504 e
= EDGE_PRED (bb
, ix
);
3505 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3507 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3508 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3512 BB_VISITED_P (bb
) = true;
3515 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3518 tm_memopt_compute_antin (basic_block bb
)
3523 /* Seed with the ANTIC_OUT of any successor. */
3524 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3526 e
= EDGE_SUCC (bb
, ix
);
3527 /* Make sure we have already visited this BB, and is thus
3529 if (BB_VISITED_P (e
->dest
))
3531 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3536 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3538 e
= EDGE_SUCC (bb
, ix
);
3539 if (BB_VISITED_P (e
->dest
))
3540 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3543 BB_VISITED_P (bb
) = true;
3546 /* Compute the AVAIL sets for every basic block in BLOCKS.
3548 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3550 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3551 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3553 This is basically what we do in lcm's compute_available(), but here
3554 we calculate two sets of sets (one for STOREs and one for READs),
3555 and we work on a region instead of the entire CFG.
3557 REGION is the TM region.
3558 BLOCKS are the basic blocks in the region. */
3561 tm_memopt_compute_available (struct tm_region
*region
,
3562 vec
<basic_block
> blocks
)
3565 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3566 unsigned int qlen
, i
;
3570 /* Allocate a worklist array/queue. Entries are only added to the
3571 list if they were not already on the list. So the size is
3572 bounded by the number of basic blocks in the region. */
3573 qlen
= blocks
.length () - 1;
3574 qin
= qout
= worklist
=
3575 XNEWVEC (basic_block
, qlen
);
3577 /* Put every block in the region on the worklist. */
3578 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3580 /* Seed AVAIL_OUT with the LOCAL set. */
3581 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3582 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3584 AVAIL_IN_WORKLIST_P (bb
) = true;
3585 /* No need to insert the entry block, since it has an AVIN of
3586 null, and an AVOUT that has already been seeded in. */
3587 if (bb
!= region
->entry_block
)
3591 /* The entry block has been initialized with the local sets. */
3592 BB_VISITED_P (region
->entry_block
) = true;
3595 qend
= &worklist
[qlen
];
3597 /* Iterate until the worklist is empty. */
3600 /* Take the first entry off the worklist. */
3607 /* This block can be added to the worklist again if necessary. */
3608 AVAIL_IN_WORKLIST_P (bb
) = false;
3609 tm_memopt_compute_avin (bb
);
3611 /* Note: We do not add the LOCAL sets here because we already
3612 seeded the AVAIL_OUT sets with them. */
3613 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3614 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3616 && (region
->exit_blocks
== NULL
3617 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3618 /* If the out state of this block changed, then we need to add
3619 its successors to the worklist if they are not already in. */
3620 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3621 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3622 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3625 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3636 dump_tm_memopt_sets (blocks
);
3639 /* Compute ANTIC sets for every basic block in BLOCKS.
3641 We compute STORE_ANTIC_OUT as follows:
3643 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3644 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3646 REGION is the TM region.
3647 BLOCKS are the basic blocks in the region. */
3650 tm_memopt_compute_antic (struct tm_region
*region
,
3651 vec
<basic_block
> blocks
)
3654 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3659 /* Allocate a worklist array/queue. Entries are only added to the
3660 list if they were not already on the list. So the size is
3661 bounded by the number of basic blocks in the region. */
3662 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3664 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3668 /* Seed ANTIC_OUT with the LOCAL set. */
3669 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3671 /* Put every block in the region on the worklist. */
3672 AVAIL_IN_WORKLIST_P (bb
) = true;
3673 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3674 and their ANTIC_OUT has already been seeded in. */
3675 if (region
->exit_blocks
3676 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3683 /* The exit blocks have been initialized with the local sets. */
3684 if (region
->exit_blocks
)
3688 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3689 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3693 qend
= &worklist
[qlen
];
3695 /* Iterate until the worklist is empty. */
3698 /* Take the first entry off the worklist. */
3705 /* This block can be added to the worklist again if necessary. */
3706 AVAIL_IN_WORKLIST_P (bb
) = false;
3707 tm_memopt_compute_antin (bb
);
3709 /* Note: We do not add the LOCAL sets here because we already
3710 seeded the ANTIC_OUT sets with them. */
3711 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3712 && bb
!= region
->entry_block
)
3713 /* If the out state of this block changed, then we need to add
3714 its predecessors to the worklist if they are not already in. */
3715 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3716 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3719 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3730 dump_tm_memopt_sets (blocks
);
3733 /* Offsets of load variants from TM_LOAD. For example,
3734 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3735 See gtm-builtins.def. */
3736 #define TRANSFORM_RAR 1
3737 #define TRANSFORM_RAW 2
3738 #define TRANSFORM_RFW 3
3739 /* Offsets of store variants from TM_STORE. */
3740 #define TRANSFORM_WAR 1
3741 #define TRANSFORM_WAW 2
3743 /* Inform about a load/store optimization. */
3746 dump_tm_memopt_transform (gimple stmt
)
3750 fprintf (dump_file
, "TM memopt: transforming: ");
3751 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3752 fprintf (dump_file
, "\n");
3756 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3757 by a builtin that is OFFSET entries down in the builtins table in
3758 gtm-builtins.def. */
3761 tm_memopt_transform_stmt (unsigned int offset
,
3763 gimple_stmt_iterator
*gsi
)
3765 tree fn
= gimple_call_fn (stmt
);
3766 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3767 TREE_OPERAND (fn
, 0)
3768 = builtin_decl_explicit ((enum built_in_function
)
3769 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3771 gimple_call_set_fn (stmt
, fn
);
3772 gsi_replace (gsi
, stmt
, true);
3773 dump_tm_memopt_transform (stmt
);
3776 /* Perform the actual TM memory optimization transformations in the
3777 basic blocks in BLOCKS. */
3780 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3784 gimple_stmt_iterator gsi
;
3786 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3788 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3790 gimple stmt
= gsi_stmt (gsi
);
3791 bitmap read_avail
= READ_AVAIL_IN (bb
);
3792 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3793 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3796 if (is_tm_simple_load (stmt
))
3798 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3799 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3800 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3801 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3803 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3804 bitmap_set_bit (store_avail
, loc
);
3806 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3807 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3809 bitmap_set_bit (read_avail
, loc
);
3811 else if (is_tm_simple_store (stmt
))
3813 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3814 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3815 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3818 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3819 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3820 bitmap_set_bit (store_avail
, loc
);
3827 /* Return a new set of bitmaps for a BB. */
3829 static struct tm_memopt_bitmaps
*
3830 tm_memopt_init_sets (void)
3832 struct tm_memopt_bitmaps
*b
3833 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3834 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3835 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3836 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3837 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3838 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3839 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3840 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3841 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3842 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3846 /* Free sets computed for each BB. */
3849 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3854 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3858 /* Clear the visited bit for every basic block in BLOCKS. */
3861 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3866 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3867 BB_VISITED_P (bb
) = false;
3870 /* Replace TM load/stores with hints for the runtime. We handle
3871 things like read-after-write, write-after-read, read-after-read,
3872 read-for-write, etc. */
3875 execute_tm_memopt (void)
3877 struct tm_region
*region
;
3878 vec
<basic_block
> bbs
;
3880 tm_memopt_value_id
= 0;
3881 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3883 for (region
= all_tm_regions
; region
; region
= region
->next
)
3885 /* All the TM stores/loads in the current region. */
3889 bitmap_obstack_initialize (&tm_memopt_obstack
);
3891 /* Save all BBs for the current region. */
3892 bbs
= get_tm_region_blocks (region
->entry_block
,
3893 region
->exit_blocks
,
3898 /* Collect all the memory operations. */
3899 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3901 bb
->aux
= tm_memopt_init_sets ();
3902 tm_memopt_accumulate_memops (bb
);
3905 /* Solve data flow equations and transform each block accordingly. */
3906 tm_memopt_clear_visited (bbs
);
3907 tm_memopt_compute_available (region
, bbs
);
3908 tm_memopt_clear_visited (bbs
);
3909 tm_memopt_compute_antic (region
, bbs
);
3910 tm_memopt_transform_blocks (bbs
);
3912 tm_memopt_free_sets (bbs
);
3914 bitmap_obstack_release (&tm_memopt_obstack
);
3915 tm_memopt_value_numbers
->empty ();
3918 delete tm_memopt_value_numbers
;
3919 tm_memopt_value_numbers
= NULL
;
3925 const pass_data pass_data_tm_memopt
=
3927 GIMPLE_PASS
, /* type */
3928 "tmmemopt", /* name */
3929 OPTGROUP_NONE
, /* optinfo_flags */
3930 TV_TRANS_MEM
, /* tv_id */
3931 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3932 0, /* properties_provided */
3933 0, /* properties_destroyed */
3934 0, /* todo_flags_start */
3935 0, /* todo_flags_finish */
3938 class pass_tm_memopt
: public gimple_opt_pass
3941 pass_tm_memopt (gcc::context
*ctxt
)
3942 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3945 /* opt_pass methods: */
3946 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3947 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3949 }; // class pass_tm_memopt
3954 make_pass_tm_memopt (gcc::context
*ctxt
)
3956 return new pass_tm_memopt (ctxt
);
3960 /* Interprocedual analysis for the creation of transactional clones.
3961 The aim of this pass is to find which functions are referenced in
3962 a non-irrevocable transaction context, and for those over which
3963 we have control (or user directive), create a version of the
3964 function which uses only the transactional interface to reference
3965 protected memories. This analysis proceeds in several steps:
3967 (1) Collect the set of all possible transactional clones:
3969 (a) For all local public functions marked tm_callable, push
3970 it onto the tm_callee queue.
3972 (b) For all local functions, scan for calls in transaction blocks.
3973 Push the caller and callee onto the tm_caller and tm_callee
3974 queues. Count the number of callers for each callee.
3976 (c) For each local function on the callee list, assume we will
3977 create a transactional clone. Push *all* calls onto the
3978 callee queues; count the number of clone callers separately
3979 to the number of original callers.
3981 (2) Propagate irrevocable status up the dominator tree:
3983 (a) Any external function on the callee list that is not marked
3984 tm_callable is irrevocable. Push all callers of such onto
3987 (b) For each function on the worklist, mark each block that
3988 contains an irrevocable call. Use the AND operator to
3989 propagate that mark up the dominator tree.
3991 (c) If we reach the entry block for a possible transactional
3992 clone, then the transactional clone is irrevocable, and
3993 we should not create the clone after all. Push all
3994 callers onto the worklist.
3996 (d) Place tm_irrevocable calls at the beginning of the relevant
3997 blocks. Special case here is the entry block for the entire
3998 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
3999 the library to begin the region in serial mode. Decrement
4000 the call count for all callees in the irrevocable region.
4002 (3) Create the transactional clones:
4004 Any tm_callee that still has a non-zero call count is cloned.
4007 /* This structure is stored in the AUX field of each cgraph_node. */
4008 struct tm_ipa_cg_data
4010 /* The clone of the function that got created. */
4011 struct cgraph_node
*clone
;
4013 /* The tm regions in the normal function. */
4014 struct tm_region
*all_tm_regions
;
4016 /* The blocks of the normal/clone functions that contain irrevocable
4017 calls, or blocks that are post-dominated by irrevocable calls. */
4018 bitmap irrevocable_blocks_normal
;
4019 bitmap irrevocable_blocks_clone
;
4021 /* The blocks of the normal function that are involved in transactions. */
4022 bitmap transaction_blocks_normal
;
4024 /* The number of callers to the transactional clone of this function
4025 from normal and transactional clones respectively. */
4026 unsigned tm_callers_normal
;
4027 unsigned tm_callers_clone
;
4029 /* True if all calls to this function's transactional clone
4030 are irrevocable. Also automatically true if the function
4031 has no transactional clone. */
4032 bool is_irrevocable
;
4034 /* Flags indicating the presence of this function in various queues. */
4035 bool in_callee_queue
;
4038 /* Flags indicating the kind of scan desired while in the worklist. */
4039 bool want_irr_scan_normal
;
4042 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4044 /* Return the ipa data associated with NODE, allocating zeroed memory
4045 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4046 and set *NODE accordingly. */
4048 static struct tm_ipa_cg_data
*
4049 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4051 struct tm_ipa_cg_data
*d
;
4053 if (traverse_aliases
&& (*node
)->alias
)
4054 *node
= (*node
)->get_alias_target ();
4056 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4060 d
= (struct tm_ipa_cg_data
*)
4061 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4062 (*node
)->aux
= (void *) d
;
4063 memset (d
, 0, sizeof (*d
));
4069 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4070 it is already present. */
4073 maybe_push_queue (struct cgraph_node
*node
,
4074 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4079 queue_p
->safe_push (node
);
4083 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4084 code path. QUEUE are the basic blocks inside the transaction
4085 represented in REGION.
4087 Later in split_code_paths() we will add the conditional to choose
4088 between the two alternatives. */
4091 ipa_uninstrument_transaction (struct tm_region
*region
,
4092 vec
<basic_block
> queue
)
4094 gimple transaction
= region
->transaction_stmt
;
4095 basic_block transaction_bb
= gimple_bb (transaction
);
4096 int n
= queue
.length ();
4097 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4099 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4101 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4102 add_phi_args_after_copy (new_bbs
, n
, e
);
4104 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4105 // a) EDGE_FALLTHRU into the transaction
4106 // b) EDGE_TM_ABORT out of the transaction
4107 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4112 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4113 Queue all callees within block BB. */
4116 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4117 basic_block bb
, bool for_clone
)
4119 gimple_stmt_iterator gsi
;
4121 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4123 gimple stmt
= gsi_stmt (gsi
);
4124 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4126 tree fndecl
= gimple_call_fndecl (stmt
);
4129 struct tm_ipa_cg_data
*d
;
4131 struct cgraph_node
*node
;
4133 if (is_tm_ending_fndecl (fndecl
))
4135 if (find_tm_replacement_function (fndecl
))
4138 node
= cgraph_node::get (fndecl
);
4139 gcc_assert (node
!= NULL
);
4140 d
= get_cg_data (&node
, true);
4142 pcallers
= (for_clone
? &d
->tm_callers_clone
4143 : &d
->tm_callers_normal
);
4146 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4152 /* Scan all calls in NODE that are within a transaction region,
4153 and push the resulting nodes into the callee queue. */
4156 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4157 cgraph_node_queue
*callees_p
)
4159 struct tm_region
*r
;
4161 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4162 d
->all_tm_regions
= all_tm_regions
;
4164 for (r
= all_tm_regions
; r
; r
= r
->next
)
4166 vec
<basic_block
> bbs
;
4170 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4171 d
->transaction_blocks_normal
, false);
4173 // Generate the uninstrumented code path for this transaction.
4174 ipa_uninstrument_transaction (r
, bbs
);
4176 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4177 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4182 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4183 // copying them, rather than forcing us to do this externally.
4184 cgraph_edge::rebuild_edges ();
4186 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4187 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4188 // Instead, just release dominators here so update_ssa recomputes them.
4189 free_dominance_info (CDI_DOMINATORS
);
4191 // When building the uninstrumented code path, copy_bbs will have invoked
4192 // create_new_def_for starting an "ssa update context". There is only one
4193 // instance of this context, so resolve ssa updates before moving on to
4194 // the next function.
4195 update_ssa (TODO_update_ssa
);
4198 /* Scan all calls in NODE as if this is the transactional clone,
4199 and push the destinations into the callee queue. */
4202 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4203 cgraph_node_queue
*callees_p
)
4205 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4208 FOR_EACH_BB_FN (bb
, fn
)
4209 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4212 /* The function NODE has been detected to be irrevocable. Push all
4213 of its callers onto WORKLIST for the purpose of re-scanning them. */
4216 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4217 cgraph_node_queue
*worklist_p
)
4219 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4220 struct cgraph_edge
*e
;
4222 d
->is_irrevocable
= true;
4224 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4227 struct cgraph_node
*caller
;
4229 /* Don't examine recursive calls. */
4230 if (e
->caller
== node
)
4232 /* Even if we think we can go irrevocable, believe the user
4234 if (is_tm_safe_or_pure (e
->caller
->decl
))
4238 d
= get_cg_data (&caller
, true);
4240 /* Check if the callee is in a transactional region. If so,
4241 schedule the function for normal re-scan as well. */
4242 bb
= gimple_bb (e
->call_stmt
);
4243 gcc_assert (bb
!= NULL
);
4244 if (d
->transaction_blocks_normal
4245 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4246 d
->want_irr_scan_normal
= true;
4248 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4252 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4253 within the block is irrevocable. */
4256 ipa_tm_scan_irr_block (basic_block bb
)
4258 gimple_stmt_iterator gsi
;
4261 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4263 gimple stmt
= gsi_stmt (gsi
);
4264 switch (gimple_code (stmt
))
4267 if (gimple_assign_single_p (stmt
))
4269 tree lhs
= gimple_assign_lhs (stmt
);
4270 tree rhs
= gimple_assign_rhs1 (stmt
);
4271 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4278 tree lhs
= gimple_call_lhs (stmt
);
4279 if (lhs
&& volatile_var_p (lhs
))
4282 if (is_tm_pure_call (stmt
))
4285 fn
= gimple_call_fn (stmt
);
4287 /* Functions with the attribute are by definition irrevocable. */
4288 if (is_tm_irrevocable (fn
))
4291 /* For direct function calls, go ahead and check for replacement
4292 functions, or transitive irrevocable functions. For indirect
4293 functions, we'll ask the runtime. */
4294 if (TREE_CODE (fn
) == ADDR_EXPR
)
4296 struct tm_ipa_cg_data
*d
;
4297 struct cgraph_node
*node
;
4299 fn
= TREE_OPERAND (fn
, 0);
4300 if (is_tm_ending_fndecl (fn
))
4302 if (find_tm_replacement_function (fn
))
4305 node
= cgraph_node::get (fn
);
4306 d
= get_cg_data (&node
, true);
4308 /* Return true if irrevocable, but above all, believe
4310 if (d
->is_irrevocable
4311 && !is_tm_safe_or_pure (fn
))
4318 /* ??? The Approved Method of indicating that an inline
4319 assembly statement is not relevant to the transaction
4320 is to wrap it in a __tm_waiver block. This is not
4321 yet implemented, so we can't check for it. */
4322 if (is_tm_safe (current_function_decl
))
4324 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4325 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4326 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4338 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4339 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4340 scanning past OLD_IRR or EXIT_BLOCKS. */
4343 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4344 bitmap old_irr
, bitmap exit_blocks
)
4346 bool any_new_irr
= false;
4349 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4353 basic_block bb
= pqueue
->pop ();
4355 /* Don't re-scan blocks we know already are irrevocable. */
4356 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4359 if (ipa_tm_scan_irr_block (bb
))
4361 bitmap_set_bit (new_irr
, bb
->index
);
4364 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4366 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4367 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4369 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4370 pqueue
->safe_push (e
->dest
);
4374 while (!pqueue
->is_empty ());
4376 BITMAP_FREE (visited_blocks
);
4381 /* Propagate the irrevocable property both up and down the dominator tree.
4382 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4383 TM regions; OLD_IRR are the results of a previous scan of the dominator
4384 tree which has been fully propagated; NEW_IRR is the set of new blocks
4385 which are gaining the irrevocable property during the current scan. */
4388 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4389 bitmap old_irr
, bitmap exit_blocks
)
4391 vec
<basic_block
> bbs
;
4392 bitmap all_region_blocks
;
4394 /* If this block is in the old set, no need to rescan. */
4395 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4398 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4399 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4400 all_region_blocks
, false);
4403 basic_block bb
= bbs
.pop ();
4404 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4405 bool all_son_irr
= false;
4409 /* Propagate up. If my children are, I am too, but we must have
4410 at least one child that is. */
4413 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4415 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4417 all_son_irr
= false;
4425 /* Add block to new_irr if it hasn't already been processed. */
4426 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4428 bitmap_set_bit (new_irr
, bb
->index
);
4434 /* Propagate down to everyone we immediately dominate. */
4438 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4440 son
= next_dom_son (CDI_DOMINATORS
, son
))
4442 /* Make sure block is actually in a TM region, and it
4443 isn't already in old_irr. */
4444 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4445 && bitmap_bit_p (all_region_blocks
, son
->index
))
4446 bitmap_set_bit (new_irr
, son
->index
);
4450 while (!bbs
.is_empty ());
4452 BITMAP_FREE (all_region_blocks
);
4457 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4459 gimple_stmt_iterator gsi
;
4461 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4463 gimple stmt
= gsi_stmt (gsi
);
4464 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4466 tree fndecl
= gimple_call_fndecl (stmt
);
4469 struct tm_ipa_cg_data
*d
;
4471 struct cgraph_node
*tnode
;
4473 if (is_tm_ending_fndecl (fndecl
))
4475 if (find_tm_replacement_function (fndecl
))
4478 tnode
= cgraph_node::get (fndecl
);
4479 d
= get_cg_data (&tnode
, true);
4481 pcallers
= (for_clone
? &d
->tm_callers_clone
4482 : &d
->tm_callers_normal
);
4484 gcc_assert (*pcallers
> 0);
4491 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4492 as well as other irrevocable actions such as inline assembly. Mark all
4493 such blocks as irrevocable and decrement the number of calls to
4494 transactional clones. Return true if, for the transactional clone, the
4495 entire function is irrevocable. */
4498 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4500 struct tm_ipa_cg_data
*d
;
4501 bitmap new_irr
, old_irr
;
4504 /* Builtin operators (operator new, and such). */
4505 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4506 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4509 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4510 calculate_dominance_info (CDI_DOMINATORS
);
4512 d
= get_cg_data (&node
, true);
4513 auto_vec
<basic_block
, 10> queue
;
4514 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4516 /* Scan each tm region, propagating irrevocable status through the tree. */
4519 old_irr
= d
->irrevocable_blocks_clone
;
4520 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4521 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4523 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4526 ret
= bitmap_bit_p (new_irr
,
4527 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4532 struct tm_region
*region
;
4534 old_irr
= d
->irrevocable_blocks_normal
;
4535 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4537 queue
.quick_push (region
->entry_block
);
4538 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4539 region
->exit_blocks
))
4540 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4541 region
->exit_blocks
);
4545 /* If we found any new irrevocable blocks, reduce the call count for
4546 transactional clones within the irrevocable blocks. Save the new
4547 set of irrevocable blocks for next time. */
4548 if (!bitmap_empty_p (new_irr
))
4550 bitmap_iterator bmi
;
4553 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4554 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4559 bitmap_ior_into (old_irr
, new_irr
);
4560 BITMAP_FREE (new_irr
);
4563 d
->irrevocable_blocks_clone
= new_irr
;
4565 d
->irrevocable_blocks_normal
= new_irr
;
4567 if (dump_file
&& new_irr
)
4570 bitmap_iterator bmi
;
4573 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4574 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4575 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4579 BITMAP_FREE (new_irr
);
4586 /* Return true if, for the transactional clone of NODE, any call
4587 may enter irrevocable mode. */
4590 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4592 struct tm_ipa_cg_data
*d
;
4596 d
= get_cg_data (&node
, true);
4598 flags
= flags_from_decl_or_type (decl
);
4600 /* Handle some TM builtins. Ordinarily these aren't actually generated
4601 at this point, but handling these functions when written in by the
4602 user makes it easier to build unit tests. */
4603 if (flags
& ECF_TM_BUILTIN
)
4606 /* Filter out all functions that are marked. */
4607 if (flags
& ECF_TM_PURE
)
4609 if (is_tm_safe (decl
))
4611 if (is_tm_irrevocable (decl
))
4613 if (is_tm_callable (decl
))
4615 if (find_tm_replacement_function (decl
))
4618 /* If we aren't seeing the final version of the function we don't
4619 know what it will contain at runtime. */
4620 if (node
->get_availability () < AVAIL_AVAILABLE
)
4623 /* If the function must go irrevocable, then of course true. */
4624 if (d
->is_irrevocable
)
4627 /* If there are any blocks marked irrevocable, then the function
4628 as a whole may enter irrevocable. */
4629 if (d
->irrevocable_blocks_clone
)
4632 /* We may have previously marked this function as tm_may_enter_irr;
4633 see pass_diagnose_tm_blocks. */
4634 if (node
->local
.tm_may_enter_irr
)
4637 /* Recurse on the main body for aliases. In general, this will
4638 result in one of the bits above being set so that we will not
4639 have to recurse next time. */
4641 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4643 /* What remains is unmarked local functions without items that force
4644 the function to go irrevocable. */
4648 /* Diagnose calls from transaction_safe functions to unmarked
4649 functions that are determined to not be safe. */
4652 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4654 struct cgraph_edge
*e
;
4656 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4657 if (!is_tm_callable (e
->callee
->decl
)
4658 && e
->callee
->local
.tm_may_enter_irr
)
4659 error_at (gimple_location (e
->call_stmt
),
4660 "unsafe function call %qD within "
4661 "%<transaction_safe%> function", e
->callee
->decl
);
4664 /* Diagnose call from atomic transactions to unmarked functions
4665 that are determined to not be safe. */
4668 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4669 struct tm_region
*all_tm_regions
)
4671 struct tm_region
*r
;
4673 for (r
= all_tm_regions
; r
; r
= r
->next
)
4674 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4676 /* Atomic transactions can be nested inside relaxed. */
4678 ipa_tm_diagnose_transaction (node
, r
->inner
);
4682 vec
<basic_block
> bbs
;
4683 gimple_stmt_iterator gsi
;
4687 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4688 r
->irr_blocks
, NULL
, false);
4690 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4691 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4693 gimple stmt
= gsi_stmt (gsi
);
4696 if (gimple_code (stmt
) == GIMPLE_ASM
)
4698 error_at (gimple_location (stmt
),
4699 "asm not allowed in atomic transaction");
4703 if (!is_gimple_call (stmt
))
4705 fndecl
= gimple_call_fndecl (stmt
);
4707 /* Indirect function calls have been diagnosed already. */
4711 /* Stop at the end of the transaction. */
4712 if (is_tm_ending_fndecl (fndecl
))
4714 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4719 /* Marked functions have been diagnosed already. */
4720 if (is_tm_pure_call (stmt
))
4722 if (is_tm_callable (fndecl
))
4725 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4726 error_at (gimple_location (stmt
),
4727 "unsafe function call %qD within "
4728 "atomic transaction", fndecl
);
4735 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4736 OLD_DECL. The returned value is a freshly malloced pointer that
4737 should be freed by the caller. */
4740 tm_mangle (tree old_asm_id
)
4742 const char *old_asm_name
;
4745 struct demangle_component
*dc
;
4748 /* Determine if the symbol is already a valid C++ mangled name. Do this
4749 even for C, which might be interfacing with C++ code via appropriately
4750 ugly identifiers. */
4751 /* ??? We could probably do just as well checking for "_Z" and be done. */
4752 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4753 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4760 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4761 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4765 old_asm_name
+= 2; /* Skip _Z */
4769 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4770 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4771 /* Don't play silly games, you! */
4774 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4775 /* I'd really like to know if we can ever be passed one of
4776 these from the C++ front end. The Logical Thing would
4777 seem that hidden-alias should be outer-most, so that we
4778 get hidden-alias of a transaction-clone and not vice-versa. */
4786 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4790 new_asm_id
= get_identifier (tm_name
);
4797 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4799 node
->mark_force_output ();
4800 node
->analyzed
= true;
4804 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4806 node
->forced_by_abi
= true;
4807 node
->analyzed
= true;
4810 /* Callback data for ipa_tm_create_version_alias. */
4811 struct create_version_alias_info
4813 struct cgraph_node
*old_node
;
4817 /* A subroutine of ipa_tm_create_version, called via
4818 cgraph_for_node_and_aliases. Create new tm clones for each of
4819 the existing aliases. */
4821 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4823 struct create_version_alias_info
*info
4824 = (struct create_version_alias_info
*)data
;
4825 tree old_decl
, new_decl
, tm_name
;
4826 struct cgraph_node
*new_node
;
4828 if (!node
->cpp_implicit_alias
)
4831 old_decl
= node
->decl
;
4832 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4833 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4834 TREE_CODE (old_decl
), tm_name
,
4835 TREE_TYPE (old_decl
));
4837 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4838 SET_DECL_RTL (new_decl
, NULL
);
4840 /* Based loosely on C++'s make_alias_for(). */
4841 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4842 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4843 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4844 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4845 DECL_EXTERNAL (new_decl
) = 0;
4846 DECL_ARTIFICIAL (new_decl
) = 1;
4847 TREE_ADDRESSABLE (new_decl
) = 1;
4848 TREE_USED (new_decl
) = 1;
4849 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4851 /* Perform the same remapping to the comdat group. */
4852 if (DECL_ONE_ONLY (new_decl
))
4853 varpool_node::get (new_decl
)->set_comdat_group
4854 (tm_mangle (decl_comdat_group_id (old_decl
)));
4856 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4857 new_node
->tm_clone
= true;
4858 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4859 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4860 /* ?? Do not traverse aliases here. */
4861 get_cg_data (&node
, false)->clone
= new_node
;
4863 record_tm_clone_pair (old_decl
, new_decl
);
4865 if (info
->old_node
->force_output
4866 || info
->old_node
->ref_list
.first_referring ())
4867 ipa_tm_mark_force_output_node (new_node
);
4868 if (info
->old_node
->forced_by_abi
)
4869 ipa_tm_mark_forced_by_abi_node (new_node
);
4873 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4874 appropriate for the transactional clone. */
4877 ipa_tm_create_version (struct cgraph_node
*old_node
)
4879 tree new_decl
, old_decl
, tm_name
;
4880 struct cgraph_node
*new_node
;
4882 old_decl
= old_node
->decl
;
4883 new_decl
= copy_node (old_decl
);
4885 /* DECL_ASSEMBLER_NAME needs to be set before we call
4886 cgraph_copy_node_for_versioning below, because cgraph_node will
4887 fill the assembler_name_hash. */
4888 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4889 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4890 SET_DECL_RTL (new_decl
, NULL
);
4891 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4893 /* Perform the same remapping to the comdat group. */
4894 if (DECL_ONE_ONLY (new_decl
))
4895 varpool_node::get (new_decl
)->set_comdat_group
4896 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4898 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4899 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4900 new_node
->local
.local
= false;
4901 new_node
->externally_visible
= old_node
->externally_visible
;
4902 new_node
->lowered
= true;
4903 new_node
->tm_clone
= 1;
4904 get_cg_data (&old_node
, true)->clone
= new_node
;
4906 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4908 /* Remap extern inline to static inline. */
4909 /* ??? Is it worth trying to use make_decl_one_only? */
4910 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4912 DECL_EXTERNAL (new_decl
) = 0;
4913 TREE_PUBLIC (new_decl
) = 0;
4914 DECL_WEAK (new_decl
) = 0;
4917 tree_function_versioning (old_decl
, new_decl
,
4922 record_tm_clone_pair (old_decl
, new_decl
);
4924 symtab
->call_cgraph_insertion_hooks (new_node
);
4925 if (old_node
->force_output
4926 || old_node
->ref_list
.first_referring ())
4927 ipa_tm_mark_force_output_node (new_node
);
4928 if (old_node
->forced_by_abi
)
4929 ipa_tm_mark_forced_by_abi_node (new_node
);
4931 /* Do the same thing, but for any aliases of the original node. */
4933 struct create_version_alias_info data
;
4934 data
.old_node
= old_node
;
4935 data
.new_decl
= new_decl
;
4936 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4941 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4944 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4947 gimple_stmt_iterator gsi
;
4950 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4952 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4953 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4955 split_block_after_labels (bb
);
4956 gsi
= gsi_after_labels (bb
);
4957 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4959 node
->create_edge (cgraph_node::get_create
4960 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4962 compute_call_stmt_bb_frequency (node
->decl
,
4966 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4969 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4970 struct tm_region
*region
,
4971 gimple_stmt_iterator
*gsi
, gimple stmt
)
4973 tree gettm_fn
, ret
, old_fn
, callfn
;
4977 old_fn
= gimple_call_fn (stmt
);
4979 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
4981 tree fndecl
= TREE_OPERAND (old_fn
, 0);
4982 tree clone
= get_tm_clone_pair (fndecl
);
4984 /* By transforming the call into a TM_GETTMCLONE, we are
4985 technically taking the address of the original function and
4986 its clone. Explain this so inlining will know this function
4988 cgraph_node::get (fndecl
)->mark_address_taken () ;
4990 cgraph_node::get (clone
)->mark_address_taken ();
4993 safe
= is_tm_safe (TREE_TYPE (old_fn
));
4994 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
4995 : BUILT_IN_TM_GETTMCLONE_IRR
);
4996 ret
= create_tmp_var (ptr_type_node
, NULL
);
4999 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5001 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5002 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5003 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5005 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5006 ret
= make_ssa_name (ret
, g
);
5007 gimple_call_set_lhs (g
, ret
);
5009 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5011 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5012 compute_call_stmt_bb_frequency (node
->decl
,
5015 /* Cast return value from tm_gettmclone* into appropriate function
5017 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
5018 g2
= gimple_build_assign (callfn
,
5019 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5020 callfn
= make_ssa_name (callfn
, g2
);
5021 gimple_assign_set_lhs (g2
, callfn
);
5022 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5024 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5025 which we would have derived from the decl. Failure to save
5026 this bit means we might have to split the basic block. */
5027 if (gimple_call_nothrow_p (stmt
))
5028 gimple_call_set_nothrow (stmt
, true);
5030 gimple_call_set_fn (stmt
, callfn
);
5032 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5033 for a call statement. Fix it. */
5035 tree lhs
= gimple_call_lhs (stmt
);
5036 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5038 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5042 temp
= create_tmp_reg (rettype
, 0);
5043 gimple_call_set_lhs (stmt
, temp
);
5045 g2
= gimple_build_assign (lhs
,
5046 fold_build1 (VIEW_CONVERT_EXPR
,
5047 TREE_TYPE (lhs
), temp
));
5048 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5053 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5054 if (e
&& e
->indirect_info
)
5055 e
->indirect_info
->polymorphic
= false;
5060 /* Helper function for ipa_tm_transform_calls*. Given a call
5061 statement in GSI which resides inside transaction REGION, redirect
5062 the call to either its wrapper function, or its clone. */
5065 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5066 struct tm_region
*region
,
5067 gimple_stmt_iterator
*gsi
,
5068 bool *need_ssa_rename_p
)
5070 gimple stmt
= gsi_stmt (*gsi
);
5071 struct cgraph_node
*new_node
;
5072 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5073 tree fndecl
= gimple_call_fndecl (stmt
);
5075 /* For indirect calls, pass the address through the runtime. */
5078 *need_ssa_rename_p
|=
5079 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5083 /* Handle some TM builtins. Ordinarily these aren't actually generated
5084 at this point, but handling these functions when written in by the
5085 user makes it easier to build unit tests. */
5086 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5089 /* Fixup recursive calls inside clones. */
5090 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5091 for recursion but not update the call statements themselves? */
5092 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5094 gimple_call_set_fndecl (stmt
, current_function_decl
);
5098 /* If there is a replacement, use it. */
5099 fndecl
= find_tm_replacement_function (fndecl
);
5102 new_node
= cgraph_node::get_create (fndecl
);
5104 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5106 We can't do this earlier in record_tm_replacement because
5107 cgraph_remove_unreachable_nodes is called before we inject
5108 references to the node. Further, we can't do this in some
5109 nice central place in ipa_tm_execute because we don't have
5110 the exact list of wrapper functions that would be used.
5111 Marking more wrappers than necessary results in the creation
5112 of unnecessary cgraph_nodes, which can cause some of the
5113 other IPA passes to crash.
5115 We do need to mark these nodes so that we get the proper
5116 result in expand_call_tm. */
5117 /* ??? This seems broken. How is it that we're marking the
5118 CALLEE as may_enter_irr? Surely we should be marking the
5119 CALLER. Also note that find_tm_replacement_function also
5120 contains mappings into the TM runtime, e.g. memcpy. These
5121 we know won't go irrevocable. */
5122 new_node
->local
.tm_may_enter_irr
= 1;
5126 struct tm_ipa_cg_data
*d
;
5127 struct cgraph_node
*tnode
= e
->callee
;
5129 d
= get_cg_data (&tnode
, true);
5130 new_node
= d
->clone
;
5132 /* As we've already skipped pure calls and appropriate builtins,
5133 and we've already marked irrevocable blocks, if we can't come
5134 up with a static replacement, then ask the runtime. */
5135 if (new_node
== NULL
)
5137 *need_ssa_rename_p
|=
5138 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5142 fndecl
= new_node
->decl
;
5145 e
->redirect_callee (new_node
);
5146 gimple_call_set_fndecl (stmt
, fndecl
);
5149 /* Helper function for ipa_tm_transform_calls. For a given BB,
5150 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5151 redirect other calls to the generated transactional clone. */
5154 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5155 basic_block bb
, bitmap irr_blocks
)
5157 gimple_stmt_iterator gsi
;
5158 bool need_ssa_rename
= false;
5160 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5162 ipa_tm_insert_irr_call (node
, region
, bb
);
5166 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5168 gimple stmt
= gsi_stmt (gsi
);
5170 if (!is_gimple_call (stmt
))
5172 if (is_tm_pure_call (stmt
))
5175 /* Redirect edges to the appropriate replacement or clone. */
5176 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5179 return need_ssa_rename
;
5182 /* Walk the CFG for REGION, beginning at BB. Install calls to
5183 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5184 the generated transactional clone. */
5187 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5188 basic_block bb
, bitmap irr_blocks
)
5190 bool need_ssa_rename
= false;
5193 auto_vec
<basic_block
> queue
;
5194 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5196 queue
.safe_push (bb
);
5202 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5204 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5207 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5210 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5211 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5213 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5214 queue
.safe_push (e
->dest
);
5217 while (!queue
.is_empty ());
5219 BITMAP_FREE (visited_blocks
);
5221 return need_ssa_rename
;
5224 /* Transform the calls within the TM regions within NODE. */
5227 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5229 struct tm_ipa_cg_data
*d
;
5230 struct tm_region
*region
;
5231 bool need_ssa_rename
= false;
5233 d
= get_cg_data (&node
, true);
5235 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5236 calculate_dominance_info (CDI_DOMINATORS
);
5238 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5240 /* If we're sure to go irrevocable, don't transform anything. */
5241 if (d
->irrevocable_blocks_normal
5242 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5243 region
->entry_block
->index
))
5245 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5246 | GTMA_MAY_ENTER_IRREVOCABLE
5247 | GTMA_HAS_NO_INSTRUMENTATION
);
5252 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5253 d
->irrevocable_blocks_normal
);
5256 if (need_ssa_rename
)
5257 update_ssa (TODO_update_ssa_only_virtuals
);
5262 /* Transform the calls within the transactional clone of NODE. */
5265 ipa_tm_transform_clone (struct cgraph_node
*node
)
5267 struct tm_ipa_cg_data
*d
;
5268 bool need_ssa_rename
;
5270 d
= get_cg_data (&node
, true);
5272 /* If this function makes no calls and has no irrevocable blocks,
5273 then there's nothing to do. */
5274 /* ??? Remove non-aborting top-level transactions. */
5275 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5278 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5279 calculate_dominance_info (CDI_DOMINATORS
);
5282 ipa_tm_transform_calls (d
->clone
, NULL
,
5283 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5284 d
->irrevocable_blocks_clone
);
5286 if (need_ssa_rename
)
5287 update_ssa (TODO_update_ssa_only_virtuals
);
5292 /* Main entry point for the transactional memory IPA pass. */
5295 ipa_tm_execute (void)
5297 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5298 /* List of functions that will go irrevocable. */
5299 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5301 struct cgraph_node
*node
;
5302 struct tm_ipa_cg_data
*d
;
5303 enum availability a
;
5306 #ifdef ENABLE_CHECKING
5307 cgraph_node::verify_cgraph_nodes ();
5310 bitmap_obstack_initialize (&tm_obstack
);
5311 initialize_original_copy_tables ();
5313 /* For all local functions marked tm_callable, queue them. */
5314 FOR_EACH_DEFINED_FUNCTION (node
)
5315 if (is_tm_callable (node
->decl
)
5316 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5318 d
= get_cg_data (&node
, true);
5319 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5322 /* For all local reachable functions... */
5323 FOR_EACH_DEFINED_FUNCTION (node
)
5325 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5327 /* ... marked tm_pure, record that fact for the runtime by
5328 indicating that the pure function is its own tm_callable.
5329 No need to do this if the function's address can't be taken. */
5330 if (is_tm_pure (node
->decl
))
5332 if (!node
->local
.local
)
5333 record_tm_clone_pair (node
->decl
, node
->decl
);
5337 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5338 calculate_dominance_info (CDI_DOMINATORS
);
5340 tm_region_init (NULL
);
5343 d
= get_cg_data (&node
, true);
5345 /* Scan for calls that are in each transaction, and
5346 generate the uninstrumented code path. */
5347 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5349 /* Put it in the worklist so we can scan the function
5350 later (ipa_tm_scan_irr_function) and mark the
5351 irrevocable blocks. */
5352 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5353 d
->want_irr_scan_normal
= true;
5359 /* For every local function on the callee list, scan as if we will be
5360 creating a transactional clone, queueing all new functions we find
5362 for (i
= 0; i
< tm_callees
.length (); ++i
)
5364 node
= tm_callees
[i
];
5365 a
= node
->get_availability ();
5366 d
= get_cg_data (&node
, true);
5368 /* Put it in the worklist so we can scan the function later
5369 (ipa_tm_scan_irr_function) and mark the irrevocable
5371 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5373 /* Some callees cannot be arbitrarily cloned. These will always be
5374 irrevocable. Mark these now, so that we need not scan them. */
5375 if (is_tm_irrevocable (node
->decl
))
5376 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5377 else if (a
<= AVAIL_NOT_AVAILABLE
5378 && !is_tm_safe_or_pure (node
->decl
))
5379 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5380 else if (a
>= AVAIL_INTERPOSABLE
)
5382 if (!tree_versionable_function_p (node
->decl
))
5383 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5384 else if (!d
->is_irrevocable
)
5386 /* If this is an alias, make sure its base is queued as well.
5387 we need not scan the callees now, as the base will do. */
5390 node
= cgraph_node::get (node
->thunk
.alias
);
5391 d
= get_cg_data (&node
, true);
5392 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5396 /* Add all nodes called by this function into
5397 tm_callees as well. */
5398 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5403 /* Iterate scans until no more work to be done. Prefer not to use
5404 vec::pop because the worklist tends to follow a breadth-first
5405 search of the callgraph, which should allow convergance with a
5406 minimum number of scans. But we also don't want the worklist
5407 array to grow without bound, so we shift the array up periodically. */
5408 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5410 if (i
> 256 && i
== irr_worklist
.length () / 8)
5412 irr_worklist
.block_remove (0, i
);
5416 node
= irr_worklist
[i
];
5417 d
= get_cg_data (&node
, true);
5418 d
->in_worklist
= false;
5420 if (d
->want_irr_scan_normal
)
5422 d
->want_irr_scan_normal
= false;
5423 ipa_tm_scan_irr_function (node
, false);
5425 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5426 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5429 /* For every function on the callee list, collect the tm_may_enter_irr
5431 irr_worklist
.truncate (0);
5432 for (i
= 0; i
< tm_callees
.length (); ++i
)
5434 node
= tm_callees
[i
];
5435 if (ipa_tm_mayenterirr_function (node
))
5437 d
= get_cg_data (&node
, true);
5438 gcc_assert (d
->in_worklist
== false);
5439 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5443 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5444 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5446 struct cgraph_node
*caller
;
5447 struct cgraph_edge
*e
;
5448 struct ipa_ref
*ref
;
5450 if (i
> 256 && i
== irr_worklist
.length () / 8)
5452 irr_worklist
.block_remove (0, i
);
5456 node
= irr_worklist
[i
];
5457 d
= get_cg_data (&node
, true);
5458 d
->in_worklist
= false;
5459 node
->local
.tm_may_enter_irr
= true;
5461 /* Propagate back to normal callers. */
5462 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5465 if (!is_tm_safe_or_pure (caller
->decl
)
5466 && !caller
->local
.tm_may_enter_irr
)
5468 d
= get_cg_data (&caller
, true);
5469 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5473 /* Propagate back to referring aliases as well. */
5474 FOR_EACH_ALIAS (node
, ref
)
5476 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5477 if (!caller
->local
.tm_may_enter_irr
)
5479 /* ?? Do not traverse aliases here. */
5480 d
= get_cg_data (&caller
, false);
5481 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5486 /* Now validate all tm_safe functions, and all atomic regions in
5488 FOR_EACH_DEFINED_FUNCTION (node
)
5490 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5492 d
= get_cg_data (&node
, true);
5493 if (is_tm_safe (node
->decl
))
5494 ipa_tm_diagnose_tm_safe (node
);
5495 else if (d
->all_tm_regions
)
5496 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5499 /* Create clones. Do those that are not irrevocable and have a
5500 positive call count. Do those publicly visible functions that
5501 the user directed us to clone. */
5502 for (i
= 0; i
< tm_callees
.length (); ++i
)
5506 node
= tm_callees
[i
];
5507 if (node
->cpp_implicit_alias
)
5510 a
= node
->get_availability ();
5511 d
= get_cg_data (&node
, true);
5513 if (a
<= AVAIL_NOT_AVAILABLE
)
5514 doit
= is_tm_callable (node
->decl
);
5515 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5517 else if (!d
->is_irrevocable
5518 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5522 ipa_tm_create_version (node
);
5525 /* Redirect calls to the new clones, and insert irrevocable marks. */
5526 for (i
= 0; i
< tm_callees
.length (); ++i
)
5528 node
= tm_callees
[i
];
5531 d
= get_cg_data (&node
, true);
5533 ipa_tm_transform_clone (node
);
5536 FOR_EACH_DEFINED_FUNCTION (node
)
5538 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5540 d
= get_cg_data (&node
, true);
5541 if (d
->all_tm_regions
)
5542 ipa_tm_transform_transaction (node
);
5545 /* Free and clear all data structures. */
5546 tm_callees
.release ();
5547 irr_worklist
.release ();
5548 bitmap_obstack_release (&tm_obstack
);
5549 free_original_copy_tables ();
5551 FOR_EACH_FUNCTION (node
)
5554 #ifdef ENABLE_CHECKING
5555 cgraph_node::verify_cgraph_nodes ();
5563 const pass_data pass_data_ipa_tm
=
5565 SIMPLE_IPA_PASS
, /* type */
5567 OPTGROUP_NONE
, /* optinfo_flags */
5568 TV_TRANS_MEM
, /* tv_id */
5569 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5570 0, /* properties_provided */
5571 0, /* properties_destroyed */
5572 0, /* todo_flags_start */
5573 0, /* todo_flags_finish */
5576 class pass_ipa_tm
: public simple_ipa_opt_pass
5579 pass_ipa_tm (gcc::context
*ctxt
)
5580 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5583 /* opt_pass methods: */
5584 virtual bool gate (function
*) { return flag_tm
; }
5585 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5587 }; // class pass_ipa_tm
5591 simple_ipa_opt_pass
*
5592 make_pass_ipa_tm (gcc::context
*ctxt
)
5594 return new pass_ipa_tm (ctxt
);
5597 #include "gt-trans-mem.h"