1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2015 Free Software Foundation, Inc.
3 Contributed by Richard Henderson <rth@redhat.com>
4 and Aldy Hernandez <aldyh@redhat.com>.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
33 #include "fold-const.h"
34 #include "internal-fn.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
44 #include "tree-into-ssa.h"
45 #include "tree-pass.h"
46 #include "tree-inline.h"
47 #include "diagnostic-core.h"
50 #include "trans-mem.h"
53 #include "langhooks.h"
54 #include "gimple-pretty-print.h"
56 #include "tree-ssa-address.h"
59 #define A_RUNINSTRUMENTEDCODE 0x0001
60 #define A_RUNUNINSTRUMENTEDCODE 0x0002
61 #define A_SAVELIVEVARIABLES 0x0004
62 #define A_RESTORELIVEVARIABLES 0x0008
63 #define A_ABORTTRANSACTION 0x0010
65 #define AR_USERABORT 0x0001
66 #define AR_USERRETRY 0x0002
67 #define AR_TMCONFLICT 0x0004
68 #define AR_EXCEPTIONBLOCKABORT 0x0008
69 #define AR_OUTERABORT 0x0010
71 #define MODE_SERIALIRREVOCABLE 0x0000
74 /* The representation of a transaction changes several times during the
75 lowering process. In the beginning, in the front-end we have the
76 GENERIC tree TRANSACTION_EXPR. For example,
84 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
85 trivially replaced with a GIMPLE_TRANSACTION node.
87 During pass_lower_tm, we examine the body of transactions looking
88 for aborts. Transactions that do not contain an abort may be
89 merged into an outer transaction. We also add a TRY-FINALLY node
90 to arrange for the transaction to be committed on any exit.
92 [??? Think about how this arrangement affects throw-with-commit
93 and throw-with-abort operations. In this case we want the TRY to
94 handle gotos, but not to catch any exceptions because the transaction
95 will already be closed.]
97 GIMPLE_TRANSACTION [label=NULL] {
104 __builtin___tm_abort ();
106 __builtin___tm_commit ();
110 During pass_lower_eh, we create EH regions for the transactions,
111 intermixed with the regular EH stuff. This gives us a nice persistent
112 mapping (all the way through rtl) from transactional memory operation
113 back to the transaction, which allows us to get the abnormal edges
114 correct to model transaction aborts and restarts:
116 GIMPLE_TRANSACTION [label=over]
122 __builtin___tm_abort ();
123 __builtin___tm_commit ();
126 This is the end of all_lowering_passes, and so is what is present
127 during the IPA passes, and through all of the optimization passes.
129 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
130 functions and mark functions for cloning.
132 At the end of gimple optimization, before exiting SSA form,
133 pass_tm_edges replaces statements that perform transactional
134 memory operations with the appropriate TM builtins, and swap
135 out function calls with their transactional clones. At this
136 point we introduce the abnormal transaction restart edges and
137 complete lowering of the GIMPLE_TRANSACTION node.
139 x = __builtin___tm_start (MAY_ABORT);
141 if (x & abort_transaction)
144 t0 = __builtin___tm_load (global);
146 __builtin___tm_store (&global, t1);
148 __builtin___tm_abort ();
149 __builtin___tm_commit ();
153 static void *expand_regions (struct tm_region
*,
154 void *(*callback
)(struct tm_region
*, void *),
158 /* Return the attributes we want to examine for X, or NULL if it's not
159 something we examine. We look at function types, but allow pointers
160 to function types and function decls and peek through. */
163 get_attrs_for (const_tree x
)
168 switch (TREE_CODE (x
))
171 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
178 if (TREE_CODE (x
) != POINTER_TYPE
)
184 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
190 return TYPE_ATTRIBUTES (x
);
194 /* Return true if X has been marked TM_PURE. */
197 is_tm_pure (const_tree x
)
201 switch (TREE_CODE (x
))
212 if (TREE_CODE (x
) != POINTER_TYPE
)
218 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
223 flags
= flags_from_decl_or_type (x
);
224 return (flags
& ECF_TM_PURE
) != 0;
227 /* Return true if X has been marked TM_IRREVOCABLE. */
230 is_tm_irrevocable (tree x
)
232 tree attrs
= get_attrs_for (x
);
234 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
237 /* A call to the irrevocable builtin is by definition,
239 if (TREE_CODE (x
) == ADDR_EXPR
)
240 x
= TREE_OPERAND (x
, 0);
241 if (TREE_CODE (x
) == FUNCTION_DECL
242 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
243 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
249 /* Return true if X has been marked TM_SAFE. */
252 is_tm_safe (const_tree x
)
256 tree attrs
= get_attrs_for (x
);
259 if (lookup_attribute ("transaction_safe", attrs
))
261 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
268 /* Return true if CALL is const, or tm_pure. */
271 is_tm_pure_call (gimple
*call
)
273 tree fn
= gimple_call_fn (call
);
275 if (TREE_CODE (fn
) == ADDR_EXPR
)
277 fn
= TREE_OPERAND (fn
, 0);
278 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
283 return is_tm_pure (fn
);
286 /* Return true if X has been marked TM_CALLABLE. */
289 is_tm_callable (tree x
)
291 tree attrs
= get_attrs_for (x
);
294 if (lookup_attribute ("transaction_callable", attrs
))
296 if (lookup_attribute ("transaction_safe", attrs
))
298 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
304 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
307 is_tm_may_cancel_outer (tree x
)
309 tree attrs
= get_attrs_for (x
);
311 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
315 /* Return true for built in functions that "end" a transaction. */
318 is_tm_ending_fndecl (tree fndecl
)
320 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
321 switch (DECL_FUNCTION_CODE (fndecl
))
323 case BUILT_IN_TM_COMMIT
:
324 case BUILT_IN_TM_COMMIT_EH
:
325 case BUILT_IN_TM_ABORT
:
326 case BUILT_IN_TM_IRREVOCABLE
:
335 /* Return true if STMT is a built in function call that "ends" a
339 is_tm_ending (gimple
*stmt
)
343 if (gimple_code (stmt
) != GIMPLE_CALL
)
346 fndecl
= gimple_call_fndecl (stmt
);
347 return (fndecl
!= NULL_TREE
348 && is_tm_ending_fndecl (fndecl
));
351 /* Return true if STMT is a TM load. */
354 is_tm_load (gimple
*stmt
)
358 if (gimple_code (stmt
) != GIMPLE_CALL
)
361 fndecl
= gimple_call_fndecl (stmt
);
362 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
363 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
366 /* Same as above, but for simple TM loads, that is, not the
367 after-write, after-read, etc optimized variants. */
370 is_tm_simple_load (gimple
*stmt
)
374 if (gimple_code (stmt
) != GIMPLE_CALL
)
377 fndecl
= gimple_call_fndecl (stmt
);
378 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
380 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
381 return (fcode
== BUILT_IN_TM_LOAD_1
382 || fcode
== BUILT_IN_TM_LOAD_2
383 || fcode
== BUILT_IN_TM_LOAD_4
384 || fcode
== BUILT_IN_TM_LOAD_8
385 || fcode
== BUILT_IN_TM_LOAD_FLOAT
386 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
387 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
388 || fcode
== BUILT_IN_TM_LOAD_M64
389 || fcode
== BUILT_IN_TM_LOAD_M128
390 || fcode
== BUILT_IN_TM_LOAD_M256
);
395 /* Return true if STMT is a TM store. */
398 is_tm_store (gimple
*stmt
)
402 if (gimple_code (stmt
) != GIMPLE_CALL
)
405 fndecl
= gimple_call_fndecl (stmt
);
406 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
407 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
410 /* Same as above, but for simple TM stores, that is, not the
411 after-write, after-read, etc optimized variants. */
414 is_tm_simple_store (gimple
*stmt
)
418 if (gimple_code (stmt
) != GIMPLE_CALL
)
421 fndecl
= gimple_call_fndecl (stmt
);
422 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
424 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
425 return (fcode
== BUILT_IN_TM_STORE_1
426 || fcode
== BUILT_IN_TM_STORE_2
427 || fcode
== BUILT_IN_TM_STORE_4
428 || fcode
== BUILT_IN_TM_STORE_8
429 || fcode
== BUILT_IN_TM_STORE_FLOAT
430 || fcode
== BUILT_IN_TM_STORE_DOUBLE
431 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
432 || fcode
== BUILT_IN_TM_STORE_M64
433 || fcode
== BUILT_IN_TM_STORE_M128
434 || fcode
== BUILT_IN_TM_STORE_M256
);
439 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
442 is_tm_abort (tree fndecl
)
445 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
446 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
449 /* Build a GENERIC tree for a user abort. This is called by front ends
450 while transforming the __tm_abort statement. */
453 build_tm_abort_call (location_t loc
, bool is_outer
)
455 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
456 build_int_cst (integer_type_node
,
458 | (is_outer
? AR_OUTERABORT
: 0)));
461 /* Map for aribtrary function replacement under TM, as created
462 by the tm_wrap attribute. */
464 struct tm_wrapper_hasher
: ggc_cache_ptr_hash
<tree_map
>
466 static inline hashval_t
hash (tree_map
*m
) { return m
->hash
; }
468 equal (tree_map
*a
, tree_map
*b
)
470 return a
->base
.from
== b
->base
.from
;
474 keep_cache_entry (tree_map
*&m
)
476 return ggc_marked_p (m
->base
.from
);
480 static GTY((cache
)) hash_table
<tm_wrapper_hasher
> *tm_wrap_map
;
483 record_tm_replacement (tree from
, tree to
)
485 struct tree_map
**slot
, *h
;
487 /* Do not inline wrapper functions that will get replaced in the TM
490 Suppose you have foo() that will get replaced into tmfoo(). Make
491 sure the inliner doesn't try to outsmart us and inline foo()
492 before we get a chance to do the TM replacement. */
493 DECL_UNINLINABLE (from
) = 1;
495 if (tm_wrap_map
== NULL
)
496 tm_wrap_map
= hash_table
<tm_wrapper_hasher
>::create_ggc (32);
498 h
= ggc_alloc
<tree_map
> ();
499 h
->hash
= htab_hash_pointer (from
);
503 slot
= tm_wrap_map
->find_slot_with_hash (h
, h
->hash
, INSERT
);
507 /* Return a TM-aware replacement function for DECL. */
510 find_tm_replacement_function (tree fndecl
)
514 struct tree_map
*h
, in
;
516 in
.base
.from
= fndecl
;
517 in
.hash
= htab_hash_pointer (fndecl
);
518 h
= tm_wrap_map
->find_with_hash (&in
, in
.hash
);
523 /* ??? We may well want TM versions of most of the common <string.h>
524 functions. For now, we've already these two defined. */
525 /* Adjust expand_call_tm() attributes as necessary for the cases
527 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
528 switch (DECL_FUNCTION_CODE (fndecl
))
530 case BUILT_IN_MEMCPY
:
531 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
532 case BUILT_IN_MEMMOVE
:
533 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
534 case BUILT_IN_MEMSET
:
535 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
543 /* When appropriate, record TM replacement for memory allocation functions.
545 FROM is the FNDECL to wrap. */
547 tm_malloc_replacement (tree from
)
552 if (TREE_CODE (from
) != FUNCTION_DECL
)
555 /* If we have a previous replacement, the user must be explicitly
556 wrapping malloc/calloc/free. They better know what they're
558 if (find_tm_replacement_function (from
))
561 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
563 if (!strcmp (str
, "malloc"))
564 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
565 else if (!strcmp (str
, "calloc"))
566 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
567 else if (!strcmp (str
, "free"))
568 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
572 TREE_NOTHROW (to
) = 0;
574 record_tm_replacement (from
, to
);
577 /* Diagnostics for tm_safe functions/regions. Called by the front end
578 once we've lowered the function to high-gimple. */
580 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
581 Process exactly one statement. WI->INFO is set to non-null when in
582 the context of a tm_safe function, and null for a __transaction block. */
584 #define DIAG_TM_OUTER 1
585 #define DIAG_TM_SAFE 2
586 #define DIAG_TM_RELAXED 4
590 unsigned int summary_flags
: 8;
591 unsigned int block_flags
: 8;
592 unsigned int func_flags
: 8;
593 unsigned int saw_volatile
: 1;
597 /* Return true if T is a volatile lvalue of some kind. */
600 volatile_lvalue_p (tree t
)
602 return ((SSA_VAR_P (t
) || REFERENCE_CLASS_P (t
))
603 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
606 /* Tree callback function for diagnose_tm pass. */
609 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
611 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
612 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
615 *walk_subtrees
= false;
616 else if (volatile_lvalue_p (*tp
)
620 if (d
->block_flags
& DIAG_TM_SAFE
)
621 error_at (gimple_location (d
->stmt
),
622 "invalid use of volatile lvalue inside transaction");
623 else if (d
->func_flags
& DIAG_TM_SAFE
)
624 error_at (gimple_location (d
->stmt
),
625 "invalid use of volatile lvalue inside %<transaction_safe%>"
633 is_tm_safe_or_pure (const_tree x
)
635 return is_tm_safe (x
) || is_tm_pure (x
);
639 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
640 struct walk_stmt_info
*wi
)
642 gimple
*stmt
= gsi_stmt (*gsi
);
643 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
645 /* Save stmt for use in leaf analysis. */
648 switch (gimple_code (stmt
))
652 tree fn
= gimple_call_fn (stmt
);
654 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
655 && is_tm_may_cancel_outer (fn
))
656 error_at (gimple_location (stmt
),
657 "%<transaction_may_cancel_outer%> function call not within"
658 " outer transaction or %<transaction_may_cancel_outer%>");
660 if (d
->summary_flags
& DIAG_TM_SAFE
)
662 bool is_safe
, direct_call_p
;
665 if (TREE_CODE (fn
) == ADDR_EXPR
666 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
668 direct_call_p
= true;
669 replacement
= TREE_OPERAND (fn
, 0);
670 replacement
= find_tm_replacement_function (replacement
);
676 direct_call_p
= false;
677 replacement
= NULL_TREE
;
680 if (is_tm_safe_or_pure (fn
))
682 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
684 /* A function explicitly marked transaction_callable as
685 opposed to transaction_safe is being defined to be
686 unsafe as part of its ABI, regardless of its contents. */
689 else if (direct_call_p
)
691 if (IS_TYPE_OR_DECL_P (fn
)
692 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
694 else if (replacement
)
696 /* ??? At present we've been considering replacements
697 merely transaction_callable, and therefore might
698 enter irrevocable. The tm_wrap attribute has not
699 yet made it into the new language spec. */
704 /* ??? Diagnostics for unmarked direct calls moved into
705 the IPA pass. Section 3.2 of the spec details how
706 functions not marked should be considered "implicitly
707 safe" based on having examined the function body. */
713 /* An unmarked indirect call. Consider it unsafe even
714 though optimization may yet figure out how to inline. */
720 if (TREE_CODE (fn
) == ADDR_EXPR
)
721 fn
= TREE_OPERAND (fn
, 0);
722 if (d
->block_flags
& DIAG_TM_SAFE
)
725 error_at (gimple_location (stmt
),
726 "unsafe function call %qD within "
727 "atomic transaction", fn
);
730 if (!DECL_P (fn
) || DECL_NAME (fn
))
731 error_at (gimple_location (stmt
),
732 "unsafe function call %qE within "
733 "atomic transaction", fn
);
735 error_at (gimple_location (stmt
),
736 "unsafe indirect function call within "
737 "atomic transaction");
743 error_at (gimple_location (stmt
),
744 "unsafe function call %qD within "
745 "%<transaction_safe%> function", fn
);
748 if (!DECL_P (fn
) || DECL_NAME (fn
))
749 error_at (gimple_location (stmt
),
750 "unsafe function call %qE within "
751 "%<transaction_safe%> function", fn
);
753 error_at (gimple_location (stmt
),
754 "unsafe indirect function call within "
755 "%<transaction_safe%> function");
764 /* ??? We ought to come up with a way to add attributes to
765 asm statements, and then add "transaction_safe" to it.
766 Either that or get the language spec to resurrect __tm_waiver. */
767 if (d
->block_flags
& DIAG_TM_SAFE
)
768 error_at (gimple_location (stmt
),
769 "asm not allowed in atomic transaction");
770 else if (d
->func_flags
& DIAG_TM_SAFE
)
771 error_at (gimple_location (stmt
),
772 "asm not allowed in %<transaction_safe%> function");
775 case GIMPLE_TRANSACTION
:
777 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
778 unsigned char inner_flags
= DIAG_TM_SAFE
;
780 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
782 if (d
->block_flags
& DIAG_TM_SAFE
)
783 error_at (gimple_location (stmt
),
784 "relaxed transaction in atomic transaction");
785 else if (d
->func_flags
& DIAG_TM_SAFE
)
786 error_at (gimple_location (stmt
),
787 "relaxed transaction in %<transaction_safe%> function");
788 inner_flags
= DIAG_TM_RELAXED
;
790 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
793 error_at (gimple_location (stmt
),
794 "outer transaction in transaction");
795 else if (d
->func_flags
& DIAG_TM_OUTER
)
796 error_at (gimple_location (stmt
),
797 "outer transaction in "
798 "%<transaction_may_cancel_outer%> function");
799 else if (d
->func_flags
& DIAG_TM_SAFE
)
800 error_at (gimple_location (stmt
),
801 "outer transaction in %<transaction_safe%> function");
802 inner_flags
|= DIAG_TM_OUTER
;
805 *handled_ops_p
= true;
806 if (gimple_transaction_body (trans_stmt
))
808 struct walk_stmt_info wi_inner
;
809 struct diagnose_tm d_inner
;
811 memset (&d_inner
, 0, sizeof (d_inner
));
812 d_inner
.func_flags
= d
->func_flags
;
813 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
814 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
816 memset (&wi_inner
, 0, sizeof (wi_inner
));
817 wi_inner
.info
= &d_inner
;
819 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
820 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
833 diagnose_tm_blocks (void)
835 struct walk_stmt_info wi
;
836 struct diagnose_tm d
;
838 memset (&d
, 0, sizeof (d
));
839 if (is_tm_may_cancel_outer (current_function_decl
))
840 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
841 else if (is_tm_safe (current_function_decl
))
842 d
.func_flags
= DIAG_TM_SAFE
;
843 d
.summary_flags
= d
.func_flags
;
845 memset (&wi
, 0, sizeof (wi
));
848 walk_gimple_seq (gimple_body (current_function_decl
),
849 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
856 const pass_data pass_data_diagnose_tm_blocks
=
858 GIMPLE_PASS
, /* type */
859 "*diagnose_tm_blocks", /* name */
860 OPTGROUP_NONE
, /* optinfo_flags */
861 TV_TRANS_MEM
, /* tv_id */
862 PROP_gimple_any
, /* properties_required */
863 0, /* properties_provided */
864 0, /* properties_destroyed */
865 0, /* todo_flags_start */
866 0, /* todo_flags_finish */
869 class pass_diagnose_tm_blocks
: public gimple_opt_pass
872 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
873 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
876 /* opt_pass methods: */
877 virtual bool gate (function
*) { return flag_tm
; }
878 virtual unsigned int execute (function
*) { return diagnose_tm_blocks (); }
880 }; // class pass_diagnose_tm_blocks
885 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
887 return new pass_diagnose_tm_blocks (ctxt
);
890 /* Instead of instrumenting thread private memory, we save the
891 addresses in a log which we later use to save/restore the addresses
892 upon transaction start/restart.
894 The log is keyed by address, where each element contains individual
895 statements among different code paths that perform the store.
897 This log is later used to generate either plain save/restore of the
898 addresses upon transaction start/restart, or calls to the ITM_L*
901 So for something like:
903 struct large { int x[1000]; };
904 struct large lala = { 0 };
910 We can either save/restore:
913 trxn = _ITM_startTransaction ();
914 if (trxn & a_saveLiveVariables)
915 tmp_lala1 = lala.x[i];
916 else if (a & a_restoreLiveVariables)
917 lala.x[i] = tmp_lala1;
919 or use the logging functions:
922 trxn = _ITM_startTransaction ();
923 _ITM_LU4 (&lala.x[i]);
925 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
926 far up the dominator tree to shadow all of the writes to a given
927 location (thus reducing the total number of logging calls), but not
928 so high as to be called on a path that does not perform a
931 /* One individual log entry. We may have multiple statements for the
932 same location if neither dominate each other (on different
936 /* Address to save. */
938 /* Entry block for the transaction this address occurs in. */
939 basic_block entry_block
;
940 /* Dominating statements the store occurs in. */
942 /* Initially, while we are building the log, we place a nonzero
943 value here to mean that this address *will* be saved with a
944 save/restore sequence. Later, when generating the save sequence
945 we place the SSA temp generated here. */
950 /* Log entry hashtable helpers. */
952 struct log_entry_hasher
: pointer_hash
<tm_log_entry
>
954 static inline hashval_t
hash (const tm_log_entry
*);
955 static inline bool equal (const tm_log_entry
*, const tm_log_entry
*);
956 static inline void remove (tm_log_entry
*);
959 /* Htab support. Return hash value for a `tm_log_entry'. */
961 log_entry_hasher::hash (const tm_log_entry
*log
)
963 return iterative_hash_expr (log
->addr
, 0);
966 /* Htab support. Return true if two log entries are the same. */
968 log_entry_hasher::equal (const tm_log_entry
*log1
, const tm_log_entry
*log2
)
972 rth: I suggest that we get rid of the component refs etc.
973 I.e. resolve the reference to base + offset.
975 We may need to actually finish a merge with mainline for this,
976 since we'd like to be presented with Richi's MEM_REF_EXPRs more
977 often than not. But in the meantime your tm_log_entry could save
978 the results of get_inner_reference.
980 See: g++.dg/tm/pr46653.C
983 /* Special case plain equality because operand_equal_p() below will
984 return FALSE if the addresses are equal but they have
985 side-effects (e.g. a volatile address). */
986 if (log1
->addr
== log2
->addr
)
989 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
992 /* Htab support. Free one tm_log_entry. */
994 log_entry_hasher::remove (tm_log_entry
*lp
)
996 lp
->stmts
.release ();
1001 /* The actual log. */
1002 static hash_table
<log_entry_hasher
> *tm_log
;
1004 /* Addresses to log with a save/restore sequence. These should be in
1006 static vec
<tree
> tm_log_save_addresses
;
1008 enum thread_memory_type
1012 mem_transaction_local
,
1016 struct tm_new_mem_map
1018 /* SSA_NAME being dereferenced. */
1020 enum thread_memory_type local_new_memory
;
1023 /* Hashtable helpers. */
1025 struct tm_mem_map_hasher
: free_ptr_hash
<tm_new_mem_map
>
1027 static inline hashval_t
hash (const tm_new_mem_map
*);
1028 static inline bool equal (const tm_new_mem_map
*, const tm_new_mem_map
*);
1032 tm_mem_map_hasher::hash (const tm_new_mem_map
*v
)
1034 return (intptr_t)v
->val
>> 4;
1038 tm_mem_map_hasher::equal (const tm_new_mem_map
*v
, const tm_new_mem_map
*c
)
1040 return v
->val
== c
->val
;
1043 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1044 of memory (malloc, alloc, etc). */
1045 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1047 /* Initialize logging data structures. */
1051 tm_log
= new hash_table
<log_entry_hasher
> (10);
1052 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1053 tm_log_save_addresses
.create (5);
1056 /* Free logging data structures. */
1058 tm_log_delete (void)
1062 delete tm_new_mem_hash
;
1063 tm_new_mem_hash
= NULL
;
1064 tm_log_save_addresses
.release ();
1067 /* Return true if MEM is a transaction invariant memory for the TM
1068 region starting at REGION_ENTRY_BLOCK. */
1070 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1072 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1073 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1077 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1078 return def_bb
!= region_entry_block
1079 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1082 mem
= strip_invariant_refs (mem
);
1083 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1086 /* Given an address ADDR in STMT, find it in the memory log or add it,
1087 making sure to keep only the addresses highest in the dominator
1090 ENTRY_BLOCK is the entry_block for the transaction.
1092 If we find the address in the log, make sure it's either the same
1093 address, or an equivalent one that dominates ADDR.
1095 If we find the address, but neither ADDR dominates the found
1096 address, nor the found one dominates ADDR, we're on different
1097 execution paths. Add it.
1099 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1102 tm_log_add (basic_block entry_block
, tree addr
, gimple
*stmt
)
1104 tm_log_entry
**slot
;
1105 struct tm_log_entry l
, *lp
;
1108 slot
= tm_log
->find_slot (&l
, INSERT
);
1111 tree type
= TREE_TYPE (addr
);
1113 lp
= XNEW (struct tm_log_entry
);
1117 /* Small invariant addresses can be handled as save/restores. */
1119 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1120 && TYPE_SIZE_UNIT (type
) != NULL
1121 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1122 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1123 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1124 /* We must be able to copy this type normally. I.e., no
1125 special constructors and the like. */
1126 && !TREE_ADDRESSABLE (type
))
1128 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1129 lp
->stmts
.create (0);
1130 lp
->entry_block
= entry_block
;
1131 /* Save addresses separately in dominator order so we don't
1132 get confused by overlapping addresses in the save/restore
1134 tm_log_save_addresses
.safe_push (lp
->addr
);
1138 /* Use the logging functions. */
1139 lp
->stmts
.create (5);
1140 lp
->stmts
.quick_push (stmt
);
1141 lp
->save_var
= NULL
;
1151 /* If we're generating a save/restore sequence, we don't care
1152 about statements. */
1156 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1158 if (stmt
== oldstmt
)
1160 /* We already have a store to the same address, higher up the
1161 dominator tree. Nothing to do. */
1162 if (dominated_by_p (CDI_DOMINATORS
,
1163 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1165 /* We should be processing blocks in dominator tree order. */
1166 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1167 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1169 /* Store is on a different code path. */
1170 lp
->stmts
.safe_push (stmt
);
1174 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1175 result, insert the new statements before GSI. */
1178 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1180 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1181 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1183 x
= build_fold_addr_expr (x
);
1184 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1187 /* Instrument one address with the logging functions.
1188 ADDR is the address to save.
1189 STMT is the statement before which to place it. */
1191 tm_log_emit_stmt (tree addr
, gimple
*stmt
)
1193 tree type
= TREE_TYPE (addr
);
1194 tree size
= TYPE_SIZE_UNIT (type
);
1195 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1197 enum built_in_function code
= BUILT_IN_TM_LOG
;
1199 if (type
== float_type_node
)
1200 code
= BUILT_IN_TM_LOG_FLOAT
;
1201 else if (type
== double_type_node
)
1202 code
= BUILT_IN_TM_LOG_DOUBLE
;
1203 else if (type
== long_double_type_node
)
1204 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1205 else if (tree_fits_uhwi_p (size
))
1207 unsigned int n
= tree_to_uhwi (size
);
1211 code
= BUILT_IN_TM_LOG_1
;
1214 code
= BUILT_IN_TM_LOG_2
;
1217 code
= BUILT_IN_TM_LOG_4
;
1220 code
= BUILT_IN_TM_LOG_8
;
1223 code
= BUILT_IN_TM_LOG
;
1224 if (TREE_CODE (type
) == VECTOR_TYPE
)
1226 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1227 code
= BUILT_IN_TM_LOG_M64
;
1228 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1229 code
= BUILT_IN_TM_LOG_M128
;
1230 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1231 code
= BUILT_IN_TM_LOG_M256
;
1237 addr
= gimplify_addr (&gsi
, addr
);
1238 if (code
== BUILT_IN_TM_LOG
)
1239 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1241 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1242 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1245 /* Go through the log and instrument address that must be instrumented
1246 with the logging functions. Leave the save/restore addresses for
1251 hash_table
<log_entry_hasher
>::iterator hi
;
1252 struct tm_log_entry
*lp
;
1254 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1261 fprintf (dump_file
, "TM thread private mem logging: ");
1262 print_generic_expr (dump_file
, lp
->addr
, 0);
1263 fprintf (dump_file
, "\n");
1269 fprintf (dump_file
, "DUMPING to variable\n");
1275 fprintf (dump_file
, "DUMPING with logging functions\n");
1276 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1277 tm_log_emit_stmt (lp
->addr
, stmt
);
1282 /* Emit the save sequence for the corresponding addresses in the log.
1283 ENTRY_BLOCK is the entry block for the transaction.
1284 BB is the basic block to insert the code in. */
1286 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1289 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1291 struct tm_log_entry l
, *lp
;
1293 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1295 l
.addr
= tm_log_save_addresses
[i
];
1296 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1297 gcc_assert (lp
->save_var
!= NULL
);
1299 /* We only care about variables in the current transaction. */
1300 if (lp
->entry_block
!= entry_block
)
1303 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1305 /* Make sure we can create an SSA_NAME for this type. For
1306 instance, aggregates aren't allowed, in which case the system
1307 will create a VOP for us and everything will just work. */
1308 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1310 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1311 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1314 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1318 /* Emit the restore sequence for the corresponding addresses in the log.
1319 ENTRY_BLOCK is the entry block for the transaction.
1320 BB is the basic block to insert the code in. */
1322 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1325 struct tm_log_entry l
, *lp
;
1326 gimple_stmt_iterator gsi
;
1329 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1331 l
.addr
= tm_log_save_addresses
[i
];
1332 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1333 gcc_assert (lp
->save_var
!= NULL
);
1335 /* We only care about variables in the current transaction. */
1336 if (lp
->entry_block
!= entry_block
)
1339 /* Restores are in LIFO order from the saves in case we have
1341 gsi
= gsi_start_bb (bb
);
1343 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1344 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1349 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1350 struct walk_stmt_info
*);
1351 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1352 struct walk_stmt_info
*);
1354 /* Evaluate an address X being dereferenced and determine if it
1355 originally points to a non aliased new chunk of memory (malloc,
1358 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1359 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1360 Return MEM_NON_LOCAL otherwise.
1362 ENTRY_BLOCK is the entry block to the transaction containing the
1363 dereference of X. */
1364 static enum thread_memory_type
1365 thread_private_new_memory (basic_block entry_block
, tree x
)
1367 gimple
*stmt
= NULL
;
1368 enum tree_code code
;
1369 tm_new_mem_map
**slot
;
1370 tm_new_mem_map elt
, *elt_p
;
1372 enum thread_memory_type retval
= mem_transaction_local
;
1375 || TREE_CODE (x
) != SSA_NAME
1376 /* Possible uninitialized use, or a function argument. In
1377 either case, we don't care. */
1378 || SSA_NAME_IS_DEFAULT_DEF (x
))
1379 return mem_non_local
;
1381 /* Look in cache first. */
1383 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1386 return elt_p
->local_new_memory
;
1388 /* Optimistically assume the memory is transaction local during
1389 processing. This catches recursion into this variable. */
1390 *slot
= elt_p
= XNEW (tm_new_mem_map
);
1392 elt_p
->local_new_memory
= mem_transaction_local
;
1394 /* Search DEF chain to find the original definition of this address. */
1397 if (ptr_deref_may_alias_global_p (x
))
1399 /* Address escapes. This is not thread-private. */
1400 retval
= mem_non_local
;
1401 goto new_memory_ret
;
1404 stmt
= SSA_NAME_DEF_STMT (x
);
1406 /* If the malloc call is outside the transaction, this is
1408 if (retval
!= mem_thread_local
1409 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1410 retval
= mem_thread_local
;
1412 if (is_gimple_assign (stmt
))
1414 code
= gimple_assign_rhs_code (stmt
);
1415 /* x = foo ==> foo */
1416 if (code
== SSA_NAME
)
1417 x
= gimple_assign_rhs1 (stmt
);
1418 /* x = foo + n ==> foo */
1419 else if (code
== POINTER_PLUS_EXPR
)
1420 x
= gimple_assign_rhs1 (stmt
);
1421 /* x = (cast*) foo ==> foo */
1422 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1423 x
= gimple_assign_rhs1 (stmt
);
1424 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1425 else if (code
== COND_EXPR
)
1427 tree op1
= gimple_assign_rhs2 (stmt
);
1428 tree op2
= gimple_assign_rhs3 (stmt
);
1429 enum thread_memory_type mem
;
1430 retval
= thread_private_new_memory (entry_block
, op1
);
1431 if (retval
== mem_non_local
)
1432 goto new_memory_ret
;
1433 mem
= thread_private_new_memory (entry_block
, op2
);
1434 retval
= MIN (retval
, mem
);
1435 goto new_memory_ret
;
1439 retval
= mem_non_local
;
1440 goto new_memory_ret
;
1445 if (gimple_code (stmt
) == GIMPLE_PHI
)
1448 enum thread_memory_type mem
;
1449 tree phi_result
= gimple_phi_result (stmt
);
1451 /* If any of the ancestors are non-local, we are sure to
1452 be non-local. Otherwise we can avoid doing anything
1453 and inherit what has already been generated. */
1455 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1457 tree op
= PHI_ARG_DEF (stmt
, i
);
1459 /* Exclude self-assignment. */
1460 if (phi_result
== op
)
1463 mem
= thread_private_new_memory (entry_block
, op
);
1464 if (mem
== mem_non_local
)
1467 goto new_memory_ret
;
1469 retval
= MIN (retval
, mem
);
1471 goto new_memory_ret
;
1476 while (TREE_CODE (x
) == SSA_NAME
);
1478 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1479 /* Thread-local or transaction-local. */
1482 retval
= mem_non_local
;
1485 elt_p
->local_new_memory
= retval
;
1489 /* Determine whether X has to be instrumented using a read
1492 ENTRY_BLOCK is the entry block for the region where stmt resides
1493 in. NULL if unknown.
1495 STMT is the statement in which X occurs in. It is used for thread
1496 private memory instrumentation. If no TPM instrumentation is
1497 desired, STMT should be null. */
1499 requires_barrier (basic_block entry_block
, tree x
, gimple
*stmt
)
1502 while (handled_component_p (x
))
1503 x
= TREE_OPERAND (x
, 0);
1505 switch (TREE_CODE (x
))
1510 enum thread_memory_type ret
;
1512 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1513 if (ret
== mem_non_local
)
1515 if (stmt
&& ret
== mem_thread_local
)
1516 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1517 tm_log_add (entry_block
, orig
, stmt
);
1519 /* Transaction-locals require nothing at all. For malloc, a
1520 transaction restart frees the memory and we reallocate.
1521 For alloca, the stack pointer gets reset by the retry and
1526 case TARGET_MEM_REF
:
1527 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1529 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1530 if (TREE_CODE (x
) == PARM_DECL
)
1532 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1538 if (DECL_BY_REFERENCE (x
))
1540 /* ??? This value is a pointer, but aggregate_value_p has been
1541 jigged to return true which confuses needs_to_live_in_memory.
1542 This ought to be cleaned up generically.
1544 FIXME: Verify this still happens after the next mainline
1545 merge. Testcase ie g++.dg/tm/pr47554.C.
1550 if (is_global_var (x
))
1551 return !TREE_READONLY (x
);
1552 if (/* FIXME: This condition should actually go below in the
1553 tm_log_add() call, however is_call_clobbered() depends on
1554 aliasing info which is not available during
1555 gimplification. Since requires_barrier() gets called
1556 during lower_sequence_tm/gimplification, leave the call
1557 to needs_to_live_in_memory until we eliminate
1558 lower_sequence_tm altogether. */
1559 needs_to_live_in_memory (x
))
1563 /* For local memory that doesn't escape (aka thread private
1564 memory), we can either save the value at the beginning of
1565 the transaction and restore on restart, or call a tm
1566 function to dynamically save and restore on restart
1569 tm_log_add (entry_block
, orig
, stmt
);
1578 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1579 a transaction region. */
1582 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1584 gimple
*stmt
= gsi_stmt (*gsi
);
1586 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1587 *state
|= GTMA_HAVE_LOAD
;
1588 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1589 *state
|= GTMA_HAVE_STORE
;
1592 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1595 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1597 gimple
*stmt
= gsi_stmt (*gsi
);
1600 if (is_tm_pure_call (stmt
))
1603 /* Check if this call is a transaction abort. */
1604 fn
= gimple_call_fndecl (stmt
);
1605 if (is_tm_abort (fn
))
1606 *state
|= GTMA_HAVE_ABORT
;
1608 /* Note that something may happen. */
1609 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1612 /* Lower a GIMPLE_TRANSACTION statement. */
1615 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1618 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1619 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1620 unsigned int this_state
= 0;
1621 struct walk_stmt_info this_wi
;
1623 /* First, lower the body. The scanning that we do inside gives
1624 us some idea of what we're dealing with. */
1625 memset (&this_wi
, 0, sizeof (this_wi
));
1626 this_wi
.info
= (void *) &this_state
;
1627 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1628 lower_sequence_tm
, NULL
, &this_wi
);
1630 /* If there was absolutely nothing transaction related inside the
1631 transaction, we may elide it. Likewise if this is a nested
1632 transaction and does not contain an abort. */
1634 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1637 *outer_state
|= this_state
;
1639 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1641 gimple_transaction_set_body (stmt
, NULL
);
1643 gsi_remove (gsi
, true);
1644 wi
->removed_stmt
= true;
1648 /* Wrap the body of the transaction in a try-finally node so that
1649 the commit call is always properly called. */
1650 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1651 if (flag_exceptions
)
1654 gimple_seq n_seq
, e_seq
;
1656 n_seq
= gimple_seq_alloc_with_stmt (g
);
1659 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1660 1, integer_zero_node
);
1661 ptr
= create_tmp_var (ptr_type_node
);
1662 gimple_call_set_lhs (g
, ptr
);
1663 gimple_seq_add_stmt (&e_seq
, g
);
1665 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1667 gimple_seq_add_stmt (&e_seq
, g
);
1669 g
= gimple_build_eh_else (n_seq
, e_seq
);
1672 g
= gimple_build_try (gimple_transaction_body (stmt
),
1673 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1674 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1676 gimple_transaction_set_body (stmt
, NULL
);
1678 /* If the transaction calls abort or if this is an outer transaction,
1679 add an "over" label afterwards. */
1680 if ((this_state
& (GTMA_HAVE_ABORT
))
1681 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1683 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1684 gimple_transaction_set_label (stmt
, label
);
1685 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1688 /* Record the set of operations found for use later. */
1689 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1690 gimple_transaction_set_subcode (stmt
, this_state
);
1693 /* Iterate through the statements in the sequence, lowering them all
1694 as appropriate for being in a transaction. */
1697 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1698 struct walk_stmt_info
*wi
)
1700 unsigned int *state
= (unsigned int *) wi
->info
;
1701 gimple
*stmt
= gsi_stmt (*gsi
);
1703 *handled_ops_p
= true;
1704 switch (gimple_code (stmt
))
1707 /* Only memory reads/writes need to be instrumented. */
1708 if (gimple_assign_single_p (stmt
))
1709 examine_assign_tm (state
, gsi
);
1713 examine_call_tm (state
, gsi
);
1717 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1720 case GIMPLE_TRANSACTION
:
1721 lower_transaction (gsi
, wi
);
1725 *handled_ops_p
= !gimple_has_substatements (stmt
);
1732 /* Iterate through the statements in the sequence, lowering them all
1733 as appropriate for being outside of a transaction. */
1736 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1737 struct walk_stmt_info
* wi
)
1739 gimple
*stmt
= gsi_stmt (*gsi
);
1741 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1743 *handled_ops_p
= true;
1744 lower_transaction (gsi
, wi
);
1747 *handled_ops_p
= !gimple_has_substatements (stmt
);
1752 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1753 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1754 been moved out, and all the data required for constructing a proper
1755 CFG has been recorded. */
1758 execute_lower_tm (void)
1760 struct walk_stmt_info wi
;
1763 /* Transactional clones aren't created until a later pass. */
1764 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1766 body
= gimple_body (current_function_decl
);
1767 memset (&wi
, 0, sizeof (wi
));
1768 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1769 gimple_set_body (current_function_decl
, body
);
1776 const pass_data pass_data_lower_tm
=
1778 GIMPLE_PASS
, /* type */
1779 "tmlower", /* name */
1780 OPTGROUP_NONE
, /* optinfo_flags */
1781 TV_TRANS_MEM
, /* tv_id */
1782 PROP_gimple_lcf
, /* properties_required */
1783 0, /* properties_provided */
1784 0, /* properties_destroyed */
1785 0, /* todo_flags_start */
1786 0, /* todo_flags_finish */
1789 class pass_lower_tm
: public gimple_opt_pass
1792 pass_lower_tm (gcc::context
*ctxt
)
1793 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1796 /* opt_pass methods: */
1797 virtual bool gate (function
*) { return flag_tm
; }
1798 virtual unsigned int execute (function
*) { return execute_lower_tm (); }
1800 }; // class pass_lower_tm
1805 make_pass_lower_tm (gcc::context
*ctxt
)
1807 return new pass_lower_tm (ctxt
);
1810 /* Collect region information for each transaction. */
1816 /* The field "transaction_stmt" is initially a gtransaction *,
1817 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1819 Helper method to get it as a gtransaction *, with code-checking
1820 in a checked-build. */
1823 get_transaction_stmt () const
1825 return as_a
<gtransaction
*> (transaction_stmt
);
1830 /* Link to the next unnested transaction. */
1831 struct tm_region
*next
;
1833 /* Link to the next inner transaction. */
1834 struct tm_region
*inner
;
1836 /* Link to the next outer transaction. */
1837 struct tm_region
*outer
;
1839 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1840 After TM_MARK, this gets replaced by a call to
1842 Hence this will be either a gtransaction *or a gcall *. */
1843 gimple
*transaction_stmt
;
1845 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1846 BUILT_IN_TM_START, this field is true if the transaction is an
1847 outer transaction. */
1848 bool original_transaction_was_outer
;
1850 /* Return value from BUILT_IN_TM_START. */
1853 /* The entry block to this region. This will always be the first
1854 block of the body of the transaction. */
1855 basic_block entry_block
;
1857 /* The first block after an expanded call to _ITM_beginTransaction. */
1858 basic_block restart_block
;
1860 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1861 These blocks are still a part of the region (i.e., the border is
1862 inclusive). Note that this set is only complete for paths in the CFG
1863 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1864 the edge to the "over" label. */
1867 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1871 /* True if there are pending edge statements to be committed for the
1872 current function being scanned in the tmmark pass. */
1873 bool pending_edge_inserts_p
;
1875 static struct tm_region
*all_tm_regions
;
1876 static bitmap_obstack tm_obstack
;
1879 /* A subroutine of tm_region_init. Record the existence of the
1880 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1882 static struct tm_region
*
1883 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1886 struct tm_region
*region
;
1888 region
= (struct tm_region
*)
1889 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1893 region
->next
= outer
->inner
;
1894 outer
->inner
= region
;
1898 region
->next
= all_tm_regions
;
1899 all_tm_regions
= region
;
1901 region
->inner
= NULL
;
1902 region
->outer
= outer
;
1904 region
->transaction_stmt
= stmt
;
1905 region
->original_transaction_was_outer
= false;
1906 region
->tm_state
= NULL
;
1908 /* There are either one or two edges out of the block containing
1909 the GIMPLE_TRANSACTION, one to the actual region and one to the
1910 "over" label if the region contains an abort. The former will
1911 always be the one marked FALLTHRU. */
1912 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1914 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1915 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1920 /* A subroutine of tm_region_init. Record all the exit and
1921 irrevocable blocks in BB into the region's exit_blocks and
1922 irr_blocks bitmaps. Returns the new region being scanned. */
1924 static struct tm_region
*
1925 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1927 gimple_stmt_iterator gsi
;
1931 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1934 /* Check to see if this is the end of a region by seeing if it
1935 contains a call to __builtin_tm_commit{,_eh}. Note that the
1936 outermost region for DECL_IS_TM_CLONE need not collect this. */
1937 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1940 if (gimple_code (g
) == GIMPLE_CALL
)
1942 tree fn
= gimple_call_fndecl (g
);
1943 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1945 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1946 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1947 && region
->exit_blocks
)
1949 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1950 region
= region
->outer
;
1953 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1954 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1961 /* Collect all of the transaction regions within the current function
1962 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1963 an "outermost" region for use by tm clones. */
1966 tm_region_init (struct tm_region
*region
)
1972 auto_vec
<basic_block
> queue
;
1973 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1974 struct tm_region
*old_region
;
1975 auto_vec
<tm_region
*> bb_regions
;
1977 all_tm_regions
= region
;
1978 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1980 /* We could store this information in bb->aux, but we may get called
1981 through get_all_tm_blocks() from another pass that may be already
1983 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
1985 queue
.safe_push (bb
);
1986 bb_regions
[bb
->index
] = region
;
1990 region
= bb_regions
[bb
->index
];
1991 bb_regions
[bb
->index
] = NULL
;
1993 /* Record exit and irrevocable blocks. */
1994 region
= tm_region_init_1 (region
, bb
);
1996 /* Check for the last statement in the block beginning a new region. */
1998 old_region
= region
;
2000 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2001 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2003 /* Process subsequent blocks. */
2004 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2005 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2007 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2008 queue
.safe_push (e
->dest
);
2010 /* If the current block started a new region, make sure that only
2011 the entry block of the new region is associated with this region.
2012 Other successors are still part of the old region. */
2013 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2014 bb_regions
[e
->dest
->index
] = old_region
;
2016 bb_regions
[e
->dest
->index
] = region
;
2019 while (!queue
.is_empty ());
2020 BITMAP_FREE (visited_blocks
);
2023 /* The "gate" function for all transactional memory expansion and optimization
2024 passes. We collect region information for each top-level transaction, and
2025 if we don't find any, we skip all of the TM passes. Each region will have
2026 all of the exit blocks recorded, and the originating statement. */
2034 calculate_dominance_info (CDI_DOMINATORS
);
2035 bitmap_obstack_initialize (&tm_obstack
);
2037 /* If the function is a TM_CLONE, then the entire function is the region. */
2038 if (decl_is_tm_clone (current_function_decl
))
2040 struct tm_region
*region
= (struct tm_region
*)
2041 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2042 memset (region
, 0, sizeof (*region
));
2043 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2044 /* For a clone, the entire function is the region. But even if
2045 we don't need to record any exit blocks, we may need to
2046 record irrevocable blocks. */
2047 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2049 tm_region_init (region
);
2053 tm_region_init (NULL
);
2055 /* If we didn't find any regions, cleanup and skip the whole tree
2056 of tm-related optimizations. */
2057 if (all_tm_regions
== NULL
)
2059 bitmap_obstack_release (&tm_obstack
);
2069 const pass_data pass_data_tm_init
=
2071 GIMPLE_PASS
, /* type */
2072 "*tminit", /* name */
2073 OPTGROUP_NONE
, /* optinfo_flags */
2074 TV_TRANS_MEM
, /* tv_id */
2075 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2076 0, /* properties_provided */
2077 0, /* properties_destroyed */
2078 0, /* todo_flags_start */
2079 0, /* todo_flags_finish */
2082 class pass_tm_init
: public gimple_opt_pass
2085 pass_tm_init (gcc::context
*ctxt
)
2086 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2089 /* opt_pass methods: */
2090 virtual bool gate (function
*) { return gate_tm_init (); }
2092 }; // class pass_tm_init
2097 make_pass_tm_init (gcc::context
*ctxt
)
2099 return new pass_tm_init (ctxt
);
2102 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2103 represented by STATE. */
2106 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2108 if (region
&& region
->transaction_stmt
)
2110 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2111 flags
|= gimple_transaction_subcode (transaction_stmt
);
2112 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2116 /* Construct a memory load in a transactional context. Return the
2117 gimple statement performing the load, or NULL if there is no
2118 TM_LOAD builtin of the appropriate size to do the load.
2120 LOC is the location to use for the new statement(s). */
2123 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2125 enum built_in_function code
= END_BUILTINS
;
2126 tree t
, type
= TREE_TYPE (rhs
), decl
;
2129 if (type
== float_type_node
)
2130 code
= BUILT_IN_TM_LOAD_FLOAT
;
2131 else if (type
== double_type_node
)
2132 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2133 else if (type
== long_double_type_node
)
2134 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2135 else if (TYPE_SIZE_UNIT (type
) != NULL
2136 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2138 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2141 code
= BUILT_IN_TM_LOAD_1
;
2144 code
= BUILT_IN_TM_LOAD_2
;
2147 code
= BUILT_IN_TM_LOAD_4
;
2150 code
= BUILT_IN_TM_LOAD_8
;
2155 if (code
== END_BUILTINS
)
2157 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2162 decl
= builtin_decl_explicit (code
);
2164 t
= gimplify_addr (gsi
, rhs
);
2165 gcall
= gimple_build_call (decl
, 1, t
);
2166 gimple_set_location (gcall
, loc
);
2168 t
= TREE_TYPE (TREE_TYPE (decl
));
2169 if (useless_type_conversion_p (type
, t
))
2171 gimple_call_set_lhs (gcall
, lhs
);
2172 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2179 temp
= create_tmp_reg (t
);
2180 gimple_call_set_lhs (gcall
, temp
);
2181 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2183 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2184 g
= gimple_build_assign (lhs
, t
);
2185 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2192 /* Similarly for storing TYPE in a transactional context. */
2195 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2197 enum built_in_function code
= END_BUILTINS
;
2198 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2201 if (type
== float_type_node
)
2202 code
= BUILT_IN_TM_STORE_FLOAT
;
2203 else if (type
== double_type_node
)
2204 code
= BUILT_IN_TM_STORE_DOUBLE
;
2205 else if (type
== long_double_type_node
)
2206 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2207 else if (TYPE_SIZE_UNIT (type
) != NULL
2208 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
)))
2210 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type
)))
2213 code
= BUILT_IN_TM_STORE_1
;
2216 code
= BUILT_IN_TM_STORE_2
;
2219 code
= BUILT_IN_TM_STORE_4
;
2222 code
= BUILT_IN_TM_STORE_8
;
2227 if (code
== END_BUILTINS
)
2229 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2234 fn
= builtin_decl_explicit (code
);
2236 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2238 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2240 /* Handle the easy initialization to zero. */
2241 if (!CONSTRUCTOR_ELTS (rhs
))
2242 rhs
= build_int_cst (simple_type
, 0);
2245 /* ...otherwise punt to the caller and probably use
2246 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2247 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2252 else if (!useless_type_conversion_p (simple_type
, type
))
2257 temp
= create_tmp_reg (simple_type
);
2258 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2259 g
= gimple_build_assign (temp
, t
);
2260 gimple_set_location (g
, loc
);
2261 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2266 t
= gimplify_addr (gsi
, lhs
);
2267 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2268 gimple_set_location (gcall
, loc
);
2269 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2275 /* Expand an assignment statement into transactional builtins. */
2278 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2280 gimple
*stmt
= gsi_stmt (*gsi
);
2281 location_t loc
= gimple_location (stmt
);
2282 tree lhs
= gimple_assign_lhs (stmt
);
2283 tree rhs
= gimple_assign_rhs1 (stmt
);
2284 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2285 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2286 gimple
*gcall
= NULL
;
2288 if (!load_p
&& !store_p
)
2290 /* Add thread private addresses to log if applicable. */
2291 requires_barrier (region
->entry_block
, lhs
, stmt
);
2296 // Remove original load/store statement.
2297 gsi_remove (gsi
, true);
2299 if (load_p
&& !store_p
)
2301 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2302 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2304 else if (store_p
&& !load_p
)
2306 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2307 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2311 tree lhs_addr
, rhs_addr
, tmp
;
2314 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2316 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2318 /* ??? Figure out if there's any possible overlap between the LHS
2319 and the RHS and if not, use MEMCPY. */
2321 if (load_p
&& is_gimple_reg (lhs
))
2323 tmp
= create_tmp_var (TREE_TYPE (lhs
));
2324 lhs_addr
= build_fold_addr_expr (tmp
);
2329 lhs_addr
= gimplify_addr (gsi
, lhs
);
2331 rhs_addr
= gimplify_addr (gsi
, rhs
);
2332 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2333 3, lhs_addr
, rhs_addr
,
2334 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2335 gimple_set_location (gcall
, loc
);
2336 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2340 gcall
= gimple_build_assign (lhs
, tmp
);
2341 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2345 /* Now that we have the load/store in its instrumented form, add
2346 thread private addresses to the log if applicable. */
2348 requires_barrier (region
->entry_block
, lhs
, gcall
);
2350 // The calls to build_tm_{store,load} above inserted the instrumented
2351 // call into the stream.
2352 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2356 /* Expand a call statement as appropriate for a transaction. That is,
2357 either verify that the call does not affect the transaction, or
2358 redirect the call to a clone that handles transactions, or change
2359 the transaction state to IRREVOCABLE. Return true if the call is
2360 one of the builtins that end a transaction. */
2363 expand_call_tm (struct tm_region
*region
,
2364 gimple_stmt_iterator
*gsi
)
2366 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2367 tree lhs
= gimple_call_lhs (stmt
);
2369 struct cgraph_node
*node
;
2370 bool retval
= false;
2372 fn_decl
= gimple_call_fndecl (stmt
);
2374 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2375 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2376 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2377 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2378 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2380 if (is_tm_pure_call (stmt
))
2384 retval
= is_tm_ending_fndecl (fn_decl
);
2387 /* Assume all non-const/pure calls write to memory, except
2388 transaction ending builtins. */
2389 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2392 /* For indirect calls, we already generated a call into the runtime. */
2395 tree fn
= gimple_call_fn (stmt
);
2397 /* We are guaranteed never to go irrevocable on a safe or pure
2398 call, and the pure call was handled above. */
2399 if (is_tm_safe (fn
))
2402 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2407 node
= cgraph_node::get (fn_decl
);
2408 /* All calls should have cgraph here. */
2411 /* We can have a nodeless call here if some pass after IPA-tm
2412 added uninstrumented calls. For example, loop distribution
2413 can transform certain loop constructs into __builtin_mem*
2414 calls. In this case, see if we have a suitable TM
2415 replacement and fill in the gaps. */
2416 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2417 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2418 gcc_assert (code
== BUILT_IN_MEMCPY
2419 || code
== BUILT_IN_MEMMOVE
2420 || code
== BUILT_IN_MEMSET
);
2422 tree repl
= find_tm_replacement_function (fn_decl
);
2425 gimple_call_set_fndecl (stmt
, repl
);
2427 node
= cgraph_node::create (repl
);
2428 node
->local
.tm_may_enter_irr
= false;
2429 return expand_call_tm (region
, gsi
);
2433 if (node
->local
.tm_may_enter_irr
)
2434 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2436 if (is_tm_abort (fn_decl
))
2438 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2442 /* Instrument the store if needed.
2444 If the assignment happens inside the function call (return slot
2445 optimization), there is no instrumentation to be done, since
2446 the callee should have done the right thing. */
2447 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2448 && !gimple_call_return_slot_opt_p (stmt
))
2450 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
));
2451 location_t loc
= gimple_location (stmt
);
2452 edge fallthru_edge
= NULL
;
2453 gassign
*assign_stmt
;
2455 /* Remember if the call was going to throw. */
2456 if (stmt_can_throw_internal (stmt
))
2460 basic_block bb
= gimple_bb (stmt
);
2462 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2463 if (e
->flags
& EDGE_FALLTHRU
)
2470 gimple_call_set_lhs (stmt
, tmp
);
2472 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2473 gimple_set_location (assign_stmt
, loc
);
2475 /* We cannot throw in the middle of a BB. If the call was going
2476 to throw, place the instrumentation on the fallthru edge, so
2477 the call remains the last statement in the block. */
2480 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2481 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2482 expand_assign_tm (region
, &fallthru_gsi
);
2483 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2484 pending_edge_inserts_p
= true;
2488 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2489 expand_assign_tm (region
, gsi
);
2492 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2499 /* Expand all statements in BB as appropriate for being inside
2503 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2505 gimple_stmt_iterator gsi
;
2507 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2509 gimple
*stmt
= gsi_stmt (gsi
);
2510 switch (gimple_code (stmt
))
2513 /* Only memory reads/writes need to be instrumented. */
2514 if (gimple_assign_single_p (stmt
)
2515 && !gimple_clobber_p (stmt
))
2517 expand_assign_tm (region
, &gsi
);
2523 if (expand_call_tm (region
, &gsi
))
2533 if (!gsi_end_p (gsi
))
2538 /* Return the list of basic-blocks in REGION.
2540 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2541 following a TM_IRREVOCABLE call.
2543 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2544 uninstrumented code path blocks in the list of basic blocks
2545 returned, false otherwise. */
2547 static vec
<basic_block
>
2548 get_tm_region_blocks (basic_block entry_block
,
2551 bitmap all_region_blocks
,
2552 bool stop_at_irrevocable_p
,
2553 bool include_uninstrumented_p
= true)
2555 vec
<basic_block
> bbs
= vNULL
;
2559 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2562 bbs
.safe_push (entry_block
);
2563 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2567 basic_block bb
= bbs
[i
++];
2570 bitmap_bit_p (exit_blocks
, bb
->index
))
2573 if (stop_at_irrevocable_p
2575 && bitmap_bit_p (irr_blocks
, bb
->index
))
2578 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2579 if ((include_uninstrumented_p
2580 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2581 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2583 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2584 bbs
.safe_push (e
->dest
);
2587 while (i
< bbs
.length ());
2589 if (all_region_blocks
)
2590 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2592 BITMAP_FREE (visited_blocks
);
2596 // Callback data for collect_bb2reg.
2599 vec
<tm_region
*> *bb2reg
;
2600 bool include_uninstrumented_p
;
2603 // Callback for expand_regions, collect innermost region data for each bb.
2605 collect_bb2reg (struct tm_region
*region
, void *data
)
2607 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2608 vec
<tm_region
*> *bb2reg
= stuff
->bb2reg
;
2609 vec
<basic_block
> queue
;
2613 queue
= get_tm_region_blocks (region
->entry_block
,
2614 region
->exit_blocks
,
2617 /*stop_at_irr_p=*/true,
2618 stuff
->include_uninstrumented_p
);
2620 // We expect expand_region to perform a post-order traversal of the region
2621 // tree. Therefore the last region seen for any bb is the innermost.
2622 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2623 (*bb2reg
)[bb
->index
] = region
;
2629 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2630 // which a basic block belongs. Note that we only consider the instrumented
2631 // code paths for the region; the uninstrumented code paths are ignored if
2632 // INCLUDE_UNINSTRUMENTED_P is false.
2634 // ??? This data is very similar to the bb_regions array that is collected
2635 // during tm_region_init. Or, rather, this data is similar to what could
2636 // be used within tm_region_init. The actual computation in tm_region_init
2637 // begins and ends with bb_regions entirely full of NULL pointers, due to
2638 // the way in which pointers are swapped in and out of the array.
2640 // ??? Our callers expect that blocks are not shared between transactions.
2641 // When the optimizers get too smart, and blocks are shared, then during
2642 // the tm_mark phase we'll add log entries to only one of the two transactions,
2643 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2644 // cycles. The symptom being SSA defs that do not dominate their uses.
2645 // Note that the optimizers were locally correct with their transformation,
2646 // as we have no info within the program that suggests that the blocks cannot
2649 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2650 // only known instance of this block sharing.
2652 static vec
<tm_region
*>
2653 get_bb_regions_instrumented (bool traverse_clones
,
2654 bool include_uninstrumented_p
)
2656 unsigned n
= last_basic_block_for_fn (cfun
);
2657 struct bb2reg_stuff stuff
;
2658 vec
<tm_region
*> ret
;
2661 ret
.safe_grow_cleared (n
);
2662 stuff
.bb2reg
= &ret
;
2663 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2664 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2669 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2673 compute_transaction_bits (void)
2675 struct tm_region
*region
;
2676 vec
<basic_block
> queue
;
2680 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2681 certainly don't need it to calculate CDI_DOMINATOR info. */
2684 FOR_EACH_BB_FN (bb
, cfun
)
2685 bb
->flags
&= ~BB_IN_TRANSACTION
;
2687 for (region
= all_tm_regions
; region
; region
= region
->next
)
2689 queue
= get_tm_region_blocks (region
->entry_block
,
2690 region
->exit_blocks
,
2693 /*stop_at_irr_p=*/true);
2694 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2695 bb
->flags
|= BB_IN_TRANSACTION
;
2700 bitmap_obstack_release (&tm_obstack
);
2703 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2704 call to BUILT_IN_TM_START. */
2707 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2709 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2710 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2711 tree tm_state
= region
->tm_state
;
2712 tree tm_state_type
= TREE_TYPE (tm_state
);
2713 edge abort_edge
= NULL
;
2714 edge inst_edge
= NULL
;
2715 edge uninst_edge
= NULL
;
2716 edge fallthru_edge
= NULL
;
2718 // Identify the various successors of the transaction start.
2722 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2724 if (e
->flags
& EDGE_TM_ABORT
)
2726 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2730 if (e
->flags
& EDGE_FALLTHRU
)
2735 /* ??? There are plenty of bits here we're not computing. */
2737 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2739 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2740 flags
|= PR_DOESGOIRREVOCABLE
;
2741 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2742 flags
|= PR_HASNOIRREVOCABLE
;
2743 /* If the transaction does not have an abort in lexical scope and is not
2744 marked as an outer transaction, then it will never abort. */
2745 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2746 flags
|= PR_HASNOABORT
;
2747 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2748 flags
|= PR_READONLY
;
2749 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2750 flags
|= PR_INSTRUMENTEDCODE
;
2752 flags
|= PR_UNINSTRUMENTEDCODE
;
2753 if (subcode
& GTMA_IS_OUTER
)
2754 region
->original_transaction_was_outer
= true;
2755 tree t
= build_int_cst (tm_state_type
, flags
);
2756 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2757 gimple_call_set_lhs (call
, tm_state
);
2758 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2760 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2761 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2762 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2763 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2764 gsi_remove (&gsi
, true);
2765 region
->transaction_stmt
= call
;
2768 // Generate log saves.
2769 if (!tm_log_save_addresses
.is_empty ())
2770 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2772 // In the beginning, we've no tests to perform on transaction restart.
2773 // Note that after this point, transaction_bb becomes the "most recent
2774 // block containing tests for the transaction".
2775 region
->restart_block
= region
->entry_block
;
2777 // Generate log restores.
2778 if (!tm_log_save_addresses
.is_empty ())
2780 basic_block test_bb
= create_empty_bb (transaction_bb
);
2781 basic_block code_bb
= create_empty_bb (test_bb
);
2782 basic_block join_bb
= create_empty_bb (code_bb
);
2783 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2784 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2785 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2786 if (region
->restart_block
== region
->entry_block
)
2787 region
->restart_block
= test_bb
;
2789 tree t1
= create_tmp_reg (tm_state_type
);
2790 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2791 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2792 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2793 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2795 t2
= build_int_cst (tm_state_type
, 0);
2796 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2797 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2799 tm_log_emit_restores (region
->entry_block
, code_bb
);
2801 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2802 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2803 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2804 redirect_edge_pred (fallthru_edge
, join_bb
);
2806 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2807 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2809 ei
->probability
= PROB_ALWAYS
;
2810 et
->probability
= PROB_LIKELY
;
2811 ef
->probability
= PROB_UNLIKELY
;
2812 et
->count
= apply_probability (test_bb
->count
, et
->probability
);
2813 ef
->count
= apply_probability (test_bb
->count
, ef
->probability
);
2815 code_bb
->count
= et
->count
;
2816 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2818 transaction_bb
= join_bb
;
2821 // If we have an ABORT edge, create a test to perform the abort.
2824 basic_block test_bb
= create_empty_bb (transaction_bb
);
2825 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2826 if (region
->restart_block
== region
->entry_block
)
2827 region
->restart_block
= test_bb
;
2829 tree t1
= create_tmp_reg (tm_state_type
);
2830 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2831 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2832 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2833 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2835 t2
= build_int_cst (tm_state_type
, 0);
2836 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2837 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2839 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2840 test_bb
->frequency
= transaction_bb
->frequency
;
2841 test_bb
->count
= transaction_bb
->count
;
2842 ei
->probability
= PROB_ALWAYS
;
2844 // Not abort edge. If both are live, chose one at random as we'll
2845 // we'll be fixing that up below.
2846 redirect_edge_pred (fallthru_edge
, test_bb
);
2847 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2848 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2849 fallthru_edge
->count
2850 = apply_probability (test_bb
->count
, fallthru_edge
->probability
);
2853 redirect_edge_pred (abort_edge
, test_bb
);
2854 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2855 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2857 = apply_probability (test_bb
->count
, abort_edge
->probability
);
2859 transaction_bb
= test_bb
;
2862 // If we have both instrumented and uninstrumented code paths, select one.
2863 if (inst_edge
&& uninst_edge
)
2865 basic_block test_bb
= create_empty_bb (transaction_bb
);
2866 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2867 if (region
->restart_block
== region
->entry_block
)
2868 region
->restart_block
= test_bb
;
2870 tree t1
= create_tmp_reg (tm_state_type
);
2871 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2873 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2874 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2875 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2877 t2
= build_int_cst (tm_state_type
, 0);
2878 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2879 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2881 // Create the edge into test_bb first, as we want to copy values
2882 // out of the fallthru edge.
2883 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2884 e
->probability
= fallthru_edge
->probability
;
2885 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2886 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2888 // Now update the edges to the inst/uninist implementations.
2889 // For now assume that the paths are equally likely. When using HTM,
2890 // we'll try the uninst path first and fallback to inst path if htm
2891 // buffers are exceeded. Without HTM we start with the inst path and
2892 // use the uninst path when falling back to serial mode.
2893 redirect_edge_pred (inst_edge
, test_bb
);
2894 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2895 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2897 = apply_probability (test_bb
->count
, inst_edge
->probability
);
2899 redirect_edge_pred (uninst_edge
, test_bb
);
2900 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2901 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2903 = apply_probability (test_bb
->count
, uninst_edge
->probability
);
2906 // If we have no previous special cases, and we have PHIs at the beginning
2907 // of the atomic region, this means we have a loop at the beginning of the
2908 // atomic region that shares the first block. This can cause problems with
2909 // the transaction restart abnormal edges to be added in the tm_edges pass.
2910 // Solve this by adding a new empty block to receive the abnormal edges.
2911 if (region
->restart_block
== region
->entry_block
2912 && phi_nodes (region
->entry_block
))
2914 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2915 region
->restart_block
= empty_bb
;
2916 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2918 redirect_edge_pred (fallthru_edge
, empty_bb
);
2919 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2925 /* Generate the temporary to be used for the return value of
2926 BUILT_IN_TM_START. */
2929 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2931 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2933 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2935 // Reset the subcode, post optimizations. We'll fill this in
2936 // again as we process blocks.
2937 if (region
->exit_blocks
)
2939 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2940 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
2942 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2943 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2944 | GTMA_MAY_ENTER_IRREVOCABLE
2945 | GTMA_HAS_NO_INSTRUMENTATION
);
2947 subcode
&= GTMA_DECLARATION_MASK
;
2948 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
2954 // Propagate flags from inner transactions outwards.
2956 propagate_tm_flags_out (struct tm_region
*region
)
2960 propagate_tm_flags_out (region
->inner
);
2962 if (region
->outer
&& region
->outer
->transaction_stmt
)
2965 = gimple_transaction_subcode (region
->get_transaction_stmt ());
2966 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2967 | GTMA_MAY_ENTER_IRREVOCABLE
);
2968 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
2969 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
2973 propagate_tm_flags_out (region
->next
);
2976 /* Entry point to the MARK phase of TM expansion. Here we replace
2977 transactional memory statements with calls to builtins, and function
2978 calls with their transactional clones (if available). But we don't
2979 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2982 execute_tm_mark (void)
2984 pending_edge_inserts_p
= false;
2986 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
2987 /*traverse_clones=*/true);
2991 vec
<tm_region
*> bb_regions
2992 = get_bb_regions_instrumented (/*traverse_clones=*/true,
2993 /*include_uninstrumented_p=*/false);
2994 struct tm_region
*r
;
2997 // Expand memory operations into calls into the runtime.
2998 // This collects log entries as well.
2999 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3003 if (r
->transaction_stmt
)
3006 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3008 /* If we're sure to go irrevocable, there won't be
3009 anything to expand, since the run-time will go
3010 irrevocable right away. */
3011 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3012 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3015 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3019 bb_regions
.release ();
3021 // Propagate flags from inner transactions outwards.
3022 propagate_tm_flags_out (all_tm_regions
);
3024 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3025 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3026 /*traverse_clones=*/false);
3031 if (pending_edge_inserts_p
)
3032 gsi_commit_edge_inserts ();
3033 free_dominance_info (CDI_DOMINATORS
);
3039 const pass_data pass_data_tm_mark
=
3041 GIMPLE_PASS
, /* type */
3042 "tmmark", /* name */
3043 OPTGROUP_NONE
, /* optinfo_flags */
3044 TV_TRANS_MEM
, /* tv_id */
3045 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3046 0, /* properties_provided */
3047 0, /* properties_destroyed */
3048 0, /* todo_flags_start */
3049 TODO_update_ssa
, /* todo_flags_finish */
3052 class pass_tm_mark
: public gimple_opt_pass
3055 pass_tm_mark (gcc::context
*ctxt
)
3056 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3059 /* opt_pass methods: */
3060 virtual unsigned int execute (function
*) { return execute_tm_mark (); }
3062 }; // class pass_tm_mark
3067 make_pass_tm_mark (gcc::context
*ctxt
)
3069 return new pass_tm_mark (ctxt
);
3073 /* Create an abnormal edge from STMT at iter, splitting the block
3074 as necessary. Adjust *PNEXT as needed for the split block. */
3077 split_bb_make_tm_edge (gimple
*stmt
, basic_block dest_bb
,
3078 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3080 basic_block bb
= gimple_bb (stmt
);
3081 if (!gsi_one_before_end_p (iter
))
3083 edge e
= split_block (bb
, stmt
);
3084 *pnext
= gsi_start_bb (e
->dest
);
3086 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3088 // Record the need for the edge for the benefit of the rtl passes.
3089 if (cfun
->gimple_df
->tm_restart
== NULL
)
3090 cfun
->gimple_df
->tm_restart
3091 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3093 struct tm_restart_node dummy
;
3095 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3097 tm_restart_node
**slot
= cfun
->gimple_df
->tm_restart
->find_slot (&dummy
,
3099 struct tm_restart_node
*n
= *slot
;
3102 n
= ggc_alloc
<tm_restart_node
> ();
3107 tree old
= n
->label_or_list
;
3108 if (TREE_CODE (old
) == LABEL_DECL
)
3109 old
= tree_cons (NULL
, old
, NULL
);
3110 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3114 /* Split block BB as necessary for every builtin function we added, and
3115 wire up the abnormal back edges implied by the transaction restart. */
3118 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3120 gimple_stmt_iterator gsi
, next_gsi
;
3122 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3124 gimple
*stmt
= gsi_stmt (gsi
);
3128 gsi_next (&next_gsi
);
3130 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3131 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3133 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3136 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt
))
3137 == BUILT_IN_TM_ABORT
)
3139 // If we have a ``_transaction_cancel [[outer]]'', there is only
3140 // one abnormal edge: to the transaction marked OUTER.
3141 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3142 // constant argument, which we can examine here. Users invoking
3143 // TM_ABORT directly get what they deserve.
3144 tree arg
= gimple_call_arg (call_stmt
, 0);
3145 if (TREE_CODE (arg
) == INTEGER_CST
3146 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3147 && !decl_is_tm_clone (current_function_decl
))
3149 // Find the GTMA_IS_OUTER transaction.
3150 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3151 if (o
->original_transaction_was_outer
)
3153 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3158 // Otherwise, the front-end should have semantically checked
3159 // outer aborts, but in either case the target region is not
3160 // within this function.
3164 // Non-outer, TM aborts have an abnormal edge to the inner-most
3165 // transaction, the one being aborted;
3166 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3170 // All TM builtins have an abnormal edge to the outer-most transaction.
3171 // We never restart inner transactions. For tm clones, we know a-priori
3172 // that the outer-most transaction is outside the function.
3173 if (decl_is_tm_clone (current_function_decl
))
3176 if (cfun
->gimple_df
->tm_restart
== NULL
)
3177 cfun
->gimple_df
->tm_restart
3178 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3180 // All TM builtins have an abnormal edge to the outer-most transaction.
3181 // We never restart inner transactions.
3182 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3185 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3189 // Delete any tail-call annotation that may have been added.
3190 // The tail-call pass may have mis-identified the commit as being
3191 // a candidate because we had not yet added this restart edge.
3192 gimple_call_set_tail (call_stmt
, false);
3196 /* Entry point to the final expansion of transactional nodes. */
3200 const pass_data pass_data_tm_edges
=
3202 GIMPLE_PASS
, /* type */
3203 "tmedge", /* name */
3204 OPTGROUP_NONE
, /* optinfo_flags */
3205 TV_TRANS_MEM
, /* tv_id */
3206 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3207 0, /* properties_provided */
3208 0, /* properties_destroyed */
3209 0, /* todo_flags_start */
3210 TODO_update_ssa
, /* todo_flags_finish */
3213 class pass_tm_edges
: public gimple_opt_pass
3216 pass_tm_edges (gcc::context
*ctxt
)
3217 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3220 /* opt_pass methods: */
3221 virtual unsigned int execute (function
*);
3223 }; // class pass_tm_edges
3226 pass_tm_edges::execute (function
*fun
)
3228 vec
<tm_region
*> bb_regions
3229 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3230 /*include_uninstrumented_p=*/true);
3231 struct tm_region
*r
;
3234 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3236 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3238 bb_regions
.release ();
3240 /* We've got to release the dominance info now, to indicate that it
3241 must be rebuilt completely. Otherwise we'll crash trying to update
3242 the SSA web in the TODO section following this pass. */
3243 free_dominance_info (CDI_DOMINATORS
);
3244 bitmap_obstack_release (&tm_obstack
);
3245 all_tm_regions
= NULL
;
3253 make_pass_tm_edges (gcc::context
*ctxt
)
3255 return new pass_tm_edges (ctxt
);
3258 /* Helper function for expand_regions. Expand REGION and recurse to
3259 the inner region. Call CALLBACK on each region. CALLBACK returns
3260 NULL to continue the traversal, otherwise a non-null value which
3261 this function will return as well. TRAVERSE_CLONES is true if we
3262 should traverse transactional clones. */
3265 expand_regions_1 (struct tm_region
*region
,
3266 void *(*callback
)(struct tm_region
*, void *),
3268 bool traverse_clones
)
3270 void *retval
= NULL
;
3271 if (region
->exit_blocks
3272 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3274 retval
= callback (region
, data
);
3280 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3287 /* Traverse the regions enclosed and including REGION. Execute
3288 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3289 continue the traversal, otherwise a non-null value which this
3290 function will return as well. TRAVERSE_CLONES is true if we should
3291 traverse transactional clones. */
3294 expand_regions (struct tm_region
*region
,
3295 void *(*callback
)(struct tm_region
*, void *),
3297 bool traverse_clones
)
3299 void *retval
= NULL
;
3302 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3305 region
= region
->next
;
3311 /* A unique TM memory operation. */
3314 /* Unique ID that all memory operations to the same location have. */
3315 unsigned int value_id
;
3316 /* Address of load/store. */
3320 /* TM memory operation hashtable helpers. */
3322 struct tm_memop_hasher
: free_ptr_hash
<tm_memop
>
3324 static inline hashval_t
hash (const tm_memop
*);
3325 static inline bool equal (const tm_memop
*, const tm_memop
*);
3328 /* Htab support. Return a hash value for a `tm_memop'. */
3330 tm_memop_hasher::hash (const tm_memop
*mem
)
3332 tree addr
= mem
->addr
;
3333 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3334 actually done with operand_equal_p (see tm_memop_eq). */
3335 if (TREE_CODE (addr
) == ADDR_EXPR
)
3336 addr
= TREE_OPERAND (addr
, 0);
3337 return iterative_hash_expr (addr
, 0);
3340 /* Htab support. Return true if two tm_memop's are the same. */
3342 tm_memop_hasher::equal (const tm_memop
*mem1
, const tm_memop
*mem2
)
3344 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3347 /* Sets for solving data flow equations in the memory optimization pass. */
3348 struct tm_memopt_bitmaps
3350 /* Stores available to this BB upon entry. Basically, stores that
3351 dominate this BB. */
3352 bitmap store_avail_in
;
3353 /* Stores available at the end of this BB. */
3354 bitmap store_avail_out
;
3355 bitmap store_antic_in
;
3356 bitmap store_antic_out
;
3357 /* Reads available to this BB upon entry. Basically, reads that
3358 dominate this BB. */
3359 bitmap read_avail_in
;
3360 /* Reads available at the end of this BB. */
3361 bitmap read_avail_out
;
3362 /* Reads performed in this BB. */
3364 /* Writes performed in this BB. */
3367 /* Temporary storage for pass. */
3368 /* Is the current BB in the worklist? */
3369 bool avail_in_worklist_p
;
3370 /* Have we visited this BB? */
3374 static bitmap_obstack tm_memopt_obstack
;
3376 /* Unique counter for TM loads and stores. Loads and stores of the
3377 same address get the same ID. */
3378 static unsigned int tm_memopt_value_id
;
3379 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3381 #define STORE_AVAIL_IN(BB) \
3382 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3383 #define STORE_AVAIL_OUT(BB) \
3384 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3385 #define STORE_ANTIC_IN(BB) \
3386 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3387 #define STORE_ANTIC_OUT(BB) \
3388 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3389 #define READ_AVAIL_IN(BB) \
3390 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3391 #define READ_AVAIL_OUT(BB) \
3392 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3393 #define READ_LOCAL(BB) \
3394 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3395 #define STORE_LOCAL(BB) \
3396 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3397 #define AVAIL_IN_WORKLIST_P(BB) \
3398 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3399 #define BB_VISITED_P(BB) \
3400 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3402 /* Given a TM load/store in STMT, return the value number for the address
3406 tm_memopt_value_number (gimple
*stmt
, enum insert_option op
)
3408 struct tm_memop tmpmem
, *mem
;
3411 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3412 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3413 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3416 else if (op
== INSERT
)
3418 mem
= XNEW (struct tm_memop
);
3420 mem
->value_id
= tm_memopt_value_id
++;
3421 mem
->addr
= tmpmem
.addr
;
3425 return mem
->value_id
;
3428 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3431 tm_memopt_accumulate_memops (basic_block bb
)
3433 gimple_stmt_iterator gsi
;
3435 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3437 gimple
*stmt
= gsi_stmt (gsi
);
3441 if (is_tm_store (stmt
))
3442 bits
= STORE_LOCAL (bb
);
3443 else if (is_tm_load (stmt
))
3444 bits
= READ_LOCAL (bb
);
3448 loc
= tm_memopt_value_number (stmt
, INSERT
);
3449 bitmap_set_bit (bits
, loc
);
3452 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3453 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3454 gimple_bb (stmt
)->index
);
3455 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3456 fprintf (dump_file
, "\n");
3461 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3464 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3468 const char *comma
= "";
3470 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3471 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3473 hash_table
<tm_memop_hasher
>::iterator hi
;
3474 struct tm_memop
*mem
= NULL
;
3476 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3477 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3478 if (mem
->value_id
== i
)
3480 gcc_assert (mem
->value_id
== i
);
3481 fprintf (dump_file
, "%s", comma
);
3483 print_generic_expr (dump_file
, mem
->addr
, 0);
3485 fprintf (dump_file
, "]\n");
3488 /* Prettily dump all of the memopt sets in BLOCKS. */
3491 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3496 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3498 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3499 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3500 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3501 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3502 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3503 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3504 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3508 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3511 tm_memopt_compute_avin (basic_block bb
)
3516 /* Seed with the AVOUT of any predecessor. */
3517 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3519 e
= EDGE_PRED (bb
, ix
);
3520 /* Make sure we have already visited this BB, and is thus
3523 If e->src->aux is NULL, this predecessor is actually on an
3524 enclosing transaction. We only care about the current
3525 transaction, so ignore it. */
3526 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3528 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3529 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3534 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3536 e
= EDGE_PRED (bb
, ix
);
3537 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3539 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3540 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3544 BB_VISITED_P (bb
) = true;
3547 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3550 tm_memopt_compute_antin (basic_block bb
)
3555 /* Seed with the ANTIC_OUT of any successor. */
3556 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3558 e
= EDGE_SUCC (bb
, ix
);
3559 /* Make sure we have already visited this BB, and is thus
3561 if (BB_VISITED_P (e
->dest
))
3563 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3568 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3570 e
= EDGE_SUCC (bb
, ix
);
3571 if (BB_VISITED_P (e
->dest
))
3572 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3575 BB_VISITED_P (bb
) = true;
3578 /* Compute the AVAIL sets for every basic block in BLOCKS.
3580 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3582 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3583 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3585 This is basically what we do in lcm's compute_available(), but here
3586 we calculate two sets of sets (one for STOREs and one for READs),
3587 and we work on a region instead of the entire CFG.
3589 REGION is the TM region.
3590 BLOCKS are the basic blocks in the region. */
3593 tm_memopt_compute_available (struct tm_region
*region
,
3594 vec
<basic_block
> blocks
)
3597 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3598 unsigned int qlen
, i
;
3602 /* Allocate a worklist array/queue. Entries are only added to the
3603 list if they were not already on the list. So the size is
3604 bounded by the number of basic blocks in the region. */
3605 qlen
= blocks
.length () - 1;
3606 qin
= qout
= worklist
=
3607 XNEWVEC (basic_block
, qlen
);
3609 /* Put every block in the region on the worklist. */
3610 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3612 /* Seed AVAIL_OUT with the LOCAL set. */
3613 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3614 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3616 AVAIL_IN_WORKLIST_P (bb
) = true;
3617 /* No need to insert the entry block, since it has an AVIN of
3618 null, and an AVOUT that has already been seeded in. */
3619 if (bb
!= region
->entry_block
)
3623 /* The entry block has been initialized with the local sets. */
3624 BB_VISITED_P (region
->entry_block
) = true;
3627 qend
= &worklist
[qlen
];
3629 /* Iterate until the worklist is empty. */
3632 /* Take the first entry off the worklist. */
3639 /* This block can be added to the worklist again if necessary. */
3640 AVAIL_IN_WORKLIST_P (bb
) = false;
3641 tm_memopt_compute_avin (bb
);
3643 /* Note: We do not add the LOCAL sets here because we already
3644 seeded the AVAIL_OUT sets with them. */
3645 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3646 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3648 && (region
->exit_blocks
== NULL
3649 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3650 /* If the out state of this block changed, then we need to add
3651 its successors to the worklist if they are not already in. */
3652 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3653 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3654 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3657 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3668 dump_tm_memopt_sets (blocks
);
3671 /* Compute ANTIC sets for every basic block in BLOCKS.
3673 We compute STORE_ANTIC_OUT as follows:
3675 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3676 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3678 REGION is the TM region.
3679 BLOCKS are the basic blocks in the region. */
3682 tm_memopt_compute_antic (struct tm_region
*region
,
3683 vec
<basic_block
> blocks
)
3686 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3691 /* Allocate a worklist array/queue. Entries are only added to the
3692 list if they were not already on the list. So the size is
3693 bounded by the number of basic blocks in the region. */
3694 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3696 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3700 /* Seed ANTIC_OUT with the LOCAL set. */
3701 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3703 /* Put every block in the region on the worklist. */
3704 AVAIL_IN_WORKLIST_P (bb
) = true;
3705 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3706 and their ANTIC_OUT has already been seeded in. */
3707 if (region
->exit_blocks
3708 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3715 /* The exit blocks have been initialized with the local sets. */
3716 if (region
->exit_blocks
)
3720 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3721 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3725 qend
= &worklist
[qlen
];
3727 /* Iterate until the worklist is empty. */
3730 /* Take the first entry off the worklist. */
3737 /* This block can be added to the worklist again if necessary. */
3738 AVAIL_IN_WORKLIST_P (bb
) = false;
3739 tm_memopt_compute_antin (bb
);
3741 /* Note: We do not add the LOCAL sets here because we already
3742 seeded the ANTIC_OUT sets with them. */
3743 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3744 && bb
!= region
->entry_block
)
3745 /* If the out state of this block changed, then we need to add
3746 its predecessors to the worklist if they are not already in. */
3747 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3748 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3751 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3762 dump_tm_memopt_sets (blocks
);
3765 /* Offsets of load variants from TM_LOAD. For example,
3766 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3767 See gtm-builtins.def. */
3768 #define TRANSFORM_RAR 1
3769 #define TRANSFORM_RAW 2
3770 #define TRANSFORM_RFW 3
3771 /* Offsets of store variants from TM_STORE. */
3772 #define TRANSFORM_WAR 1
3773 #define TRANSFORM_WAW 2
3775 /* Inform about a load/store optimization. */
3778 dump_tm_memopt_transform (gimple
*stmt
)
3782 fprintf (dump_file
, "TM memopt: transforming: ");
3783 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3784 fprintf (dump_file
, "\n");
3788 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3789 by a builtin that is OFFSET entries down in the builtins table in
3790 gtm-builtins.def. */
3793 tm_memopt_transform_stmt (unsigned int offset
,
3795 gimple_stmt_iterator
*gsi
)
3797 tree fn
= gimple_call_fn (stmt
);
3798 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3799 TREE_OPERAND (fn
, 0)
3800 = builtin_decl_explicit ((enum built_in_function
)
3801 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3803 gimple_call_set_fn (stmt
, fn
);
3804 gsi_replace (gsi
, stmt
, true);
3805 dump_tm_memopt_transform (stmt
);
3808 /* Perform the actual TM memory optimization transformations in the
3809 basic blocks in BLOCKS. */
3812 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3816 gimple_stmt_iterator gsi
;
3818 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3820 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3822 gimple
*stmt
= gsi_stmt (gsi
);
3823 bitmap read_avail
= READ_AVAIL_IN (bb
);
3824 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3825 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3828 if (is_tm_simple_load (stmt
))
3830 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3831 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3832 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3833 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3834 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3836 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3837 bitmap_set_bit (store_avail
, loc
);
3839 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3840 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3842 bitmap_set_bit (read_avail
, loc
);
3844 else if (is_tm_simple_store (stmt
))
3846 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3847 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3848 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3849 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3852 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3853 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3854 bitmap_set_bit (store_avail
, loc
);
3861 /* Return a new set of bitmaps for a BB. */
3863 static struct tm_memopt_bitmaps
*
3864 tm_memopt_init_sets (void)
3866 struct tm_memopt_bitmaps
*b
3867 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3868 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3869 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3870 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3871 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3872 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3873 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3874 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3875 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3876 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3880 /* Free sets computed for each BB. */
3883 tm_memopt_free_sets (vec
<basic_block
> blocks
)
3888 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3892 /* Clear the visited bit for every basic block in BLOCKS. */
3895 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
3900 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3901 BB_VISITED_P (bb
) = false;
3904 /* Replace TM load/stores with hints for the runtime. We handle
3905 things like read-after-write, write-after-read, read-after-read,
3906 read-for-write, etc. */
3909 execute_tm_memopt (void)
3911 struct tm_region
*region
;
3912 vec
<basic_block
> bbs
;
3914 tm_memopt_value_id
= 0;
3915 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
3917 for (region
= all_tm_regions
; region
; region
= region
->next
)
3919 /* All the TM stores/loads in the current region. */
3923 bitmap_obstack_initialize (&tm_memopt_obstack
);
3925 /* Save all BBs for the current region. */
3926 bbs
= get_tm_region_blocks (region
->entry_block
,
3927 region
->exit_blocks
,
3932 /* Collect all the memory operations. */
3933 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
3935 bb
->aux
= tm_memopt_init_sets ();
3936 tm_memopt_accumulate_memops (bb
);
3939 /* Solve data flow equations and transform each block accordingly. */
3940 tm_memopt_clear_visited (bbs
);
3941 tm_memopt_compute_available (region
, bbs
);
3942 tm_memopt_clear_visited (bbs
);
3943 tm_memopt_compute_antic (region
, bbs
);
3944 tm_memopt_transform_blocks (bbs
);
3946 tm_memopt_free_sets (bbs
);
3948 bitmap_obstack_release (&tm_memopt_obstack
);
3949 tm_memopt_value_numbers
->empty ();
3952 delete tm_memopt_value_numbers
;
3953 tm_memopt_value_numbers
= NULL
;
3959 const pass_data pass_data_tm_memopt
=
3961 GIMPLE_PASS
, /* type */
3962 "tmmemopt", /* name */
3963 OPTGROUP_NONE
, /* optinfo_flags */
3964 TV_TRANS_MEM
, /* tv_id */
3965 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3966 0, /* properties_provided */
3967 0, /* properties_destroyed */
3968 0, /* todo_flags_start */
3969 0, /* todo_flags_finish */
3972 class pass_tm_memopt
: public gimple_opt_pass
3975 pass_tm_memopt (gcc::context
*ctxt
)
3976 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
3979 /* opt_pass methods: */
3980 virtual bool gate (function
*) { return flag_tm
&& optimize
> 0; }
3981 virtual unsigned int execute (function
*) { return execute_tm_memopt (); }
3983 }; // class pass_tm_memopt
3988 make_pass_tm_memopt (gcc::context
*ctxt
)
3990 return new pass_tm_memopt (ctxt
);
3994 /* Interprocedual analysis for the creation of transactional clones.
3995 The aim of this pass is to find which functions are referenced in
3996 a non-irrevocable transaction context, and for those over which
3997 we have control (or user directive), create a version of the
3998 function which uses only the transactional interface to reference
3999 protected memories. This analysis proceeds in several steps:
4001 (1) Collect the set of all possible transactional clones:
4003 (a) For all local public functions marked tm_callable, push
4004 it onto the tm_callee queue.
4006 (b) For all local functions, scan for calls in transaction blocks.
4007 Push the caller and callee onto the tm_caller and tm_callee
4008 queues. Count the number of callers for each callee.
4010 (c) For each local function on the callee list, assume we will
4011 create a transactional clone. Push *all* calls onto the
4012 callee queues; count the number of clone callers separately
4013 to the number of original callers.
4015 (2) Propagate irrevocable status up the dominator tree:
4017 (a) Any external function on the callee list that is not marked
4018 tm_callable is irrevocable. Push all callers of such onto
4021 (b) For each function on the worklist, mark each block that
4022 contains an irrevocable call. Use the AND operator to
4023 propagate that mark up the dominator tree.
4025 (c) If we reach the entry block for a possible transactional
4026 clone, then the transactional clone is irrevocable, and
4027 we should not create the clone after all. Push all
4028 callers onto the worklist.
4030 (d) Place tm_irrevocable calls at the beginning of the relevant
4031 blocks. Special case here is the entry block for the entire
4032 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4033 the library to begin the region in serial mode. Decrement
4034 the call count for all callees in the irrevocable region.
4036 (3) Create the transactional clones:
4038 Any tm_callee that still has a non-zero call count is cloned.
4041 /* This structure is stored in the AUX field of each cgraph_node. */
4042 struct tm_ipa_cg_data
4044 /* The clone of the function that got created. */
4045 struct cgraph_node
*clone
;
4047 /* The tm regions in the normal function. */
4048 struct tm_region
*all_tm_regions
;
4050 /* The blocks of the normal/clone functions that contain irrevocable
4051 calls, or blocks that are post-dominated by irrevocable calls. */
4052 bitmap irrevocable_blocks_normal
;
4053 bitmap irrevocable_blocks_clone
;
4055 /* The blocks of the normal function that are involved in transactions. */
4056 bitmap transaction_blocks_normal
;
4058 /* The number of callers to the transactional clone of this function
4059 from normal and transactional clones respectively. */
4060 unsigned tm_callers_normal
;
4061 unsigned tm_callers_clone
;
4063 /* True if all calls to this function's transactional clone
4064 are irrevocable. Also automatically true if the function
4065 has no transactional clone. */
4066 bool is_irrevocable
;
4068 /* Flags indicating the presence of this function in various queues. */
4069 bool in_callee_queue
;
4072 /* Flags indicating the kind of scan desired while in the worklist. */
4073 bool want_irr_scan_normal
;
4076 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4078 /* Return the ipa data associated with NODE, allocating zeroed memory
4079 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4080 and set *NODE accordingly. */
4082 static struct tm_ipa_cg_data
*
4083 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4085 struct tm_ipa_cg_data
*d
;
4087 if (traverse_aliases
&& (*node
)->alias
)
4088 *node
= (*node
)->get_alias_target ();
4090 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4094 d
= (struct tm_ipa_cg_data
*)
4095 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4096 (*node
)->aux
= (void *) d
;
4097 memset (d
, 0, sizeof (*d
));
4103 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4104 it is already present. */
4107 maybe_push_queue (struct cgraph_node
*node
,
4108 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4113 queue_p
->safe_push (node
);
4117 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
4118 code path. QUEUE are the basic blocks inside the transaction
4119 represented in REGION.
4121 Later in split_code_paths() we will add the conditional to choose
4122 between the two alternatives. */
4125 ipa_uninstrument_transaction (struct tm_region
*region
,
4126 vec
<basic_block
> queue
)
4128 gimple
*transaction
= region
->transaction_stmt
;
4129 basic_block transaction_bb
= gimple_bb (transaction
);
4130 int n
= queue
.length ();
4131 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
4133 copy_bbs (queue
.address (), n
, new_bbs
, NULL
, 0, NULL
, NULL
, transaction_bb
,
4135 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
4136 add_phi_args_after_copy (new_bbs
, n
, e
);
4138 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
4139 // a) EDGE_FALLTHRU into the transaction
4140 // b) EDGE_TM_ABORT out of the transaction
4141 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
4146 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4147 Queue all callees within block BB. */
4150 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4151 basic_block bb
, bool for_clone
)
4153 gimple_stmt_iterator gsi
;
4155 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4157 gimple
*stmt
= gsi_stmt (gsi
);
4158 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4160 tree fndecl
= gimple_call_fndecl (stmt
);
4163 struct tm_ipa_cg_data
*d
;
4165 struct cgraph_node
*node
;
4167 if (is_tm_ending_fndecl (fndecl
))
4169 if (find_tm_replacement_function (fndecl
))
4172 node
= cgraph_node::get (fndecl
);
4173 gcc_assert (node
!= NULL
);
4174 d
= get_cg_data (&node
, true);
4176 pcallers
= (for_clone
? &d
->tm_callers_clone
4177 : &d
->tm_callers_normal
);
4180 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4186 /* Scan all calls in NODE that are within a transaction region,
4187 and push the resulting nodes into the callee queue. */
4190 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4191 cgraph_node_queue
*callees_p
)
4193 struct tm_region
*r
;
4195 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4196 d
->all_tm_regions
= all_tm_regions
;
4198 for (r
= all_tm_regions
; r
; r
= r
->next
)
4200 vec
<basic_block
> bbs
;
4204 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4205 d
->transaction_blocks_normal
, false);
4207 // Generate the uninstrumented code path for this transaction.
4208 ipa_uninstrument_transaction (r
, bbs
);
4210 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4211 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4216 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
4217 // copying them, rather than forcing us to do this externally.
4218 cgraph_edge::rebuild_edges ();
4220 // ??? In ipa_uninstrument_transaction we don't try to update dominators
4221 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
4222 // Instead, just release dominators here so update_ssa recomputes them.
4223 free_dominance_info (CDI_DOMINATORS
);
4225 // When building the uninstrumented code path, copy_bbs will have invoked
4226 // create_new_def_for starting an "ssa update context". There is only one
4227 // instance of this context, so resolve ssa updates before moving on to
4228 // the next function.
4229 update_ssa (TODO_update_ssa
);
4232 /* Scan all calls in NODE as if this is the transactional clone,
4233 and push the destinations into the callee queue. */
4236 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4237 cgraph_node_queue
*callees_p
)
4239 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4242 FOR_EACH_BB_FN (bb
, fn
)
4243 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4246 /* The function NODE has been detected to be irrevocable. Push all
4247 of its callers onto WORKLIST for the purpose of re-scanning them. */
4250 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4251 cgraph_node_queue
*worklist_p
)
4253 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4254 struct cgraph_edge
*e
;
4256 d
->is_irrevocable
= true;
4258 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4261 struct cgraph_node
*caller
;
4263 /* Don't examine recursive calls. */
4264 if (e
->caller
== node
)
4266 /* Even if we think we can go irrevocable, believe the user
4268 if (is_tm_safe_or_pure (e
->caller
->decl
))
4272 d
= get_cg_data (&caller
, true);
4274 /* Check if the callee is in a transactional region. If so,
4275 schedule the function for normal re-scan as well. */
4276 bb
= gimple_bb (e
->call_stmt
);
4277 gcc_assert (bb
!= NULL
);
4278 if (d
->transaction_blocks_normal
4279 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4280 d
->want_irr_scan_normal
= true;
4282 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4286 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4287 within the block is irrevocable. */
4290 ipa_tm_scan_irr_block (basic_block bb
)
4292 gimple_stmt_iterator gsi
;
4295 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4297 gimple
*stmt
= gsi_stmt (gsi
);
4298 switch (gimple_code (stmt
))
4301 if (gimple_assign_single_p (stmt
))
4303 tree lhs
= gimple_assign_lhs (stmt
);
4304 tree rhs
= gimple_assign_rhs1 (stmt
);
4305 if (volatile_lvalue_p (lhs
) || volatile_lvalue_p (rhs
))
4312 tree lhs
= gimple_call_lhs (stmt
);
4313 if (lhs
&& volatile_lvalue_p (lhs
))
4316 if (is_tm_pure_call (stmt
))
4319 fn
= gimple_call_fn (stmt
);
4321 /* Functions with the attribute are by definition irrevocable. */
4322 if (is_tm_irrevocable (fn
))
4325 /* For direct function calls, go ahead and check for replacement
4326 functions, or transitive irrevocable functions. For indirect
4327 functions, we'll ask the runtime. */
4328 if (TREE_CODE (fn
) == ADDR_EXPR
)
4330 struct tm_ipa_cg_data
*d
;
4331 struct cgraph_node
*node
;
4333 fn
= TREE_OPERAND (fn
, 0);
4334 if (is_tm_ending_fndecl (fn
))
4336 if (find_tm_replacement_function (fn
))
4339 node
= cgraph_node::get (fn
);
4340 d
= get_cg_data (&node
, true);
4342 /* Return true if irrevocable, but above all, believe
4344 if (d
->is_irrevocable
4345 && !is_tm_safe_or_pure (fn
))
4352 /* ??? The Approved Method of indicating that an inline
4353 assembly statement is not relevant to the transaction
4354 is to wrap it in a __tm_waiver block. This is not
4355 yet implemented, so we can't check for it. */
4356 if (is_tm_safe (current_function_decl
))
4358 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4359 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4360 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4372 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4373 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4374 scanning past OLD_IRR or EXIT_BLOCKS. */
4377 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4378 bitmap old_irr
, bitmap exit_blocks
)
4380 bool any_new_irr
= false;
4383 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4387 basic_block bb
= pqueue
->pop ();
4389 /* Don't re-scan blocks we know already are irrevocable. */
4390 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4393 if (ipa_tm_scan_irr_block (bb
))
4395 bitmap_set_bit (new_irr
, bb
->index
);
4398 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4400 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4401 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4403 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4404 pqueue
->safe_push (e
->dest
);
4408 while (!pqueue
->is_empty ());
4410 BITMAP_FREE (visited_blocks
);
4415 /* Propagate the irrevocable property both up and down the dominator tree.
4416 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4417 TM regions; OLD_IRR are the results of a previous scan of the dominator
4418 tree which has been fully propagated; NEW_IRR is the set of new blocks
4419 which are gaining the irrevocable property during the current scan. */
4422 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4423 bitmap old_irr
, bitmap exit_blocks
)
4425 vec
<basic_block
> bbs
;
4426 bitmap all_region_blocks
;
4428 /* If this block is in the old set, no need to rescan. */
4429 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4432 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4433 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4434 all_region_blocks
, false);
4437 basic_block bb
= bbs
.pop ();
4438 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4439 bool all_son_irr
= false;
4443 /* Propagate up. If my children are, I am too, but we must have
4444 at least one child that is. */
4447 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4449 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4451 all_son_irr
= false;
4459 /* Add block to new_irr if it hasn't already been processed. */
4460 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4462 bitmap_set_bit (new_irr
, bb
->index
);
4468 /* Propagate down to everyone we immediately dominate. */
4472 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4474 son
= next_dom_son (CDI_DOMINATORS
, son
))
4476 /* Make sure block is actually in a TM region, and it
4477 isn't already in old_irr. */
4478 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4479 && bitmap_bit_p (all_region_blocks
, son
->index
))
4480 bitmap_set_bit (new_irr
, son
->index
);
4484 while (!bbs
.is_empty ());
4486 BITMAP_FREE (all_region_blocks
);
4491 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4493 gimple_stmt_iterator gsi
;
4495 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4497 gimple
*stmt
= gsi_stmt (gsi
);
4498 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4500 tree fndecl
= gimple_call_fndecl (stmt
);
4503 struct tm_ipa_cg_data
*d
;
4505 struct cgraph_node
*tnode
;
4507 if (is_tm_ending_fndecl (fndecl
))
4509 if (find_tm_replacement_function (fndecl
))
4512 tnode
= cgraph_node::get (fndecl
);
4513 d
= get_cg_data (&tnode
, true);
4515 pcallers
= (for_clone
? &d
->tm_callers_clone
4516 : &d
->tm_callers_normal
);
4518 gcc_assert (*pcallers
> 0);
4525 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4526 as well as other irrevocable actions such as inline assembly. Mark all
4527 such blocks as irrevocable and decrement the number of calls to
4528 transactional clones. Return true if, for the transactional clone, the
4529 entire function is irrevocable. */
4532 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4534 struct tm_ipa_cg_data
*d
;
4535 bitmap new_irr
, old_irr
;
4538 /* Builtin operators (operator new, and such). */
4539 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4540 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4543 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4544 calculate_dominance_info (CDI_DOMINATORS
);
4546 d
= get_cg_data (&node
, true);
4547 auto_vec
<basic_block
, 10> queue
;
4548 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4550 /* Scan each tm region, propagating irrevocable status through the tree. */
4553 old_irr
= d
->irrevocable_blocks_clone
;
4554 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4555 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4557 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4560 ret
= bitmap_bit_p (new_irr
,
4561 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4566 struct tm_region
*region
;
4568 old_irr
= d
->irrevocable_blocks_normal
;
4569 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4571 queue
.quick_push (region
->entry_block
);
4572 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4573 region
->exit_blocks
))
4574 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4575 region
->exit_blocks
);
4579 /* If we found any new irrevocable blocks, reduce the call count for
4580 transactional clones within the irrevocable blocks. Save the new
4581 set of irrevocable blocks for next time. */
4582 if (!bitmap_empty_p (new_irr
))
4584 bitmap_iterator bmi
;
4587 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4588 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4593 bitmap_ior_into (old_irr
, new_irr
);
4594 BITMAP_FREE (new_irr
);
4597 d
->irrevocable_blocks_clone
= new_irr
;
4599 d
->irrevocable_blocks_normal
= new_irr
;
4601 if (dump_file
&& new_irr
)
4604 bitmap_iterator bmi
;
4607 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4608 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4609 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4613 BITMAP_FREE (new_irr
);
4620 /* Return true if, for the transactional clone of NODE, any call
4621 may enter irrevocable mode. */
4624 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4626 struct tm_ipa_cg_data
*d
;
4630 d
= get_cg_data (&node
, true);
4632 flags
= flags_from_decl_or_type (decl
);
4634 /* Handle some TM builtins. Ordinarily these aren't actually generated
4635 at this point, but handling these functions when written in by the
4636 user makes it easier to build unit tests. */
4637 if (flags
& ECF_TM_BUILTIN
)
4640 /* Filter out all functions that are marked. */
4641 if (flags
& ECF_TM_PURE
)
4643 if (is_tm_safe (decl
))
4645 if (is_tm_irrevocable (decl
))
4647 if (is_tm_callable (decl
))
4649 if (find_tm_replacement_function (decl
))
4652 /* If we aren't seeing the final version of the function we don't
4653 know what it will contain at runtime. */
4654 if (node
->get_availability () < AVAIL_AVAILABLE
)
4657 /* If the function must go irrevocable, then of course true. */
4658 if (d
->is_irrevocable
)
4661 /* If there are any blocks marked irrevocable, then the function
4662 as a whole may enter irrevocable. */
4663 if (d
->irrevocable_blocks_clone
)
4666 /* We may have previously marked this function as tm_may_enter_irr;
4667 see pass_diagnose_tm_blocks. */
4668 if (node
->local
.tm_may_enter_irr
)
4671 /* Recurse on the main body for aliases. In general, this will
4672 result in one of the bits above being set so that we will not
4673 have to recurse next time. */
4675 return ipa_tm_mayenterirr_function (cgraph_node::get (node
->thunk
.alias
));
4677 /* What remains is unmarked local functions without items that force
4678 the function to go irrevocable. */
4682 /* Diagnose calls from transaction_safe functions to unmarked
4683 functions that are determined to not be safe. */
4686 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4688 struct cgraph_edge
*e
;
4690 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4691 if (!is_tm_callable (e
->callee
->decl
)
4692 && e
->callee
->local
.tm_may_enter_irr
)
4693 error_at (gimple_location (e
->call_stmt
),
4694 "unsafe function call %qD within "
4695 "%<transaction_safe%> function", e
->callee
->decl
);
4698 /* Diagnose call from atomic transactions to unmarked functions
4699 that are determined to not be safe. */
4702 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4703 struct tm_region
*all_tm_regions
)
4705 struct tm_region
*r
;
4707 for (r
= all_tm_regions
; r
; r
= r
->next
)
4708 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4711 /* Atomic transactions can be nested inside relaxed. */
4713 ipa_tm_diagnose_transaction (node
, r
->inner
);
4717 vec
<basic_block
> bbs
;
4718 gimple_stmt_iterator gsi
;
4722 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4723 r
->irr_blocks
, NULL
, false);
4725 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4726 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4728 gimple
*stmt
= gsi_stmt (gsi
);
4731 if (gimple_code (stmt
) == GIMPLE_ASM
)
4733 error_at (gimple_location (stmt
),
4734 "asm not allowed in atomic transaction");
4738 if (!is_gimple_call (stmt
))
4740 fndecl
= gimple_call_fndecl (stmt
);
4742 /* Indirect function calls have been diagnosed already. */
4746 /* Stop at the end of the transaction. */
4747 if (is_tm_ending_fndecl (fndecl
))
4749 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4754 /* Marked functions have been diagnosed already. */
4755 if (is_tm_pure_call (stmt
))
4757 if (is_tm_callable (fndecl
))
4760 if (cgraph_node::local_info (fndecl
)->tm_may_enter_irr
)
4761 error_at (gimple_location (stmt
),
4762 "unsafe function call %qD within "
4763 "atomic transaction", fndecl
);
4770 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4771 OLD_DECL. The returned value is a freshly malloced pointer that
4772 should be freed by the caller. */
4775 tm_mangle (tree old_asm_id
)
4777 const char *old_asm_name
;
4780 struct demangle_component
*dc
;
4783 /* Determine if the symbol is already a valid C++ mangled name. Do this
4784 even for C, which might be interfacing with C++ code via appropriately
4785 ugly identifiers. */
4786 /* ??? We could probably do just as well checking for "_Z" and be done. */
4787 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4788 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4795 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4796 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4800 old_asm_name
+= 2; /* Skip _Z */
4804 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4805 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4806 /* Don't play silly games, you! */
4809 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4810 /* I'd really like to know if we can ever be passed one of
4811 these from the C++ front end. The Logical Thing would
4812 seem that hidden-alias should be outer-most, so that we
4813 get hidden-alias of a transaction-clone and not vice-versa. */
4821 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4825 new_asm_id
= get_identifier (tm_name
);
4832 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4834 node
->mark_force_output ();
4835 node
->analyzed
= true;
4839 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4841 node
->forced_by_abi
= true;
4842 node
->analyzed
= true;
4845 /* Callback data for ipa_tm_create_version_alias. */
4846 struct create_version_alias_info
4848 struct cgraph_node
*old_node
;
4852 /* A subroutine of ipa_tm_create_version, called via
4853 cgraph_for_node_and_aliases. Create new tm clones for each of
4854 the existing aliases. */
4856 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4858 struct create_version_alias_info
*info
4859 = (struct create_version_alias_info
*)data
;
4860 tree old_decl
, new_decl
, tm_name
;
4861 struct cgraph_node
*new_node
;
4863 if (!node
->cpp_implicit_alias
)
4866 old_decl
= node
->decl
;
4867 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4868 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4869 TREE_CODE (old_decl
), tm_name
,
4870 TREE_TYPE (old_decl
));
4872 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4873 SET_DECL_RTL (new_decl
, NULL
);
4875 /* Based loosely on C++'s make_alias_for(). */
4876 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4877 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4878 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4879 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4880 DECL_EXTERNAL (new_decl
) = 0;
4881 DECL_ARTIFICIAL (new_decl
) = 1;
4882 TREE_ADDRESSABLE (new_decl
) = 1;
4883 TREE_USED (new_decl
) = 1;
4884 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4886 /* Perform the same remapping to the comdat group. */
4887 if (DECL_ONE_ONLY (new_decl
))
4888 varpool_node::get (new_decl
)->set_comdat_group
4889 (tm_mangle (decl_comdat_group_id (old_decl
)));
4891 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4892 new_node
->tm_clone
= true;
4893 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4894 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4895 /* ?? Do not traverse aliases here. */
4896 get_cg_data (&node
, false)->clone
= new_node
;
4898 record_tm_clone_pair (old_decl
, new_decl
);
4900 if (info
->old_node
->force_output
4901 || info
->old_node
->ref_list
.first_referring ())
4902 ipa_tm_mark_force_output_node (new_node
);
4903 if (info
->old_node
->forced_by_abi
)
4904 ipa_tm_mark_forced_by_abi_node (new_node
);
4908 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4909 appropriate for the transactional clone. */
4912 ipa_tm_create_version (struct cgraph_node
*old_node
)
4914 tree new_decl
, old_decl
, tm_name
;
4915 struct cgraph_node
*new_node
;
4917 old_decl
= old_node
->decl
;
4918 new_decl
= copy_node (old_decl
);
4920 /* DECL_ASSEMBLER_NAME needs to be set before we call
4921 cgraph_copy_node_for_versioning below, because cgraph_node will
4922 fill the assembler_name_hash. */
4923 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4924 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4925 SET_DECL_RTL (new_decl
, NULL
);
4926 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4928 /* Perform the same remapping to the comdat group. */
4929 if (DECL_ONE_ONLY (new_decl
))
4930 varpool_node::get (new_decl
)->set_comdat_group
4931 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
4933 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
4934 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
4935 new_node
->local
.local
= false;
4936 new_node
->externally_visible
= old_node
->externally_visible
;
4937 new_node
->lowered
= true;
4938 new_node
->tm_clone
= 1;
4939 if (!old_node
->implicit_section
)
4940 new_node
->set_section (old_node
->get_section ());
4941 get_cg_data (&old_node
, true)->clone
= new_node
;
4943 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
4945 /* Remap extern inline to static inline. */
4946 /* ??? Is it worth trying to use make_decl_one_only? */
4947 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4949 DECL_EXTERNAL (new_decl
) = 0;
4950 TREE_PUBLIC (new_decl
) = 0;
4951 DECL_WEAK (new_decl
) = 0;
4954 tree_function_versioning (old_decl
, new_decl
,
4959 record_tm_clone_pair (old_decl
, new_decl
);
4961 symtab
->call_cgraph_insertion_hooks (new_node
);
4962 if (old_node
->force_output
4963 || old_node
->ref_list
.first_referring ())
4964 ipa_tm_mark_force_output_node (new_node
);
4965 if (old_node
->forced_by_abi
)
4966 ipa_tm_mark_forced_by_abi_node (new_node
);
4968 /* Do the same thing, but for any aliases of the original node. */
4970 struct create_version_alias_info data
;
4971 data
.old_node
= old_node
;
4972 data
.new_decl
= new_decl
;
4973 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
4978 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4981 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4984 gimple_stmt_iterator gsi
;
4987 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4989 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4990 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4992 split_block_after_labels (bb
);
4993 gsi
= gsi_after_labels (bb
);
4994 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4996 node
->create_edge (cgraph_node::get_create
4997 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4999 compute_call_stmt_bb_frequency (node
->decl
,
5003 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5006 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5007 struct tm_region
*region
,
5008 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5010 tree gettm_fn
, ret
, old_fn
, callfn
;
5015 old_fn
= gimple_call_fn (stmt
);
5017 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5019 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5020 tree clone
= get_tm_clone_pair (fndecl
);
5022 /* By transforming the call into a TM_GETTMCLONE, we are
5023 technically taking the address of the original function and
5024 its clone. Explain this so inlining will know this function
5026 cgraph_node::get (fndecl
)->mark_address_taken () ;
5028 cgraph_node::get (clone
)->mark_address_taken ();
5031 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5032 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5033 : BUILT_IN_TM_GETTMCLONE_IRR
);
5034 ret
= create_tmp_var (ptr_type_node
);
5037 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5039 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5040 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5041 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5043 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5044 ret
= make_ssa_name (ret
, g
);
5045 gimple_call_set_lhs (g
, ret
);
5047 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5049 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, 0,
5050 compute_call_stmt_bb_frequency (node
->decl
,
5053 /* Cast return value from tm_gettmclone* into appropriate function
5055 callfn
= create_tmp_var (TREE_TYPE (old_fn
));
5056 g2
= gimple_build_assign (callfn
,
5057 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5058 callfn
= make_ssa_name (callfn
, g2
);
5059 gimple_assign_set_lhs (g2
, callfn
);
5060 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5062 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5063 which we would have derived from the decl. Failure to save
5064 this bit means we might have to split the basic block. */
5065 if (gimple_call_nothrow_p (stmt
))
5066 gimple_call_set_nothrow (stmt
, true);
5068 gimple_call_set_fn (stmt
, callfn
);
5070 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5071 for a call statement. Fix it. */
5073 tree lhs
= gimple_call_lhs (stmt
);
5074 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5076 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5080 temp
= create_tmp_reg (rettype
);
5081 gimple_call_set_lhs (stmt
, temp
);
5083 g2
= gimple_build_assign (lhs
,
5084 fold_build1 (VIEW_CONVERT_EXPR
,
5085 TREE_TYPE (lhs
), temp
));
5086 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5091 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5092 if (e
&& e
->indirect_info
)
5093 e
->indirect_info
->polymorphic
= false;
5098 /* Helper function for ipa_tm_transform_calls*. Given a call
5099 statement in GSI which resides inside transaction REGION, redirect
5100 the call to either its wrapper function, or its clone. */
5103 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5104 struct tm_region
*region
,
5105 gimple_stmt_iterator
*gsi
,
5106 bool *need_ssa_rename_p
)
5108 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5109 struct cgraph_node
*new_node
;
5110 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5111 tree fndecl
= gimple_call_fndecl (stmt
);
5113 /* For indirect calls, pass the address through the runtime. */
5116 *need_ssa_rename_p
|=
5117 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5121 /* Handle some TM builtins. Ordinarily these aren't actually generated
5122 at this point, but handling these functions when written in by the
5123 user makes it easier to build unit tests. */
5124 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5127 /* Fixup recursive calls inside clones. */
5128 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5129 for recursion but not update the call statements themselves? */
5130 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5132 gimple_call_set_fndecl (stmt
, current_function_decl
);
5136 /* If there is a replacement, use it. */
5137 fndecl
= find_tm_replacement_function (fndecl
);
5140 new_node
= cgraph_node::get_create (fndecl
);
5142 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5144 We can't do this earlier in record_tm_replacement because
5145 cgraph_remove_unreachable_nodes is called before we inject
5146 references to the node. Further, we can't do this in some
5147 nice central place in ipa_tm_execute because we don't have
5148 the exact list of wrapper functions that would be used.
5149 Marking more wrappers than necessary results in the creation
5150 of unnecessary cgraph_nodes, which can cause some of the
5151 other IPA passes to crash.
5153 We do need to mark these nodes so that we get the proper
5154 result in expand_call_tm. */
5155 /* ??? This seems broken. How is it that we're marking the
5156 CALLEE as may_enter_irr? Surely we should be marking the
5157 CALLER. Also note that find_tm_replacement_function also
5158 contains mappings into the TM runtime, e.g. memcpy. These
5159 we know won't go irrevocable. */
5160 new_node
->local
.tm_may_enter_irr
= 1;
5164 struct tm_ipa_cg_data
*d
;
5165 struct cgraph_node
*tnode
= e
->callee
;
5167 d
= get_cg_data (&tnode
, true);
5168 new_node
= d
->clone
;
5170 /* As we've already skipped pure calls and appropriate builtins,
5171 and we've already marked irrevocable blocks, if we can't come
5172 up with a static replacement, then ask the runtime. */
5173 if (new_node
== NULL
)
5175 *need_ssa_rename_p
|=
5176 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5180 fndecl
= new_node
->decl
;
5183 e
->redirect_callee (new_node
);
5184 gimple_call_set_fndecl (stmt
, fndecl
);
5187 /* Helper function for ipa_tm_transform_calls. For a given BB,
5188 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5189 redirect other calls to the generated transactional clone. */
5192 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5193 basic_block bb
, bitmap irr_blocks
)
5195 gimple_stmt_iterator gsi
;
5196 bool need_ssa_rename
= false;
5198 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5200 ipa_tm_insert_irr_call (node
, region
, bb
);
5204 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5206 gimple
*stmt
= gsi_stmt (gsi
);
5208 if (!is_gimple_call (stmt
))
5210 if (is_tm_pure_call (stmt
))
5213 /* Redirect edges to the appropriate replacement or clone. */
5214 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5217 return need_ssa_rename
;
5220 /* Walk the CFG for REGION, beginning at BB. Install calls to
5221 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5222 the generated transactional clone. */
5225 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5226 basic_block bb
, bitmap irr_blocks
)
5228 bool need_ssa_rename
= false;
5231 auto_vec
<basic_block
> queue
;
5232 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5234 queue
.safe_push (bb
);
5240 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5242 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5245 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5248 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5249 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5251 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5252 queue
.safe_push (e
->dest
);
5255 while (!queue
.is_empty ());
5257 BITMAP_FREE (visited_blocks
);
5259 return need_ssa_rename
;
5262 /* Transform the calls within the TM regions within NODE. */
5265 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5267 struct tm_ipa_cg_data
*d
;
5268 struct tm_region
*region
;
5269 bool need_ssa_rename
= false;
5271 d
= get_cg_data (&node
, true);
5273 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5274 calculate_dominance_info (CDI_DOMINATORS
);
5276 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5278 /* If we're sure to go irrevocable, don't transform anything. */
5279 if (d
->irrevocable_blocks_normal
5280 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5281 region
->entry_block
->index
))
5283 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5284 | GTMA_MAY_ENTER_IRREVOCABLE
5285 | GTMA_HAS_NO_INSTRUMENTATION
);
5290 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5291 d
->irrevocable_blocks_normal
);
5294 if (need_ssa_rename
)
5295 update_ssa (TODO_update_ssa_only_virtuals
);
5300 /* Transform the calls within the transactional clone of NODE. */
5303 ipa_tm_transform_clone (struct cgraph_node
*node
)
5305 struct tm_ipa_cg_data
*d
;
5306 bool need_ssa_rename
;
5308 d
= get_cg_data (&node
, true);
5310 /* If this function makes no calls and has no irrevocable blocks,
5311 then there's nothing to do. */
5312 /* ??? Remove non-aborting top-level transactions. */
5313 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5316 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5317 calculate_dominance_info (CDI_DOMINATORS
);
5320 ipa_tm_transform_calls (d
->clone
, NULL
,
5321 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5322 d
->irrevocable_blocks_clone
);
5324 if (need_ssa_rename
)
5325 update_ssa (TODO_update_ssa_only_virtuals
);
5330 /* Main entry point for the transactional memory IPA pass. */
5333 ipa_tm_execute (void)
5335 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5336 /* List of functions that will go irrevocable. */
5337 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5339 struct cgraph_node
*node
;
5340 struct tm_ipa_cg_data
*d
;
5341 enum availability a
;
5344 #ifdef ENABLE_CHECKING
5345 cgraph_node::verify_cgraph_nodes ();
5348 bitmap_obstack_initialize (&tm_obstack
);
5349 initialize_original_copy_tables ();
5351 /* For all local functions marked tm_callable, queue them. */
5352 FOR_EACH_DEFINED_FUNCTION (node
)
5353 if (is_tm_callable (node
->decl
)
5354 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5356 d
= get_cg_data (&node
, true);
5357 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5360 /* For all local reachable functions... */
5361 FOR_EACH_DEFINED_FUNCTION (node
)
5363 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5365 /* ... marked tm_pure, record that fact for the runtime by
5366 indicating that the pure function is its own tm_callable.
5367 No need to do this if the function's address can't be taken. */
5368 if (is_tm_pure (node
->decl
))
5370 if (!node
->local
.local
)
5371 record_tm_clone_pair (node
->decl
, node
->decl
);
5375 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5376 calculate_dominance_info (CDI_DOMINATORS
);
5378 tm_region_init (NULL
);
5381 d
= get_cg_data (&node
, true);
5383 /* Scan for calls that are in each transaction, and
5384 generate the uninstrumented code path. */
5385 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5387 /* Put it in the worklist so we can scan the function
5388 later (ipa_tm_scan_irr_function) and mark the
5389 irrevocable blocks. */
5390 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5391 d
->want_irr_scan_normal
= true;
5397 /* For every local function on the callee list, scan as if we will be
5398 creating a transactional clone, queueing all new functions we find
5400 for (i
= 0; i
< tm_callees
.length (); ++i
)
5402 node
= tm_callees
[i
];
5403 a
= node
->get_availability ();
5404 d
= get_cg_data (&node
, true);
5406 /* Put it in the worklist so we can scan the function later
5407 (ipa_tm_scan_irr_function) and mark the irrevocable
5409 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5411 /* Some callees cannot be arbitrarily cloned. These will always be
5412 irrevocable. Mark these now, so that we need not scan them. */
5413 if (is_tm_irrevocable (node
->decl
))
5414 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5415 else if (a
<= AVAIL_NOT_AVAILABLE
5416 && !is_tm_safe_or_pure (node
->decl
))
5417 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5418 else if (a
>= AVAIL_INTERPOSABLE
)
5420 if (!tree_versionable_function_p (node
->decl
))
5421 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5422 else if (!d
->is_irrevocable
)
5424 /* If this is an alias, make sure its base is queued as well.
5425 we need not scan the callees now, as the base will do. */
5428 node
= cgraph_node::get (node
->thunk
.alias
);
5429 d
= get_cg_data (&node
, true);
5430 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5434 /* Add all nodes called by this function into
5435 tm_callees as well. */
5436 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5441 /* Iterate scans until no more work to be done. Prefer not to use
5442 vec::pop because the worklist tends to follow a breadth-first
5443 search of the callgraph, which should allow convergance with a
5444 minimum number of scans. But we also don't want the worklist
5445 array to grow without bound, so we shift the array up periodically. */
5446 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5448 if (i
> 256 && i
== irr_worklist
.length () / 8)
5450 irr_worklist
.block_remove (0, i
);
5454 node
= irr_worklist
[i
];
5455 d
= get_cg_data (&node
, true);
5456 d
->in_worklist
= false;
5458 if (d
->want_irr_scan_normal
)
5460 d
->want_irr_scan_normal
= false;
5461 ipa_tm_scan_irr_function (node
, false);
5463 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5464 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5467 /* For every function on the callee list, collect the tm_may_enter_irr
5469 irr_worklist
.truncate (0);
5470 for (i
= 0; i
< tm_callees
.length (); ++i
)
5472 node
= tm_callees
[i
];
5473 if (ipa_tm_mayenterirr_function (node
))
5475 d
= get_cg_data (&node
, true);
5476 gcc_assert (d
->in_worklist
== false);
5477 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5481 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5482 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5484 struct cgraph_node
*caller
;
5485 struct cgraph_edge
*e
;
5486 struct ipa_ref
*ref
;
5488 if (i
> 256 && i
== irr_worklist
.length () / 8)
5490 irr_worklist
.block_remove (0, i
);
5494 node
= irr_worklist
[i
];
5495 d
= get_cg_data (&node
, true);
5496 d
->in_worklist
= false;
5497 node
->local
.tm_may_enter_irr
= true;
5499 /* Propagate back to normal callers. */
5500 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5503 if (!is_tm_safe_or_pure (caller
->decl
)
5504 && !caller
->local
.tm_may_enter_irr
)
5506 d
= get_cg_data (&caller
, true);
5507 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5511 /* Propagate back to referring aliases as well. */
5512 FOR_EACH_ALIAS (node
, ref
)
5514 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5515 if (!caller
->local
.tm_may_enter_irr
)
5517 /* ?? Do not traverse aliases here. */
5518 d
= get_cg_data (&caller
, false);
5519 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5524 /* Now validate all tm_safe functions, and all atomic regions in
5526 FOR_EACH_DEFINED_FUNCTION (node
)
5528 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5530 d
= get_cg_data (&node
, true);
5531 if (is_tm_safe (node
->decl
))
5532 ipa_tm_diagnose_tm_safe (node
);
5533 else if (d
->all_tm_regions
)
5534 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5537 /* Create clones. Do those that are not irrevocable and have a
5538 positive call count. Do those publicly visible functions that
5539 the user directed us to clone. */
5540 for (i
= 0; i
< tm_callees
.length (); ++i
)
5544 node
= tm_callees
[i
];
5545 if (node
->cpp_implicit_alias
)
5548 a
= node
->get_availability ();
5549 d
= get_cg_data (&node
, true);
5551 if (a
<= AVAIL_NOT_AVAILABLE
)
5552 doit
= is_tm_callable (node
->decl
);
5553 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5555 else if (!d
->is_irrevocable
5556 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5560 ipa_tm_create_version (node
);
5563 /* Redirect calls to the new clones, and insert irrevocable marks. */
5564 for (i
= 0; i
< tm_callees
.length (); ++i
)
5566 node
= tm_callees
[i
];
5569 d
= get_cg_data (&node
, true);
5571 ipa_tm_transform_clone (node
);
5574 FOR_EACH_DEFINED_FUNCTION (node
)
5576 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5578 d
= get_cg_data (&node
, true);
5579 if (d
->all_tm_regions
)
5580 ipa_tm_transform_transaction (node
);
5583 /* Free and clear all data structures. */
5584 tm_callees
.release ();
5585 irr_worklist
.release ();
5586 bitmap_obstack_release (&tm_obstack
);
5587 free_original_copy_tables ();
5589 FOR_EACH_FUNCTION (node
)
5592 #ifdef ENABLE_CHECKING
5593 cgraph_node::verify_cgraph_nodes ();
5601 const pass_data pass_data_ipa_tm
=
5603 SIMPLE_IPA_PASS
, /* type */
5605 OPTGROUP_NONE
, /* optinfo_flags */
5606 TV_TRANS_MEM
, /* tv_id */
5607 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5608 0, /* properties_provided */
5609 0, /* properties_destroyed */
5610 0, /* todo_flags_start */
5611 0, /* todo_flags_finish */
5614 class pass_ipa_tm
: public simple_ipa_opt_pass
5617 pass_ipa_tm (gcc::context
*ctxt
)
5618 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5621 /* opt_pass methods: */
5622 virtual bool gate (function
*) { return flag_tm
; }
5623 virtual unsigned int execute (function
*) { return ipa_tm_execute (); }
5625 }; // class pass_ipa_tm
5629 simple_ipa_opt_pass
*
5630 make_pass_ipa_tm (gcc::context
*ctxt
)
5632 return new pass_ipa_tm (ctxt
);
5635 #include "gt-trans-mem.h"