1 /* Passes for transactional memory support.
2 Copyright (C) 2008-2023 Free Software Foundation, Inc.
3 Contributed by Richard Henderson <rth@redhat.com>
4 and Aldy Hernandez <aldyh@redhat.com>.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "tree-pass.h"
34 #include "gimple-pretty-print.h"
35 #include "diagnostic-core.h"
36 #include "fold-const.h"
40 #include "gimple-iterator.h"
41 #include "gimplify-me.h"
42 #include "gimple-walk.h"
44 #include "tree-into-ssa.h"
45 #include "tree-inline.h"
48 #include "trans-mem.h"
49 #include "langhooks.h"
51 #include "tree-ssa-address.h"
52 #include "stringpool.h"
54 #include "alloc-pool.h"
55 #include "symbol-summary.h"
56 #include "symtab-thunks.h"
58 #define A_RUNINSTRUMENTEDCODE 0x0001
59 #define A_RUNUNINSTRUMENTEDCODE 0x0002
60 #define A_SAVELIVEVARIABLES 0x0004
61 #define A_RESTORELIVEVARIABLES 0x0008
62 #define A_ABORTTRANSACTION 0x0010
64 #define AR_USERABORT 0x0001
65 #define AR_USERRETRY 0x0002
66 #define AR_TMCONFLICT 0x0004
67 #define AR_EXCEPTIONBLOCKABORT 0x0008
68 #define AR_OUTERABORT 0x0010
70 #define MODE_SERIALIRREVOCABLE 0x0000
73 /* The representation of a transaction changes several times during the
74 lowering process. In the beginning, in the front-end we have the
75 GENERIC tree TRANSACTION_EXPR. For example,
83 During initial gimplification (gimplify.cc) the TRANSACTION_EXPR node is
84 trivially replaced with a GIMPLE_TRANSACTION node.
86 During pass_lower_tm, we examine the body of transactions looking
87 for aborts. Transactions that do not contain an abort may be
88 merged into an outer transaction. We also add a TRY-FINALLY node
89 to arrange for the transaction to be committed on any exit.
91 [??? Think about how this arrangement affects throw-with-commit
92 and throw-with-abort operations. In this case we want the TRY to
93 handle gotos, but not to catch any exceptions because the transaction
94 will already be closed.]
96 GIMPLE_TRANSACTION [label=NULL] {
103 __builtin___tm_abort ();
105 __builtin___tm_commit ();
109 During pass_lower_eh, we create EH regions for the transactions,
110 intermixed with the regular EH stuff. This gives us a nice persistent
111 mapping (all the way through rtl) from transactional memory operation
112 back to the transaction, which allows us to get the abnormal edges
113 correct to model transaction aborts and restarts:
115 GIMPLE_TRANSACTION [label=over]
121 __builtin___tm_abort ();
122 __builtin___tm_commit ();
125 This is the end of all_lowering_passes, and so is what is present
126 during the IPA passes, and through all of the optimization passes.
128 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
129 functions and mark functions for cloning.
131 At the end of gimple optimization, before exiting SSA form,
132 pass_tm_edges replaces statements that perform transactional
133 memory operations with the appropriate TM builtins, and swap
134 out function calls with their transactional clones. At this
135 point we introduce the abnormal transaction restart edges and
136 complete lowering of the GIMPLE_TRANSACTION node.
138 x = __builtin___tm_start (MAY_ABORT);
140 if (x & abort_transaction)
143 t0 = __builtin___tm_load (global);
145 __builtin___tm_store (&global, t1);
147 __builtin___tm_abort ();
148 __builtin___tm_commit ();
152 static void *expand_regions (struct tm_region
*,
153 void *(*callback
)(struct tm_region
*, void *),
157 /* Return the attributes we want to examine for X, or NULL if it's not
158 something we examine. We look at function types, but allow pointers
159 to function types and function decls and peek through. */
162 get_attrs_for (const_tree x
)
167 switch (TREE_CODE (x
))
170 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
176 if (TREE_CODE (x
) != POINTER_TYPE
)
182 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
188 return TYPE_ATTRIBUTES (x
);
192 /* Return true if X has been marked TM_PURE. */
195 is_tm_pure (const_tree x
)
199 switch (TREE_CODE (x
))
210 if (TREE_CODE (x
) != POINTER_TYPE
)
216 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
221 flags
= flags_from_decl_or_type (x
);
222 return (flags
& ECF_TM_PURE
) != 0;
225 /* Return true if X has been marked TM_IRREVOCABLE. */
228 is_tm_irrevocable (tree x
)
230 tree attrs
= get_attrs_for (x
);
232 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
235 /* A call to the irrevocable builtin is by definition,
237 if (TREE_CODE (x
) == ADDR_EXPR
)
238 x
= TREE_OPERAND (x
, 0);
239 if (TREE_CODE (x
) == FUNCTION_DECL
240 && fndecl_built_in_p (x
, BUILT_IN_TM_IRREVOCABLE
))
246 /* Return true if X has been marked TM_SAFE. */
249 is_tm_safe (const_tree x
)
253 tree attrs
= get_attrs_for (x
);
256 if (lookup_attribute ("transaction_safe", attrs
))
258 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
265 /* Return true if CALL is const, or tm_pure. */
268 is_tm_pure_call (gimple
*call
)
270 return (gimple_call_flags (call
) & (ECF_CONST
| ECF_TM_PURE
)) != 0;
273 /* Return true if X has been marked TM_CALLABLE. */
276 is_tm_callable (tree x
)
278 tree attrs
= get_attrs_for (x
);
281 if (lookup_attribute ("transaction_callable", attrs
))
283 if (lookup_attribute ("transaction_safe", attrs
))
285 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
291 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
294 is_tm_may_cancel_outer (tree x
)
296 tree attrs
= get_attrs_for (x
);
298 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
302 /* Return true for built in functions that "end" a transaction. */
305 is_tm_ending_fndecl (tree fndecl
)
307 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
308 switch (DECL_FUNCTION_CODE (fndecl
))
310 case BUILT_IN_TM_COMMIT
:
311 case BUILT_IN_TM_COMMIT_EH
:
312 case BUILT_IN_TM_ABORT
:
313 case BUILT_IN_TM_IRREVOCABLE
:
322 /* Return true if STMT is a built in function call that "ends" a
326 is_tm_ending (gimple
*stmt
)
330 if (gimple_code (stmt
) != GIMPLE_CALL
)
333 fndecl
= gimple_call_fndecl (stmt
);
334 return (fndecl
!= NULL_TREE
335 && is_tm_ending_fndecl (fndecl
));
338 /* Return true if STMT is a TM load. */
341 is_tm_load (gimple
*stmt
)
345 if (gimple_code (stmt
) != GIMPLE_CALL
)
348 fndecl
= gimple_call_fndecl (stmt
);
350 && fndecl_built_in_p (fndecl
, BUILT_IN_NORMAL
)
351 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
354 /* Same as above, but for simple TM loads, that is, not the
355 after-write, after-read, etc optimized variants. */
358 is_tm_simple_load (gimple
*stmt
)
362 if (gimple_code (stmt
) != GIMPLE_CALL
)
365 fndecl
= gimple_call_fndecl (stmt
);
366 if (fndecl
&& fndecl_built_in_p (fndecl
, BUILT_IN_NORMAL
))
368 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
369 return (fcode
== BUILT_IN_TM_LOAD_1
370 || fcode
== BUILT_IN_TM_LOAD_2
371 || fcode
== BUILT_IN_TM_LOAD_4
372 || fcode
== BUILT_IN_TM_LOAD_8
373 || fcode
== BUILT_IN_TM_LOAD_FLOAT
374 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
375 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
376 || fcode
== BUILT_IN_TM_LOAD_M64
377 || fcode
== BUILT_IN_TM_LOAD_M128
378 || fcode
== BUILT_IN_TM_LOAD_M256
);
383 /* Return true if STMT is a TM store. */
386 is_tm_store (gimple
*stmt
)
390 if (gimple_code (stmt
) != GIMPLE_CALL
)
393 fndecl
= gimple_call_fndecl (stmt
);
395 && fndecl_built_in_p (fndecl
, BUILT_IN_NORMAL
)
396 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
399 /* Same as above, but for simple TM stores, that is, not the
400 after-write, after-read, etc optimized variants. */
403 is_tm_simple_store (gimple
*stmt
)
407 if (gimple_code (stmt
) != GIMPLE_CALL
)
410 fndecl
= gimple_call_fndecl (stmt
);
412 && fndecl_built_in_p (fndecl
, BUILT_IN_NORMAL
))
414 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
415 return (fcode
== BUILT_IN_TM_STORE_1
416 || fcode
== BUILT_IN_TM_STORE_2
417 || fcode
== BUILT_IN_TM_STORE_4
418 || fcode
== BUILT_IN_TM_STORE_8
419 || fcode
== BUILT_IN_TM_STORE_FLOAT
420 || fcode
== BUILT_IN_TM_STORE_DOUBLE
421 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
422 || fcode
== BUILT_IN_TM_STORE_M64
423 || fcode
== BUILT_IN_TM_STORE_M128
424 || fcode
== BUILT_IN_TM_STORE_M256
);
429 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
432 is_tm_abort (tree fndecl
)
434 return (fndecl
&& fndecl_built_in_p (fndecl
, BUILT_IN_TM_ABORT
));
437 /* Build a GENERIC tree for a user abort. This is called by front ends
438 while transforming the __tm_abort statement. */
441 build_tm_abort_call (location_t loc
, bool is_outer
)
443 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
444 build_int_cst (integer_type_node
,
446 | (is_outer
? AR_OUTERABORT
: 0)));
449 /* Map for arbitrary function replacement under TM, as created
450 by the tm_wrap attribute. */
452 struct tm_wrapper_hasher
: ggc_cache_ptr_hash
<tree_map
>
454 static inline hashval_t
hash (tree_map
*m
) { return m
->hash
; }
456 equal (tree_map
*a
, tree_map
*b
)
458 return a
->base
.from
== b
->base
.from
;
462 keep_cache_entry (tree_map
*&m
)
464 return ggc_marked_p (m
->base
.from
);
468 static GTY((cache
)) hash_table
<tm_wrapper_hasher
> *tm_wrap_map
;
471 record_tm_replacement (tree from
, tree to
)
473 struct tree_map
**slot
, *h
;
475 /* Do not inline wrapper functions that will get replaced in the TM
478 Suppose you have foo() that will get replaced into tmfoo(). Make
479 sure the inliner doesn't try to outsmart us and inline foo()
480 before we get a chance to do the TM replacement. */
481 DECL_UNINLINABLE (from
) = 1;
483 if (tm_wrap_map
== NULL
)
484 tm_wrap_map
= hash_table
<tm_wrapper_hasher
>::create_ggc (32);
486 h
= ggc_alloc
<tree_map
> ();
487 h
->hash
= htab_hash_pointer (from
);
491 slot
= tm_wrap_map
->find_slot_with_hash (h
, h
->hash
, INSERT
);
495 /* Return a TM-aware replacement function for DECL. */
498 find_tm_replacement_function (tree fndecl
)
502 struct tree_map
*h
, in
;
504 in
.base
.from
= fndecl
;
505 in
.hash
= htab_hash_pointer (fndecl
);
506 h
= tm_wrap_map
->find_with_hash (&in
, in
.hash
);
511 /* ??? We may well want TM versions of most of the common <string.h>
512 functions. For now, we've already these two defined. */
513 /* Adjust expand_call_tm() attributes as necessary for the cases
515 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
516 switch (DECL_FUNCTION_CODE (fndecl
))
518 case BUILT_IN_MEMCPY
:
519 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
520 case BUILT_IN_MEMMOVE
:
521 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
522 case BUILT_IN_MEMSET
:
523 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
531 /* When appropriate, record TM replacement for memory allocation functions.
533 FROM is the FNDECL to wrap. */
535 tm_malloc_replacement (tree from
)
540 if (TREE_CODE (from
) != FUNCTION_DECL
)
543 /* If we have a previous replacement, the user must be explicitly
544 wrapping malloc/calloc/free. They better know what they're
546 if (find_tm_replacement_function (from
))
549 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
551 if (!strcmp (str
, "malloc"))
552 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
553 else if (!strcmp (str
, "calloc"))
554 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
555 else if (!strcmp (str
, "free"))
556 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
560 TREE_NOTHROW (to
) = 0;
562 record_tm_replacement (from
, to
);
565 /* Diagnostics for tm_safe functions/regions. Called by the front end
566 once we've lowered the function to high-gimple. */
568 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
569 Process exactly one statement. WI->INFO is set to non-null when in
570 the context of a tm_safe function, and null for a __transaction block. */
572 #define DIAG_TM_OUTER 1
573 #define DIAG_TM_SAFE 2
574 #define DIAG_TM_RELAXED 4
578 unsigned int summary_flags
: 8;
579 unsigned int block_flags
: 8;
580 unsigned int func_flags
: 8;
581 unsigned int saw_volatile
: 1;
585 /* Return true if T is a volatile lvalue of some kind. */
588 volatile_lvalue_p (tree t
)
590 return ((SSA_VAR_P (t
) || REFERENCE_CLASS_P (t
))
591 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
594 /* Tree callback function for diagnose_tm pass. */
597 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
599 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
600 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
603 *walk_subtrees
= false;
604 else if (volatile_lvalue_p (*tp
)
608 if (d
->block_flags
& DIAG_TM_SAFE
)
609 error_at (gimple_location (d
->stmt
),
610 "invalid use of volatile lvalue inside transaction");
611 else if (d
->func_flags
& DIAG_TM_SAFE
)
612 error_at (gimple_location (d
->stmt
),
613 "invalid use of volatile lvalue inside %<transaction_safe%> "
621 is_tm_safe_or_pure (const_tree x
)
623 return is_tm_safe (x
) || is_tm_pure (x
);
627 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
628 struct walk_stmt_info
*wi
)
630 gimple
*stmt
= gsi_stmt (*gsi
);
631 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
633 /* Save stmt for use in leaf analysis. */
636 switch (gimple_code (stmt
))
640 tree fn
= gimple_call_fn (stmt
);
642 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
643 && is_tm_may_cancel_outer (fn
))
644 error_at (gimple_location (stmt
),
645 "%<transaction_may_cancel_outer%> function call not within"
646 " outer transaction or %<transaction_may_cancel_outer%>");
648 if (d
->summary_flags
& DIAG_TM_SAFE
)
650 bool is_safe
, direct_call_p
;
653 if (TREE_CODE (fn
) == ADDR_EXPR
654 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
656 direct_call_p
= true;
657 replacement
= TREE_OPERAND (fn
, 0);
658 replacement
= find_tm_replacement_function (replacement
);
664 direct_call_p
= false;
665 replacement
= NULL_TREE
;
668 if (is_tm_safe_or_pure (fn
))
670 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
672 /* A function explicitly marked transaction_callable as
673 opposed to transaction_safe is being defined to be
674 unsafe as part of its ABI, regardless of its contents. */
677 else if (direct_call_p
)
679 if (IS_TYPE_OR_DECL_P (fn
)
680 && flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
682 else if (replacement
)
684 /* ??? At present we've been considering replacements
685 merely transaction_callable, and therefore might
686 enter irrevocable. The tm_wrap attribute has not
687 yet made it into the new language spec. */
692 /* ??? Diagnostics for unmarked direct calls moved into
693 the IPA pass. Section 3.2 of the spec details how
694 functions not marked should be considered "implicitly
695 safe" based on having examined the function body. */
701 /* An unmarked indirect call. Consider it unsafe even
702 though optimization may yet figure out how to inline. */
708 if (TREE_CODE (fn
) == ADDR_EXPR
)
709 fn
= TREE_OPERAND (fn
, 0);
710 if (d
->block_flags
& DIAG_TM_SAFE
)
713 error_at (gimple_location (stmt
),
714 "unsafe function call %qD within "
715 "atomic transaction", fn
);
718 if ((!DECL_P (fn
) || DECL_NAME (fn
))
719 && TREE_CODE (fn
) != SSA_NAME
)
720 error_at (gimple_location (stmt
),
721 "unsafe function call %qE within "
722 "atomic transaction", fn
);
724 error_at (gimple_location (stmt
),
725 "unsafe indirect function call within "
726 "atomic transaction");
732 error_at (gimple_location (stmt
),
733 "unsafe function call %qD within "
734 "%<transaction_safe%> function", fn
);
737 if ((!DECL_P (fn
) || DECL_NAME (fn
))
738 && TREE_CODE (fn
) != SSA_NAME
)
739 error_at (gimple_location (stmt
),
740 "unsafe function call %qE within "
741 "%<transaction_safe%> function", fn
);
743 error_at (gimple_location (stmt
),
744 "unsafe indirect function call within "
745 "%<transaction_safe%> function");
754 /* ??? We ought to come up with a way to add attributes to
755 asm statements, and then add "transaction_safe" to it.
756 Either that or get the language spec to resurrect __tm_waiver. */
757 if (d
->block_flags
& DIAG_TM_SAFE
)
758 error_at (gimple_location (stmt
),
759 "%<asm%> not allowed in atomic transaction");
760 else if (d
->func_flags
& DIAG_TM_SAFE
)
761 error_at (gimple_location (stmt
),
762 "%<asm%> not allowed in %<transaction_safe%> function");
765 case GIMPLE_TRANSACTION
:
767 gtransaction
*trans_stmt
= as_a
<gtransaction
*> (stmt
);
768 unsigned char inner_flags
= DIAG_TM_SAFE
;
770 if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_RELAXED
)
772 if (d
->block_flags
& DIAG_TM_SAFE
)
773 error_at (gimple_location (stmt
),
774 "relaxed transaction in atomic transaction");
775 else if (d
->func_flags
& DIAG_TM_SAFE
)
776 error_at (gimple_location (stmt
),
777 "relaxed transaction in %<transaction_safe%> function");
778 inner_flags
= DIAG_TM_RELAXED
;
780 else if (gimple_transaction_subcode (trans_stmt
) & GTMA_IS_OUTER
)
783 error_at (gimple_location (stmt
),
784 "outer transaction in transaction");
785 else if (d
->func_flags
& DIAG_TM_OUTER
)
786 error_at (gimple_location (stmt
),
787 "outer transaction in "
788 "%<transaction_may_cancel_outer%> function");
789 else if (d
->func_flags
& DIAG_TM_SAFE
)
790 error_at (gimple_location (stmt
),
791 "outer transaction in %<transaction_safe%> function");
792 inner_flags
|= DIAG_TM_OUTER
;
795 *handled_ops_p
= true;
796 if (gimple_transaction_body (trans_stmt
))
798 struct walk_stmt_info wi_inner
;
799 struct diagnose_tm d_inner
;
801 memset (&d_inner
, 0, sizeof (d_inner
));
802 d_inner
.func_flags
= d
->func_flags
;
803 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
804 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
806 memset (&wi_inner
, 0, sizeof (wi_inner
));
807 wi_inner
.info
= &d_inner
;
809 walk_gimple_seq (gimple_transaction_body (trans_stmt
),
810 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
823 diagnose_tm_blocks (void)
825 struct walk_stmt_info wi
;
826 struct diagnose_tm d
;
828 memset (&d
, 0, sizeof (d
));
829 if (is_tm_may_cancel_outer (current_function_decl
))
830 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
831 else if (is_tm_safe (current_function_decl
))
832 d
.func_flags
= DIAG_TM_SAFE
;
833 d
.summary_flags
= d
.func_flags
;
835 memset (&wi
, 0, sizeof (wi
));
838 walk_gimple_seq (gimple_body (current_function_decl
),
839 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
846 const pass_data pass_data_diagnose_tm_blocks
=
848 GIMPLE_PASS
, /* type */
849 "*diagnose_tm_blocks", /* name */
850 OPTGROUP_NONE
, /* optinfo_flags */
851 TV_TRANS_MEM
, /* tv_id */
852 PROP_gimple_any
, /* properties_required */
853 0, /* properties_provided */
854 0, /* properties_destroyed */
855 0, /* todo_flags_start */
856 0, /* todo_flags_finish */
859 class pass_diagnose_tm_blocks
: public gimple_opt_pass
862 pass_diagnose_tm_blocks (gcc::context
*ctxt
)
863 : gimple_opt_pass (pass_data_diagnose_tm_blocks
, ctxt
)
866 /* opt_pass methods: */
867 bool gate (function
*) final override
{ return flag_tm
; }
868 unsigned int execute (function
*) final override
870 return diagnose_tm_blocks ();
873 }; // class pass_diagnose_tm_blocks
878 make_pass_diagnose_tm_blocks (gcc::context
*ctxt
)
880 return new pass_diagnose_tm_blocks (ctxt
);
883 /* Instead of instrumenting thread private memory, we save the
884 addresses in a log which we later use to save/restore the addresses
885 upon transaction start/restart.
887 The log is keyed by address, where each element contains individual
888 statements among different code paths that perform the store.
890 This log is later used to generate either plain save/restore of the
891 addresses upon transaction start/restart, or calls to the ITM_L*
894 So for something like:
896 struct large { int x[1000]; };
897 struct large lala = { 0 };
903 We can either save/restore:
906 trxn = _ITM_startTransaction ();
907 if (trxn & a_saveLiveVariables)
908 tmp_lala1 = lala.x[i];
909 else if (a & a_restoreLiveVariables)
910 lala.x[i] = tmp_lala1;
912 or use the logging functions:
915 trxn = _ITM_startTransaction ();
916 _ITM_LU4 (&lala.x[i]);
918 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
919 far up the dominator tree to shadow all of the writes to a given
920 location (thus reducing the total number of logging calls), but not
921 so high as to be called on a path that does not perform a
924 /* One individual log entry. We may have multiple statements for the
925 same location if neither dominate each other (on different
929 /* Address to save. */
931 /* Entry block for the transaction this address occurs in. */
932 basic_block entry_block
;
933 /* Dominating statements the store occurs in. */
935 /* Initially, while we are building the log, we place a nonzero
936 value here to mean that this address *will* be saved with a
937 save/restore sequence. Later, when generating the save sequence
938 we place the SSA temp generated here. */
943 /* Log entry hashtable helpers. */
945 struct log_entry_hasher
: pointer_hash
<tm_log_entry
>
947 static inline hashval_t
hash (const tm_log_entry
*);
948 static inline bool equal (const tm_log_entry
*, const tm_log_entry
*);
949 static inline void remove (tm_log_entry
*);
952 /* Htab support. Return hash value for a `tm_log_entry'. */
954 log_entry_hasher::hash (const tm_log_entry
*log
)
956 return iterative_hash_expr (log
->addr
, 0);
959 /* Htab support. Return true if two log entries are the same. */
961 log_entry_hasher::equal (const tm_log_entry
*log1
, const tm_log_entry
*log2
)
965 rth: I suggest that we get rid of the component refs etc.
966 I.e. resolve the reference to base + offset.
968 We may need to actually finish a merge with mainline for this,
969 since we'd like to be presented with Richi's MEM_REF_EXPRs more
970 often than not. But in the meantime your tm_log_entry could save
971 the results of get_inner_reference.
973 See: g++.dg/tm/pr46653.C
976 /* Special case plain equality because operand_equal_p() below will
977 return FALSE if the addresses are equal but they have
978 side-effects (e.g. a volatile address). */
979 if (log1
->addr
== log2
->addr
)
982 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
985 /* Htab support. Free one tm_log_entry. */
987 log_entry_hasher::remove (tm_log_entry
*lp
)
989 lp
->stmts
.release ();
994 /* The actual log. */
995 static hash_table
<log_entry_hasher
> *tm_log
;
997 /* Addresses to log with a save/restore sequence. These should be in
999 static vec
<tree
> tm_log_save_addresses
;
1001 enum thread_memory_type
1005 mem_transaction_local
,
1009 struct tm_new_mem_map
1011 /* SSA_NAME being dereferenced. */
1013 enum thread_memory_type local_new_memory
;
1016 /* Hashtable helpers. */
1018 struct tm_mem_map_hasher
: free_ptr_hash
<tm_new_mem_map
>
1020 static inline hashval_t
hash (const tm_new_mem_map
*);
1021 static inline bool equal (const tm_new_mem_map
*, const tm_new_mem_map
*);
1025 tm_mem_map_hasher::hash (const tm_new_mem_map
*v
)
1027 return (intptr_t)v
->val
>> 4;
1031 tm_mem_map_hasher::equal (const tm_new_mem_map
*v
, const tm_new_mem_map
*c
)
1033 return v
->val
== c
->val
;
1036 /* Map for an SSA_NAME originally pointing to a non aliased new piece
1037 of memory (malloc, alloc, etc). */
1038 static hash_table
<tm_mem_map_hasher
> *tm_new_mem_hash
;
1040 /* Initialize logging data structures. */
1044 tm_log
= new hash_table
<log_entry_hasher
> (10);
1045 tm_new_mem_hash
= new hash_table
<tm_mem_map_hasher
> (5);
1046 tm_log_save_addresses
.create (5);
1049 /* Free logging data structures. */
1051 tm_log_delete (void)
1055 delete tm_new_mem_hash
;
1056 tm_new_mem_hash
= NULL
;
1057 tm_log_save_addresses
.release ();
1060 /* Return true if MEM is a transaction invariant memory for the TM
1061 region starting at REGION_ENTRY_BLOCK. */
1063 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
1065 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
1066 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
1070 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
1071 return def_bb
!= region_entry_block
1072 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
1075 mem
= strip_invariant_refs (mem
);
1076 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
1079 /* Given an address ADDR in STMT, find it in the memory log or add it,
1080 making sure to keep only the addresses highest in the dominator
1083 ENTRY_BLOCK is the entry_block for the transaction.
1085 If we find the address in the log, make sure it's either the same
1086 address, or an equivalent one that dominates ADDR.
1088 If we find the address, but neither ADDR dominates the found
1089 address, nor the found one dominates ADDR, we're on different
1090 execution paths. Add it.
1092 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1095 tm_log_add (basic_block entry_block
, tree addr
, gimple
*stmt
)
1097 tm_log_entry
**slot
;
1098 struct tm_log_entry l
, *lp
;
1101 slot
= tm_log
->find_slot (&l
, INSERT
);
1104 tree type
= TREE_TYPE (addr
);
1106 lp
= XNEW (struct tm_log_entry
);
1110 /* Small invariant addresses can be handled as save/restores. */
1112 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1113 && TYPE_SIZE_UNIT (type
) != NULL
1114 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type
))
1115 && ((HOST_WIDE_INT
) tree_to_uhwi (TYPE_SIZE_UNIT (type
))
1116 < param_tm_max_aggregate_size
)
1117 /* We must be able to copy this type normally. I.e., no
1118 special constructors and the like. */
1119 && !TREE_ADDRESSABLE (type
))
1121 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1122 lp
->stmts
.create (0);
1123 lp
->entry_block
= entry_block
;
1124 /* Save addresses separately in dominator order so we don't
1125 get confused by overlapping addresses in the save/restore
1127 tm_log_save_addresses
.safe_push (lp
->addr
);
1131 /* Use the logging functions. */
1132 lp
->stmts
.create (5);
1133 lp
->stmts
.quick_push (stmt
);
1134 lp
->save_var
= NULL
;
1144 /* If we're generating a save/restore sequence, we don't care
1145 about statements. */
1149 for (i
= 0; lp
->stmts
.iterate (i
, &oldstmt
); ++i
)
1151 if (stmt
== oldstmt
)
1153 /* We already have a store to the same address, higher up the
1154 dominator tree. Nothing to do. */
1155 if (dominated_by_p (CDI_DOMINATORS
,
1156 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1158 /* We should be processing blocks in dominator tree order. */
1159 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1160 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1162 /* Store is on a different code path. */
1163 lp
->stmts
.safe_push (stmt
);
1167 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1168 result, insert the new statements before GSI. */
1171 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1173 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1174 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1176 x
= build_fold_addr_expr (x
);
1177 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1180 /* Instrument one address with the logging functions.
1181 ADDR is the address to save.
1182 STMT is the statement before which to place it. */
1184 tm_log_emit_stmt (tree addr
, gimple
*stmt
)
1186 tree type
= TREE_TYPE (addr
);
1187 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1189 enum built_in_function code
= BUILT_IN_TM_LOG
;
1191 if (type
== float_type_node
)
1192 code
= BUILT_IN_TM_LOG_FLOAT
;
1193 else if (type
== double_type_node
)
1194 code
= BUILT_IN_TM_LOG_DOUBLE
;
1195 else if (type
== long_double_type_node
)
1196 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1197 else if (TYPE_SIZE (type
) != NULL
1198 && tree_fits_uhwi_p (TYPE_SIZE (type
)))
1200 unsigned HOST_WIDE_INT type_size
= tree_to_uhwi (TYPE_SIZE (type
));
1202 if (TREE_CODE (type
) == VECTOR_TYPE
)
1207 code
= BUILT_IN_TM_LOG_M64
;
1210 code
= BUILT_IN_TM_LOG_M128
;
1213 code
= BUILT_IN_TM_LOG_M256
;
1218 if (!builtin_decl_explicit_p (code
))
1227 code
= BUILT_IN_TM_LOG_1
;
1230 code
= BUILT_IN_TM_LOG_2
;
1233 code
= BUILT_IN_TM_LOG_4
;
1236 code
= BUILT_IN_TM_LOG_8
;
1242 if (code
!= BUILT_IN_TM_LOG
&& !builtin_decl_explicit_p (code
))
1243 code
= BUILT_IN_TM_LOG
;
1244 tree decl
= builtin_decl_explicit (code
);
1246 addr
= gimplify_addr (&gsi
, addr
);
1247 if (code
== BUILT_IN_TM_LOG
)
1248 log
= gimple_build_call (decl
, 2, addr
, TYPE_SIZE_UNIT (type
));
1250 log
= gimple_build_call (decl
, 1, addr
);
1251 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1254 /* Go through the log and instrument address that must be instrumented
1255 with the logging functions. Leave the save/restore addresses for
1260 hash_table
<log_entry_hasher
>::iterator hi
;
1261 struct tm_log_entry
*lp
;
1263 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log
, lp
, tm_log_entry_t
, hi
)
1270 fprintf (dump_file
, "TM thread private mem logging: ");
1271 print_generic_expr (dump_file
, lp
->addr
);
1272 fprintf (dump_file
, "\n");
1278 fprintf (dump_file
, "DUMPING to variable\n");
1284 fprintf (dump_file
, "DUMPING with logging functions\n");
1285 for (i
= 0; lp
->stmts
.iterate (i
, &stmt
); ++i
)
1286 tm_log_emit_stmt (lp
->addr
, stmt
);
1291 /* Emit the save sequence for the corresponding addresses in the log.
1292 ENTRY_BLOCK is the entry block for the transaction.
1293 BB is the basic block to insert the code in. */
1295 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1298 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1300 struct tm_log_entry l
, *lp
;
1302 for (i
= 0; i
< tm_log_save_addresses
.length (); ++i
)
1304 l
.addr
= tm_log_save_addresses
[i
];
1305 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1306 gcc_assert (lp
->save_var
!= NULL
);
1308 /* We only care about variables in the current transaction. */
1309 if (lp
->entry_block
!= entry_block
)
1312 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1314 /* Make sure we can create an SSA_NAME for this type. For
1315 instance, aggregates aren't allowed, in which case the system
1316 will create a VOP for us and everything will just work. */
1317 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1319 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1320 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1323 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1327 /* Emit the restore sequence for the corresponding addresses in the log.
1328 ENTRY_BLOCK is the entry block for the transaction.
1329 BB is the basic block to insert the code in. */
1331 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1334 struct tm_log_entry l
, *lp
;
1335 gimple_stmt_iterator gsi
;
1338 for (i
= tm_log_save_addresses
.length () - 1; i
>= 0; i
--)
1340 l
.addr
= tm_log_save_addresses
[i
];
1341 lp
= *(tm_log
->find_slot (&l
, NO_INSERT
));
1342 gcc_assert (lp
->save_var
!= NULL
);
1344 /* We only care about variables in the current transaction. */
1345 if (lp
->entry_block
!= entry_block
)
1348 /* Restores are in LIFO order from the saves in case we have
1350 gsi
= gsi_start_bb (bb
);
1352 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1353 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1358 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1359 struct walk_stmt_info
*);
1360 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1361 struct walk_stmt_info
*);
1363 /* Evaluate an address X being dereferenced and determine if it
1364 originally points to a non aliased new chunk of memory (malloc,
1367 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1368 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1369 Return MEM_NON_LOCAL otherwise.
1371 ENTRY_BLOCK is the entry block to the transaction containing the
1372 dereference of X. */
1373 static enum thread_memory_type
1374 thread_private_new_memory (basic_block entry_block
, tree x
)
1376 gimple
*stmt
= NULL
;
1377 enum tree_code code
;
1378 tm_new_mem_map
**slot
;
1379 tm_new_mem_map elt
, *elt_p
;
1381 enum thread_memory_type retval
= mem_transaction_local
;
1384 || TREE_CODE (x
) != SSA_NAME
1385 /* Possible uninitialized use, or a function argument. In
1386 either case, we don't care. */
1387 || SSA_NAME_IS_DEFAULT_DEF (x
))
1388 return mem_non_local
;
1390 /* Look in cache first. */
1392 slot
= tm_new_mem_hash
->find_slot (&elt
, INSERT
);
1395 return elt_p
->local_new_memory
;
1397 /* Optimistically assume the memory is transaction local during
1398 processing. This catches recursion into this variable. */
1399 *slot
= elt_p
= XNEW (tm_new_mem_map
);
1401 elt_p
->local_new_memory
= mem_transaction_local
;
1403 /* Search DEF chain to find the original definition of this address. */
1406 if (ptr_deref_may_alias_global_p (x
, true))
1408 /* Address escapes. This is not thread-private. */
1409 retval
= mem_non_local
;
1410 goto new_memory_ret
;
1413 stmt
= SSA_NAME_DEF_STMT (x
);
1415 /* If the malloc call is outside the transaction, this is
1417 if (retval
!= mem_thread_local
1418 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1419 retval
= mem_thread_local
;
1421 if (is_gimple_assign (stmt
))
1423 code
= gimple_assign_rhs_code (stmt
);
1424 /* x = foo ==> foo */
1425 if (code
== SSA_NAME
)
1426 x
= gimple_assign_rhs1 (stmt
);
1427 /* x = foo + n ==> foo */
1428 else if (code
== POINTER_PLUS_EXPR
)
1429 x
= gimple_assign_rhs1 (stmt
);
1430 /* x = (cast*) foo ==> foo */
1431 else if (code
== VIEW_CONVERT_EXPR
|| CONVERT_EXPR_CODE_P (code
))
1432 x
= gimple_assign_rhs1 (stmt
);
1433 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1434 else if (code
== COND_EXPR
)
1436 tree op1
= gimple_assign_rhs2 (stmt
);
1437 tree op2
= gimple_assign_rhs3 (stmt
);
1438 enum thread_memory_type mem
;
1439 retval
= thread_private_new_memory (entry_block
, op1
);
1440 if (retval
== mem_non_local
)
1441 goto new_memory_ret
;
1442 mem
= thread_private_new_memory (entry_block
, op2
);
1443 retval
= MIN (retval
, mem
);
1444 goto new_memory_ret
;
1448 retval
= mem_non_local
;
1449 goto new_memory_ret
;
1454 if (gimple_code (stmt
) == GIMPLE_PHI
)
1457 enum thread_memory_type mem
;
1458 tree phi_result
= gimple_phi_result (stmt
);
1460 /* If any of the ancestors are non-local, we are sure to
1461 be non-local. Otherwise we can avoid doing anything
1462 and inherit what has already been generated. */
1464 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1466 tree op
= PHI_ARG_DEF (stmt
, i
);
1468 /* Exclude self-assignment. */
1469 if (phi_result
== op
)
1472 mem
= thread_private_new_memory (entry_block
, op
);
1473 if (mem
== mem_non_local
)
1476 goto new_memory_ret
;
1478 retval
= MIN (retval
, mem
);
1480 goto new_memory_ret
;
1485 while (TREE_CODE (x
) == SSA_NAME
);
1487 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1488 /* Thread-local or transaction-local. */
1491 retval
= mem_non_local
;
1494 elt_p
->local_new_memory
= retval
;
1498 /* Determine whether X has to be instrumented using a read
1501 ENTRY_BLOCK is the entry block for the region where stmt resides
1502 in. NULL if unknown.
1504 STMT is the statement in which X occurs in. It is used for thread
1505 private memory instrumentation. If no TPM instrumentation is
1506 desired, STMT should be null. */
1508 requires_barrier (basic_block entry_block
, tree x
, gimple
*stmt
)
1511 while (handled_component_p (x
))
1512 x
= TREE_OPERAND (x
, 0);
1514 switch (TREE_CODE (x
))
1519 enum thread_memory_type ret
;
1521 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1522 if (ret
== mem_non_local
)
1524 if (stmt
&& ret
== mem_thread_local
)
1525 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1526 tm_log_add (entry_block
, orig
, stmt
);
1528 /* Transaction-locals require nothing at all. For malloc, a
1529 transaction restart frees the memory and we reallocate.
1530 For alloca, the stack pointer gets reset by the retry and
1535 case TARGET_MEM_REF
:
1536 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1538 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1539 if (TREE_CODE (x
) == PARM_DECL
)
1541 gcc_assert (VAR_P (x
));
1547 if (DECL_BY_REFERENCE (x
))
1549 /* ??? This value is a pointer, but aggregate_value_p has been
1550 jigged to return true which confuses needs_to_live_in_memory.
1551 This ought to be cleaned up generically.
1553 FIXME: Verify this still happens after the next mainline
1554 merge. Testcase ie g++.dg/tm/pr47554.C.
1559 if (is_global_var (x
))
1560 return !TREE_READONLY (x
);
1561 if (/* FIXME: This condition should actually go below in the
1562 tm_log_add() call, however is_call_clobbered() depends on
1563 aliasing info which is not available during
1564 gimplification. Since requires_barrier() gets called
1565 during lower_sequence_tm/gimplification, leave the call
1566 to needs_to_live_in_memory until we eliminate
1567 lower_sequence_tm altogether. */
1568 needs_to_live_in_memory (x
))
1572 /* For local memory that doesn't escape (aka thread private
1573 memory), we can either save the value at the beginning of
1574 the transaction and restore on restart, or call a tm
1575 function to dynamically save and restore on restart
1578 tm_log_add (entry_block
, orig
, stmt
);
1587 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1588 a transaction region. */
1591 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1593 gimple
*stmt
= gsi_stmt (*gsi
);
1595 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1596 *state
|= GTMA_HAVE_LOAD
;
1597 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1598 *state
|= GTMA_HAVE_STORE
;
1601 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1604 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1606 gimple
*stmt
= gsi_stmt (*gsi
);
1609 if (is_tm_pure_call (stmt
))
1612 /* Check if this call is a transaction abort. */
1613 fn
= gimple_call_fndecl (stmt
);
1614 if (is_tm_abort (fn
))
1615 *state
|= GTMA_HAVE_ABORT
;
1617 /* Note that something may happen. */
1618 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1621 /* Iterate through the statements in the sequence, moving labels
1622 (and thus edges) of transactions from "label_norm" to "label_uninst". */
1625 make_tm_uninst (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1626 struct walk_stmt_info
*)
1628 gimple
*stmt
= gsi_stmt (*gsi
);
1630 if (gtransaction
*txn
= dyn_cast
<gtransaction
*> (stmt
))
1632 *handled_ops_p
= true;
1633 txn
->label_uninst
= txn
->label_norm
;
1634 txn
->label_norm
= NULL
;
1637 *handled_ops_p
= !gimple_has_substatements (stmt
);
1642 /* Lower a GIMPLE_TRANSACTION statement. */
1645 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1648 gtransaction
*stmt
= as_a
<gtransaction
*> (gsi_stmt (*gsi
));
1649 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1650 unsigned int this_state
= 0;
1651 struct walk_stmt_info this_wi
;
1653 /* First, lower the body. The scanning that we do inside gives
1654 us some idea of what we're dealing with. */
1655 memset (&this_wi
, 0, sizeof (this_wi
));
1656 this_wi
.info
= (void *) &this_state
;
1657 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1658 lower_sequence_tm
, NULL
, &this_wi
);
1660 /* If there was absolutely nothing transaction related inside the
1661 transaction, we may elide it. Likewise if this is a nested
1662 transaction and does not contain an abort. */
1664 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1667 *outer_state
|= this_state
;
1669 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1671 gimple_transaction_set_body (stmt
, NULL
);
1673 gsi_remove (gsi
, true);
1674 wi
->removed_stmt
= true;
1678 /* Wrap the body of the transaction in a try-finally node so that
1679 the commit call is always properly called. */
1680 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1681 if (flag_exceptions
)
1684 gimple_seq n_seq
, e_seq
;
1686 n_seq
= gimple_seq_alloc_with_stmt (g
);
1689 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1690 1, integer_zero_node
);
1691 ptr
= create_tmp_var (ptr_type_node
);
1692 gimple_call_set_lhs (g
, ptr
);
1693 gimple_seq_add_stmt (&e_seq
, g
);
1695 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1697 gimple_seq_add_stmt (&e_seq
, g
);
1699 g
= gimple_build_eh_else (n_seq
, e_seq
);
1702 g
= gimple_build_try (gimple_transaction_body (stmt
),
1703 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1705 /* For a (potentially) outer transaction, create two paths. */
1706 gimple_seq uninst
= NULL
;
1707 if (outer_state
== NULL
)
1709 uninst
= copy_gimple_seq_and_replace_locals (g
);
1710 /* In the uninstrumented copy, reset inner transactions to have only
1711 an uninstrumented code path. */
1712 memset (&this_wi
, 0, sizeof (this_wi
));
1713 walk_gimple_seq (uninst
, make_tm_uninst
, NULL
, &this_wi
);
1716 tree label1
= create_artificial_label (UNKNOWN_LOCATION
);
1717 gsi_insert_after (gsi
, gimple_build_label (label1
), GSI_CONTINUE_LINKING
);
1718 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1719 gimple_transaction_set_label_norm (stmt
, label1
);
1721 /* If the transaction calls abort or if this is an outer transaction,
1722 add an "over" label afterwards. */
1724 if ((this_state
& GTMA_HAVE_ABORT
)
1725 || outer_state
== NULL
1726 || (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
))
1728 label3
= create_artificial_label (UNKNOWN_LOCATION
);
1729 gimple_transaction_set_label_over (stmt
, label3
);
1734 gsi_insert_after (gsi
, gimple_build_goto (label3
), GSI_CONTINUE_LINKING
);
1736 tree label2
= create_artificial_label (UNKNOWN_LOCATION
);
1737 gsi_insert_after (gsi
, gimple_build_label (label2
), GSI_CONTINUE_LINKING
);
1738 gsi_insert_seq_after (gsi
, uninst
, GSI_CONTINUE_LINKING
);
1739 gimple_transaction_set_label_uninst (stmt
, label2
);
1743 gsi_insert_after (gsi
, gimple_build_label (label3
), GSI_CONTINUE_LINKING
);
1745 gimple_transaction_set_body (stmt
, NULL
);
1747 /* Record the set of operations found for use later. */
1748 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1749 gimple_transaction_set_subcode (stmt
, this_state
);
1752 /* Iterate through the statements in the sequence, lowering them all
1753 as appropriate for being in a transaction. */
1756 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1757 struct walk_stmt_info
*wi
)
1759 unsigned int *state
= (unsigned int *) wi
->info
;
1760 gimple
*stmt
= gsi_stmt (*gsi
);
1762 *handled_ops_p
= true;
1763 switch (gimple_code (stmt
))
1766 /* Only memory reads/writes need to be instrumented. */
1767 if (gimple_assign_single_p (stmt
))
1768 examine_assign_tm (state
, gsi
);
1772 examine_call_tm (state
, gsi
);
1776 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1779 case GIMPLE_TRANSACTION
:
1780 lower_transaction (gsi
, wi
);
1784 *handled_ops_p
= !gimple_has_substatements (stmt
);
1791 /* Iterate through the statements in the sequence, lowering them all
1792 as appropriate for being outside of a transaction. */
1795 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1796 struct walk_stmt_info
* wi
)
1798 gimple
*stmt
= gsi_stmt (*gsi
);
1800 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1802 *handled_ops_p
= true;
1803 lower_transaction (gsi
, wi
);
1806 *handled_ops_p
= !gimple_has_substatements (stmt
);
1811 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1812 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1813 been moved out, and all the data required for constructing a proper
1814 CFG has been recorded. */
1817 execute_lower_tm (void)
1819 struct walk_stmt_info wi
;
1822 /* Transactional clones aren't created until a later pass. */
1823 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1825 body
= gimple_body (current_function_decl
);
1826 memset (&wi
, 0, sizeof (wi
));
1827 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1828 gimple_set_body (current_function_decl
, body
);
1835 const pass_data pass_data_lower_tm
=
1837 GIMPLE_PASS
, /* type */
1838 "tmlower", /* name */
1839 OPTGROUP_NONE
, /* optinfo_flags */
1840 TV_TRANS_MEM
, /* tv_id */
1841 PROP_gimple_lcf
, /* properties_required */
1842 0, /* properties_provided */
1843 0, /* properties_destroyed */
1844 0, /* todo_flags_start */
1845 0, /* todo_flags_finish */
1848 class pass_lower_tm
: public gimple_opt_pass
1851 pass_lower_tm (gcc::context
*ctxt
)
1852 : gimple_opt_pass (pass_data_lower_tm
, ctxt
)
1855 /* opt_pass methods: */
1856 bool gate (function
*) final override
{ return flag_tm
; }
1857 unsigned int execute (function
*) final override
1859 return execute_lower_tm ();
1862 }; // class pass_lower_tm
1867 make_pass_lower_tm (gcc::context
*ctxt
)
1869 return new pass_lower_tm (ctxt
);
1872 /* Collect region information for each transaction. */
1878 /* The field "transaction_stmt" is initially a gtransaction *,
1879 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1881 Helper method to get it as a gtransaction *, with code-checking
1882 in a checked-build. */
1885 get_transaction_stmt () const
1887 return as_a
<gtransaction
*> (transaction_stmt
);
1892 /* Link to the next unnested transaction. */
1893 struct tm_region
*next
;
1895 /* Link to the next inner transaction. */
1896 struct tm_region
*inner
;
1898 /* Link to the next outer transaction. */
1899 struct tm_region
*outer
;
1901 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1902 After TM_MARK, this gets replaced by a call to
1904 Hence this will be either a gtransaction *or a gcall *. */
1905 gimple
*transaction_stmt
;
1907 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1908 BUILT_IN_TM_START, this field is true if the transaction is an
1909 outer transaction. */
1910 bool original_transaction_was_outer
;
1912 /* Return value from BUILT_IN_TM_START. */
1915 /* The entry block to this region. This will always be the first
1916 block of the body of the transaction. */
1917 basic_block entry_block
;
1919 /* The first block after an expanded call to _ITM_beginTransaction. */
1920 basic_block restart_block
;
1922 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1923 These blocks are still a part of the region (i.e., the border is
1924 inclusive). Note that this set is only complete for paths in the CFG
1925 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1926 the edge to the "over" label. */
1929 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1933 /* True if there are pending edge statements to be committed for the
1934 current function being scanned in the tmmark pass. */
1935 bool pending_edge_inserts_p
;
1937 static struct tm_region
*all_tm_regions
;
1938 static bitmap_obstack tm_obstack
;
1941 /* A subroutine of tm_region_init. Record the existence of the
1942 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1944 static struct tm_region
*
1945 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
,
1948 struct tm_region
*region
;
1950 region
= (struct tm_region
*)
1951 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1955 region
->next
= outer
->inner
;
1956 outer
->inner
= region
;
1960 region
->next
= all_tm_regions
;
1961 all_tm_regions
= region
;
1963 region
->inner
= NULL
;
1964 region
->outer
= outer
;
1966 region
->transaction_stmt
= stmt
;
1967 region
->original_transaction_was_outer
= false;
1968 region
->tm_state
= NULL
;
1970 /* There are either one or two edges out of the block containing
1971 the GIMPLE_TRANSACTION, one to the actual region and one to the
1972 "over" label if the region contains an abort. The former will
1973 always be the one marked FALLTHRU. */
1974 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1976 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1977 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1982 /* A subroutine of tm_region_init. Record all the exit and
1983 irrevocable blocks in BB into the region's exit_blocks and
1984 irr_blocks bitmaps. Returns the new region being scanned. */
1986 static struct tm_region
*
1987 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1989 gimple_stmt_iterator gsi
;
1993 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1996 /* Check to see if this is the end of a region by seeing if it
1997 contains a call to __builtin_tm_commit{,_eh}. Note that the
1998 outermost region for DECL_IS_TM_CLONE need not collect this. */
1999 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
2002 if (gimple_code (g
) == GIMPLE_CALL
)
2004 tree fn
= gimple_call_fndecl (g
);
2005 if (fn
&& fndecl_built_in_p (fn
, BUILT_IN_NORMAL
))
2007 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
2008 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
2009 && region
->exit_blocks
)
2011 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
2012 region
= region
->outer
;
2015 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
2016 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
2023 /* Collect all of the transaction regions within the current function
2024 and record them in ALL_TM_REGIONS. The REGION parameter may specify
2025 an "outermost" region for use by tm clones. */
2028 tm_region_init (struct tm_region
*region
)
2034 auto_vec
<basic_block
> queue
;
2035 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2036 struct tm_region
*old_region
;
2037 auto_vec
<tm_region
*> bb_regions
;
2039 /* We could store this information in bb->aux, but we may get called
2040 through get_all_tm_blocks() from another pass that may be already
2042 bb_regions
.safe_grow_cleared (last_basic_block_for_fn (cfun
), true);
2044 all_tm_regions
= region
;
2045 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2046 queue
.safe_push (bb
);
2047 bitmap_set_bit (visited_blocks
, bb
->index
);
2048 bb_regions
[bb
->index
] = region
;
2053 region
= bb_regions
[bb
->index
];
2054 bb_regions
[bb
->index
] = NULL
;
2056 /* Record exit and irrevocable blocks. */
2057 region
= tm_region_init_1 (region
, bb
);
2059 /* Check for the last statement in the block beginning a new region. */
2060 g
= last_nondebug_stmt (bb
);
2061 old_region
= region
;
2063 if (gtransaction
*trans_stmt
= dyn_cast
<gtransaction
*> (g
))
2064 region
= tm_region_init_0 (region
, bb
, trans_stmt
);
2066 /* Process subsequent blocks. */
2067 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2068 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2070 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2071 queue
.safe_push (e
->dest
);
2073 /* If the current block started a new region, make sure that only
2074 the entry block of the new region is associated with this region.
2075 Other successors are still part of the old region. */
2076 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
2077 bb_regions
[e
->dest
->index
] = old_region
;
2079 bb_regions
[e
->dest
->index
] = region
;
2082 while (!queue
.is_empty ());
2083 BITMAP_FREE (visited_blocks
);
2086 /* The "gate" function for all transactional memory expansion and optimization
2087 passes. We collect region information for each top-level transaction, and
2088 if we don't find any, we skip all of the TM passes. Each region will have
2089 all of the exit blocks recorded, and the originating statement. */
2097 calculate_dominance_info (CDI_DOMINATORS
);
2098 bitmap_obstack_initialize (&tm_obstack
);
2100 /* If the function is a TM_CLONE, then the entire function is the region. */
2101 if (decl_is_tm_clone (current_function_decl
))
2103 struct tm_region
*region
= (struct tm_region
*)
2104 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
2105 memset (region
, 0, sizeof (*region
));
2106 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
2107 /* For a clone, the entire function is the region. But even if
2108 we don't need to record any exit blocks, we may need to
2109 record irrevocable blocks. */
2110 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
2112 tm_region_init (region
);
2116 tm_region_init (NULL
);
2118 /* If we didn't find any regions, cleanup and skip the whole tree
2119 of tm-related optimizations. */
2120 if (all_tm_regions
== NULL
)
2122 bitmap_obstack_release (&tm_obstack
);
2132 const pass_data pass_data_tm_init
=
2134 GIMPLE_PASS
, /* type */
2135 "*tminit", /* name */
2136 OPTGROUP_NONE
, /* optinfo_flags */
2137 TV_TRANS_MEM
, /* tv_id */
2138 ( PROP_ssa
| PROP_cfg
), /* properties_required */
2139 0, /* properties_provided */
2140 0, /* properties_destroyed */
2141 0, /* todo_flags_start */
2142 0, /* todo_flags_finish */
2145 class pass_tm_init
: public gimple_opt_pass
2148 pass_tm_init (gcc::context
*ctxt
)
2149 : gimple_opt_pass (pass_data_tm_init
, ctxt
)
2152 /* opt_pass methods: */
2153 bool gate (function
*) final override
{ return gate_tm_init (); }
2155 }; // class pass_tm_init
2160 make_pass_tm_init (gcc::context
*ctxt
)
2162 return new pass_tm_init (ctxt
);
2165 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2166 represented by STATE. */
2169 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
2171 if (region
&& region
->transaction_stmt
)
2173 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
2174 flags
|= gimple_transaction_subcode (transaction_stmt
);
2175 gimple_transaction_set_subcode (transaction_stmt
, flags
);
2179 /* Construct a memory load in a transactional context. Return the
2180 gimple statement performing the load, or NULL if there is no
2181 TM_LOAD builtin of the appropriate size to do the load.
2183 LOC is the location to use for the new statement(s). */
2186 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2188 tree t
, type
= TREE_TYPE (rhs
);
2191 built_in_function code
;
2192 if (type
== float_type_node
)
2193 code
= BUILT_IN_TM_LOAD_FLOAT
;
2194 else if (type
== double_type_node
)
2195 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2196 else if (type
== long_double_type_node
)
2197 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2200 if (TYPE_SIZE (type
) == NULL
|| !tree_fits_uhwi_p (TYPE_SIZE (type
)))
2202 unsigned HOST_WIDE_INT type_size
= tree_to_uhwi (TYPE_SIZE (type
));
2204 if (TREE_CODE (type
) == VECTOR_TYPE
)
2209 code
= BUILT_IN_TM_LOAD_M64
;
2212 code
= BUILT_IN_TM_LOAD_M128
;
2215 code
= BUILT_IN_TM_LOAD_M256
;
2220 if (!builtin_decl_explicit_p (code
))
2229 code
= BUILT_IN_TM_LOAD_1
;
2232 code
= BUILT_IN_TM_LOAD_2
;
2235 code
= BUILT_IN_TM_LOAD_4
;
2238 code
= BUILT_IN_TM_LOAD_8
;
2246 tree decl
= builtin_decl_explicit (code
);
2249 t
= gimplify_addr (gsi
, rhs
);
2250 gcall
= gimple_build_call (decl
, 1, t
);
2251 gimple_set_location (gcall
, loc
);
2253 t
= TREE_TYPE (TREE_TYPE (decl
));
2254 if (useless_type_conversion_p (type
, t
))
2256 gimple_call_set_lhs (gcall
, lhs
);
2257 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2264 temp
= create_tmp_reg (t
);
2265 gimple_call_set_lhs (gcall
, temp
);
2266 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2268 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2269 g
= gimple_build_assign (lhs
, t
);
2270 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2277 /* Similarly for storing TYPE in a transactional context. */
2280 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2282 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2285 built_in_function code
;
2286 if (type
== float_type_node
)
2287 code
= BUILT_IN_TM_STORE_FLOAT
;
2288 else if (type
== double_type_node
)
2289 code
= BUILT_IN_TM_STORE_DOUBLE
;
2290 else if (type
== long_double_type_node
)
2291 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2294 if (TYPE_SIZE (type
) == NULL
|| !tree_fits_uhwi_p (TYPE_SIZE (type
)))
2296 unsigned HOST_WIDE_INT type_size
= tree_to_uhwi (TYPE_SIZE (type
));
2298 if (TREE_CODE (type
) == VECTOR_TYPE
)
2303 code
= BUILT_IN_TM_STORE_M64
;
2306 code
= BUILT_IN_TM_STORE_M128
;
2309 code
= BUILT_IN_TM_STORE_M256
;
2314 if (!builtin_decl_explicit_p (code
))
2323 code
= BUILT_IN_TM_STORE_1
;
2326 code
= BUILT_IN_TM_STORE_2
;
2329 code
= BUILT_IN_TM_STORE_4
;
2332 code
= BUILT_IN_TM_STORE_8
;
2340 fn
= builtin_decl_explicit (code
);
2343 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2345 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2347 /* Handle the easy initialization to zero. */
2348 if (!CONSTRUCTOR_ELTS (rhs
))
2349 rhs
= build_int_cst (simple_type
, 0);
2352 /* ...otherwise punt to the caller and probably use
2353 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2354 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2359 else if (!useless_type_conversion_p (simple_type
, type
))
2364 temp
= create_tmp_reg (simple_type
);
2365 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2366 g
= gimple_build_assign (temp
, t
);
2367 gimple_set_location (g
, loc
);
2368 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2373 t
= gimplify_addr (gsi
, lhs
);
2374 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2375 gimple_set_location (gcall
, loc
);
2376 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2382 /* Expand an assignment statement into transactional builtins. */
2385 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2387 gimple
*stmt
= gsi_stmt (*gsi
);
2388 location_t loc
= gimple_location (stmt
);
2389 tree lhs
= gimple_assign_lhs (stmt
);
2390 tree rhs
= gimple_assign_rhs1 (stmt
);
2391 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2392 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2393 gimple
*gcall
= NULL
;
2395 if (!load_p
&& !store_p
)
2397 /* Add thread private addresses to log if applicable. */
2398 requires_barrier (region
->entry_block
, lhs
, stmt
);
2404 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2406 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2408 // Remove original load/store statement.
2409 gsi_remove (gsi
, true);
2411 // Attempt to use a simple load/store helper function.
2412 if (load_p
&& !store_p
)
2413 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2414 else if (store_p
&& !load_p
)
2415 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2417 // If gcall has not been set, then we do not have a simple helper
2418 // function available for the type. This may be true of larger
2419 // structures, vectors, and non-standard float types.
2422 tree lhs_addr
, rhs_addr
, ltmp
= NULL
, copy_fn
;
2424 // If this is a type that we couldn't handle above, but it's
2425 // in a register, we must spill it to memory for the copy.
2426 if (is_gimple_reg (lhs
))
2428 ltmp
= create_tmp_var (TREE_TYPE (lhs
));
2429 lhs_addr
= build_fold_addr_expr (ltmp
);
2432 lhs_addr
= gimplify_addr (gsi
, lhs
);
2433 if (is_gimple_reg (rhs
))
2435 tree rtmp
= create_tmp_var (TREE_TYPE (rhs
));
2436 TREE_ADDRESSABLE (rtmp
) = 1;
2437 rhs_addr
= build_fold_addr_expr (rtmp
);
2438 gcall
= gimple_build_assign (rtmp
, rhs
);
2439 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2442 rhs_addr
= gimplify_addr (gsi
, rhs
);
2444 // Choose the appropriate memory transfer function.
2445 if (load_p
&& store_p
)
2447 // ??? Figure out if there's any possible overlap between
2448 // the LHS and the RHS and if not, use MEMCPY.
2449 copy_fn
= builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
2453 // Note that the store is non-transactional and cannot overlap.
2454 copy_fn
= builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RTWN
);
2458 // Note that the load is non-transactional and cannot overlap.
2459 copy_fn
= builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RNWT
);
2462 gcall
= gimple_build_call (copy_fn
, 3, lhs_addr
, rhs_addr
,
2463 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2464 gimple_set_location (gcall
, loc
);
2465 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2469 gcall
= gimple_build_assign (lhs
, ltmp
);
2470 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2474 // Now that we have the load/store in its instrumented form, add
2475 // thread private addresses to the log if applicable.
2477 requires_barrier (region
->entry_block
, lhs
, gcall
);
2481 /* Expand a call statement as appropriate for a transaction. That is,
2482 either verify that the call does not affect the transaction, or
2483 redirect the call to a clone that handles transactions, or change
2484 the transaction state to IRREVOCABLE. Return true if the call is
2485 one of the builtins that end a transaction. */
2488 expand_call_tm (struct tm_region
*region
,
2489 gimple_stmt_iterator
*gsi
)
2491 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
2492 tree lhs
= gimple_call_lhs (stmt
);
2494 struct cgraph_node
*node
;
2495 bool retval
= false;
2497 fn_decl
= gimple_call_fndecl (stmt
);
2499 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2500 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2501 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2502 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2503 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2505 if (is_tm_pure_call (stmt
))
2509 retval
= is_tm_ending_fndecl (fn_decl
);
2512 /* Assume all non-const/pure calls write to memory, except
2513 transaction ending builtins. */
2514 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2517 /* For indirect calls, we already generated a call into the runtime. */
2520 tree fn
= gimple_call_fn (stmt
);
2522 /* We are guaranteed never to go irrevocable on a safe or pure
2523 call, and the pure call was handled above. */
2524 if (is_tm_safe (fn
))
2527 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2532 node
= cgraph_node::get (fn_decl
);
2533 /* All calls should have cgraph here. */
2536 /* We can have a nodeless call here if some pass after IPA-tm
2537 added uninstrumented calls. For example, loop distribution
2538 can transform certain loop constructs into __builtin_mem*
2539 calls. In this case, see if we have a suitable TM
2540 replacement and fill in the gaps. */
2541 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2542 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2543 gcc_assert (code
== BUILT_IN_MEMCPY
2544 || code
== BUILT_IN_MEMMOVE
2545 || code
== BUILT_IN_MEMSET
);
2547 tree repl
= find_tm_replacement_function (fn_decl
);
2550 gimple_call_set_fndecl (stmt
, repl
);
2552 node
= cgraph_node::create (repl
);
2553 node
->tm_may_enter_irr
= false;
2554 return expand_call_tm (region
, gsi
);
2558 if (node
->tm_may_enter_irr
)
2559 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2561 if (is_tm_abort (fn_decl
))
2563 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2567 /* Instrument the store if needed.
2569 If the assignment happens inside the function call (return slot
2570 optimization), there is no instrumentation to be done, since
2571 the callee should have done the right thing. */
2572 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2573 && !gimple_call_return_slot_opt_p (stmt
))
2575 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
));
2576 location_t loc
= gimple_location (stmt
);
2577 edge fallthru_edge
= NULL
;
2578 gassign
*assign_stmt
;
2580 /* Remember if the call was going to throw. */
2581 if (stmt_can_throw_internal (cfun
, stmt
))
2585 basic_block bb
= gimple_bb (stmt
);
2587 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2588 if (e
->flags
& EDGE_FALLTHRU
)
2595 gimple_call_set_lhs (stmt
, tmp
);
2597 assign_stmt
= gimple_build_assign (lhs
, tmp
);
2598 gimple_set_location (assign_stmt
, loc
);
2600 /* We cannot throw in the middle of a BB. If the call was going
2601 to throw, place the instrumentation on the fallthru edge, so
2602 the call remains the last statement in the block. */
2605 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (assign_stmt
);
2606 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2607 expand_assign_tm (region
, &fallthru_gsi
);
2608 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2609 pending_edge_inserts_p
= true;
2613 gsi_insert_after (gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
2614 expand_assign_tm (region
, gsi
);
2617 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2624 /* Expand all statements in BB as appropriate for being inside
2628 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2630 gimple_stmt_iterator gsi
;
2632 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2634 gimple
*stmt
= gsi_stmt (gsi
);
2635 switch (gimple_code (stmt
))
2638 /* Only memory reads/writes need to be instrumented. */
2639 if (gimple_assign_single_p (stmt
)
2640 && !gimple_clobber_p (stmt
))
2642 expand_assign_tm (region
, &gsi
);
2648 if (expand_call_tm (region
, &gsi
))
2658 if (!gsi_end_p (gsi
))
2663 /* Return the list of basic-blocks in REGION.
2665 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2666 following a TM_IRREVOCABLE call.
2668 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2669 uninstrumented code path blocks in the list of basic blocks
2670 returned, false otherwise. */
2672 static vec
<basic_block
>
2673 get_tm_region_blocks (basic_block entry_block
,
2676 bitmap all_region_blocks
,
2677 bool stop_at_irrevocable_p
,
2678 bool include_uninstrumented_p
= true)
2680 vec
<basic_block
> bbs
= vNULL
;
2684 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2687 bbs
.safe_push (entry_block
);
2688 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2692 basic_block bb
= bbs
[i
++];
2695 bitmap_bit_p (exit_blocks
, bb
->index
))
2698 if (stop_at_irrevocable_p
2700 && bitmap_bit_p (irr_blocks
, bb
->index
))
2703 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2704 if ((include_uninstrumented_p
2705 || !(e
->flags
& EDGE_TM_UNINSTRUMENTED
))
2706 && !bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2708 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2709 bbs
.safe_push (e
->dest
);
2712 while (i
< bbs
.length ());
2714 if (all_region_blocks
)
2715 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2717 BITMAP_FREE (visited_blocks
);
2721 // Callback data for collect_bb2reg.
2724 vec
<tm_region
*> *bb2reg
;
2725 bool include_uninstrumented_p
;
2728 // Callback for expand_regions, collect innermost region data for each bb.
2730 collect_bb2reg (struct tm_region
*region
, void *data
)
2732 struct bb2reg_stuff
*stuff
= (struct bb2reg_stuff
*)data
;
2733 vec
<tm_region
*> *bb2reg
= stuff
->bb2reg
;
2734 vec
<basic_block
> queue
;
2738 queue
= get_tm_region_blocks (region
->entry_block
,
2739 region
->exit_blocks
,
2742 /*stop_at_irr_p=*/true,
2743 stuff
->include_uninstrumented_p
);
2745 // We expect expand_region to perform a post-order traversal of the region
2746 // tree. Therefore the last region seen for any bb is the innermost.
2747 FOR_EACH_VEC_ELT (queue
, i
, bb
)
2748 (*bb2reg
)[bb
->index
] = region
;
2754 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2755 // which a basic block belongs. Note that we only consider the instrumented
2756 // code paths for the region; the uninstrumented code paths are ignored if
2757 // INCLUDE_UNINSTRUMENTED_P is false.
2759 // ??? This data is very similar to the bb_regions array that is collected
2760 // during tm_region_init. Or, rather, this data is similar to what could
2761 // be used within tm_region_init. The actual computation in tm_region_init
2762 // begins and ends with bb_regions entirely full of NULL pointers, due to
2763 // the way in which pointers are swapped in and out of the array.
2765 // ??? Our callers expect that blocks are not shared between transactions.
2766 // When the optimizers get too smart, and blocks are shared, then during
2767 // the tm_mark phase we'll add log entries to only one of the two transactions,
2768 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2769 // cycles. The symptom being SSA defs that do not dominate their uses.
2770 // Note that the optimizers were locally correct with their transformation,
2771 // as we have no info within the program that suggests that the blocks cannot
2774 // ??? There is currently a hack inside tree-ssa-pre.cc to work around the
2775 // only known instance of this block sharing.
2777 static vec
<tm_region
*>
2778 get_bb_regions_instrumented (bool traverse_clones
,
2779 bool include_uninstrumented_p
)
2781 unsigned n
= last_basic_block_for_fn (cfun
);
2782 struct bb2reg_stuff stuff
;
2783 vec
<tm_region
*> ret
;
2786 ret
.safe_grow_cleared (n
, true);
2787 stuff
.bb2reg
= &ret
;
2788 stuff
.include_uninstrumented_p
= include_uninstrumented_p
;
2789 expand_regions (all_tm_regions
, collect_bb2reg
, &stuff
, traverse_clones
);
2794 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2798 compute_transaction_bits (void)
2800 struct tm_region
*region
;
2801 vec
<basic_block
> queue
;
2805 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2806 certainly don't need it to calculate CDI_DOMINATOR info. */
2809 FOR_EACH_BB_FN (bb
, cfun
)
2810 bb
->flags
&= ~BB_IN_TRANSACTION
;
2812 for (region
= all_tm_regions
; region
; region
= region
->next
)
2814 queue
= get_tm_region_blocks (region
->entry_block
,
2815 region
->exit_blocks
,
2818 /*stop_at_irr_p=*/true);
2819 for (i
= 0; queue
.iterate (i
, &bb
); ++i
)
2820 bb
->flags
|= BB_IN_TRANSACTION
;
2825 bitmap_obstack_release (&tm_obstack
);
2828 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2829 call to BUILT_IN_TM_START. */
2832 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2834 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2835 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2836 tree tm_state
= region
->tm_state
;
2837 tree tm_state_type
= TREE_TYPE (tm_state
);
2838 edge abort_edge
= NULL
;
2839 edge inst_edge
= NULL
;
2840 edge uninst_edge
= NULL
;
2841 edge fallthru_edge
= NULL
;
2843 // Identify the various successors of the transaction start.
2847 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2849 if (e
->flags
& EDGE_TM_ABORT
)
2851 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2855 if (e
->flags
& EDGE_FALLTHRU
)
2860 /* ??? There are plenty of bits here we're not computing. */
2862 int subcode
= gimple_transaction_subcode (region
->get_transaction_stmt ());
2864 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2865 flags
|= PR_DOESGOIRREVOCABLE
;
2866 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2867 flags
|= PR_HASNOIRREVOCABLE
;
2868 /* If the transaction does not have an abort in lexical scope and is not
2869 marked as an outer transaction, then it will never abort. */
2870 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2871 flags
|= PR_HASNOABORT
;
2872 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2873 flags
|= PR_READONLY
;
2874 if (inst_edge
&& !(subcode
& GTMA_HAS_NO_INSTRUMENTATION
))
2875 flags
|= PR_INSTRUMENTEDCODE
;
2877 flags
|= PR_UNINSTRUMENTEDCODE
;
2878 if (subcode
& GTMA_IS_OUTER
)
2879 region
->original_transaction_was_outer
= true;
2880 tree t
= build_int_cst (tm_state_type
, flags
);
2881 gcall
*call
= gimple_build_call (tm_start
, 1, t
);
2882 gimple_call_set_lhs (call
, tm_state
);
2883 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2885 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2886 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2887 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2888 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2889 gsi_remove (&gsi
, true);
2890 region
->transaction_stmt
= call
;
2893 // Generate log saves.
2894 if (!tm_log_save_addresses
.is_empty ())
2895 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2897 // In the beginning, we've no tests to perform on transaction restart.
2898 // Note that after this point, transaction_bb becomes the "most recent
2899 // block containing tests for the transaction".
2900 region
->restart_block
= region
->entry_block
;
2902 // Generate log restores.
2903 if (!tm_log_save_addresses
.is_empty ())
2905 basic_block test_bb
= create_empty_bb (transaction_bb
);
2906 basic_block code_bb
= create_empty_bb (test_bb
);
2907 basic_block join_bb
= create_empty_bb (code_bb
);
2908 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2909 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2910 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2911 if (region
->restart_block
== region
->entry_block
)
2912 region
->restart_block
= test_bb
;
2914 tree t1
= create_tmp_reg (tm_state_type
);
2915 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2916 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2917 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2918 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2920 t2
= build_int_cst (tm_state_type
, 0);
2921 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2922 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2924 tm_log_emit_restores (region
->entry_block
, code_bb
);
2926 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2927 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2928 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2929 redirect_edge_pred (fallthru_edge
, join_bb
);
2931 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2933 ei
->probability
= profile_probability::always ();
2934 et
->probability
= profile_probability::likely ();
2935 ef
->probability
= profile_probability::unlikely ();
2937 code_bb
->count
= et
->count ();
2939 transaction_bb
= join_bb
;
2942 // If we have an ABORT edge, create a test to perform the abort.
2945 basic_block test_bb
= create_empty_bb (transaction_bb
);
2946 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2947 if (region
->restart_block
== region
->entry_block
)
2948 region
->restart_block
= test_bb
;
2950 tree t1
= create_tmp_reg (tm_state_type
);
2951 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2952 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2953 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2954 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2956 t2
= build_int_cst (tm_state_type
, 0);
2957 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2958 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2960 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2961 test_bb
->count
= transaction_bb
->count
;
2962 ei
->probability
= profile_probability::always ();
2964 // Not abort edge. If both are live, chose one at random as we'll
2965 // we'll be fixing that up below.
2966 redirect_edge_pred (fallthru_edge
, test_bb
);
2967 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2968 fallthru_edge
->probability
= profile_probability::very_likely ();
2971 redirect_edge_pred (abort_edge
, test_bb
);
2972 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2973 abort_edge
->probability
= profile_probability::unlikely ();
2975 transaction_bb
= test_bb
;
2978 // If we have both instrumented and uninstrumented code paths, select one.
2979 if (inst_edge
&& uninst_edge
)
2981 basic_block test_bb
= create_empty_bb (transaction_bb
);
2982 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2983 if (region
->restart_block
== region
->entry_block
)
2984 region
->restart_block
= test_bb
;
2986 tree t1
= create_tmp_reg (tm_state_type
);
2987 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2989 gimple
*stmt
= gimple_build_assign (t1
, BIT_AND_EXPR
, tm_state
, t2
);
2990 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2991 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2993 t2
= build_int_cst (tm_state_type
, 0);
2994 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2995 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2997 // Create the edge into test_bb first, as we want to copy values
2998 // out of the fallthru edge.
2999 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
3000 e
->probability
= fallthru_edge
->probability
;
3001 test_bb
->count
= fallthru_edge
->count ();
3003 // Now update the edges to the inst/uninist implementations.
3004 // For now assume that the paths are equally likely. When using HTM,
3005 // we'll try the uninst path first and fallback to inst path if htm
3006 // buffers are exceeded. Without HTM we start with the inst path and
3007 // use the uninst path when falling back to serial mode.
3008 redirect_edge_pred (inst_edge
, test_bb
);
3009 inst_edge
->flags
= EDGE_FALSE_VALUE
;
3010 inst_edge
->probability
= profile_probability::even ();
3012 redirect_edge_pred (uninst_edge
, test_bb
);
3013 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
3014 uninst_edge
->probability
= profile_probability::even ();
3017 // If we have no previous special cases, and we have PHIs at the beginning
3018 // of the atomic region, this means we have a loop at the beginning of the
3019 // atomic region that shares the first block. This can cause problems with
3020 // the transaction restart abnormal edges to be added in the tm_edges pass.
3021 // Solve this by adding a new empty block to receive the abnormal edges.
3022 if (region
->restart_block
== region
->entry_block
3023 && phi_nodes (region
->entry_block
))
3025 basic_block empty_bb
= create_empty_bb (transaction_bb
);
3026 region
->restart_block
= empty_bb
;
3027 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
3029 redirect_edge_pred (fallthru_edge
, empty_bb
);
3030 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
3036 /* Generate the temporary to be used for the return value of
3037 BUILT_IN_TM_START. */
3040 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
3042 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
3044 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
3046 // Reset the subcode, post optimizations. We'll fill this in
3047 // again as we process blocks.
3048 if (region
->exit_blocks
)
3050 gtransaction
*transaction_stmt
= region
->get_transaction_stmt ();
3051 unsigned int subcode
= gimple_transaction_subcode (transaction_stmt
);
3053 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
3054 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
3055 | GTMA_MAY_ENTER_IRREVOCABLE
3056 | GTMA_HAS_NO_INSTRUMENTATION
);
3058 subcode
&= GTMA_DECLARATION_MASK
;
3059 gimple_transaction_set_subcode (transaction_stmt
, subcode
);
3065 // Propagate flags from inner transactions outwards.
3067 propagate_tm_flags_out (struct tm_region
*region
)
3071 propagate_tm_flags_out (region
->inner
);
3073 if (region
->outer
&& region
->outer
->transaction_stmt
)
3076 = gimple_transaction_subcode (region
->get_transaction_stmt ());
3077 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
3078 | GTMA_MAY_ENTER_IRREVOCABLE
);
3079 s
|= gimple_transaction_subcode (region
->outer
->get_transaction_stmt ());
3080 gimple_transaction_set_subcode (region
->outer
->get_transaction_stmt (),
3084 propagate_tm_flags_out (region
->next
);
3087 /* Entry point to the MARK phase of TM expansion. Here we replace
3088 transactional memory statements with calls to builtins, and function
3089 calls with their transactional clones (if available). But we don't
3090 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
3093 execute_tm_mark (void)
3095 pending_edge_inserts_p
= false;
3097 expand_regions (all_tm_regions
, generate_tm_state
, NULL
,
3098 /*traverse_clones=*/true);
3102 vec
<tm_region
*> bb_regions
3103 = get_bb_regions_instrumented (/*traverse_clones=*/true,
3104 /*include_uninstrumented_p=*/false);
3105 struct tm_region
*r
;
3108 // Expand memory operations into calls into the runtime.
3109 // This collects log entries as well.
3110 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3114 if (r
->transaction_stmt
)
3117 = gimple_transaction_subcode (r
->get_transaction_stmt ());
3119 /* If we're sure to go irrevocable, there won't be
3120 anything to expand, since the run-time will go
3121 irrevocable right away. */
3122 if (sub
& GTMA_DOES_GO_IRREVOCABLE
3123 && sub
& GTMA_MAY_ENTER_IRREVOCABLE
)
3126 expand_block_tm (r
, BASIC_BLOCK_FOR_FN (cfun
, i
));
3130 bb_regions
.release ();
3132 // Propagate flags from inner transactions outwards.
3133 propagate_tm_flags_out (all_tm_regions
);
3135 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3136 expand_regions (all_tm_regions
, expand_transaction
, NULL
,
3137 /*traverse_clones=*/false);
3142 if (pending_edge_inserts_p
)
3143 gsi_commit_edge_inserts ();
3144 free_dominance_info (CDI_DOMINATORS
);
3150 const pass_data pass_data_tm_mark
=
3152 GIMPLE_PASS
, /* type */
3153 "tmmark", /* name */
3154 OPTGROUP_NONE
, /* optinfo_flags */
3155 TV_TRANS_MEM
, /* tv_id */
3156 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3157 0, /* properties_provided */
3158 0, /* properties_destroyed */
3159 0, /* todo_flags_start */
3160 TODO_update_ssa
, /* todo_flags_finish */
3163 class pass_tm_mark
: public gimple_opt_pass
3166 pass_tm_mark (gcc::context
*ctxt
)
3167 : gimple_opt_pass (pass_data_tm_mark
, ctxt
)
3170 /* opt_pass methods: */
3171 unsigned int execute (function
*) final override
3173 return execute_tm_mark ();
3176 }; // class pass_tm_mark
3181 make_pass_tm_mark (gcc::context
*ctxt
)
3183 return new pass_tm_mark (ctxt
);
3187 /* Create an abnormal edge from STMT at iter, splitting the block
3188 as necessary. Adjust *PNEXT as needed for the split block. */
3191 split_bb_make_tm_edge (gimple
*stmt
, basic_block dest_bb
,
3192 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
3194 basic_block bb
= gimple_bb (stmt
);
3195 if (!gsi_one_before_end_p (iter
))
3197 edge e
= split_block (bb
, stmt
);
3198 *pnext
= gsi_start_bb (e
->dest
);
3200 edge e
= make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
3202 e
->probability
= profile_probability::guessed_never ();
3204 // Record the need for the edge for the benefit of the rtl passes.
3205 if (cfun
->gimple_df
->tm_restart
== NULL
)
3206 cfun
->gimple_df
->tm_restart
3207 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3209 struct tm_restart_node dummy
;
3211 dummy
.label_or_list
= gimple_block_label (dest_bb
);
3213 tm_restart_node
**slot
= cfun
->gimple_df
->tm_restart
->find_slot (&dummy
,
3215 struct tm_restart_node
*n
= *slot
;
3218 *slot
= n
= ggc_alloc
<tm_restart_node
> ();
3223 tree old
= n
->label_or_list
;
3224 if (TREE_CODE (old
) == LABEL_DECL
)
3225 old
= tree_cons (NULL
, old
, NULL
);
3226 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
3230 /* Split block BB as necessary for every builtin function we added, and
3231 wire up the abnormal back edges implied by the transaction restart. */
3234 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
3236 gimple_stmt_iterator gsi
, next_gsi
;
3238 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
3240 gimple
*stmt
= gsi_stmt (gsi
);
3244 gsi_next (&next_gsi
);
3246 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3247 call_stmt
= dyn_cast
<gcall
*> (stmt
);
3249 || (gimple_call_flags (call_stmt
) & ECF_TM_BUILTIN
) == 0)
3252 if (gimple_call_builtin_p (call_stmt
, BUILT_IN_TM_ABORT
))
3254 // If we have a ``_transaction_cancel [[outer]]'', there is only
3255 // one abnormal edge: to the transaction marked OUTER.
3256 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3257 // constant argument, which we can examine here. Users invoking
3258 // TM_ABORT directly get what they deserve.
3259 tree arg
= gimple_call_arg (call_stmt
, 0);
3260 if (TREE_CODE (arg
) == INTEGER_CST
3261 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
3262 && !decl_is_tm_clone (current_function_decl
))
3264 // Find the GTMA_IS_OUTER transaction.
3265 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3266 if (o
->original_transaction_was_outer
)
3268 split_bb_make_tm_edge (call_stmt
, o
->restart_block
,
3273 // Otherwise, the front-end should have semantically checked
3274 // outer aborts, but in either case the target region is not
3275 // within this function.
3279 // Non-outer, TM aborts have an abnormal edge to the inner-most
3280 // transaction, the one being aborted;
3281 split_bb_make_tm_edge (call_stmt
, region
->restart_block
, gsi
,
3285 // All TM builtins have an abnormal edge to the outer-most transaction.
3286 // We never restart inner transactions. For tm clones, we know a-priori
3287 // that the outer-most transaction is outside the function.
3288 if (decl_is_tm_clone (current_function_decl
))
3291 if (cfun
->gimple_df
->tm_restart
== NULL
)
3292 cfun
->gimple_df
->tm_restart
3293 = hash_table
<tm_restart_hasher
>::create_ggc (31);
3295 // All TM builtins have an abnormal edge to the outer-most transaction.
3296 // We never restart inner transactions.
3297 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
3300 split_bb_make_tm_edge (call_stmt
, o
->restart_block
, gsi
, &next_gsi
);
3304 // Delete any tail-call annotation that may have been added.
3305 // The tail-call pass may have mis-identified the commit as being
3306 // a candidate because we had not yet added this restart edge.
3307 gimple_call_set_tail (call_stmt
, false);
3311 /* Entry point to the final expansion of transactional nodes. */
3315 const pass_data pass_data_tm_edges
=
3317 GIMPLE_PASS
, /* type */
3318 "tmedge", /* name */
3319 OPTGROUP_NONE
, /* optinfo_flags */
3320 TV_TRANS_MEM
, /* tv_id */
3321 ( PROP_ssa
| PROP_cfg
), /* properties_required */
3322 0, /* properties_provided */
3323 0, /* properties_destroyed */
3324 0, /* todo_flags_start */
3325 TODO_update_ssa
, /* todo_flags_finish */
3328 class pass_tm_edges
: public gimple_opt_pass
3331 pass_tm_edges (gcc::context
*ctxt
)
3332 : gimple_opt_pass (pass_data_tm_edges
, ctxt
)
3335 /* opt_pass methods: */
3336 unsigned int execute (function
*) final override
;
3338 }; // class pass_tm_edges
3341 pass_tm_edges::execute (function
*fun
)
3343 vec
<tm_region
*> bb_regions
3344 = get_bb_regions_instrumented (/*traverse_clones=*/false,
3345 /*include_uninstrumented_p=*/true);
3346 struct tm_region
*r
;
3349 FOR_EACH_VEC_ELT (bb_regions
, i
, r
)
3351 expand_block_edges (r
, BASIC_BLOCK_FOR_FN (fun
, i
));
3353 bb_regions
.release ();
3355 /* We've got to release the dominance info now, to indicate that it
3356 must be rebuilt completely. Otherwise we'll crash trying to update
3357 the SSA web in the TODO section following this pass. */
3358 free_dominance_info (CDI_DOMINATORS
);
3359 /* We'ge also wrecked loops badly with inserting of abnormal edges. */
3360 loops_state_set (LOOPS_NEED_FIXUP
);
3361 bitmap_obstack_release (&tm_obstack
);
3362 all_tm_regions
= NULL
;
3370 make_pass_tm_edges (gcc::context
*ctxt
)
3372 return new pass_tm_edges (ctxt
);
3375 /* Helper function for expand_regions. Expand REGION and recurse to
3376 the inner region. Call CALLBACK on each region. CALLBACK returns
3377 NULL to continue the traversal, otherwise a non-null value which
3378 this function will return as well. TRAVERSE_CLONES is true if we
3379 should traverse transactional clones. */
3382 expand_regions_1 (struct tm_region
*region
,
3383 void *(*callback
)(struct tm_region
*, void *),
3385 bool traverse_clones
)
3387 void *retval
= NULL
;
3388 if (region
->exit_blocks
3389 || (traverse_clones
&& decl_is_tm_clone (current_function_decl
)))
3391 retval
= callback (region
, data
);
3397 retval
= expand_regions (region
->inner
, callback
, data
, traverse_clones
);
3404 /* Traverse the regions enclosed and including REGION. Execute
3405 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3406 continue the traversal, otherwise a non-null value which this
3407 function will return as well. TRAVERSE_CLONES is true if we should
3408 traverse transactional clones. */
3411 expand_regions (struct tm_region
*region
,
3412 void *(*callback
)(struct tm_region
*, void *),
3414 bool traverse_clones
)
3416 void *retval
= NULL
;
3419 retval
= expand_regions_1 (region
, callback
, data
, traverse_clones
);
3422 region
= region
->next
;
3428 /* A unique TM memory operation. */
3431 /* Unique ID that all memory operations to the same location have. */
3432 unsigned int value_id
;
3433 /* Address of load/store. */
3437 /* TM memory operation hashtable helpers. */
3439 struct tm_memop_hasher
: free_ptr_hash
<tm_memop
>
3441 static inline hashval_t
hash (const tm_memop
*);
3442 static inline bool equal (const tm_memop
*, const tm_memop
*);
3445 /* Htab support. Return a hash value for a `tm_memop'. */
3447 tm_memop_hasher::hash (const tm_memop
*mem
)
3449 tree addr
= mem
->addr
;
3450 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3451 actually done with operand_equal_p (see tm_memop_eq). */
3452 if (TREE_CODE (addr
) == ADDR_EXPR
)
3453 addr
= TREE_OPERAND (addr
, 0);
3454 return iterative_hash_expr (addr
, 0);
3457 /* Htab support. Return true if two tm_memop's are the same. */
3459 tm_memop_hasher::equal (const tm_memop
*mem1
, const tm_memop
*mem2
)
3461 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3464 /* Sets for solving data flow equations in the memory optimization pass. */
3465 struct tm_memopt_bitmaps
3467 /* Stores available to this BB upon entry. Basically, stores that
3468 dominate this BB. */
3469 bitmap store_avail_in
;
3470 /* Stores available at the end of this BB. */
3471 bitmap store_avail_out
;
3472 bitmap store_antic_in
;
3473 bitmap store_antic_out
;
3474 /* Reads available to this BB upon entry. Basically, reads that
3475 dominate this BB. */
3476 bitmap read_avail_in
;
3477 /* Reads available at the end of this BB. */
3478 bitmap read_avail_out
;
3479 /* Reads performed in this BB. */
3481 /* Writes performed in this BB. */
3484 /* Temporary storage for pass. */
3485 /* Is the current BB in the worklist? */
3486 bool avail_in_worklist_p
;
3487 /* Have we visited this BB? */
3491 static bitmap_obstack tm_memopt_obstack
;
3493 /* Unique counter for TM loads and stores. Loads and stores of the
3494 same address get the same ID. */
3495 static unsigned int tm_memopt_value_id
;
3496 static hash_table
<tm_memop_hasher
> *tm_memopt_value_numbers
;
3498 #define STORE_AVAIL_IN(BB) \
3499 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3500 #define STORE_AVAIL_OUT(BB) \
3501 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3502 #define STORE_ANTIC_IN(BB) \
3503 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3504 #define STORE_ANTIC_OUT(BB) \
3505 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3506 #define READ_AVAIL_IN(BB) \
3507 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3508 #define READ_AVAIL_OUT(BB) \
3509 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3510 #define READ_LOCAL(BB) \
3511 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3512 #define STORE_LOCAL(BB) \
3513 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3514 #define AVAIL_IN_WORKLIST_P(BB) \
3515 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3516 #define BB_VISITED_P(BB) \
3517 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3519 /* Given a TM load/store in STMT, return the value number for the address
3523 tm_memopt_value_number (gimple
*stmt
, enum insert_option op
)
3525 struct tm_memop tmpmem
, *mem
;
3528 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3529 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3530 slot
= tm_memopt_value_numbers
->find_slot (&tmpmem
, op
);
3533 else if (op
== INSERT
)
3535 mem
= XNEW (struct tm_memop
);
3537 mem
->value_id
= tm_memopt_value_id
++;
3538 mem
->addr
= tmpmem
.addr
;
3542 return mem
->value_id
;
3545 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3548 tm_memopt_accumulate_memops (basic_block bb
)
3550 gimple_stmt_iterator gsi
;
3552 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3554 gimple
*stmt
= gsi_stmt (gsi
);
3558 if (is_tm_store (stmt
))
3559 bits
= STORE_LOCAL (bb
);
3560 else if (is_tm_load (stmt
))
3561 bits
= READ_LOCAL (bb
);
3565 loc
= tm_memopt_value_number (stmt
, INSERT
);
3566 bitmap_set_bit (bits
, loc
);
3569 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3570 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3571 gimple_bb (stmt
)->index
);
3572 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0));
3573 fprintf (dump_file
, "\n");
3578 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3581 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3585 const char *comma
= "";
3587 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3588 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3590 hash_table
<tm_memop_hasher
>::iterator hi
;
3591 struct tm_memop
*mem
= NULL
;
3593 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3594 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3595 if (mem
->value_id
== i
)
3597 gcc_assert (mem
->value_id
== i
);
3598 fprintf (dump_file
, "%s", comma
);
3600 print_generic_expr (dump_file
, mem
->addr
);
3602 fprintf (dump_file
, "]\n");
3605 /* Prettily dump all of the memopt sets in BLOCKS. */
3608 dump_tm_memopt_sets (vec
<basic_block
> blocks
)
3613 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3615 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3616 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3617 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3618 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3619 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3620 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3621 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3625 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3628 tm_memopt_compute_avin (basic_block bb
)
3633 /* Seed with the AVOUT of any predecessor. */
3634 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3636 e
= EDGE_PRED (bb
, ix
);
3637 /* Make sure we have already visited this BB, and is thus
3640 If e->src->aux is NULL, this predecessor is actually on an
3641 enclosing transaction. We only care about the current
3642 transaction, so ignore it. */
3643 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3645 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3646 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3651 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3653 e
= EDGE_PRED (bb
, ix
);
3654 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3656 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3657 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3661 BB_VISITED_P (bb
) = true;
3664 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3667 tm_memopt_compute_antin (basic_block bb
)
3672 /* Seed with the ANTIC_OUT of any successor. */
3673 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3675 e
= EDGE_SUCC (bb
, ix
);
3676 /* Make sure we have already visited this BB, and is thus
3678 if (BB_VISITED_P (e
->dest
))
3680 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3685 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3687 e
= EDGE_SUCC (bb
, ix
);
3688 if (BB_VISITED_P (e
->dest
))
3689 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3692 BB_VISITED_P (bb
) = true;
3695 /* Compute the AVAIL sets for every basic block in BLOCKS.
3697 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3699 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3700 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3702 This is basically what we do in lcm's compute_available(), but here
3703 we calculate two sets of sets (one for STOREs and one for READs),
3704 and we work on a region instead of the entire CFG.
3706 REGION is the TM region.
3707 BLOCKS are the basic blocks in the region. */
3710 tm_memopt_compute_available (struct tm_region
*region
,
3711 vec
<basic_block
> blocks
)
3714 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3715 unsigned int qlen
, i
;
3719 /* Allocate a worklist array/queue. Entries are only added to the
3720 list if they were not already on the list. So the size is
3721 bounded by the number of basic blocks in the region. */
3722 gcc_assert (!blocks
.is_empty ());
3723 qlen
= blocks
.length () - 1;
3724 qin
= qout
= worklist
= XNEWVEC (basic_block
, qlen
);
3726 /* Put every block in the region on the worklist. */
3727 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3729 /* Seed AVAIL_OUT with the LOCAL set. */
3730 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3731 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3733 AVAIL_IN_WORKLIST_P (bb
) = true;
3734 /* No need to insert the entry block, since it has an AVIN of
3735 null, and an AVOUT that has already been seeded in. */
3736 if (bb
!= region
->entry_block
)
3740 /* The entry block has been initialized with the local sets. */
3741 BB_VISITED_P (region
->entry_block
) = true;
3744 qend
= &worklist
[qlen
];
3746 /* Iterate until the worklist is empty. */
3749 /* Take the first entry off the worklist. */
3756 /* This block can be added to the worklist again if necessary. */
3757 AVAIL_IN_WORKLIST_P (bb
) = false;
3758 tm_memopt_compute_avin (bb
);
3760 /* Note: We do not add the LOCAL sets here because we already
3761 seeded the AVAIL_OUT sets with them. */
3762 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3763 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3765 && (region
->exit_blocks
== NULL
3766 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3767 /* If the out state of this block changed, then we need to add
3768 its successors to the worklist if they are not already in. */
3769 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3770 if (!AVAIL_IN_WORKLIST_P (e
->dest
)
3771 && e
->dest
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
3774 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3785 dump_tm_memopt_sets (blocks
);
3788 /* Compute ANTIC sets for every basic block in BLOCKS.
3790 We compute STORE_ANTIC_OUT as follows:
3792 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3793 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3795 REGION is the TM region.
3796 BLOCKS are the basic blocks in the region. */
3799 tm_memopt_compute_antic (struct tm_region
*region
,
3800 vec
<basic_block
> blocks
)
3803 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3808 /* Allocate a worklist array/queue. Entries are only added to the
3809 list if they were not already on the list. So the size is
3810 bounded by the number of basic blocks in the region. */
3811 qin
= qout
= worklist
= XNEWVEC (basic_block
, blocks
.length ());
3813 for (qlen
= 0, i
= blocks
.length () - 1; i
>= 0; --i
)
3817 /* Seed ANTIC_OUT with the LOCAL set. */
3818 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3820 /* Put every block in the region on the worklist. */
3821 AVAIL_IN_WORKLIST_P (bb
) = true;
3822 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3823 and their ANTIC_OUT has already been seeded in. */
3824 if (region
->exit_blocks
3825 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3832 /* The exit blocks have been initialized with the local sets. */
3833 if (region
->exit_blocks
)
3837 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3838 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun
, i
)) = true;
3842 qend
= &worklist
[qlen
];
3844 /* Iterate until the worklist is empty. */
3847 /* Take the first entry off the worklist. */
3854 /* This block can be added to the worklist again if necessary. */
3855 AVAIL_IN_WORKLIST_P (bb
) = false;
3856 tm_memopt_compute_antin (bb
);
3858 /* Note: We do not add the LOCAL sets here because we already
3859 seeded the ANTIC_OUT sets with them. */
3860 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3861 && bb
!= region
->entry_block
)
3862 /* If the out state of this block changed, then we need to add
3863 its predecessors to the worklist if they are not already in. */
3864 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3865 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3868 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3879 dump_tm_memopt_sets (blocks
);
3882 /* Offsets of load variants from TM_LOAD. For example,
3883 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3884 See gtm-builtins.def. */
3885 #define TRANSFORM_RAR 1
3886 #define TRANSFORM_RAW 2
3887 #define TRANSFORM_RFW 3
3888 /* Offsets of store variants from TM_STORE. */
3889 #define TRANSFORM_WAR 1
3890 #define TRANSFORM_WAW 2
3892 /* Inform about a load/store optimization. */
3895 dump_tm_memopt_transform (gimple
*stmt
)
3899 fprintf (dump_file
, "TM memopt: transforming: ");
3900 print_gimple_stmt (dump_file
, stmt
, 0);
3901 fprintf (dump_file
, "\n");
3905 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3906 by a builtin that is OFFSET entries down in the builtins table in
3907 gtm-builtins.def. */
3910 tm_memopt_transform_stmt (unsigned int offset
,
3912 gimple_stmt_iterator
*gsi
)
3914 tree fn
= gimple_call_fn (stmt
);
3915 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3916 TREE_OPERAND (fn
, 0)
3917 = builtin_decl_explicit ((enum built_in_function
)
3918 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3920 gimple_call_set_fn (stmt
, fn
);
3921 gsi_replace (gsi
, stmt
, true);
3922 dump_tm_memopt_transform (stmt
);
3925 /* Perform the actual TM memory optimization transformations in the
3926 basic blocks in BLOCKS. */
3929 tm_memopt_transform_blocks (vec
<basic_block
> blocks
)
3933 gimple_stmt_iterator gsi
;
3935 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
3937 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3939 gimple
*stmt
= gsi_stmt (gsi
);
3940 bitmap read_avail
= READ_AVAIL_IN (bb
);
3941 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3942 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3945 if (is_tm_simple_load (stmt
))
3947 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3948 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3949 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3950 tm_memopt_transform_stmt (TRANSFORM_RAW
, call_stmt
, &gsi
);
3951 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3953 tm_memopt_transform_stmt (TRANSFORM_RFW
, call_stmt
, &gsi
);
3954 bitmap_set_bit (store_avail
, loc
);
3956 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3957 tm_memopt_transform_stmt (TRANSFORM_RAR
, call_stmt
, &gsi
);
3959 bitmap_set_bit (read_avail
, loc
);
3961 else if (is_tm_simple_store (stmt
))
3963 gcall
*call_stmt
= as_a
<gcall
*> (stmt
);
3964 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3965 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3966 tm_memopt_transform_stmt (TRANSFORM_WAW
, call_stmt
, &gsi
);
3969 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3970 tm_memopt_transform_stmt (TRANSFORM_WAR
, call_stmt
, &gsi
);
3971 bitmap_set_bit (store_avail
, loc
);
3978 /* Return a new set of bitmaps for a BB. */
3980 static struct tm_memopt_bitmaps
*
3981 tm_memopt_init_sets (void)
3983 struct tm_memopt_bitmaps
*b
3984 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3985 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3986 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3987 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3988 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3989 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3990 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3991 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3992 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3993 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3997 /* Free sets computed for each BB. */
4000 tm_memopt_free_sets (vec
<basic_block
> blocks
)
4005 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
4009 /* Clear the visited bit for every basic block in BLOCKS. */
4012 tm_memopt_clear_visited (vec
<basic_block
> blocks
)
4017 for (i
= 0; blocks
.iterate (i
, &bb
); ++i
)
4018 BB_VISITED_P (bb
) = false;
4021 /* Replace TM load/stores with hints for the runtime. We handle
4022 things like read-after-write, write-after-read, read-after-read,
4023 read-for-write, etc. */
4026 execute_tm_memopt (void)
4028 struct tm_region
*region
;
4029 vec
<basic_block
> bbs
;
4031 tm_memopt_value_id
= 0;
4032 tm_memopt_value_numbers
= new hash_table
<tm_memop_hasher
> (10);
4034 for (region
= all_tm_regions
; region
; region
= region
->next
)
4036 /* All the TM stores/loads in the current region. */
4040 bitmap_obstack_initialize (&tm_memopt_obstack
);
4042 /* Save all BBs for the current region. */
4043 bbs
= get_tm_region_blocks (region
->entry_block
,
4044 region
->exit_blocks
,
4049 /* Collect all the memory operations. */
4050 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4052 bb
->aux
= tm_memopt_init_sets ();
4053 tm_memopt_accumulate_memops (bb
);
4056 /* Solve data flow equations and transform each block accordingly. */
4057 tm_memopt_clear_visited (bbs
);
4058 tm_memopt_compute_available (region
, bbs
);
4059 tm_memopt_clear_visited (bbs
);
4060 tm_memopt_compute_antic (region
, bbs
);
4061 tm_memopt_transform_blocks (bbs
);
4063 tm_memopt_free_sets (bbs
);
4065 bitmap_obstack_release (&tm_memopt_obstack
);
4066 tm_memopt_value_numbers
->empty ();
4069 delete tm_memopt_value_numbers
;
4070 tm_memopt_value_numbers
= NULL
;
4076 const pass_data pass_data_tm_memopt
=
4078 GIMPLE_PASS
, /* type */
4079 "tmmemopt", /* name */
4080 OPTGROUP_NONE
, /* optinfo_flags */
4081 TV_TRANS_MEM
, /* tv_id */
4082 ( PROP_ssa
| PROP_cfg
), /* properties_required */
4083 0, /* properties_provided */
4084 0, /* properties_destroyed */
4085 0, /* todo_flags_start */
4086 0, /* todo_flags_finish */
4089 class pass_tm_memopt
: public gimple_opt_pass
4092 pass_tm_memopt (gcc::context
*ctxt
)
4093 : gimple_opt_pass (pass_data_tm_memopt
, ctxt
)
4096 /* opt_pass methods: */
4097 bool gate (function
*) final override
{ return flag_tm
&& optimize
> 0; }
4098 unsigned int execute (function
*) final override
4100 return execute_tm_memopt ();
4103 }; // class pass_tm_memopt
4108 make_pass_tm_memopt (gcc::context
*ctxt
)
4110 return new pass_tm_memopt (ctxt
);
4114 /* Interprocedual analysis for the creation of transactional clones.
4115 The aim of this pass is to find which functions are referenced in
4116 a non-irrevocable transaction context, and for those over which
4117 we have control (or user directive), create a version of the
4118 function which uses only the transactional interface to reference
4119 protected memories. This analysis proceeds in several steps:
4121 (1) Collect the set of all possible transactional clones:
4123 (a) For all local public functions marked tm_callable, push
4124 it onto the tm_callee queue.
4126 (b) For all local functions, scan for calls in transaction blocks.
4127 Push the caller and callee onto the tm_caller and tm_callee
4128 queues. Count the number of callers for each callee.
4130 (c) For each local function on the callee list, assume we will
4131 create a transactional clone. Push *all* calls onto the
4132 callee queues; count the number of clone callers separately
4133 to the number of original callers.
4135 (2) Propagate irrevocable status up the dominator tree:
4137 (a) Any external function on the callee list that is not marked
4138 tm_callable is irrevocable. Push all callers of such onto
4141 (b) For each function on the worklist, mark each block that
4142 contains an irrevocable call. Use the AND operator to
4143 propagate that mark up the dominator tree.
4145 (c) If we reach the entry block for a possible transactional
4146 clone, then the transactional clone is irrevocable, and
4147 we should not create the clone after all. Push all
4148 callers onto the worklist.
4150 (d) Place tm_irrevocable calls at the beginning of the relevant
4151 blocks. Special case here is the entry block for the entire
4152 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4153 the library to begin the region in serial mode. Decrement
4154 the call count for all callees in the irrevocable region.
4156 (3) Create the transactional clones:
4158 Any tm_callee that still has a non-zero call count is cloned.
4161 /* This structure is stored in the AUX field of each cgraph_node. */
4162 struct tm_ipa_cg_data
4164 /* The clone of the function that got created. */
4165 struct cgraph_node
*clone
;
4167 /* The tm regions in the normal function. */
4168 struct tm_region
*all_tm_regions
;
4170 /* The blocks of the normal/clone functions that contain irrevocable
4171 calls, or blocks that are post-dominated by irrevocable calls. */
4172 bitmap irrevocable_blocks_normal
;
4173 bitmap irrevocable_blocks_clone
;
4175 /* The blocks of the normal function that are involved in transactions. */
4176 bitmap transaction_blocks_normal
;
4178 /* The number of callers to the transactional clone of this function
4179 from normal and transactional clones respectively. */
4180 unsigned tm_callers_normal
;
4181 unsigned tm_callers_clone
;
4183 /* True if all calls to this function's transactional clone
4184 are irrevocable. Also automatically true if the function
4185 has no transactional clone. */
4186 bool is_irrevocable
;
4188 /* Flags indicating the presence of this function in various queues. */
4189 bool in_callee_queue
;
4192 /* Flags indicating the kind of scan desired while in the worklist. */
4193 bool want_irr_scan_normal
;
4196 typedef vec
<cgraph_node
*> cgraph_node_queue
;
4198 /* Return the ipa data associated with NODE, allocating zeroed memory
4199 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4200 and set *NODE accordingly. */
4202 static struct tm_ipa_cg_data
*
4203 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
4205 struct tm_ipa_cg_data
*d
;
4207 if (traverse_aliases
&& (*node
)->alias
)
4208 *node
= (*node
)->get_alias_target ();
4210 d
= (struct tm_ipa_cg_data
*) (*node
)->aux
;
4214 d
= (struct tm_ipa_cg_data
*)
4215 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
4216 (*node
)->aux
= (void *) d
;
4217 memset (d
, 0, sizeof (*d
));
4223 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4224 it is already present. */
4227 maybe_push_queue (struct cgraph_node
*node
,
4228 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
4233 queue_p
->safe_push (node
);
4237 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4238 Queue all callees within block BB. */
4241 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
4242 basic_block bb
, bool for_clone
)
4244 gimple_stmt_iterator gsi
;
4246 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4248 gimple
*stmt
= gsi_stmt (gsi
);
4249 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4251 tree fndecl
= gimple_call_fndecl (stmt
);
4254 struct tm_ipa_cg_data
*d
;
4256 struct cgraph_node
*node
;
4258 if (is_tm_ending_fndecl (fndecl
))
4260 if (find_tm_replacement_function (fndecl
))
4263 node
= cgraph_node::get (fndecl
);
4264 gcc_assert (node
!= NULL
);
4265 d
= get_cg_data (&node
, true);
4267 pcallers
= (for_clone
? &d
->tm_callers_clone
4268 : &d
->tm_callers_normal
);
4271 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
4277 /* Scan all calls in NODE that are within a transaction region,
4278 and push the resulting nodes into the callee queue. */
4281 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
4282 cgraph_node_queue
*callees_p
)
4284 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
4285 d
->all_tm_regions
= all_tm_regions
;
4287 for (tm_region
*r
= all_tm_regions
; r
; r
= r
->next
)
4289 vec
<basic_block
> bbs
;
4293 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
4294 d
->transaction_blocks_normal
, false, false);
4296 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
4297 ipa_tm_scan_calls_block (callees_p
, bb
, false);
4303 /* Scan all calls in NODE as if this is the transactional clone,
4304 and push the destinations into the callee queue. */
4307 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4308 cgraph_node_queue
*callees_p
)
4310 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->decl
);
4313 FOR_EACH_BB_FN (bb
, fn
)
4314 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4317 /* The function NODE has been detected to be irrevocable. Push all
4318 of its callers onto WORKLIST for the purpose of re-scanning them. */
4321 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4322 cgraph_node_queue
*worklist_p
)
4324 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4325 struct cgraph_edge
*e
;
4327 d
->is_irrevocable
= true;
4329 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4332 struct cgraph_node
*caller
;
4334 /* Don't examine recursive calls. */
4335 if (e
->caller
== node
)
4337 /* Even if we think we can go irrevocable, believe the user
4339 if (is_tm_safe_or_pure (e
->caller
->decl
))
4343 d
= get_cg_data (&caller
, true);
4345 /* Check if the callee is in a transactional region. If so,
4346 schedule the function for normal re-scan as well. */
4347 bb
= gimple_bb (e
->call_stmt
);
4348 gcc_assert (bb
!= NULL
);
4349 if (d
->transaction_blocks_normal
4350 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4351 d
->want_irr_scan_normal
= true;
4353 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4357 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4358 within the block is irrevocable. */
4361 ipa_tm_scan_irr_block (basic_block bb
)
4363 gimple_stmt_iterator gsi
;
4366 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4368 gimple
*stmt
= gsi_stmt (gsi
);
4369 switch (gimple_code (stmt
))
4372 if (gimple_assign_single_p (stmt
))
4374 tree lhs
= gimple_assign_lhs (stmt
);
4375 tree rhs
= gimple_assign_rhs1 (stmt
);
4376 if (volatile_lvalue_p (lhs
) || volatile_lvalue_p (rhs
))
4383 tree lhs
= gimple_call_lhs (stmt
);
4384 if (lhs
&& volatile_lvalue_p (lhs
))
4387 if (is_tm_pure_call (stmt
))
4390 fn
= gimple_call_fn (stmt
);
4392 /* Functions with the attribute are by definition irrevocable. */
4393 if (is_tm_irrevocable (fn
))
4396 /* For direct function calls, go ahead and check for replacement
4397 functions, or transitive irrevocable functions. For indirect
4398 functions, we'll ask the runtime. */
4399 if (TREE_CODE (fn
) == ADDR_EXPR
)
4401 struct tm_ipa_cg_data
*d
;
4402 struct cgraph_node
*node
;
4404 fn
= TREE_OPERAND (fn
, 0);
4405 if (is_tm_ending_fndecl (fn
))
4407 if (find_tm_replacement_function (fn
))
4410 node
= cgraph_node::get (fn
);
4411 d
= get_cg_data (&node
, true);
4413 /* Return true if irrevocable, but above all, believe
4415 if (d
->is_irrevocable
4416 && !is_tm_safe_or_pure (fn
))
4423 /* ??? The Approved Method of indicating that an inline
4424 assembly statement is not relevant to the transaction
4425 is to wrap it in a __tm_waiver block. This is not
4426 yet implemented, so we can't check for it. */
4427 if (is_tm_safe (current_function_decl
))
4428 error_at (gimple_location (stmt
),
4429 "%<asm%> not allowed in %<transaction_safe%> function");
4440 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4441 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4442 scanning past OLD_IRR or EXIT_BLOCKS. */
4445 ipa_tm_scan_irr_blocks (vec
<basic_block
> *pqueue
, bitmap new_irr
,
4446 bitmap old_irr
, bitmap exit_blocks
)
4448 bool any_new_irr
= false;
4451 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4455 basic_block bb
= pqueue
->pop ();
4457 /* Don't re-scan blocks we know already are irrevocable. */
4458 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4461 if (ipa_tm_scan_irr_block (bb
))
4463 bitmap_set_bit (new_irr
, bb
->index
);
4466 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4468 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4469 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4471 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4472 pqueue
->safe_push (e
->dest
);
4476 while (!pqueue
->is_empty ());
4478 BITMAP_FREE (visited_blocks
);
4483 /* Propagate the irrevocable property both up and down the dominator tree.
4484 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4485 TM regions; OLD_IRR are the results of a previous scan of the dominator
4486 tree which has been fully propagated; NEW_IRR is the set of new blocks
4487 which are gaining the irrevocable property during the current scan. */
4490 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4491 bitmap old_irr
, bitmap exit_blocks
)
4493 vec
<basic_block
> bbs
;
4494 bitmap all_region_blocks
;
4496 /* If this block is in the old set, no need to rescan. */
4497 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4500 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4501 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4502 all_region_blocks
, false);
4505 basic_block bb
= bbs
.pop ();
4506 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4507 bool all_son_irr
= false;
4511 /* Propagate up. If my children are, I am too, but we must have
4512 at least one child that is. */
4515 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4517 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4519 all_son_irr
= false;
4527 /* Add block to new_irr if it hasn't already been processed. */
4528 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4530 bitmap_set_bit (new_irr
, bb
->index
);
4536 /* Propagate down to everyone we immediately dominate. */
4540 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4542 son
= next_dom_son (CDI_DOMINATORS
, son
))
4544 /* Make sure block is actually in a TM region, and it
4545 isn't already in old_irr. */
4546 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4547 && bitmap_bit_p (all_region_blocks
, son
->index
))
4548 bitmap_set_bit (new_irr
, son
->index
);
4552 while (!bbs
.is_empty ());
4554 BITMAP_FREE (all_region_blocks
);
4559 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4561 gimple_stmt_iterator gsi
;
4563 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4565 gimple
*stmt
= gsi_stmt (gsi
);
4566 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4568 tree fndecl
= gimple_call_fndecl (stmt
);
4571 struct tm_ipa_cg_data
*d
;
4573 struct cgraph_node
*tnode
;
4575 if (is_tm_ending_fndecl (fndecl
))
4577 if (find_tm_replacement_function (fndecl
))
4580 tnode
= cgraph_node::get (fndecl
);
4581 d
= get_cg_data (&tnode
, true);
4583 pcallers
= (for_clone
? &d
->tm_callers_clone
4584 : &d
->tm_callers_normal
);
4586 gcc_assert (*pcallers
> 0);
4593 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4594 as well as other irrevocable actions such as inline assembly. Mark all
4595 such blocks as irrevocable and decrement the number of calls to
4596 transactional clones. Return true if, for the transactional clone, the
4597 entire function is irrevocable. */
4600 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4602 struct tm_ipa_cg_data
*d
;
4603 bitmap new_irr
, old_irr
;
4606 /* Builtin operators (operator new, and such). */
4607 if (DECL_STRUCT_FUNCTION (node
->decl
) == NULL
4608 || DECL_STRUCT_FUNCTION (node
->decl
)->cfg
== NULL
)
4611 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
4612 calculate_dominance_info (CDI_DOMINATORS
);
4614 d
= get_cg_data (&node
, true);
4615 auto_vec
<basic_block
, 10> queue
;
4616 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4618 /* Scan each tm region, propagating irrevocable status through the tree. */
4621 old_irr
= d
->irrevocable_blocks_clone
;
4622 queue
.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
4623 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4625 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
4628 ret
= bitmap_bit_p (new_irr
,
4629 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
))->index
);
4634 struct tm_region
*region
;
4636 old_irr
= d
->irrevocable_blocks_normal
;
4637 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4639 queue
.quick_push (region
->entry_block
);
4640 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4641 region
->exit_blocks
))
4642 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4643 region
->exit_blocks
);
4647 /* If we found any new irrevocable blocks, reduce the call count for
4648 transactional clones within the irrevocable blocks. Save the new
4649 set of irrevocable blocks for next time. */
4650 if (!bitmap_empty_p (new_irr
))
4652 bitmap_iterator bmi
;
4655 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4656 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun
, i
),
4661 bitmap_ior_into (old_irr
, new_irr
);
4662 BITMAP_FREE (new_irr
);
4665 d
->irrevocable_blocks_clone
= new_irr
;
4667 d
->irrevocable_blocks_normal
= new_irr
;
4669 if (dump_file
&& new_irr
)
4672 bitmap_iterator bmi
;
4675 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4676 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4677 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4681 BITMAP_FREE (new_irr
);
4688 /* Return true if, for the transactional clone of NODE, any call
4689 may enter irrevocable mode. */
4692 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4694 struct tm_ipa_cg_data
*d
;
4698 d
= get_cg_data (&node
, true);
4700 flags
= flags_from_decl_or_type (decl
);
4702 /* Handle some TM builtins. Ordinarily these aren't actually generated
4703 at this point, but handling these functions when written in by the
4704 user makes it easier to build unit tests. */
4705 if (flags
& ECF_TM_BUILTIN
)
4708 /* Filter out all functions that are marked. */
4709 if (flags
& ECF_TM_PURE
)
4711 if (is_tm_safe (decl
))
4713 if (is_tm_irrevocable (decl
))
4715 if (is_tm_callable (decl
))
4717 if (find_tm_replacement_function (decl
))
4720 /* If we aren't seeing the final version of the function we don't
4721 know what it will contain at runtime. */
4722 if (node
->get_availability () < AVAIL_AVAILABLE
)
4725 /* If the function must go irrevocable, then of course true. */
4726 if (d
->is_irrevocable
)
4729 /* If there are any blocks marked irrevocable, then the function
4730 as a whole may enter irrevocable. */
4731 if (d
->irrevocable_blocks_clone
)
4734 /* We may have previously marked this function as tm_may_enter_irr;
4735 see pass_diagnose_tm_blocks. */
4736 if (node
->tm_may_enter_irr
)
4739 /* Recurse on the main body for aliases. In general, this will
4740 result in one of the bits above being set so that we will not
4741 have to recurse next time. */
4743 return ipa_tm_mayenterirr_function
4744 (cgraph_node::get (thunk_info::get (node
)->alias
));
4746 /* What remains is unmarked local functions without items that force
4747 the function to go irrevocable. */
4751 /* Diagnose calls from transaction_safe functions to unmarked
4752 functions that are determined to not be safe. */
4755 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4757 struct cgraph_edge
*e
;
4759 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4760 if (!is_tm_callable (e
->callee
->decl
)
4761 && e
->callee
->tm_may_enter_irr
)
4762 error_at (gimple_location (e
->call_stmt
),
4763 "unsafe function call %qD within "
4764 "%<transaction_safe%> function", e
->callee
->decl
);
4767 /* Diagnose call from atomic transactions to unmarked functions
4768 that are determined to not be safe. */
4771 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4772 struct tm_region
*all_tm_regions
)
4774 struct tm_region
*r
;
4776 for (r
= all_tm_regions
; r
; r
= r
->next
)
4777 if (gimple_transaction_subcode (r
->get_transaction_stmt ())
4780 /* Atomic transactions can be nested inside relaxed. */
4782 ipa_tm_diagnose_transaction (node
, r
->inner
);
4786 vec
<basic_block
> bbs
;
4787 gimple_stmt_iterator gsi
;
4791 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4792 r
->irr_blocks
, NULL
, false);
4794 for (i
= 0; bbs
.iterate (i
, &bb
); ++i
)
4795 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4797 gimple
*stmt
= gsi_stmt (gsi
);
4800 if (gimple_code (stmt
) == GIMPLE_ASM
)
4802 error_at (gimple_location (stmt
),
4803 "%<asm%> not allowed in atomic transaction");
4807 if (!is_gimple_call (stmt
))
4809 fndecl
= gimple_call_fndecl (stmt
);
4811 /* Indirect function calls have been diagnosed already. */
4815 /* Stop at the end of the transaction. */
4816 if (is_tm_ending_fndecl (fndecl
))
4818 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4823 /* Marked functions have been diagnosed already. */
4824 if (is_tm_pure_call (stmt
))
4826 if (is_tm_callable (fndecl
))
4829 if (cgraph_node::local_info_node (fndecl
)->tm_may_enter_irr
)
4830 error_at (gimple_location (stmt
),
4831 "unsafe function call %qD within "
4832 "atomic transaction", fndecl
);
4839 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4840 OLD_DECL. The returned value is a freshly malloced pointer that
4841 should be freed by the caller. */
4844 tm_mangle (tree old_asm_id
)
4846 const char *old_asm_name
;
4849 struct demangle_component
*dc
;
4852 /* Determine if the symbol is already a valid C++ mangled name. Do this
4853 even for C, which might be interfacing with C++ code via appropriately
4854 ugly identifiers. */
4855 /* ??? We could probably do just as well checking for "_Z" and be done. */
4856 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4857 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4864 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4865 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4869 old_asm_name
+= 2; /* Skip _Z */
4873 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4874 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4875 /* Don't play silly games, you! */
4878 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4879 /* I'd really like to know if we can ever be passed one of
4880 these from the C++ front end. The Logical Thing would
4881 seem that hidden-alias should be outer-most, so that we
4882 get hidden-alias of a transaction-clone and not vice-versa. */
4890 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4894 new_asm_id
= get_identifier (tm_name
);
4901 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4903 node
->mark_force_output ();
4904 node
->analyzed
= true;
4908 ipa_tm_mark_forced_by_abi_node (struct cgraph_node
*node
)
4910 node
->forced_by_abi
= true;
4911 node
->analyzed
= true;
4914 /* Callback data for ipa_tm_create_version_alias. */
4915 struct create_version_alias_info
4917 struct cgraph_node
*old_node
;
4921 /* A subroutine of ipa_tm_create_version, called via
4922 cgraph_for_node_and_aliases. Create new tm clones for each of
4923 the existing aliases. */
4925 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4927 struct create_version_alias_info
*info
4928 = (struct create_version_alias_info
*)data
;
4929 tree old_decl
, new_decl
, tm_name
;
4930 struct cgraph_node
*new_node
;
4932 if (!node
->cpp_implicit_alias
)
4935 old_decl
= node
->decl
;
4936 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4937 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4938 TREE_CODE (old_decl
), tm_name
,
4939 TREE_TYPE (old_decl
));
4941 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4942 SET_DECL_RTL (new_decl
, NULL
);
4944 /* Based loosely on C++'s make_alias_for(). */
4945 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4946 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4947 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4948 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4949 DECL_EXTERNAL (new_decl
) = 0;
4950 DECL_ARTIFICIAL (new_decl
) = 1;
4951 TREE_ADDRESSABLE (new_decl
) = 1;
4952 TREE_USED (new_decl
) = 1;
4953 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4955 /* Perform the same remapping to the comdat group. */
4956 if (DECL_ONE_ONLY (new_decl
))
4957 varpool_node::get (new_decl
)->set_comdat_group
4958 (tm_mangle (decl_comdat_group_id (old_decl
)));
4960 new_node
= cgraph_node::create_same_body_alias (new_decl
, info
->new_decl
);
4961 new_node
->tm_clone
= true;
4962 new_node
->externally_visible
= info
->old_node
->externally_visible
;
4963 new_node
->no_reorder
= info
->old_node
->no_reorder
;
4964 /* ?? Do not traverse aliases here. */
4965 get_cg_data (&node
, false)->clone
= new_node
;
4967 record_tm_clone_pair (old_decl
, new_decl
);
4969 if (info
->old_node
->force_output
4970 || info
->old_node
->ref_list
.first_referring ())
4971 ipa_tm_mark_force_output_node (new_node
);
4972 if (info
->old_node
->forced_by_abi
)
4973 ipa_tm_mark_forced_by_abi_node (new_node
);
4977 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4978 appropriate for the transactional clone. */
4981 ipa_tm_create_version (struct cgraph_node
*old_node
)
4983 tree new_decl
, old_decl
, tm_name
;
4984 struct cgraph_node
*new_node
;
4986 old_decl
= old_node
->decl
;
4987 new_decl
= copy_node (old_decl
);
4989 /* DECL_ASSEMBLER_NAME needs to be set before we call
4990 cgraph_copy_node_for_versioning below, because cgraph_node will
4991 fill the assembler_name_hash. */
4992 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4993 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4994 SET_DECL_RTL (new_decl
, NULL
);
4995 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4997 /* Perform the same remapping to the comdat group. */
4998 if (DECL_ONE_ONLY (new_decl
))
4999 varpool_node::get (new_decl
)->set_comdat_group
5000 (tm_mangle (DECL_COMDAT_GROUP (old_decl
)));
5002 gcc_assert (!old_node
->ipa_transforms_to_apply
.exists ());
5003 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
5004 new_node
->local
= false;
5005 new_node
->externally_visible
= old_node
->externally_visible
;
5006 new_node
->lowered
= true;
5007 new_node
->tm_clone
= 1;
5008 if (!old_node
->implicit_section
)
5009 new_node
->set_section (*old_node
);
5010 get_cg_data (&old_node
, true)->clone
= new_node
;
5012 if (old_node
->get_availability () >= AVAIL_INTERPOSABLE
)
5014 /* Remap extern inline to static inline. */
5015 /* ??? Is it worth trying to use make_decl_one_only? */
5016 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
5018 DECL_EXTERNAL (new_decl
) = 0;
5019 TREE_PUBLIC (new_decl
) = 0;
5020 DECL_WEAK (new_decl
) = 0;
5023 tree_function_versioning (old_decl
, new_decl
,
5024 NULL
, NULL
, false, NULL
, NULL
);
5027 record_tm_clone_pair (old_decl
, new_decl
);
5029 symtab
->call_cgraph_insertion_hooks (new_node
);
5030 if (old_node
->force_output
5031 || old_node
->ref_list
.first_referring ())
5032 ipa_tm_mark_force_output_node (new_node
);
5033 if (old_node
->forced_by_abi
)
5034 ipa_tm_mark_forced_by_abi_node (new_node
);
5036 /* Do the same thing, but for any aliases of the original node. */
5038 struct create_version_alias_info data
;
5039 data
.old_node
= old_node
;
5040 data
.new_decl
= new_decl
;
5041 old_node
->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias
,
5046 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
5049 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
5052 gimple_stmt_iterator gsi
;
5055 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5057 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
5058 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
5060 split_block_after_labels (bb
);
5061 gsi
= gsi_after_labels (bb
);
5062 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
5064 node
->create_edge (cgraph_node::get_create
5065 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
5066 g
, gimple_bb (g
)->count
);
5069 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5072 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
5073 struct tm_region
*region
,
5074 gimple_stmt_iterator
*gsi
, gcall
*stmt
)
5076 tree gettm_fn
, ret
, old_fn
, callfn
;
5081 old_fn
= gimple_call_fn (stmt
);
5083 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
5085 tree fndecl
= TREE_OPERAND (old_fn
, 0);
5086 tree clone
= get_tm_clone_pair (fndecl
);
5088 /* By transforming the call into a TM_GETTMCLONE, we are
5089 technically taking the address of the original function and
5090 its clone. Explain this so inlining will know this function
5092 cgraph_node::get (fndecl
)->mark_address_taken () ;
5094 cgraph_node::get (clone
)->mark_address_taken ();
5097 safe
= is_tm_safe (TREE_TYPE (old_fn
));
5098 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
5099 : BUILT_IN_TM_GETTMCLONE_IRR
);
5100 ret
= create_tmp_var (ptr_type_node
);
5103 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5105 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5106 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
5107 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
5109 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
5110 ret
= make_ssa_name (ret
, g
);
5111 gimple_call_set_lhs (g
, ret
);
5113 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
5115 node
->create_edge (cgraph_node::get_create (gettm_fn
), g
, gimple_bb (g
)->count
);
5117 /* Cast return value from tm_gettmclone* into appropriate function
5119 callfn
= create_tmp_var (TREE_TYPE (old_fn
));
5120 g2
= gimple_build_assign (callfn
,
5121 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
5122 callfn
= make_ssa_name (callfn
, g2
);
5123 gimple_assign_set_lhs (g2
, callfn
);
5124 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
5126 /* ??? This is a hack to preserve the NOTHROW bit on the call,
5127 which we would have derived from the decl. Failure to save
5128 this bit means we might have to split the basic block. */
5129 if (gimple_call_nothrow_p (stmt
))
5130 gimple_call_set_nothrow (stmt
, true);
5132 gimple_call_set_fn (stmt
, callfn
);
5134 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5135 for a call statement. Fix it. */
5137 tree lhs
= gimple_call_lhs (stmt
);
5138 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
5140 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
5144 temp
= create_tmp_reg (rettype
);
5145 gimple_call_set_lhs (stmt
, temp
);
5147 g2
= gimple_build_assign (lhs
,
5148 fold_build1 (VIEW_CONVERT_EXPR
,
5149 TREE_TYPE (lhs
), temp
));
5150 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
5155 cgraph_edge
*e
= cgraph_node::get (current_function_decl
)->get_edge (stmt
);
5156 if (e
&& e
->indirect_info
)
5157 e
->indirect_info
->polymorphic
= false;
5162 /* Helper function for ipa_tm_transform_calls*. Given a call
5163 statement in GSI which resides inside transaction REGION, redirect
5164 the call to either its wrapper function, or its clone. */
5167 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
5168 struct tm_region
*region
,
5169 gimple_stmt_iterator
*gsi
,
5170 bool *need_ssa_rename_p
)
5172 gcall
*stmt
= as_a
<gcall
*> (gsi_stmt (*gsi
));
5173 struct cgraph_node
*new_node
;
5174 struct cgraph_edge
*e
= node
->get_edge (stmt
);
5175 tree fndecl
= gimple_call_fndecl (stmt
);
5177 /* For indirect calls, pass the address through the runtime. */
5180 *need_ssa_rename_p
|=
5181 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5185 /* Handle some TM builtins. Ordinarily these aren't actually generated
5186 at this point, but handling these functions when written in by the
5187 user makes it easier to build unit tests. */
5188 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
5191 /* Fixup recursive calls inside clones. */
5192 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5193 for recursion but not update the call statements themselves? */
5194 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
5196 gimple_call_set_fndecl (stmt
, current_function_decl
);
5200 /* If there is a replacement, use it. */
5201 fndecl
= find_tm_replacement_function (fndecl
);
5204 new_node
= cgraph_node::get_create (fndecl
);
5206 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5208 We can't do this earlier in record_tm_replacement because
5209 cgraph_remove_unreachable_nodes is called before we inject
5210 references to the node. Further, we can't do this in some
5211 nice central place in ipa_tm_execute because we don't have
5212 the exact list of wrapper functions that would be used.
5213 Marking more wrappers than necessary results in the creation
5214 of unnecessary cgraph_nodes, which can cause some of the
5215 other IPA passes to crash.
5217 We do need to mark these nodes so that we get the proper
5218 result in expand_call_tm. */
5219 /* ??? This seems broken. How is it that we're marking the
5220 CALLEE as may_enter_irr? Surely we should be marking the
5221 CALLER. Also note that find_tm_replacement_function also
5222 contains mappings into the TM runtime, e.g. memcpy. These
5223 we know won't go irrevocable. */
5224 new_node
->tm_may_enter_irr
= 1;
5228 struct tm_ipa_cg_data
*d
;
5229 struct cgraph_node
*tnode
= e
->callee
;
5231 d
= get_cg_data (&tnode
, true);
5232 new_node
= d
->clone
;
5234 /* As we've already skipped pure calls and appropriate builtins,
5235 and we've already marked irrevocable blocks, if we can't come
5236 up with a static replacement, then ask the runtime. */
5237 if (new_node
== NULL
)
5239 *need_ssa_rename_p
|=
5240 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
5244 fndecl
= new_node
->decl
;
5247 e
->redirect_callee (new_node
);
5248 gimple_call_set_fndecl (stmt
, fndecl
);
5251 /* Helper function for ipa_tm_transform_calls. For a given BB,
5252 install calls to tm_irrevocable when IRR_BLOCKS are reached,
5253 redirect other calls to the generated transactional clone. */
5256 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
5257 basic_block bb
, bitmap irr_blocks
)
5259 gimple_stmt_iterator gsi
;
5260 bool need_ssa_rename
= false;
5262 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5264 ipa_tm_insert_irr_call (node
, region
, bb
);
5268 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5270 gimple
*stmt
= gsi_stmt (gsi
);
5272 if (!is_gimple_call (stmt
))
5274 if (is_tm_pure_call (stmt
))
5277 /* Redirect edges to the appropriate replacement or clone. */
5278 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
5281 return need_ssa_rename
;
5284 /* Walk the CFG for REGION, beginning at BB. Install calls to
5285 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5286 the generated transactional clone. */
5289 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
5290 basic_block bb
, bitmap irr_blocks
)
5292 bool need_ssa_rename
= false;
5295 auto_vec
<basic_block
> queue
;
5296 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
5298 queue
.safe_push (bb
);
5304 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
5306 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
5309 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
5312 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
5313 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
5315 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
5316 queue
.safe_push (e
->dest
);
5319 while (!queue
.is_empty ());
5321 BITMAP_FREE (visited_blocks
);
5323 return need_ssa_rename
;
5326 /* Transform the calls within the TM regions within NODE. */
5329 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5331 struct tm_ipa_cg_data
*d
;
5332 struct tm_region
*region
;
5333 bool need_ssa_rename
= false;
5335 d
= get_cg_data (&node
, true);
5337 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5338 calculate_dominance_info (CDI_DOMINATORS
);
5340 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5342 /* If we're sure to go irrevocable, don't transform anything. */
5343 if (d
->irrevocable_blocks_normal
5344 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5345 region
->entry_block
->index
))
5347 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
5348 | GTMA_MAY_ENTER_IRREVOCABLE
5349 | GTMA_HAS_NO_INSTRUMENTATION
);
5354 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5355 d
->irrevocable_blocks_normal
);
5358 if (need_ssa_rename
)
5359 update_ssa (TODO_update_ssa_only_virtuals
);
5364 /* Transform the calls within the transactional clone of NODE. */
5367 ipa_tm_transform_clone (struct cgraph_node
*node
)
5369 struct tm_ipa_cg_data
*d
;
5370 bool need_ssa_rename
;
5372 d
= get_cg_data (&node
, true);
5374 /* If this function makes no calls and has no irrevocable blocks,
5375 then there's nothing to do. */
5376 /* ??? Remove non-aborting top-level transactions. */
5377 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5380 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->decl
));
5381 calculate_dominance_info (CDI_DOMINATORS
);
5384 ipa_tm_transform_calls (d
->clone
, NULL
,
5385 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)),
5386 d
->irrevocable_blocks_clone
);
5388 if (need_ssa_rename
)
5389 update_ssa (TODO_update_ssa_only_virtuals
);
5394 /* Main entry point for the transactional memory IPA pass. */
5397 ipa_tm_execute (void)
5399 cgraph_node_queue tm_callees
= cgraph_node_queue ();
5400 /* List of functions that will go irrevocable. */
5401 cgraph_node_queue irr_worklist
= cgraph_node_queue ();
5403 struct cgraph_node
*node
;
5404 struct tm_ipa_cg_data
*d
;
5405 enum availability a
;
5408 cgraph_node::checking_verify_cgraph_nodes ();
5410 bitmap_obstack_initialize (&tm_obstack
);
5411 initialize_original_copy_tables ();
5413 /* For all local functions marked tm_callable, queue them. */
5414 FOR_EACH_DEFINED_FUNCTION (node
)
5415 if (is_tm_callable (node
->decl
)
5416 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5418 d
= get_cg_data (&node
, true);
5419 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5422 /* For all local reachable functions... */
5423 FOR_EACH_DEFINED_FUNCTION (node
)
5425 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5427 /* ... marked tm_pure, record that fact for the runtime by
5428 indicating that the pure function is its own tm_callable.
5429 No need to do this if the function's address can't be taken. */
5430 if (is_tm_pure (node
->decl
))
5433 record_tm_clone_pair (node
->decl
, node
->decl
);
5437 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
5438 calculate_dominance_info (CDI_DOMINATORS
);
5440 tm_region_init (NULL
);
5443 d
= get_cg_data (&node
, true);
5445 /* Scan for calls that are in each transaction, and
5446 generate the uninstrumented code path. */
5447 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5449 /* Put it in the worklist so we can scan the function
5450 later (ipa_tm_scan_irr_function) and mark the
5451 irrevocable blocks. */
5452 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5453 d
->want_irr_scan_normal
= true;
5459 /* For every local function on the callee list, scan as if we will be
5460 creating a transactional clone, queueing all new functions we find
5462 for (i
= 0; i
< tm_callees
.length (); ++i
)
5464 node
= tm_callees
[i
];
5465 a
= node
->get_availability ();
5466 d
= get_cg_data (&node
, true);
5468 /* Put it in the worklist so we can scan the function later
5469 (ipa_tm_scan_irr_function) and mark the irrevocable
5471 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5473 /* Some callees cannot be arbitrarily cloned. These will always be
5474 irrevocable. Mark these now, so that we need not scan them. */
5475 if (is_tm_irrevocable (node
->decl
))
5476 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5477 else if (a
<= AVAIL_NOT_AVAILABLE
5478 && !is_tm_safe_or_pure (node
->decl
))
5479 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5480 else if (a
>= AVAIL_INTERPOSABLE
)
5482 if (!tree_versionable_function_p (node
->decl
))
5483 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5484 else if (!d
->is_irrevocable
)
5486 /* If this is an alias, make sure its base is queued as well.
5487 we need not scan the callees now, as the base will do. */
5490 node
= cgraph_node::get (thunk_info::get (node
)->alias
);
5491 d
= get_cg_data (&node
, true);
5492 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5496 /* Add all nodes called by this function into
5497 tm_callees as well. */
5498 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5503 /* Iterate scans until no more work to be done. Prefer not to use
5504 vec::pop because the worklist tends to follow a breadth-first
5505 search of the callgraph, which should allow convergance with a
5506 minimum number of scans. But we also don't want the worklist
5507 array to grow without bound, so we shift the array up periodically. */
5508 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5510 if (i
> 256 && i
== irr_worklist
.length () / 8)
5512 irr_worklist
.block_remove (0, i
);
5516 node
= irr_worklist
[i
];
5517 d
= get_cg_data (&node
, true);
5518 d
->in_worklist
= false;
5520 if (d
->want_irr_scan_normal
)
5522 d
->want_irr_scan_normal
= false;
5523 ipa_tm_scan_irr_function (node
, false);
5525 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5526 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5529 /* For every function on the callee list, collect the tm_may_enter_irr
5531 irr_worklist
.truncate (0);
5532 for (i
= 0; i
< tm_callees
.length (); ++i
)
5534 node
= tm_callees
[i
];
5535 if (ipa_tm_mayenterirr_function (node
))
5537 d
= get_cg_data (&node
, true);
5538 gcc_assert (d
->in_worklist
== false);
5539 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5543 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5544 for (i
= 0; i
< irr_worklist
.length (); ++i
)
5546 struct cgraph_node
*caller
;
5547 struct cgraph_edge
*e
;
5548 struct ipa_ref
*ref
;
5550 if (i
> 256 && i
== irr_worklist
.length () / 8)
5552 irr_worklist
.block_remove (0, i
);
5556 node
= irr_worklist
[i
];
5557 d
= get_cg_data (&node
, true);
5558 d
->in_worklist
= false;
5559 node
->tm_may_enter_irr
= true;
5561 /* Propagate back to normal callers. */
5562 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5565 if (!is_tm_safe_or_pure (caller
->decl
)
5566 && !caller
->tm_may_enter_irr
)
5568 d
= get_cg_data (&caller
, true);
5569 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5573 /* Propagate back to referring aliases as well. */
5574 FOR_EACH_ALIAS (node
, ref
)
5576 caller
= dyn_cast
<cgraph_node
*> (ref
->referring
);
5577 if (!caller
->tm_may_enter_irr
)
5579 /* ?? Do not traverse aliases here. */
5580 d
= get_cg_data (&caller
, false);
5581 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5586 /* Now validate all tm_safe functions, and all atomic regions in
5588 FOR_EACH_DEFINED_FUNCTION (node
)
5590 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5592 d
= get_cg_data (&node
, true);
5593 if (is_tm_safe (node
->decl
))
5594 ipa_tm_diagnose_tm_safe (node
);
5595 else if (d
->all_tm_regions
)
5596 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5599 /* Create clones. Do those that are not irrevocable and have a
5600 positive call count. Do those publicly visible functions that
5601 the user directed us to clone. */
5602 for (i
= 0; i
< tm_callees
.length (); ++i
)
5606 node
= tm_callees
[i
];
5607 if (node
->cpp_implicit_alias
)
5610 a
= node
->get_availability ();
5611 d
= get_cg_data (&node
, true);
5613 if (a
<= AVAIL_NOT_AVAILABLE
)
5614 doit
= is_tm_callable (node
->decl
);
5615 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->decl
))
5617 else if (!d
->is_irrevocable
5618 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5622 ipa_tm_create_version (node
);
5625 /* Redirect calls to the new clones, and insert irrevocable marks. */
5626 for (i
= 0; i
< tm_callees
.length (); ++i
)
5628 node
= tm_callees
[i
];
5631 d
= get_cg_data (&node
, true);
5633 ipa_tm_transform_clone (node
);
5636 FOR_EACH_DEFINED_FUNCTION (node
)
5638 && node
->get_availability () >= AVAIL_INTERPOSABLE
)
5640 d
= get_cg_data (&node
, true);
5641 if (d
->all_tm_regions
)
5642 ipa_tm_transform_transaction (node
);
5645 /* Free and clear all data structures. */
5646 tm_callees
.release ();
5647 irr_worklist
.release ();
5648 bitmap_obstack_release (&tm_obstack
);
5649 free_original_copy_tables ();
5651 FOR_EACH_FUNCTION (node
)
5654 cgraph_node::checking_verify_cgraph_nodes ();
5661 const pass_data pass_data_ipa_tm
=
5663 SIMPLE_IPA_PASS
, /* type */
5665 OPTGROUP_NONE
, /* optinfo_flags */
5666 TV_TRANS_MEM
, /* tv_id */
5667 ( PROP_ssa
| PROP_cfg
), /* properties_required */
5668 0, /* properties_provided */
5669 0, /* properties_destroyed */
5670 0, /* todo_flags_start */
5671 0, /* todo_flags_finish */
5674 class pass_ipa_tm
: public simple_ipa_opt_pass
5677 pass_ipa_tm (gcc::context
*ctxt
)
5678 : simple_ipa_opt_pass (pass_data_ipa_tm
, ctxt
)
5681 /* opt_pass methods: */
5682 bool gate (function
*) final override
{ return flag_tm
; }
5683 unsigned int execute (function
*) final override
{ return ipa_tm_execute (); }
5685 }; // class pass_ipa_tm
5689 simple_ipa_opt_pass
*
5690 make_pass_ipa_tm (gcc::context
*ctxt
)
5692 return new pass_ipa_tm (ctxt
);
5695 #include "gt-trans-mem.h"