6 * Rodrigo Kumpera (kumpera@gmail.com)
8 * Copyright 2015 Xamarin, Inc (http://www.xamarin.com)
9 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
14 /* enable pthread extensions */
16 #define _DARWIN_C_SOURCE
19 #include <mono/utils/mono-compiler.h>
20 #include <mono/utils/mono-threads.h>
21 #include <mono/utils/mono-tls.h>
22 #include <mono/utils/hazard-pointer.h>
23 #include <mono/utils/mono-memory-model.h>
24 #include <mono/utils/mono-mmap.h>
25 #include <mono/utils/atomic.h>
26 #include <mono/utils/mono-time.h>
27 #include <mono/utils/mono-counters.h>
28 #include <mono/utils/mono-threads-coop.h>
29 #include <mono/utils/mono-threads-api.h>
30 #include <mono/utils/checked-build.h>
31 #include <mono/utils/mono-threads-debug.h>
34 #include <mono/utils/mach-support.h>
38 // TODO: Find MSVC replacement for __builtin_unwind_init
39 #define SAVE_REGS_ON_STACK g_assert_not_reached ();
40 #elif defined (HOST_WASM)
41 //TODO: figure out wasm stack scanning
42 #define SAVE_REGS_ON_STACK do {} while (0)
44 #define SAVE_REGS_ON_STACK __builtin_unwind_init ();
47 volatile size_t mono_polling_required
;
49 // FIXME: This would be more efficient if instead of instantiating the stack it just pushed a simple depth counter up and down,
50 // perhaps with a per-thread cookie in the high bits.
51 #ifdef ENABLE_CHECKED_BUILD_GC
53 // Maintains a single per-thread stack of ints, used to ensure nesting is not violated
54 static MonoNativeTlsKey coop_reset_count_stack_key
;
57 coop_tls_push (gpointer cookie
)
61 stack
= (GArray
*)mono_native_tls_get_value (coop_reset_count_stack_key
);
63 stack
= g_array_new (FALSE
, FALSE
, sizeof(gpointer
));
64 mono_native_tls_set_value (coop_reset_count_stack_key
, stack
);
67 g_array_append_val (stack
, cookie
);
71 coop_tls_pop (gpointer received_cookie
)
74 gpointer expected_cookie
;
76 stack
= (GArray
*)mono_native_tls_get_value (coop_reset_count_stack_key
);
77 if (!stack
|| 0 == stack
->len
)
78 mono_fatal_with_history ("Received cookie %p but found no stack at all\n", received_cookie
);
80 expected_cookie
= g_array_index (stack
, gpointer
, stack
->len
- 1);
83 if (0 == stack
->len
) {
84 g_array_free (stack
,TRUE
);
85 mono_native_tls_set_value (coop_reset_count_stack_key
, NULL
);
88 if (expected_cookie
!= received_cookie
)
89 mono_fatal_with_history ("Received cookie %p but expected %p\n", received_cookie
, expected_cookie
);
95 check_info (MonoThreadInfo
*info
, const gchar
*action
, const gchar
*state
, const char *func
)
98 g_error ("%s Cannot %s GC %s region if the thread is not attached", func
, action
, state
);
99 if (!mono_thread_info_is_current (info
))
100 g_error ("%s [%p] Cannot %s GC %s region on a different thread", func
, mono_thread_info_get_tid (info
), action
, state
);
101 if (!mono_thread_info_is_live (info
))
102 g_error ("%s [%p] Cannot %s GC %s region if the thread is not live", func
, mono_thread_info_get_tid (info
), action
, state
);
105 static int coop_reset_blocking_count
;
106 static int coop_try_blocking_count
;
107 static int coop_do_blocking_count
;
108 static int coop_do_polling_count
;
109 static int coop_save_count
;
112 mono_threads_state_poll_with_info (MonoThreadInfo
*info
);
115 mono_threads_state_poll (void)
117 mono_threads_state_poll_with_info (mono_thread_info_current_unchecked ());
121 mono_threads_state_poll_with_info (MonoThreadInfo
*info
)
123 g_assert (mono_threads_is_blocking_transition_enabled ());
125 ++coop_do_polling_count
;
130 THREADS_SUSPEND_DEBUG ("FINISH SELF SUSPEND OF %p\n", mono_thread_info_get_tid (info
));
132 /* Fast fail if no_safepoints is set */
133 g_assert (!(info
->thread_state
& THREAD_SUSPEND_NO_SAFEPOINTS_MASK
));
135 /* Fast check for pending suspend requests */
136 if (!(info
->thread_state
& STATE_ASYNC_SUSPEND_REQUESTED
))
140 mono_threads_get_runtime_callbacks ()->thread_state_init (&info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
]);
142 /* commit the saved state and notify others if needed */
143 switch (mono_threads_transition_state_poll (info
)) {
144 case SelfSuspendResumed
:
146 case SelfSuspendNotifyAndWait
:
147 mono_threads_notify_initiator_of_suspend (info
);
148 mono_thread_info_wait_for_resume (info
);
152 if (info
->async_target
) {
153 info
->async_target (info
->user_data
);
154 info
->async_target
= NULL
;
155 info
->user_data
= NULL
;
159 static volatile gpointer
* dummy_global
;
161 static MONO_NEVER_INLINE
163 return_stack_ptr (gpointer
*i
)
170 copy_stack_data (MonoThreadInfo
*info
, MonoStackData
*stackdata_begin
)
172 MonoThreadUnwindState
*state
;
175 void* stackdata_end
= return_stack_ptr (&dummy
);
179 state
= &info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
];
181 stackdata_size
= (char*)mono_stackdata_get_stackpointer (stackdata_begin
) - (char*)stackdata_end
;
183 const char *function_name
= mono_stackdata_get_function_name (stackdata_begin
);
185 if (((gsize
) stackdata_begin
& (SIZEOF_VOID_P
- 1)) != 0)
186 g_error ("%s stackdata_begin (%p) must be %d-byte aligned", function_name
, stackdata_begin
, SIZEOF_VOID_P
);
187 if (((gsize
) stackdata_end
& (SIZEOF_VOID_P
- 1)) != 0)
188 g_error ("%s stackdata_end (%p) must be %d-byte aligned", function_name
, stackdata_end
, SIZEOF_VOID_P
);
190 if (stackdata_size
<= 0)
191 g_error ("%s stackdata_size = %d, but must be > 0, stackdata_begin = %p, stackdata_end = %p", function_name
, stackdata_size
, stackdata_begin
, stackdata_end
);
193 g_byte_array_set_size (info
->stackdata
, stackdata_size
);
194 state
->gc_stackdata
= info
->stackdata
->data
;
195 memcpy (state
->gc_stackdata
, stackdata_end
, stackdata_size
);
197 state
->gc_stackdata_size
= stackdata_size
;
201 mono_threads_enter_gc_safe_region_unbalanced_with_info (MonoThreadInfo
*info
, MonoStackData
*stackdata
);
204 mono_threads_enter_gc_safe_region_internal (MonoStackData
*stackdata
)
206 return mono_threads_enter_gc_safe_region_with_info (mono_thread_info_current_unchecked (), stackdata
);
210 mono_threads_enter_gc_safe_region (gpointer
*stackpointer
)
212 MONO_STACKDATA (stackdata
);
213 stackdata
.stackpointer
= stackpointer
;
214 return mono_threads_enter_gc_safe_region_internal (&stackdata
);
218 mono_threads_enter_gc_safe_region_with_info (MonoThreadInfo
*info
, MonoStackData
*stackdata
)
222 if (!mono_threads_is_blocking_transition_enabled ())
225 cookie
= mono_threads_enter_gc_safe_region_unbalanced_with_info (info
, stackdata
);
227 #ifdef ENABLE_CHECKED_BUILD_GC
228 if (mono_check_mode_enabled (MONO_CHECK_MODE_GC
))
229 coop_tls_push (cookie
);
236 mono_threads_enter_gc_safe_region_unbalanced_internal (MonoStackData
*stackdata
)
238 return mono_threads_enter_gc_safe_region_unbalanced_with_info (mono_thread_info_current_unchecked (), stackdata
);
242 mono_threads_enter_gc_safe_region_unbalanced (gpointer
*stackpointer
)
244 MONO_STACKDATA (stackdata
);
245 stackdata
.stackpointer
= stackpointer
;
246 return mono_threads_enter_gc_safe_region_unbalanced_internal (&stackdata
);
250 mono_threads_enter_gc_safe_region_unbalanced_with_info (MonoThreadInfo
*info
, MonoStackData
*stackdata
)
252 if (!mono_threads_is_blocking_transition_enabled ())
255 ++coop_do_blocking_count
;
257 const char *function_name
= mono_stackdata_get_function_name (stackdata
);
259 check_info (info
, "enter", "safe", function_name
);
261 copy_stack_data (info
, stackdata
);
265 mono_threads_get_runtime_callbacks ()->thread_state_init (&info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
]);
267 switch (mono_threads_transition_do_blocking (info
, function_name
)) {
268 case DoBlockingContinue
:
270 case DoBlockingPollAndRetry
:
271 mono_threads_state_poll_with_info (info
);
279 mono_threads_exit_gc_safe_region_internal (gpointer cookie
, MonoStackData
*stackdata
)
281 if (!mono_threads_is_blocking_transition_enabled ())
284 #ifdef ENABLE_CHECKED_BUILD_GC
285 if (mono_check_mode_enabled (MONO_CHECK_MODE_GC
))
286 coop_tls_pop (cookie
);
289 mono_threads_exit_gc_safe_region_unbalanced_internal (cookie
, stackdata
);
293 mono_threads_exit_gc_safe_region (gpointer cookie
, gpointer
*stackpointer
)
295 MONO_STACKDATA (stackdata
);
296 stackdata
.stackpointer
= stackpointer
;
297 mono_threads_exit_gc_safe_region_internal (cookie
, &stackdata
);
301 mono_threads_exit_gc_safe_region_unbalanced_internal (gpointer cookie
, MonoStackData
*stackdata
)
303 MonoThreadInfo
*info
;
305 if (!mono_threads_is_blocking_transition_enabled ())
308 info
= (MonoThreadInfo
*)cookie
;
310 const char *function_name
= mono_stackdata_get_function_name (stackdata
);
312 check_info (info
, "exit", "safe", function_name
);
314 switch (mono_threads_transition_done_blocking (info
, function_name
)) {
316 info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
].valid
= FALSE
;
318 case DoneBlockingWait
:
319 /* If full coop suspend, we're just waiting for the initiator
320 * to resume us. If hybrid suspend, we were either self
321 * suspended cooperatively from async_suspend_requested (same
322 * as full coop), or we were suspended preemptively while in
323 * blocking and we're waiting for two things: the suspend
324 * signal handler to run and notify the initiator and
325 * immediately return, and then for the resume. */
326 THREADS_SUSPEND_DEBUG ("state polling done, notifying of resume\n");
327 mono_thread_info_wait_for_resume (info
);
330 g_error ("Unknown thread state");
333 if (info
->async_target
) {
334 info
->async_target (info
->user_data
);
335 info
->async_target
= NULL
;
336 info
->user_data
= NULL
;
341 mono_threads_exit_gc_safe_region_unbalanced (gpointer cookie
, gpointer
*stackpointer
)
343 MONO_STACKDATA (stackdata
);
344 stackdata
.stackpointer
= stackpointer
;
345 mono_threads_exit_gc_safe_region_unbalanced_internal (cookie
, &stackdata
);
349 mono_threads_assert_gc_safe_region (void)
351 MONO_REQ_GC_SAFE_MODE
;
355 mono_threads_enter_gc_unsafe_region_internal (MonoStackData
*stackdata
)
357 return mono_threads_enter_gc_unsafe_region_with_info (mono_thread_info_current_unchecked (), stackdata
);
361 mono_threads_enter_gc_unsafe_region (gpointer
*stackpointer
)
363 MONO_STACKDATA (stackdata
);
364 stackdata
.stackpointer
= stackpointer
;
365 return mono_threads_enter_gc_unsafe_region_internal (&stackdata
);
369 mono_threads_enter_gc_unsafe_region_with_info (THREAD_INFO_TYPE
*info
, MonoStackData
*stackdata
)
373 if (!mono_threads_is_blocking_transition_enabled ())
376 cookie
= mono_threads_enter_gc_unsafe_region_unbalanced_with_info (info
, stackdata
);
378 #ifdef ENABLE_CHECKED_BUILD_GC
379 if (mono_check_mode_enabled (MONO_CHECK_MODE_GC
))
380 coop_tls_push (cookie
);
387 mono_threads_enter_gc_unsafe_region_unbalanced_internal (MonoStackData
*stackdata
)
389 return mono_threads_enter_gc_unsafe_region_unbalanced_with_info (mono_thread_info_current_unchecked (), stackdata
);
393 mono_threads_enter_gc_unsafe_region_unbalanced (gpointer
*stackpointer
)
395 MONO_STACKDATA (stackdata
);
396 stackdata
.stackpointer
= stackpointer
;
397 return mono_threads_enter_gc_unsafe_region_unbalanced_internal (&stackdata
);
401 mono_threads_enter_gc_unsafe_region_unbalanced_with_info (MonoThreadInfo
*info
, MonoStackData
*stackdata
)
403 if (!mono_threads_is_blocking_transition_enabled ())
406 ++coop_reset_blocking_count
;
408 const char *function_name
= mono_stackdata_get_function_name (stackdata
);
410 check_info (info
, "enter", "unsafe", function_name
);
412 copy_stack_data (info
, stackdata
);
414 switch (mono_threads_transition_abort_blocking (info
, function_name
)) {
415 case AbortBlockingIgnore
:
416 info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
].valid
= FALSE
;
418 case AbortBlockingIgnoreAndPoll
:
419 mono_threads_state_poll_with_info (info
);
421 case AbortBlockingOk
:
422 info
->thread_saved_state
[SELF_SUSPEND_STATE_INDEX
].valid
= FALSE
;
424 case AbortBlockingWait
:
425 /* If full coop suspend, we're just waiting for the initiator
426 * to resume us. If hybrid suspend, we were either self
427 * suspended cooperatively from async_suspend_requested (same
428 * as full coop), or we were suspended preemptively while in
429 * blocking and we're waiting for two things: the suspend
430 * signal handler to run and notify the initiator and
431 * immediately return, and then for the resume. */
432 mono_thread_info_wait_for_resume (info
);
435 g_error ("Unknown thread state %s", function_name
);
438 if (info
->async_target
) {
439 info
->async_target (info
->user_data
);
440 info
->async_target
= NULL
;
441 info
->user_data
= NULL
;
448 mono_threads_enter_gc_unsafe_region_cookie (void)
450 MonoThreadInfo
*info
;
452 g_assert (mono_threads_is_blocking_transition_enabled ());
454 info
= mono_thread_info_current_unchecked ();
456 check_info (info
, "enter (cookie)", "unsafe", "");
458 #ifdef ENABLE_CHECKED_BUILD_GC
459 if (mono_check_mode_enabled (MONO_CHECK_MODE_GC
))
460 coop_tls_push (info
);
467 mono_threads_exit_gc_unsafe_region_internal (gpointer cookie
, MonoStackData
*stackdata
)
469 if (!mono_threads_is_blocking_transition_enabled ())
472 #ifdef ENABLE_CHECKED_BUILD_GC
473 if (mono_check_mode_enabled (MONO_CHECK_MODE_GC
))
474 coop_tls_pop (cookie
);
477 mono_threads_exit_gc_unsafe_region_unbalanced_internal (cookie
, stackdata
);
481 mono_threads_exit_gc_unsafe_region (gpointer cookie
, gpointer
*stackpointer
)
483 MONO_STACKDATA (stackdata
);
484 stackdata
.stackpointer
= stackpointer
;
485 mono_threads_exit_gc_unsafe_region_internal (cookie
, &stackdata
);
489 mono_threads_exit_gc_unsafe_region_unbalanced_internal (gpointer cookie
, MonoStackData
*stackdata
)
491 if (!mono_threads_is_blocking_transition_enabled ())
497 mono_threads_enter_gc_safe_region_unbalanced_internal (stackdata
);
501 mono_threads_exit_gc_unsafe_region_unbalanced (gpointer cookie
, gpointer
*stackpointer
)
503 MONO_STACKDATA (stackdata
);
504 stackdata
.stackpointer
= stackpointer
;
505 mono_threads_exit_gc_unsafe_region_unbalanced_internal (cookie
, &stackdata
);
509 mono_threads_assert_gc_unsafe_region (void)
511 MONO_REQ_GC_UNSAFE_MODE
;
515 threads_suspend_policy_default (void)
517 #if defined (ENABLE_COOP_SUSPEND)
518 return MONO_THREADS_SUSPEND_FULL_COOP
;
519 #elif defined (ENABLE_HYBRID_SUSPEND)
520 return MONO_THREADS_SUSPEND_HYBRID
;
522 return 0; /* unset */
526 /* Look up whether an env var is set, warn that it's obsolete and offer a new
530 hasenv_obsolete (const char *name
, const char* newval
)
532 // If they already set MONO_THREADS_SUSPEND to something, maybe they're keeping
533 // the old var set for compatability with old Mono - in that case don't nag.
534 // FIXME: but maybe nag if MONO_THREADS_SUSPEND isn't set to "newval"?
535 static int quiet
= -1;
536 if (g_hasenv (name
)) {
537 if (G_UNLIKELY (quiet
== -1))
538 quiet
= g_hasenv ("MONO_THREADS_SUSPEND");
540 g_warning ("%s environment variable is obsolete. Use MONO_THREADS_SUSPEND=%s", name
, newval
);
547 threads_suspend_policy_getenv_compat (void)
550 if (hasenv_obsolete ("MONO_ENABLE_COOP", "coop") || hasenv_obsolete ("MONO_ENABLE_COOP_SUSPEND", "coop")) {
551 g_assertf (!hasenv_obsolete ("MONO_ENABLE_HYBRID_SUSPEND", "hybrid"),
552 "Environment variables set to enable both hybrid and cooperative suspend simultaneously");
553 policy
= MONO_THREADS_SUSPEND_FULL_COOP
;
554 } else if (hasenv_obsolete ("MONO_ENABLE_HYBRID_SUSPEND", "hybrid"))
555 policy
= MONO_THREADS_SUSPEND_HYBRID
;
560 threads_suspend_policy_getenv (void)
563 if (g_hasenv ("MONO_THREADS_SUSPEND")) {
564 gchar
*str
= g_getenv ("MONO_THREADS_SUSPEND");
565 if (!strcmp (str
, "coop"))
566 policy
= MONO_THREADS_SUSPEND_FULL_COOP
;
567 else if (!strcmp (str
, "hybrid"))
568 policy
= MONO_THREADS_SUSPEND_HYBRID
;
569 else if (!strcmp (str
, "preemptive"))
570 policy
= MONO_THREADS_SUSPEND_FULL_PREEMPTIVE
;
572 g_error ("MONO_THREADS_SUSPEND environment variable set to '%s', must be one of coop, hybrid, preemptive.", str
);
578 static char threads_suspend_policy
;
580 static MonoThreadsSuspendPolicy
581 mono_threads_suspend_policy (void)
583 int policy
= threads_suspend_policy
;
584 if (G_UNLIKELY (policy
== 0)) {
585 // thread suspend policy:
586 // if the MONO_THREADS_SUSPEND env is set, use it.
587 // otherwise if there's a compiled-in default, use it.
588 // otherwise if one of the old environment variables is set, use that.
589 // otherwise use full preemptive suspend.
590 (policy
= threads_suspend_policy_getenv ())
591 || (policy
= threads_suspend_policy_default ())
592 || (policy
= threads_suspend_policy_getenv_compat ())
593 || (policy
= MONO_THREADS_SUSPEND_FULL_PREEMPTIVE
);
595 threads_suspend_policy
= (char)policy
;
597 return (MonoThreadsSuspendPolicy
)policy
;
600 static MonoThreadsSuspendPolicy
601 mono_threads_suspend_validate_policy (MonoThreadsSuspendPolicy policy
)
604 case MONO_THREADS_SUSPEND_FULL_COOP
:
605 case MONO_THREADS_SUSPEND_FULL_PREEMPTIVE
:
606 case MONO_THREADS_SUSPEND_HYBRID
:
609 g_error ("Invalid suspend policy %d.", (int)policy
);
614 * mono_threads_suspend_override_policy:
616 * Don't use this. Provides a last resort escape hatch to override configure
617 * and environment settings and use the given thread suspend policy.
621 mono_threads_suspend_override_policy (MonoThreadsSuspendPolicy new_policy
)
623 threads_suspend_policy
= (char)mono_threads_suspend_validate_policy (new_policy
);
624 g_warning ("Overriding suspend policy. Using %s suspend.", mono_threads_suspend_policy_name ());
628 mono_threads_suspend_policy_name (void)
630 MonoThreadsSuspendPolicy policy
= mono_threads_suspend_policy ();
632 case MONO_THREADS_SUSPEND_FULL_COOP
:
633 return "cooperative";
634 case MONO_THREADS_SUSPEND_FULL_PREEMPTIVE
:
636 case MONO_THREADS_SUSPEND_HYBRID
:
639 g_assert_not_reached ();
644 mono_threads_is_cooperative_suspension_enabled (void)
646 return (mono_threads_suspend_policy () == MONO_THREADS_SUSPEND_FULL_COOP
);
650 mono_threads_is_blocking_transition_enabled (void)
652 static int is_blocking_transition_enabled
= -1;
653 if (G_UNLIKELY (is_blocking_transition_enabled
== -1)) {
654 if (g_hasenv ("MONO_ENABLE_BLOCKING_TRANSITION"))
655 is_blocking_transition_enabled
= 1;
657 switch (mono_threads_suspend_policy ()) {
658 case MONO_THREADS_SUSPEND_FULL_COOP
:
659 case MONO_THREADS_SUSPEND_HYBRID
:
660 is_blocking_transition_enabled
= 1;
662 case MONO_THREADS_SUSPEND_FULL_PREEMPTIVE
:
663 is_blocking_transition_enabled
= 0;
666 g_assert_not_reached ();
670 return is_blocking_transition_enabled
== 1;
674 mono_threads_is_hybrid_suspension_enabled (void)
676 return (mono_threads_suspend_policy () == MONO_THREADS_SUSPEND_HYBRID
);
680 mono_threads_coop_init (void)
682 if (!mono_threads_are_safepoints_enabled () && !mono_threads_is_blocking_transition_enabled ())
685 mono_counters_register ("Coop Reset Blocking", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &coop_reset_blocking_count
);
686 mono_counters_register ("Coop Try Blocking", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &coop_try_blocking_count
);
687 mono_counters_register ("Coop Do Blocking", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &coop_do_blocking_count
);
688 mono_counters_register ("Coop Do Polling", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &coop_do_polling_count
);
689 mono_counters_register ("Coop Save Count", MONO_COUNTER_GC
| MONO_COUNTER_INT
, &coop_save_count
);
690 //See the above for what's wrong here.
692 #ifdef ENABLE_CHECKED_BUILD_GC
693 mono_native_tls_alloc (&coop_reset_count_stack_key
, NULL
);
698 mono_threads_coop_begin_global_suspend (void)
700 if (mono_threads_are_safepoints_enabled ())
701 mono_polling_required
= 1;
705 mono_threads_coop_end_global_suspend (void)
707 if (mono_threads_are_safepoints_enabled ())
708 mono_polling_required
= 0;
712 mono_threads_enter_no_safepoints_region (const char *func
)
714 MONO_REQ_GC_UNSAFE_MODE
;
715 mono_threads_transition_begin_no_safepoints (mono_thread_info_current (), func
);
719 mono_threads_exit_no_safepoints_region (const char *func
)
721 MONO_REQ_GC_UNSAFE_MODE
;
722 mono_threads_transition_end_no_safepoints (mono_thread_info_current (), func
);