mono/utils/mono-threads-coop.c

   1 /**
   2  * \file
   3  * Coop threading
   4  *
   5  * Author:
   6  *      Rodrigo Kumpera (kumpera@gmail.com)
   7  *
   8  * Copyright 2015 Xamarin, Inc (http://www.xamarin.com)
   9  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
  10  */
  11
  12 #include <config.h>
  13
  14 /* enable pthread extensions */
  15 #ifdef TARGET_MACH
  16 #define _DARWIN_C_SOURCE
  17 #endif
  18
  19 #include <mono/utils/mono-compiler.h>
  20 #include <mono/utils/mono-threads.h>
  21 #include <mono/utils/mono-tls.h>
  22 #include <mono/utils/hazard-pointer.h>
  23 #include <mono/utils/mono-memory-model.h>
  24 #include <mono/utils/mono-mmap.h>
  25 #include <mono/utils/atomic.h>
  26 #include <mono/utils/mono-time.h>
  27 #include <mono/utils/mono-counters.h>
  28 #include <mono/utils/mono-threads-coop.h>
  29 #include <mono/utils/mono-threads-api.h>
  30 #include <mono/utils/checked-build.h>
  31 #include <mono/utils/mono-threads-debug.h>
  32
  33 #ifdef TARGET_OSX
  34 #include <mono/utils/mach-support.h>
  35 #endif
  36
  37 #ifdef _MSC_VER
  38 // TODO: Find MSVC replacement for __builtin_unwind_init
  39 #define SAVE_REGS_ON_STACK g_assert_not_reached ();
  40 #elif defined (HOST_WASM)
  41 //TODO: figure out wasm stack scanning
  42 #define SAVE_REGS_ON_STACK do {} while (0)
  43 #else
  44 #define SAVE_REGS_ON_STACK __builtin_unwind_init ();
  45 #endif
  46
  47 volatile size_t mono_polling_required;
  48
  49 // FIXME: This would be more efficient if instead of instantiating the stack it just pushed a simple depth counter up and down,
  50 // perhaps with a per-thread cookie in the high bits.
  51 #ifdef ENABLE_CHECKED_BUILD_GC
  52
  53 // Maintains a single per-thread stack of ints, used to ensure nesting is not violated
  54 static MonoNativeTlsKey coop_reset_count_stack_key;
  55
  56 static void
  57 coop_tls_push (gpointer cookie)
  58 {
  59         GArray *stack;
  60
  61         stack = mono_native_tls_get_value (coop_reset_count_stack_key);
  62         if (!stack) {
  63                 stack = g_array_new (FALSE, FALSE, sizeof(gpointer));
  64                 mono_native_tls_set_value (coop_reset_count_stack_key, stack);
  65         }
  66
  67         g_array_append_val (stack, cookie);
  68 }
  69
  70 static void
  71 coop_tls_pop (gpointer received_cookie)
  72 {
  73         GArray *stack;
  74         gpointer expected_cookie;
  75
  76         stack = mono_native_tls_get_value (coop_reset_count_stack_key);
  77         if (!stack || 0 == stack->len)
  78                 mono_fatal_with_history ("Received cookie %p but found no stack at all\n", received_cookie);
  79
  80         expected_cookie = g_array_index (stack, gpointer, stack->len - 1);
  81         stack->len --;
  82
  83         if (0 == stack->len) {
  84                 g_array_free (stack,TRUE);
  85                 mono_native_tls_set_value (coop_reset_count_stack_key, NULL);
  86         }
  87
  88         if (expected_cookie != received_cookie)
  89                 mono_fatal_with_history ("Received cookie %p but expected %p\n", received_cookie, expected_cookie);
  90 }
  91
  92 #endif
  93
  94 static void
  95 check_info (MonoThreadInfo *info, const gchar *action, const gchar *state, const char *func)
  96 {
  97         if (!info)
  98                 g_error ("%s Cannot %s GC %s region if the thread is not attached", func, action, state);
  99         if (!mono_thread_info_is_current (info))
 100                 g_error ("%s [%p] Cannot %s GC %s region on a different thread", func, mono_thread_info_get_tid (info), action, state);
 101         if (!mono_thread_info_is_live (info))
 102                 g_error ("%s [%p] Cannot %s GC %s region if the thread is not live", func, mono_thread_info_get_tid (info), action, state);
 103 }
 104
 105 static int coop_reset_blocking_count;
 106 static int coop_try_blocking_count;
 107 static int coop_do_blocking_count;
 108 static int coop_do_polling_count;
 109 static int coop_save_count;
 110
 111 static void
 112 mono_threads_state_poll_with_info (MonoThreadInfo *info);
 113
 114 void
 115 mono_threads_state_poll (void)
 116 {
 117         mono_threads_state_poll_with_info (mono_thread_info_current_unchecked ());
 118 }
 119
 120 static void
 121 mono_threads_state_poll_with_info (MonoThreadInfo *info)
 122 {
 123         g_assert (mono_threads_is_blocking_transition_enabled ());
 124
 125         ++coop_do_polling_count;
 126
 127         if (!info)
 128                 return;
 129
 130         THREADS_SUSPEND_DEBUG ("FINISH SELF SUSPEND OF %p\n", mono_thread_info_get_tid (info));
 131
 132         /* Fast check for pending suspend requests */
 133         if (!(info->thread_state & STATE_ASYNC_SUSPEND_REQUESTED))
 134                 return;
 135
 136         ++coop_save_count;
 137         mono_threads_get_runtime_callbacks ()->thread_state_init (&info->thread_saved_state [SELF_SUSPEND_STATE_INDEX]);
 138
 139         /* commit the saved state and notify others if needed */
 140         switch (mono_threads_transition_state_poll (info)) {
 141         case SelfSuspendResumed:
 142                 break;
 143         case SelfSuspendNotifyAndWait:
 144                 mono_threads_notify_initiator_of_suspend (info);
 145                 mono_thread_info_wait_for_resume (info);
 146                 break;
 147         }
 148
 149         if (info->async_target) {
 150                 info->async_target (info->user_data);
 151                 info->async_target = NULL;
 152                 info->user_data = NULL;
 153         }
 154 }
 155
 156 static volatile gpointer* dummy_global;
 157
 158 static MONO_NEVER_INLINE
 159 void*
 160 return_stack_ptr (gpointer *i)
 161 {
 162         dummy_global = i;
 163         return i;
 164 }
 165
 166 static void
 167 copy_stack_data (MonoThreadInfo *info, MonoStackData *stackdata_begin)
 168 {
 169         MonoThreadUnwindState *state;
 170         int stackdata_size;
 171         gpointer dummy;
 172         void* stackdata_end = return_stack_ptr (&dummy);
 173
 174         SAVE_REGS_ON_STACK;
 175
 176         state = &info->thread_saved_state [SELF_SUSPEND_STATE_INDEX];
 177
 178         stackdata_size = (char*)mono_stackdata_get_stackpointer (stackdata_begin) - (char*)stackdata_end;
 179
 180         const char *function_name = mono_stackdata_get_function_name (stackdata_begin);
 181
 182         if (((gsize) stackdata_begin & (SIZEOF_VOID_P - 1)) != 0)
 183                 g_error ("%s stackdata_begin (%p) must be %d-byte aligned", function_name, stackdata_begin, SIZEOF_VOID_P);
 184         if (((gsize) stackdata_end & (SIZEOF_VOID_P - 1)) != 0)
 185                 g_error ("%s stackdata_end (%p) must be %d-byte aligned", function_name, stackdata_end, SIZEOF_VOID_P);
 186
 187         if (stackdata_size <= 0)
 188                 g_error ("%s stackdata_size = %d, but must be > 0, stackdata_begin = %p, stackdata_end = %p", function_name, stackdata_size, stackdata_begin, stackdata_end);
 189
 190         g_byte_array_set_size (info->stackdata, stackdata_size);
 191         state->gc_stackdata = info->stackdata->data;
 192         memcpy (state->gc_stackdata, stackdata_end, stackdata_size);
 193
 194         state->gc_stackdata_size = stackdata_size;
 195 }
 196
 197 static gpointer
 198 mono_threads_enter_gc_safe_region_unbalanced_with_info (MonoThreadInfo *info, MonoStackData *stackdata);
 199
 200 gpointer
 201 mono_threads_enter_gc_safe_region_internal (MonoStackData *stackdata)
 202 {
 203         return mono_threads_enter_gc_safe_region_with_info (mono_thread_info_current_unchecked (), stackdata);
 204 }
 205
 206 gpointer
 207 mono_threads_enter_gc_safe_region (gpointer *stackpointer)
 208 {
 209         MONO_STACKDATA (stackdata);
 210         stackdata.stackpointer = stackpointer;
 211         return mono_threads_enter_gc_safe_region_internal (&stackdata);
 212 }
 213
 214 gpointer
 215 mono_threads_enter_gc_safe_region_with_info (MonoThreadInfo *info, MonoStackData *stackdata)
 216 {
 217         gpointer cookie;
 218
 219         if (!mono_threads_is_blocking_transition_enabled ())
 220                 return NULL;
 221
 222         cookie = mono_threads_enter_gc_safe_region_unbalanced_with_info (info, stackdata);
 223
 224 #ifdef ENABLE_CHECKED_BUILD_GC
 225         if (mono_check_mode_enabled (MONO_CHECK_MODE_GC))
 226                 coop_tls_push (cookie);
 227 #endif
 228
 229         return cookie;
 230 }
 231
 232 gpointer
 233 mono_threads_enter_gc_safe_region_unbalanced_internal (MonoStackData *stackdata)
 234 {
 235         return mono_threads_enter_gc_safe_region_unbalanced_with_info (mono_thread_info_current_unchecked (), stackdata);
 236 }
 237
 238 gpointer
 239 mono_threads_enter_gc_safe_region_unbalanced (gpointer *stackpointer)
 240 {
 241         MONO_STACKDATA (stackdata);
 242         stackdata.stackpointer = stackpointer;
 243         return mono_threads_enter_gc_safe_region_unbalanced_internal (&stackdata);
 244 }
 245
 246 static gpointer
 247 mono_threads_enter_gc_safe_region_unbalanced_with_info (MonoThreadInfo *info, MonoStackData *stackdata)
 248 {
 249         if (!mono_threads_is_blocking_transition_enabled ())
 250                 return NULL;
 251
 252         ++coop_do_blocking_count;
 253
 254         const char *function_name = mono_stackdata_get_function_name (stackdata);
 255
 256         check_info (info, "enter", "safe", function_name);
 257
 258         copy_stack_data (info, stackdata);
 259
 260 retry:
 261         ++coop_save_count;
 262         mono_threads_get_runtime_callbacks ()->thread_state_init (&info->thread_saved_state [SELF_SUSPEND_STATE_INDEX]);
 263
 264         switch (mono_threads_transition_do_blocking (info, function_name)) {
 265         case DoBlockingContinue:
 266                 break;
 267         case DoBlockingPollAndRetry:
 268                 mono_threads_state_poll_with_info (info);
 269                 goto retry;
 270         }
 271
 272         return info;
 273 }
 274
 275 void
 276 mono_threads_exit_gc_safe_region_internal (gpointer cookie, MonoStackData *stackdata)
 277 {
 278         if (!mono_threads_is_blocking_transition_enabled ())
 279                 return;
 280
 281 #ifdef ENABLE_CHECKED_BUILD_GC
 282         if (mono_check_mode_enabled (MONO_CHECK_MODE_GC))
 283                 coop_tls_pop (cookie);
 284 #endif
 285
 286         mono_threads_exit_gc_safe_region_unbalanced_internal (cookie, stackdata);
 287 }
 288
 289 void
 290 mono_threads_exit_gc_safe_region (gpointer cookie, gpointer *stackpointer)
 291 {
 292         MONO_STACKDATA (stackdata);
 293         stackdata.stackpointer = stackpointer;
 294         mono_threads_exit_gc_safe_region_internal (cookie, &stackdata);
 295 }
 296
 297 void
 298 mono_threads_exit_gc_safe_region_unbalanced_internal (gpointer cookie, MonoStackData *stackdata)
 299 {
 300         MonoThreadInfo *info;
 301
 302         if (!mono_threads_is_blocking_transition_enabled ())
 303                 return;
 304
 305         info = (MonoThreadInfo *)cookie;
 306
 307         const char *function_name = mono_stackdata_get_function_name (stackdata);
 308
 309         check_info (info, "exit", "safe", function_name);
 310
 311         switch (mono_threads_transition_done_blocking (info, function_name)) {
 312         case DoneBlockingOk:
 313                 info->thread_saved_state [SELF_SUSPEND_STATE_INDEX].valid = FALSE;
 314                 break;
 315         case DoneBlockingWait:
 316                 /* If full coop suspend, we're just waiting for the initiator
 317                  * to resume us.  If hybrid suspend, we were either self
 318                  * suspended cooperatively from async_suspend_requested (same
 319                  * as full coop), or we were suspended preemptively while in
 320                  * blocking and we're waiting for two things: the suspend
 321                  * signal handler to run and notify the initiator and
 322                  * immediately return, and then for the resume. */
 323                 THREADS_SUSPEND_DEBUG ("state polling done, notifying of resume\n");
 324                 mono_thread_info_wait_for_resume (info);
 325                 break;
 326         default:
 327                 g_error ("Unknown thread state");
 328         }
 329
 330         if (info->async_target) {
 331                 info->async_target (info->user_data);
 332                 info->async_target = NULL;
 333                 info->user_data = NULL;
 334         }
 335 }
 336
 337 void
 338 mono_threads_exit_gc_safe_region_unbalanced (gpointer cookie, gpointer *stackpointer)
 339 {
 340         MONO_STACKDATA (stackdata);
 341         stackdata.stackpointer = stackpointer;
 342         mono_threads_exit_gc_safe_region_unbalanced_internal (cookie, &stackdata);
 343 }
 344
 345 void
 346 mono_threads_assert_gc_safe_region (void)
 347 {
 348         MONO_REQ_GC_SAFE_MODE;
 349 }
 350
 351 gpointer
 352 mono_threads_enter_gc_unsafe_region_internal (MonoStackData *stackdata)
 353 {
 354         return mono_threads_enter_gc_unsafe_region_with_info (mono_thread_info_current_unchecked (), stackdata);
 355 }
 356
 357 gpointer
 358 mono_threads_enter_gc_unsafe_region (gpointer *stackpointer)
 359 {
 360         MONO_STACKDATA (stackdata);
 361         stackdata.stackpointer = stackpointer;
 362         return mono_threads_enter_gc_unsafe_region_internal (&stackdata);
 363 }
 364
 365 gpointer
 366 mono_threads_enter_gc_unsafe_region_with_info (THREAD_INFO_TYPE *info, MonoStackData *stackdata)
 367 {
 368         gpointer cookie;
 369
 370         if (!mono_threads_is_blocking_transition_enabled ())
 371                 return NULL;
 372
 373         cookie = mono_threads_enter_gc_unsafe_region_unbalanced_with_info (info, stackdata);
 374
 375 #ifdef ENABLE_CHECKED_BUILD_GC
 376         if (mono_check_mode_enabled (MONO_CHECK_MODE_GC))
 377                 coop_tls_push (cookie);
 378 #endif
 379
 380         return cookie;
 381 }
 382
 383 gpointer
 384 mono_threads_enter_gc_unsafe_region_unbalanced_internal (MonoStackData *stackdata)
 385 {
 386         return mono_threads_enter_gc_unsafe_region_unbalanced_with_info (mono_thread_info_current_unchecked (), stackdata);
 387 }
 388
 389 gpointer
 390 mono_threads_enter_gc_unsafe_region_unbalanced (gpointer *stackpointer)
 391 {
 392         MONO_STACKDATA (stackdata);
 393         stackdata.stackpointer = stackpointer;
 394         return mono_threads_enter_gc_unsafe_region_unbalanced_internal (&stackdata);
 395 }
 396
 397 gpointer
 398 mono_threads_enter_gc_unsafe_region_unbalanced_with_info (MonoThreadInfo *info, MonoStackData *stackdata)
 399 {
 400         if (!mono_threads_is_blocking_transition_enabled ())
 401                 return NULL;
 402
 403         ++coop_reset_blocking_count;
 404
 405         const char *function_name = mono_stackdata_get_function_name (stackdata);
 406
 407         check_info (info, "enter", "unsafe", function_name);
 408
 409         copy_stack_data (info, stackdata);
 410
 411         switch (mono_threads_transition_abort_blocking (info, function_name)) {
 412         case AbortBlockingIgnore:
 413                 info->thread_saved_state [SELF_SUSPEND_STATE_INDEX].valid = FALSE;
 414                 return NULL;
 415         case AbortBlockingIgnoreAndPoll:
 416                 mono_threads_state_poll_with_info (info);
 417                 return NULL;
 418         case AbortBlockingOk:
 419                 info->thread_saved_state [SELF_SUSPEND_STATE_INDEX].valid = FALSE;
 420                 break;
 421         case AbortBlockingWait:
 422                 /* If full coop suspend, we're just waiting for the initiator
 423                  * to resume us.  If hybrid suspend, we were either self
 424                  * suspended cooperatively from async_suspend_requested (same
 425                  * as full coop), or we were suspended preemptively while in
 426                  * blocking and we're waiting for two things: the suspend
 427                  * signal handler to run and notify the initiator and
 428                  * immediately return, and then for the resume. */
 429                 mono_thread_info_wait_for_resume (info);
 430                 break;
 431         default:
 432                 g_error ("Unknown thread state %s", function_name);
 433         }
 434
 435         if (info->async_target) {
 436                 info->async_target (info->user_data);
 437                 info->async_target = NULL;
 438                 info->user_data = NULL;
 439         }
 440
 441         return info;
 442 }
 443
 444 gpointer
 445 mono_threads_enter_gc_unsafe_region_cookie (void)
 446 {
 447         MonoThreadInfo *info;
 448
 449         g_assert (mono_threads_is_blocking_transition_enabled ());
 450
 451         info = mono_thread_info_current_unchecked ();
 452
 453         check_info (info, "enter (cookie)", "unsafe", "");
 454
 455 #ifdef ENABLE_CHECKED_BUILD_GC
 456         if (mono_check_mode_enabled (MONO_CHECK_MODE_GC))
 457                 coop_tls_push (info);
 458 #endif
 459
 460         return info;
 461 }
 462
 463 void
 464 mono_threads_exit_gc_unsafe_region_internal (gpointer cookie, MonoStackData *stackdata)
 465 {
 466         if (!mono_threads_is_blocking_transition_enabled ())
 467                 return;
 468
 469 #ifdef ENABLE_CHECKED_BUILD_GC
 470         if (mono_check_mode_enabled (MONO_CHECK_MODE_GC))
 471                 coop_tls_pop (cookie);
 472 #endif
 473
 474         mono_threads_exit_gc_unsafe_region_unbalanced_internal (cookie, stackdata);
 475 }
 476
 477 void
 478 mono_threads_exit_gc_unsafe_region (gpointer cookie, gpointer *stackpointer)
 479 {
 480         MONO_STACKDATA (stackdata);
 481         stackdata.stackpointer = stackpointer;
 482         mono_threads_exit_gc_unsafe_region_internal (cookie, &stackdata);
 483 }
 484
 485 void
 486 mono_threads_exit_gc_unsafe_region_unbalanced_internal (gpointer cookie, MonoStackData *stackdata)
 487 {
 488         if (!mono_threads_is_blocking_transition_enabled ())
 489                 return;
 490
 491         if (!cookie)
 492                 return;
 493
 494         mono_threads_enter_gc_safe_region_unbalanced_internal (stackdata);
 495 }
 496
 497 void
 498 mono_threads_exit_gc_unsafe_region_unbalanced (gpointer cookie, gpointer *stackpointer)
 499 {
 500         MONO_STACKDATA (stackdata);
 501         stackdata.stackpointer = stackpointer;
 502         mono_threads_exit_gc_unsafe_region_unbalanced_internal (cookie, &stackdata);
 503 }
 504
 505 void
 506 mono_threads_assert_gc_unsafe_region (void)
 507 {
 508         MONO_REQ_GC_UNSAFE_MODE;
 509 }
 510
 511 gboolean
 512 mono_threads_is_cooperative_suspension_enabled (void)
 513 {
 514 #if defined(ENABLE_COOP_SUSPEND)
 515         return TRUE;
 516 #else
 517         static int is_coop_enabled = -1;
 518         if (G_UNLIKELY (is_coop_enabled == -1))
 519                 is_coop_enabled = (g_hasenv ("MONO_ENABLE_COOP") || g_hasenv ("MONO_ENABLE_COOP_SUSPEND")) ? 1 : 0;
 520         return is_coop_enabled == 1;
 521 #endif
 522 }
 523
 524 gboolean
 525 mono_threads_is_blocking_transition_enabled (void)
 526 {
 527 #if defined(ENABLE_COOP_SUSPEND) || defined(ENABLE_HYBRID_SUSPEND)
 528         return TRUE;
 529 #else
 530         static int is_blocking_transition_enabled = -1;
 531         if (G_UNLIKELY (is_blocking_transition_enabled == -1))
 532                 is_blocking_transition_enabled = (g_hasenv ("MONO_ENABLE_COOP") || g_hasenv ("MONO_ENABLE_COOP_SUSPEND") || g_hasenv ("MONO_ENABLE_HYBRID_SUSPEND") || g_hasenv ("MONO_ENABLE_BLOCKING_TRANSITION")) ? 1 : 0;
 533         return is_blocking_transition_enabled == 1;
 534 #endif
 535 }
 536
 537 gboolean
 538 mono_threads_is_hybrid_suspension_enabled (void)
 539 {
 540 #if defined(ENABLE_HYBRID_SUSPEND)
 541         return TRUE;
 542 #else
 543         static int is_hybrid_suspension_enabled = -1;
 544         if (G_UNLIKELY (is_hybrid_suspension_enabled == -1))
 545                 is_hybrid_suspension_enabled = (g_hasenv ("MONO_ENABLE_HYBRID_SUSPEND")) ? 1 : 0;
 546         return is_hybrid_suspension_enabled == 1;
 547 #endif
 548 }
 549
 550
 551 void
 552 mono_threads_coop_init (void)
 553 {
 554         if (!mono_threads_are_safepoints_enabled () && !mono_threads_is_blocking_transition_enabled ())
 555                 return;
 556
 557         mono_counters_register ("Coop Reset Blocking", MONO_COUNTER_GC | MONO_COUNTER_INT, &coop_reset_blocking_count);
 558         mono_counters_register ("Coop Try Blocking", MONO_COUNTER_GC | MONO_COUNTER_INT, &coop_try_blocking_count);
 559         mono_counters_register ("Coop Do Blocking", MONO_COUNTER_GC | MONO_COUNTER_INT, &coop_do_blocking_count);
 560         mono_counters_register ("Coop Do Polling", MONO_COUNTER_GC | MONO_COUNTER_INT, &coop_do_polling_count);
 561         mono_counters_register ("Coop Save Count", MONO_COUNTER_GC | MONO_COUNTER_INT, &coop_save_count);
 562         //See the above for what's wrong here.
 563
 564 #ifdef ENABLE_CHECKED_BUILD_GC
 565         mono_native_tls_alloc (&coop_reset_count_stack_key, NULL);
 566 #endif
 567 }
 568
 569 void
 570 mono_threads_coop_begin_global_suspend (void)
 571 {
 572         if (mono_threads_are_safepoints_enabled ())
 573                 mono_polling_required = 1;
 574 }
 575
 576 void
 577 mono_threads_coop_end_global_suspend (void)
 578 {
 579         if (mono_threads_are_safepoints_enabled ())
 580                 mono_polling_required = 0;
 581 }