libsanitizer/tsan/tsan_interceptors_libdispatch.cpp

   1 //===-- tsan_interceptors_libdispatch.cpp ---------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Support for intercepting libdispatch (GCD).
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "sanitizer_common/sanitizer_common.h"
  15 #include "interception/interception.h"
  16 #include "tsan_interceptors.h"
  17 #include "tsan_rtl.h"
  18
  19 #include "BlocksRuntime/Block.h"
  20 #include "tsan_dispatch_defs.h"
  21
  22 namespace __tsan {
  23   typedef u16 uint16_t;
  24
  25 typedef struct {
  26   dispatch_queue_t queue;
  27   void *orig_context;
  28   dispatch_function_t orig_work;
  29   bool free_context_in_callback;
  30   bool submitted_synchronously;
  31   bool is_barrier_block;
  32   uptr non_queue_sync_object;
  33 } block_context_t;
  34
  35 // The offsets of different fields of the dispatch_queue_t structure, exported
  36 // by libdispatch.dylib.
  37 extern "C" struct dispatch_queue_offsets_s {
  38   const uint16_t dqo_version;
  39   const uint16_t dqo_label;
  40   const uint16_t dqo_label_size;
  41   const uint16_t dqo_flags;
  42   const uint16_t dqo_flags_size;
  43   const uint16_t dqo_serialnum;
  44   const uint16_t dqo_serialnum_size;
  45   const uint16_t dqo_width;
  46   const uint16_t dqo_width_size;
  47   const uint16_t dqo_running;
  48   const uint16_t dqo_running_size;
  49   const uint16_t dqo_suspend_cnt;
  50   const uint16_t dqo_suspend_cnt_size;
  51   const uint16_t dqo_target_queue;
  52   const uint16_t dqo_target_queue_size;
  53   const uint16_t dqo_priority;
  54   const uint16_t dqo_priority_size;
  55 } dispatch_queue_offsets;
  56
  57 static bool IsQueueSerial(dispatch_queue_t q) {
  58   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
  59   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
  60   CHECK_NE(width, 0);
  61   return width == 1;
  62 }
  63
  64 static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
  65   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
  66   dispatch_queue_t tq = *(
  67       dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
  68   return tq;
  69 }
  70
  71 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
  72   dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
  73   CHECK_NE(tq, 0);
  74   return tq;
  75 }
  76
  77 static block_context_t *AllocContext(ThreadState *thr, uptr pc,
  78                                      dispatch_queue_t queue, void *orig_context,
  79                                      dispatch_function_t orig_work) {
  80   block_context_t *new_context =
  81       (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));
  82   new_context->queue = queue;
  83   new_context->orig_context = orig_context;
  84   new_context->orig_work = orig_work;
  85   new_context->free_context_in_callback = true;
  86   new_context->submitted_synchronously = false;
  87   new_context->is_barrier_block = false;
  88   new_context->non_queue_sync_object = 0;
  89   return new_context;
  90 }
  91
  92 #define GET_QUEUE_SYNC_VARS(context, q)                                  \
  93   bool is_queue_serial = q && IsQueueSerial(q);                          \
  94   uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object;             \
  95   uptr serial_sync = (uptr)sync_ptr;                                     \
  96   uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
  97   bool serial_task = context->is_barrier_block || is_queue_serial
  98
  99 static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
 100                                       block_context_t *context) {
 101   uptr submit_sync = (uptr)context;
 102   Acquire(thr, pc, submit_sync);
 103
 104   dispatch_queue_t q = context->queue;
 105   do {
 106     GET_QUEUE_SYNC_VARS(context, q);
 107     if (serial_sync) Acquire(thr, pc, serial_sync);
 108     if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
 109
 110     if (q) q = GetTargetQueueFromQueue(q);
 111   } while (q);
 112 }
 113
 114 static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
 115                                        block_context_t *context) {
 116   uptr submit_sync = (uptr)context;
 117   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
 118
 119   dispatch_queue_t q = context->queue;
 120   do {
 121     GET_QUEUE_SYNC_VARS(context, q);
 122     if (serial_task && serial_sync) Release(thr, pc, serial_sync);
 123     if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
 124
 125     if (q) q = GetTargetQueueFromQueue(q);
 126   } while (q);
 127 }
 128
 129 static void dispatch_callback_wrap(void *param) {
 130   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
 131   block_context_t *context = (block_context_t *)param;
 132
 133   dispatch_sync_pre_execute(thr, pc, context);
 134
 135   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 136   context->orig_work(context->orig_context);
 137   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 138
 139   dispatch_sync_post_execute(thr, pc, context);
 140
 141   if (context->free_context_in_callback) user_free(thr, pc, context);
 142 }
 143
 144 static void invoke_block(void *param) {
 145   dispatch_block_t block = (dispatch_block_t)param;
 146   block();
 147 }
 148
 149 static void invoke_and_release_block(void *param) {
 150   dispatch_block_t block = (dispatch_block_t)param;
 151   block();
 152   Block_release(block);
 153 }
 154
 155 #define DISPATCH_INTERCEPT_ASYNC_B(name, barrier)                            \
 156   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 157     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 158     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 159     dispatch_block_t heap_block = Block_copy(block);                         \
 160     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 161     block_context_t *new_context =                                           \
 162         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
 163     new_context->is_barrier_block = barrier;                                 \
 164     Release(thr, pc, (uptr)new_context);                                     \
 165     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 166     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
 167     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 168   }
 169
 170 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
 171   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q,                           \
 172                    DISPATCH_NOESCAPE dispatch_block_t block) {               \
 173     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 174     block_context_t new_context = {                                          \
 175         q, block, &invoke_block, false, true, barrier, 0};                   \
 176     Release(thr, pc, (uptr)&new_context);                                    \
 177     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 178     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
 179     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 180     Acquire(thr, pc, (uptr)&new_context);                                    \
 181   }
 182
 183 #define DISPATCH_INTERCEPT_ASYNC_F(name, barrier)                 \
 184   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
 185                    dispatch_function_t work) {                    \
 186     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
 187     block_context_t *new_context =                                \
 188         AllocContext(thr, pc, q, context, work);                  \
 189     new_context->is_barrier_block = barrier;                      \
 190     Release(thr, pc, (uptr)new_context);                          \
 191     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
 192     REAL(name)(q, new_context, dispatch_callback_wrap);           \
 193     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
 194   }
 195
 196 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
 197   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
 198                    dispatch_function_t work) {                                \
 199     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
 200     block_context_t new_context = {                                           \
 201         q, context, work, false, true, barrier, 0};                           \
 202     Release(thr, pc, (uptr)&new_context);                                     \
 203     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
 204     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
 205     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
 206     Acquire(thr, pc, (uptr)&new_context);                                     \
 207   }
 208
 209 #define DISPATCH_INTERCEPT(name, barrier)             \
 210   DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
 211   DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier)   \
 212   DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier)   \
 213   DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
 214
 215 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
 216 // context, which is used to synchronize (we release the context before
 217 // submitting, and the callback acquires it before executing the original
 218 // callback).
 219 DISPATCH_INTERCEPT(dispatch, false)
 220 DISPATCH_INTERCEPT(dispatch_barrier, true)
 221
 222 DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
 223              dispatch_queue_t queue, void *context, dispatch_function_t work)
 224
 225 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
 226                  dispatch_queue_t queue, dispatch_block_t block) {
 227   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
 228   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 229   dispatch_block_t heap_block = Block_copy(block);
 230   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 231   block_context_t *new_context =
 232       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
 233   Release(thr, pc, (uptr)new_context);
 234   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 235   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
 236   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 237 }
 238
 239 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 240                  dispatch_queue_t queue, void *context,
 241                  dispatch_function_t work) {
 242   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
 243   WRAP(dispatch_after)(when, queue, ^(void) {
 244     work(context);
 245   });
 246 }
 247
 248 // GCD's dispatch_once implementation has a fast path that contains a racy read
 249 // and it's inlined into user's code. Furthermore, this fast path doesn't
 250 // establish a proper happens-before relations between the initialization and
 251 // code following the call to dispatch_once. We could deal with this in
 252 // instrumented code, but there's not much we can do about it in system
 253 // libraries. Let's disable the fast path (by never storing the value ~0 to
 254 // predicate), so the interceptor is always called, and let's add proper release
 255 // and acquire semantics. Since TSan does not see its own atomic stores, the
 256 // race on predicate won't be reported - the only accesses to it that TSan sees
 257 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
 258 // both a macro and a real function, we want to intercept the function, so we
 259 // need to undefine the macro.
 260 #undef dispatch_once
 261 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
 262                  DISPATCH_NOESCAPE dispatch_block_t block) {
 263   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
 264   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
 265   u32 v = atomic_load(a, memory_order_acquire);
 266   if (v == 0 &&
 267       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
 268     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 269     block();
 270     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 271     Release(thr, pc, (uptr)a);
 272     atomic_store(a, 2, memory_order_release);
 273   } else {
 274     while (v != 2) {
 275       internal_sched_yield();
 276       v = atomic_load(a, memory_order_acquire);
 277     }
 278     Acquire(thr, pc, (uptr)a);
 279   }
 280 }
 281
 282 #undef dispatch_once_f
 283 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
 284                  void *context, dispatch_function_t function) {
 285   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
 286   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 287   WRAP(dispatch_once)(predicate, ^(void) {
 288     function(context);
 289   });
 290   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 291 }
 292
 293 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
 294                  dispatch_semaphore_t dsema) {
 295   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
 296   Release(thr, pc, (uptr)dsema);
 297   return REAL(dispatch_semaphore_signal)(dsema);
 298 }
 299
 300 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
 301                  dispatch_time_t timeout) {
 302   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
 303   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
 304   if (result == 0) Acquire(thr, pc, (uptr)dsema);
 305   return result;
 306 }
 307
 308 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
 309                  dispatch_time_t timeout) {
 310   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
 311   long_t result = REAL(dispatch_group_wait)(group, timeout);
 312   if (result == 0) Acquire(thr, pc, (uptr)group);
 313   return result;
 314 }
 315
 316 // Used, but not intercepted.
 317 extern "C" void dispatch_group_enter(dispatch_group_t group);
 318
 319 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
 320   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
 321   // Acquired in the group notification callback in dispatch_group_notify[_f].
 322   Release(thr, pc, (uptr)group);
 323   REAL(dispatch_group_leave)(group);
 324 }
 325
 326 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
 327                  dispatch_queue_t queue, dispatch_block_t block) {
 328   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
 329   dispatch_retain(group);
 330   dispatch_group_enter(group);
 331   __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
 332   WRAP(dispatch_async)(queue, ^(void) {
 333     block_copy();
 334     Block_release(block_copy);
 335     WRAP(dispatch_group_leave)(group);
 336     dispatch_release(group);
 337   });
 338 }
 339
 340 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
 341                  dispatch_queue_t queue, void *context,
 342                  dispatch_function_t work) {
 343   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
 344   dispatch_retain(group);
 345   dispatch_group_enter(group);
 346   WRAP(dispatch_async)(queue, ^(void) {
 347     work(context);
 348     WRAP(dispatch_group_leave)(group);
 349     dispatch_release(group);
 350   });
 351 }
 352
 353 DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
 354              dispatch_queue_t q, void *context, dispatch_function_t work)
 355
 356 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
 357                  dispatch_queue_t q, dispatch_block_t block) {
 358   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
 359
 360   // To make sure the group is still available in the callback (otherwise
 361   // it can be already destroyed).  Will be released in the callback.
 362   dispatch_retain(group);
 363
 364   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 365   dispatch_block_t heap_block = Block_copy(^(void) {
 366     {
 367       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
 368       // Released when leaving the group (dispatch_group_leave).
 369       Acquire(thr, pc, (uptr)group);
 370     }
 371     dispatch_release(group);
 372     block();
 373   });
 374   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 375   block_context_t *new_context =
 376       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 377   new_context->is_barrier_block = true;
 378   Release(thr, pc, (uptr)new_context);
 379   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
 380 }
 381
 382 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
 383                  dispatch_queue_t q, void *context, dispatch_function_t work) {
 384   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
 385 }
 386
 387 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
 388                  dispatch_source_t source, dispatch_block_t handler) {
 389   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
 390   if (handler == nullptr)
 391     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 392   dispatch_queue_t q = GetTargetQueueFromSource(source);
 393   __block block_context_t new_context = {
 394       q, handler, &invoke_block, false, false, false, 0 };
 395   dispatch_block_t new_handler = Block_copy(^(void) {
 396     new_context.orig_context = handler;  // To explicitly capture "handler".
 397     dispatch_callback_wrap(&new_context);
 398   });
 399   uptr submit_sync = (uptr)&new_context;
 400   Release(thr, pc, submit_sync);
 401   REAL(dispatch_source_set_event_handler)(source, new_handler);
 402   Block_release(new_handler);
 403 }
 404
 405 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
 406                  dispatch_source_t source, dispatch_function_t handler) {
 407   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
 408   if (handler == nullptr)
 409     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 410   dispatch_block_t block = ^(void) {
 411     handler(dispatch_get_context(source));
 412   };
 413   WRAP(dispatch_source_set_event_handler)(source, block);
 414 }
 415
 416 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
 417                  dispatch_source_t source, dispatch_block_t handler) {
 418   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
 419   if (handler == nullptr)
 420     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 421   dispatch_queue_t q = GetTargetQueueFromSource(source);
 422   __block block_context_t new_context = {
 423       q, handler, &invoke_block, false, false, false, 0};
 424   dispatch_block_t new_handler = Block_copy(^(void) {
 425     new_context.orig_context = handler;  // To explicitly capture "handler".
 426     dispatch_callback_wrap(&new_context);
 427   });
 428   uptr submit_sync = (uptr)&new_context;
 429   Release(thr, pc, submit_sync);
 430   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
 431   Block_release(new_handler);
 432 }
 433
 434 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
 435                  dispatch_source_t source, dispatch_function_t handler) {
 436   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
 437                           handler);
 438   if (handler == nullptr)
 439     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 440   dispatch_block_t block = ^(void) {
 441     handler(dispatch_get_context(source));
 442   };
 443   WRAP(dispatch_source_set_cancel_handler)(source, block);
 444 }
 445
 446 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
 447                  dispatch_source_t source, dispatch_block_t handler) {
 448   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
 449                           handler);
 450   if (handler == nullptr)
 451     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 452   dispatch_queue_t q = GetTargetQueueFromSource(source);
 453   __block block_context_t new_context = {
 454       q, handler, &invoke_block, false, false, false, 0};
 455   dispatch_block_t new_handler = Block_copy(^(void) {
 456     new_context.orig_context = handler;  // To explicitly capture "handler".
 457     dispatch_callback_wrap(&new_context);
 458   });
 459   uptr submit_sync = (uptr)&new_context;
 460   Release(thr, pc, submit_sync);
 461   REAL(dispatch_source_set_registration_handler)(source, new_handler);
 462   Block_release(new_handler);
 463 }
 464
 465 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 466                  dispatch_source_t source, dispatch_function_t handler) {
 467   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
 468                           handler);
 469   if (handler == nullptr)
 470     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 471   dispatch_block_t block = ^(void) {
 472     handler(dispatch_get_context(source));
 473   };
 474   WRAP(dispatch_source_set_registration_handler)(source, block);
 475 }
 476
 477 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
 478                  dispatch_queue_t queue,
 479                  DISPATCH_NOESCAPE void (^block)(size_t)) {
 480   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 481
 482   u8 sync1, sync2;
 483   uptr parent_to_child_sync = (uptr)&sync1;
 484   uptr child_to_parent_sync = (uptr)&sync2;
 485
 486   Release(thr, pc, parent_to_child_sync);
 487   void (^new_block)(size_t) = ^(size_t iteration) {
 488     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
 489     Acquire(thr, pc, parent_to_child_sync);
 490     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 491     block(iteration);
 492     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 493     Release(thr, pc, child_to_parent_sync);
 494   };
 495   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 496   REAL(dispatch_apply)(iterations, queue, new_block);
 497   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 498   Acquire(thr, pc, child_to_parent_sync);
 499 }
 500
 501 static void invoke_block_iteration(void *param, size_t iteration) {
 502   auto block = (void (^)(size_t)) param;
 503   block(iteration);
 504 }
 505
 506 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
 507                  dispatch_queue_t queue, void *context,
 508                  void (*work)(void *, size_t)) {
 509   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
 510
 511   // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
 512   // implements dispatch_apply in terms of dispatch_apply_f.
 513   u8 sync1, sync2;
 514   uptr parent_to_child_sync = (uptr)&sync1;
 515   uptr child_to_parent_sync = (uptr)&sync2;
 516
 517   Release(thr, pc, parent_to_child_sync);
 518   void (^new_block)(size_t) = ^(size_t iteration) {
 519     SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
 520     Acquire(thr, pc, parent_to_child_sync);
 521     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 522     work(context, iteration);
 523     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 524     Release(thr, pc, child_to_parent_sync);
 525   };
 526   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 527   REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
 528   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 529   Acquire(thr, pc, child_to_parent_sync);
 530 }
 531
 532 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 533 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
 534
 535 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
 536                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
 537   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
 538   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
 539     return REAL(dispatch_data_create)(buffer, size, q, destructor);
 540
 541   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
 542     destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
 543   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
 544     destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
 545
 546   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 547   dispatch_block_t heap_block = Block_copy(destructor);
 548   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 549   block_context_t *new_context =
 550       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 551   uptr submit_sync = (uptr)new_context;
 552   Release(thr, pc, submit_sync);
 553   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
 554     dispatch_callback_wrap(new_context);
 555   });
 556 }
 557
 558 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
 559 typedef void (^cleanup_handler_t)(int error);
 560
 561 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
 562                  dispatch_queue_t q, fd_handler_t h) {
 563   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
 564   __block block_context_t new_context = {
 565       q, nullptr, &invoke_block, false, false, false, 0};
 566   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 567     new_context.orig_context = ^(void) {
 568       h(data, error);
 569     };
 570     dispatch_callback_wrap(&new_context);
 571   });
 572   uptr submit_sync = (uptr)&new_context;
 573   Release(thr, pc, submit_sync);
 574   REAL(dispatch_read)(fd, length, q, new_h);
 575   Block_release(new_h);
 576 }
 577
 578 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
 579                  dispatch_queue_t q, fd_handler_t h) {
 580   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
 581   __block block_context_t new_context = {
 582       q, nullptr, &invoke_block, false, false, false, 0};
 583   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 584     new_context.orig_context = ^(void) {
 585       h(data, error);
 586     };
 587     dispatch_callback_wrap(&new_context);
 588   });
 589   uptr submit_sync = (uptr)&new_context;
 590   Release(thr, pc, submit_sync);
 591   REAL(dispatch_write)(fd, data, q, new_h);
 592   Block_release(new_h);
 593 }
 594
 595 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
 596                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
 597   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
 598   __block block_context_t new_context = {
 599       q, nullptr, &invoke_block, false, false, false, 0};
 600   dispatch_io_handler_t new_h =
 601       Block_copy(^(bool done, dispatch_data_t data, int error) {
 602         new_context.orig_context = ^(void) {
 603           h(done, data, error);
 604         };
 605         dispatch_callback_wrap(&new_context);
 606       });
 607   uptr submit_sync = (uptr)&new_context;
 608   Release(thr, pc, submit_sync);
 609   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
 610   Block_release(new_h);
 611 }
 612
 613 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
 614                  dispatch_data_t data, dispatch_queue_t q,
 615                  dispatch_io_handler_t h) {
 616   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
 617   __block block_context_t new_context = {
 618       q, nullptr, &invoke_block, false, false, false, 0};
 619   dispatch_io_handler_t new_h =
 620       Block_copy(^(bool done, dispatch_data_t data, int error) {
 621         new_context.orig_context = ^(void) {
 622           h(done, data, error);
 623         };
 624         dispatch_callback_wrap(&new_context);
 625       });
 626   uptr submit_sync = (uptr)&new_context;
 627   Release(thr, pc, submit_sync);
 628   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
 629   Block_release(new_h);
 630 }
 631
 632 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
 633                  dispatch_block_t barrier) {
 634   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
 635   __block block_context_t new_context = {
 636       nullptr, nullptr, &invoke_block, false, false, false, 0};
 637   new_context.non_queue_sync_object = (uptr)channel;
 638   new_context.is_barrier_block = true;
 639   dispatch_block_t new_block = Block_copy(^(void) {
 640     new_context.orig_context = ^(void) {
 641       barrier();
 642     };
 643     dispatch_callback_wrap(&new_context);
 644   });
 645   uptr submit_sync = (uptr)&new_context;
 646   Release(thr, pc, submit_sync);
 647   REAL(dispatch_io_barrier)(channel, new_block);
 648   Block_release(new_block);
 649 }
 650
 651 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
 652                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
 653   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
 654   __block dispatch_io_t new_channel = nullptr;
 655   __block block_context_t new_context = {
 656       q, nullptr, &invoke_block, false, false, false, 0};
 657   cleanup_handler_t new_h = Block_copy(^(int error) {
 658     {
 659       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 660       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 661     }
 662     new_context.orig_context = ^(void) {
 663       h(error);
 664     };
 665     dispatch_callback_wrap(&new_context);
 666   });
 667   uptr submit_sync = (uptr)&new_context;
 668   Release(thr, pc, submit_sync);
 669   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
 670   Block_release(new_h);
 671   return new_channel;
 672 }
 673
 674 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
 675                  dispatch_io_type_t type, const char *path, int oflag,
 676                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
 677   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
 678                           q, h);
 679   __block dispatch_io_t new_channel = nullptr;
 680   __block block_context_t new_context = {
 681       q, nullptr, &invoke_block, false, false, false, 0};
 682   cleanup_handler_t new_h = Block_copy(^(int error) {
 683     {
 684       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 685       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 686     }
 687     new_context.orig_context = ^(void) {
 688       h(error);
 689     };
 690     dispatch_callback_wrap(&new_context);
 691   });
 692   uptr submit_sync = (uptr)&new_context;
 693   Release(thr, pc, submit_sync);
 694   new_channel =
 695       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
 696   Block_release(new_h);
 697   return new_channel;
 698 }
 699
 700 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
 701                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
 702                  cleanup_handler_t h) {
 703   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
 704   __block dispatch_io_t new_channel = nullptr;
 705   __block block_context_t new_context = {
 706       q, nullptr, &invoke_block, false, false, false, 0};
 707   cleanup_handler_t new_h = Block_copy(^(int error) {
 708     {
 709       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 710       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 711     }
 712     new_context.orig_context = ^(void) {
 713       h(error);
 714     };
 715     dispatch_callback_wrap(&new_context);
 716   });
 717   uptr submit_sync = (uptr)&new_context;
 718   Release(thr, pc, submit_sync);
 719   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
 720   Block_release(new_h);
 721   return new_channel;
 722 }
 723
 724 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
 725                  dispatch_io_close_flags_t flags) {
 726   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
 727   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
 728   return REAL(dispatch_io_close)(channel, flags);
 729 }
 730
 731 // Resuming a suspended queue needs to synchronize with all subsequent
 732 // executions of blocks in that queue.
 733 TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
 734   SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
 735   Release(thr, pc, (uptr)o);  // Synchronizes with the Acquire() on serial_sync
 736                               // in dispatch_sync_pre_execute
 737   return REAL(dispatch_resume)(o);
 738 }
 739
 740 void InitializeLibdispatchInterceptors() {
 741   INTERCEPT_FUNCTION(dispatch_async);
 742   INTERCEPT_FUNCTION(dispatch_async_f);
 743   INTERCEPT_FUNCTION(dispatch_sync);
 744   INTERCEPT_FUNCTION(dispatch_sync_f);
 745   INTERCEPT_FUNCTION(dispatch_barrier_async);
 746   INTERCEPT_FUNCTION(dispatch_barrier_async_f);
 747   INTERCEPT_FUNCTION(dispatch_barrier_sync);
 748   INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
 749   INTERCEPT_FUNCTION(dispatch_after);
 750   INTERCEPT_FUNCTION(dispatch_after_f);
 751   INTERCEPT_FUNCTION(dispatch_once);
 752   INTERCEPT_FUNCTION(dispatch_once_f);
 753   INTERCEPT_FUNCTION(dispatch_semaphore_signal);
 754   INTERCEPT_FUNCTION(dispatch_semaphore_wait);
 755   INTERCEPT_FUNCTION(dispatch_group_wait);
 756   INTERCEPT_FUNCTION(dispatch_group_leave);
 757   INTERCEPT_FUNCTION(dispatch_group_async);
 758   INTERCEPT_FUNCTION(dispatch_group_async_f);
 759   INTERCEPT_FUNCTION(dispatch_group_notify);
 760   INTERCEPT_FUNCTION(dispatch_group_notify_f);
 761   INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
 762   INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
 763   INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
 764   INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
 765   INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
 766   INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
 767   INTERCEPT_FUNCTION(dispatch_apply);
 768   INTERCEPT_FUNCTION(dispatch_apply_f);
 769   INTERCEPT_FUNCTION(dispatch_data_create);
 770   INTERCEPT_FUNCTION(dispatch_read);
 771   INTERCEPT_FUNCTION(dispatch_write);
 772   INTERCEPT_FUNCTION(dispatch_io_read);
 773   INTERCEPT_FUNCTION(dispatch_io_write);
 774   INTERCEPT_FUNCTION(dispatch_io_barrier);
 775   INTERCEPT_FUNCTION(dispatch_io_create);
 776   INTERCEPT_FUNCTION(dispatch_io_create_with_path);
 777   INTERCEPT_FUNCTION(dispatch_io_create_with_io);
 778   INTERCEPT_FUNCTION(dispatch_io_close);
 779   INTERCEPT_FUNCTION(dispatch_resume);
 780 }
 781
 782 }  // namespace __tsan