libsanitizer/tsan/tsan_libdispatch_mac.cc

   1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Mac-specific libdispatch (GCD) support.
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "sanitizer_common/sanitizer_platform.h"
  14 #if SANITIZER_MAC
  15
  16 #include "sanitizer_common/sanitizer_common.h"
  17 #include "interception/interception.h"
  18 #include "tsan_interceptors.h"
  19 #include "tsan_platform.h"
  20 #include "tsan_rtl.h"
  21
  22 #include <Block.h>
  23 #include <dispatch/dispatch.h>
  24 #include <pthread.h>
  25
  26 typedef long long_t;  // NOLINT
  27
  28 namespace __tsan {
  29
  30 typedef struct {
  31   dispatch_queue_t queue;
  32   void *orig_context;
  33   dispatch_function_t orig_work;
  34   bool free_context_in_callback;
  35   bool submitted_synchronously;
  36   bool is_barrier_block;
  37   uptr non_queue_sync_object;
  38 } tsan_block_context_t;
  39
  40 // The offsets of different fields of the dispatch_queue_t structure, exported
  41 // by libdispatch.dylib.
  42 extern "C" struct dispatch_queue_offsets_s {
  43   const uint16_t dqo_version;
  44   const uint16_t dqo_label;
  45   const uint16_t dqo_label_size;
  46   const uint16_t dqo_flags;
  47   const uint16_t dqo_flags_size;
  48   const uint16_t dqo_serialnum;
  49   const uint16_t dqo_serialnum_size;
  50   const uint16_t dqo_width;
  51   const uint16_t dqo_width_size;
  52   const uint16_t dqo_running;
  53   const uint16_t dqo_running_size;
  54   const uint16_t dqo_suspend_cnt;
  55   const uint16_t dqo_suspend_cnt_size;
  56   const uint16_t dqo_target_queue;
  57   const uint16_t dqo_target_queue_size;
  58   const uint16_t dqo_priority;
  59   const uint16_t dqo_priority_size;
  60 } dispatch_queue_offsets;
  61
  62 static bool IsQueueSerial(dispatch_queue_t q) {
  63   CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
  64   uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
  65   CHECK_NE(width, 0);
  66   return width == 1;
  67 }
  68
  69 static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
  70   CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
  71   dispatch_queue_t target_queue =
  72       *(dispatch_queue_t *)(((uptr)source) +
  73                             dispatch_queue_offsets.dqo_target_queue);
  74   CHECK_NE(target_queue, 0);
  75   return target_queue;
  76 }
  77
  78 static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
  79                                           dispatch_queue_t queue,
  80                                           void *orig_context,
  81                                           dispatch_function_t orig_work) {
  82   tsan_block_context_t *new_context =
  83       (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
  84   new_context->queue = queue;
  85   new_context->orig_context = orig_context;
  86   new_context->orig_work = orig_work;
  87   new_context->free_context_in_callback = true;
  88   new_context->submitted_synchronously = false;
  89   new_context->is_barrier_block = false;
  90   return new_context;
  91 }
  92
  93 static void dispatch_callback_wrap(void *param) {
  94   SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
  95   tsan_block_context_t *context = (tsan_block_context_t *)param;
  96   bool is_queue_serial = context->queue && IsQueueSerial(context->queue);
  97   uptr sync_ptr = (uptr)context->queue ?: context->non_queue_sync_object;
  98
  99   uptr serial_sync = (uptr)sync_ptr;
 100   uptr concurrent_sync = ((uptr)sync_ptr) + sizeof(uptr);
 101   uptr submit_sync = (uptr)context;
 102   bool serial_task = context->is_barrier_block || is_queue_serial;
 103
 104   Acquire(thr, pc, submit_sync);
 105   Acquire(thr, pc, serial_sync);
 106   if (serial_task) Acquire(thr, pc, concurrent_sync);
 107
 108   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 109   context->orig_work(context->orig_context);
 110   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 111
 112   Release(thr, pc, serial_task ? serial_sync : concurrent_sync);
 113   if (context->submitted_synchronously) Release(thr, pc, submit_sync);
 114
 115   if (context->free_context_in_callback) user_free(thr, pc, context);
 116 }
 117
 118 static void invoke_block(void *param) {
 119   dispatch_block_t block = (dispatch_block_t)param;
 120   block();
 121 }
 122
 123 static void invoke_and_release_block(void *param) {
 124   dispatch_block_t block = (dispatch_block_t)param;
 125   block();
 126   Block_release(block);
 127 }
 128
 129 #define DISPATCH_INTERCEPT_B(name, barrier)                                  \
 130   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 131     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 132     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 133     dispatch_block_t heap_block = Block_copy(block);                         \
 134     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 135     tsan_block_context_t *new_context =                                      \
 136         AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
 137     new_context->is_barrier_block = barrier;                                 \
 138     Release(thr, pc, (uptr)new_context);                                     \
 139     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 140     REAL(name##_f)(q, new_context, dispatch_callback_wrap);                  \
 141     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 142   }
 143
 144 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier)                             \
 145   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
 146     SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
 147     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 148     dispatch_block_t heap_block = Block_copy(block);                         \
 149     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 150     tsan_block_context_t new_context = {                                     \
 151         q, heap_block, &invoke_and_release_block, false, true, barrier, 0};  \
 152     Release(thr, pc, (uptr)&new_context);                                    \
 153     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                           \
 154     REAL(name##_f)(q, &new_context, dispatch_callback_wrap);                 \
 155     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                             \
 156     Acquire(thr, pc, (uptr)&new_context);                                    \
 157   }
 158
 159 #define DISPATCH_INTERCEPT_F(name, barrier)                       \
 160   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
 161                    dispatch_function_t work) {                    \
 162     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
 163     tsan_block_context_t *new_context =                           \
 164         AllocContext(thr, pc, q, context, work);                  \
 165     new_context->is_barrier_block = barrier;                      \
 166     Release(thr, pc, (uptr)new_context);                          \
 167     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                \
 168     REAL(name)(q, new_context, dispatch_callback_wrap);           \
 169     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                  \
 170   }
 171
 172 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier)                              \
 173   TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context,             \
 174                    dispatch_function_t work) {                                \
 175     SCOPED_TSAN_INTERCEPTOR(name, q, context, work);                          \
 176     tsan_block_context_t new_context = {                                      \
 177         q, context, work, false, true, barrier, 0};                           \
 178     Release(thr, pc, (uptr)&new_context);                                     \
 179     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();                            \
 180     REAL(name)(q, &new_context, dispatch_callback_wrap);                      \
 181     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();                              \
 182     Acquire(thr, pc, (uptr)&new_context);                                     \
 183   }
 184
 185 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
 186 // context, which is used to synchronize (we release the context before
 187 // submitting, and the callback acquires it before executing the original
 188 // callback).
 189 DISPATCH_INTERCEPT_B(dispatch_async, false)
 190 DISPATCH_INTERCEPT_B(dispatch_barrier_async, true)
 191 DISPATCH_INTERCEPT_F(dispatch_async_f, false)
 192 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f, true)
 193 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync, false)
 194 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync, true)
 195 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f, false)
 196 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f, true)
 197
 198 TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
 199                  dispatch_queue_t queue, dispatch_block_t block) {
 200   SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
 201   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 202   dispatch_block_t heap_block = Block_copy(block);
 203   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 204   tsan_block_context_t *new_context =
 205       AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
 206   Release(thr, pc, (uptr)new_context);
 207   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 208   REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
 209   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 210 }
 211
 212 TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
 213                  dispatch_queue_t queue, void *context,
 214                  dispatch_function_t work) {
 215   SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
 216   WRAP(dispatch_after)(when, queue, ^(void) {
 217     work(context);
 218   });
 219 }
 220
 221 // GCD's dispatch_once implementation has a fast path that contains a racy read
 222 // and it's inlined into user's code. Furthermore, this fast path doesn't
 223 // establish a proper happens-before relations between the initialization and
 224 // code following the call to dispatch_once. We could deal with this in
 225 // instrumented code, but there's not much we can do about it in system
 226 // libraries. Let's disable the fast path (by never storing the value ~0 to
 227 // predicate), so the interceptor is always called, and let's add proper release
 228 // and acquire semantics. Since TSan does not see its own atomic stores, the
 229 // race on predicate won't be reported - the only accesses to it that TSan sees
 230 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
 231 // both a macro and a real function, we want to intercept the function, so we
 232 // need to undefine the macro.
 233 #undef dispatch_once
 234 TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
 235                  dispatch_block_t block) {
 236   SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
 237   atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
 238   u32 v = atomic_load(a, memory_order_acquire);
 239   if (v == 0 &&
 240       atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
 241     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 242     block();
 243     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 244     Release(thr, pc, (uptr)a);
 245     atomic_store(a, 2, memory_order_release);
 246   } else {
 247     while (v != 2) {
 248       internal_sched_yield();
 249       v = atomic_load(a, memory_order_acquire);
 250     }
 251     Acquire(thr, pc, (uptr)a);
 252   }
 253 }
 254
 255 #undef dispatch_once_f
 256 TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
 257                  void *context, dispatch_function_t function) {
 258   SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
 259   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 260   WRAP(dispatch_once)(predicate, ^(void) {
 261     function(context);
 262   });
 263   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 264 }
 265
 266 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
 267                  dispatch_semaphore_t dsema) {
 268   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
 269   Release(thr, pc, (uptr)dsema);
 270   return REAL(dispatch_semaphore_signal)(dsema);
 271 }
 272
 273 TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
 274                  dispatch_time_t timeout) {
 275   SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
 276   long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
 277   if (result == 0) Acquire(thr, pc, (uptr)dsema);
 278   return result;
 279 }
 280
 281 TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
 282                  dispatch_time_t timeout) {
 283   SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
 284   long_t result = REAL(dispatch_group_wait)(group, timeout);
 285   if (result == 0) Acquire(thr, pc, (uptr)group);
 286   return result;
 287 }
 288
 289 TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
 290   SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
 291   // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
 292   Release(thr, pc, (uptr)group);
 293   REAL(dispatch_group_leave)(group);
 294 }
 295
 296 TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
 297                  dispatch_queue_t queue, dispatch_block_t block) {
 298   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
 299   dispatch_retain(group);
 300   dispatch_group_enter(group);
 301   __block dispatch_block_t block_copy = (dispatch_block_t)_Block_copy(block);
 302   WRAP(dispatch_async)(queue, ^(void) {
 303     block_copy();
 304     _Block_release(block_copy);
 305     WRAP(dispatch_group_leave)(group);
 306     dispatch_release(group);
 307   });
 308 }
 309
 310 TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
 311                  dispatch_queue_t queue, void *context,
 312                  dispatch_function_t work) {
 313   SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
 314   dispatch_retain(group);
 315   dispatch_group_enter(group);
 316   WRAP(dispatch_async)(queue, ^(void) {
 317     work(context);
 318     WRAP(dispatch_group_leave)(group);
 319     dispatch_release(group);
 320   });
 321 }
 322
 323 TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
 324                  dispatch_queue_t q, dispatch_block_t block) {
 325   SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
 326
 327   // To make sure the group is still available in the callback (otherwise
 328   // it can be already destroyed).  Will be released in the callback.
 329   dispatch_retain(group);
 330
 331   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 332   dispatch_block_t heap_block = Block_copy(^(void) {
 333     {
 334       SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
 335       // Released when leaving the group (dispatch_group_leave).
 336       Acquire(thr, pc, (uptr)group);
 337     }
 338     dispatch_release(group);
 339     block();
 340   });
 341   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 342   tsan_block_context_t *new_context =
 343       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 344   new_context->is_barrier_block = true;
 345   Release(thr, pc, (uptr)new_context);
 346   REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
 347 }
 348
 349 TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
 350                  dispatch_queue_t q, void *context, dispatch_function_t work) {
 351   WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
 352 }
 353
 354 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
 355                  dispatch_source_t source, dispatch_block_t handler) {
 356   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
 357   if (handler == nullptr)
 358     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 359   dispatch_queue_t q = GetTargetQueueFromSource(source);
 360   __block tsan_block_context_t new_context = {
 361       q, handler, &invoke_block, false, false, false, 0 };
 362   dispatch_block_t new_handler = Block_copy(^(void) {
 363     new_context.orig_context = handler;  // To explicitly capture "handler".
 364     dispatch_callback_wrap(&new_context);
 365   });
 366   uptr submit_sync = (uptr)&new_context;
 367   Release(thr, pc, submit_sync);
 368   REAL(dispatch_source_set_event_handler)(source, new_handler);
 369   Block_release(new_handler);
 370 }
 371
 372 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
 373                  dispatch_source_t source, dispatch_function_t handler) {
 374   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
 375   if (handler == nullptr)
 376     return REAL(dispatch_source_set_event_handler)(source, nullptr);
 377   dispatch_block_t block = ^(void) {
 378     handler(dispatch_get_context(source));
 379   };
 380   WRAP(dispatch_source_set_event_handler)(source, block);
 381 }
 382
 383 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
 384                  dispatch_source_t source, dispatch_block_t handler) {
 385   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
 386   if (handler == nullptr)
 387     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 388   dispatch_queue_t q = GetTargetQueueFromSource(source);
 389   __block tsan_block_context_t new_context = {
 390       q, handler, &invoke_block, false, false, false, 0};
 391   dispatch_block_t new_handler = Block_copy(^(void) {
 392     new_context.orig_context = handler;  // To explicitly capture "handler".
 393     dispatch_callback_wrap(&new_context);
 394   });
 395   uptr submit_sync = (uptr)&new_context;
 396   Release(thr, pc, submit_sync);
 397   REAL(dispatch_source_set_cancel_handler)(source, new_handler);
 398   Block_release(new_handler);
 399 }
 400
 401 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
 402                  dispatch_source_t source, dispatch_function_t handler) {
 403   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
 404                           handler);
 405   if (handler == nullptr)
 406     return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
 407   dispatch_block_t block = ^(void) {
 408     handler(dispatch_get_context(source));
 409   };
 410   WRAP(dispatch_source_set_cancel_handler)(source, block);
 411 }
 412
 413 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
 414                  dispatch_source_t source, dispatch_block_t handler) {
 415   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
 416                           handler);
 417   if (handler == nullptr)
 418     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 419   dispatch_queue_t q = GetTargetQueueFromSource(source);
 420   __block tsan_block_context_t new_context = {
 421       q, handler, &invoke_block, false, false, false, 0};
 422   dispatch_block_t new_handler = Block_copy(^(void) {
 423     new_context.orig_context = handler;  // To explicitly capture "handler".
 424     dispatch_callback_wrap(&new_context);
 425   });
 426   uptr submit_sync = (uptr)&new_context;
 427   Release(thr, pc, submit_sync);
 428   REAL(dispatch_source_set_registration_handler)(source, new_handler);
 429   Block_release(new_handler);
 430 }
 431
 432 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
 433                  dispatch_source_t source, dispatch_function_t handler) {
 434   SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
 435                           handler);
 436   if (handler == nullptr)
 437     return REAL(dispatch_source_set_registration_handler)(source, nullptr);
 438   dispatch_block_t block = ^(void) {
 439     handler(dispatch_get_context(source));
 440   };
 441   WRAP(dispatch_source_set_registration_handler)(source, block);
 442 }
 443
 444 TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
 445                  dispatch_queue_t queue, void (^block)(size_t)) {
 446   SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
 447
 448   void *parent_to_child_sync = nullptr;
 449   uptr parent_to_child_sync_uptr = (uptr)&parent_to_child_sync;
 450   void *child_to_parent_sync = nullptr;
 451   uptr child_to_parent_sync_uptr = (uptr)&child_to_parent_sync;
 452
 453   Release(thr, pc, parent_to_child_sync_uptr);
 454   void (^new_block)(size_t) = ^(size_t iteration) {
 455     SCOPED_INTERCEPTOR_RAW(dispatch_apply);
 456     Acquire(thr, pc, parent_to_child_sync_uptr);
 457     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 458     block(iteration);
 459     SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 460     Release(thr, pc, child_to_parent_sync_uptr);
 461   };
 462   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 463   REAL(dispatch_apply)(iterations, queue, new_block);
 464   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 465   Acquire(thr, pc, child_to_parent_sync_uptr);
 466 }
 467
 468 TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
 469                  dispatch_queue_t queue, void *context,
 470                  void (*work)(void *, size_t)) {
 471   SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
 472   void (^new_block)(size_t) = ^(size_t iteration) {
 473     work(context, iteration);
 474   };
 475   WRAP(dispatch_apply)(iterations, queue, new_block);
 476 }
 477
 478 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 479 DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
 480
 481 TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
 482                  size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
 483   SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
 484   if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
 485     return REAL(dispatch_data_create)(buffer, size, q, destructor);
 486
 487   if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
 488     destructor = ^(void) { WRAP(free)((void *)buffer); };
 489   else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
 490     destructor = ^(void) { WRAP(munmap)((void *)buffer, size); };
 491
 492   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
 493   dispatch_block_t heap_block = Block_copy(destructor);
 494   SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
 495   tsan_block_context_t *new_context =
 496       AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
 497   uptr submit_sync = (uptr)new_context;
 498   Release(thr, pc, submit_sync);
 499   return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
 500     dispatch_callback_wrap(new_context);
 501   });
 502 }
 503
 504 typedef void (^fd_handler_t)(dispatch_data_t data, int error);
 505 typedef void (^cleanup_handler_t)(int error);
 506
 507 TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
 508                  dispatch_queue_t q, fd_handler_t h) {
 509   SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
 510   __block tsan_block_context_t new_context = {
 511       q, nullptr, &invoke_block, false, false, false, 0};
 512   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 513     new_context.orig_context = ^(void) {
 514       h(data, error);
 515     };
 516     dispatch_callback_wrap(&new_context);
 517   });
 518   uptr submit_sync = (uptr)&new_context;
 519   Release(thr, pc, submit_sync);
 520   REAL(dispatch_read)(fd, length, q, new_h);
 521   Block_release(new_h);
 522 }
 523
 524 TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
 525                  dispatch_queue_t q, fd_handler_t h) {
 526   SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
 527   __block tsan_block_context_t new_context = {
 528       q, nullptr, &invoke_block, false, false, false, 0};
 529   fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
 530     new_context.orig_context = ^(void) {
 531       h(data, error);
 532     };
 533     dispatch_callback_wrap(&new_context);
 534   });
 535   uptr submit_sync = (uptr)&new_context;
 536   Release(thr, pc, submit_sync);
 537   REAL(dispatch_write)(fd, data, q, new_h);
 538   Block_release(new_h);
 539 }
 540
 541 TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
 542                  size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
 543   SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
 544   __block tsan_block_context_t new_context = {
 545       q, nullptr, &invoke_block, false, false, false, 0};
 546   dispatch_io_handler_t new_h =
 547       Block_copy(^(bool done, dispatch_data_t data, int error) {
 548         new_context.orig_context = ^(void) {
 549           h(done, data, error);
 550         };
 551         dispatch_callback_wrap(&new_context);
 552       });
 553   uptr submit_sync = (uptr)&new_context;
 554   Release(thr, pc, submit_sync);
 555   REAL(dispatch_io_read)(channel, offset, length, q, new_h);
 556   Block_release(new_h);
 557 }
 558
 559 TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
 560                  dispatch_data_t data, dispatch_queue_t q,
 561                  dispatch_io_handler_t h) {
 562   SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
 563   __block tsan_block_context_t new_context = {
 564       q, nullptr, &invoke_block, false, false, false, 0};
 565   dispatch_io_handler_t new_h =
 566       Block_copy(^(bool done, dispatch_data_t data, int error) {
 567         new_context.orig_context = ^(void) {
 568           h(done, data, error);
 569         };
 570         dispatch_callback_wrap(&new_context);
 571       });
 572   uptr submit_sync = (uptr)&new_context;
 573   Release(thr, pc, submit_sync);
 574   REAL(dispatch_io_write)(channel, offset, data, q, new_h);
 575   Block_release(new_h);
 576 }
 577
 578 TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
 579                  dispatch_block_t barrier) {
 580   SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
 581   __block tsan_block_context_t new_context = {
 582       nullptr, nullptr, &invoke_block, false, false, false, 0};
 583   new_context.non_queue_sync_object = (uptr)channel;
 584   new_context.is_barrier_block = true;
 585   dispatch_block_t new_block = Block_copy(^(void) {
 586     new_context.orig_context = ^(void) {
 587       barrier();
 588     };
 589     dispatch_callback_wrap(&new_context);
 590   });
 591   uptr submit_sync = (uptr)&new_context;
 592   Release(thr, pc, submit_sync);
 593   REAL(dispatch_io_barrier)(channel, new_block);
 594   Block_release(new_block);
 595 }
 596
 597 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
 598                  dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
 599   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
 600   __block dispatch_io_t new_channel = nullptr;
 601   __block tsan_block_context_t new_context = {
 602       q, nullptr, &invoke_block, false, false, false, 0};
 603   cleanup_handler_t new_h = Block_copy(^(int error) {
 604     {
 605       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 606       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 607     }
 608     new_context.orig_context = ^(void) {
 609       h(error);
 610     };
 611     dispatch_callback_wrap(&new_context);
 612   });
 613   uptr submit_sync = (uptr)&new_context;
 614   Release(thr, pc, submit_sync);
 615   new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
 616   Block_release(new_h);
 617   return new_channel;
 618 }
 619
 620 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
 621                  dispatch_io_type_t type, const char *path, int oflag,
 622                  mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
 623   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
 624                           q, h);
 625   __block dispatch_io_t new_channel = nullptr;
 626   __block tsan_block_context_t new_context = {
 627       q, nullptr, &invoke_block, false, false, false, 0};
 628   cleanup_handler_t new_h = Block_copy(^(int error) {
 629     {
 630       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 631       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 632     }
 633     new_context.orig_context = ^(void) {
 634       h(error);
 635     };
 636     dispatch_callback_wrap(&new_context);
 637   });
 638   uptr submit_sync = (uptr)&new_context;
 639   Release(thr, pc, submit_sync);
 640   new_channel =
 641       REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
 642   Block_release(new_h);
 643   return new_channel;
 644 }
 645
 646 TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
 647                  dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
 648                  cleanup_handler_t h) {
 649   SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
 650   __block dispatch_io_t new_channel = nullptr;
 651   __block tsan_block_context_t new_context = {
 652       q, nullptr, &invoke_block, false, false, false, 0};
 653   cleanup_handler_t new_h = Block_copy(^(int error) {
 654     {
 655       SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
 656       Acquire(thr, pc, (uptr)new_channel);  // Release() in dispatch_io_close.
 657     }
 658     new_context.orig_context = ^(void) {
 659       h(error);
 660     };
 661     dispatch_callback_wrap(&new_context);
 662   });
 663   uptr submit_sync = (uptr)&new_context;
 664   Release(thr, pc, submit_sync);
 665   new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
 666   Block_release(new_h);
 667   return new_channel;
 668 }
 669
 670 TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
 671                  dispatch_io_close_flags_t flags) {
 672   SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
 673   Release(thr, pc, (uptr)channel);  // Acquire() in dispatch_io_create[_*].
 674   return REAL(dispatch_io_close)(channel, flags);
 675 }
 676
 677 }  // namespace __tsan
 678
 679 #endif  // SANITIZER_MAC