1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Mac-specific libdispatch (GCD) support.
11 //===----------------------------------------------------------------------===//
13 #include "sanitizer_common/sanitizer_platform.h"
16 #include "sanitizer_common/sanitizer_common.h"
17 #include "interception/interception.h"
18 #include "tsan_interceptors.h"
19 #include "tsan_platform.h"
23 #include <dispatch/dispatch.h>
26 // DISPATCH_NOESCAPE is not defined prior to XCode 8.
27 #ifndef DISPATCH_NOESCAPE
28 #define DISPATCH_NOESCAPE
31 typedef long long_t
; // NOLINT
36 dispatch_queue_t queue
;
38 dispatch_function_t orig_work
;
39 bool free_context_in_callback
;
40 bool submitted_synchronously
;
41 bool is_barrier_block
;
42 uptr non_queue_sync_object
;
43 } tsan_block_context_t
;
45 // The offsets of different fields of the dispatch_queue_t structure, exported
46 // by libdispatch.dylib.
47 extern "C" struct dispatch_queue_offsets_s
{
48 const uint16_t dqo_version
;
49 const uint16_t dqo_label
;
50 const uint16_t dqo_label_size
;
51 const uint16_t dqo_flags
;
52 const uint16_t dqo_flags_size
;
53 const uint16_t dqo_serialnum
;
54 const uint16_t dqo_serialnum_size
;
55 const uint16_t dqo_width
;
56 const uint16_t dqo_width_size
;
57 const uint16_t dqo_running
;
58 const uint16_t dqo_running_size
;
59 const uint16_t dqo_suspend_cnt
;
60 const uint16_t dqo_suspend_cnt_size
;
61 const uint16_t dqo_target_queue
;
62 const uint16_t dqo_target_queue_size
;
63 const uint16_t dqo_priority
;
64 const uint16_t dqo_priority_size
;
65 } dispatch_queue_offsets
;
67 static bool IsQueueSerial(dispatch_queue_t q
) {
68 CHECK_EQ(dispatch_queue_offsets
.dqo_width_size
, 2);
69 uptr width
= *(uint16_t *)(((uptr
)q
) + dispatch_queue_offsets
.dqo_width
);
74 static dispatch_queue_t
GetTargetQueueFromQueue(dispatch_queue_t q
) {
75 CHECK_EQ(dispatch_queue_offsets
.dqo_target_queue_size
, 8);
76 dispatch_queue_t tq
= *(
77 dispatch_queue_t
*)(((uptr
)q
) + dispatch_queue_offsets
.dqo_target_queue
);
81 static dispatch_queue_t
GetTargetQueueFromSource(dispatch_source_t source
) {
82 dispatch_queue_t tq
= GetTargetQueueFromQueue((dispatch_queue_t
)source
);
87 static tsan_block_context_t
*AllocContext(ThreadState
*thr
, uptr pc
,
88 dispatch_queue_t queue
,
90 dispatch_function_t orig_work
) {
91 tsan_block_context_t
*new_context
=
92 (tsan_block_context_t
*)user_alloc_internal(thr
, pc
,
93 sizeof(tsan_block_context_t
));
94 new_context
->queue
= queue
;
95 new_context
->orig_context
= orig_context
;
96 new_context
->orig_work
= orig_work
;
97 new_context
->free_context_in_callback
= true;
98 new_context
->submitted_synchronously
= false;
99 new_context
->is_barrier_block
= false;
100 new_context
->non_queue_sync_object
= 0;
104 #define GET_QUEUE_SYNC_VARS(context, q) \
105 bool is_queue_serial = q && IsQueueSerial(q); \
106 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
107 uptr serial_sync = (uptr)sync_ptr; \
108 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
109 bool serial_task = context->is_barrier_block || is_queue_serial
111 static void dispatch_sync_pre_execute(ThreadState
*thr
, uptr pc
,
112 tsan_block_context_t
*context
) {
113 uptr submit_sync
= (uptr
)context
;
114 Acquire(thr
, pc
, submit_sync
);
116 dispatch_queue_t q
= context
->queue
;
118 GET_QUEUE_SYNC_VARS(context
, q
);
119 if (serial_sync
) Acquire(thr
, pc
, serial_sync
);
120 if (serial_task
&& concurrent_sync
) Acquire(thr
, pc
, concurrent_sync
);
122 if (q
) q
= GetTargetQueueFromQueue(q
);
126 static void dispatch_sync_post_execute(ThreadState
*thr
, uptr pc
,
127 tsan_block_context_t
*context
) {
128 uptr submit_sync
= (uptr
)context
;
129 if (context
->submitted_synchronously
) Release(thr
, pc
, submit_sync
);
131 dispatch_queue_t q
= context
->queue
;
133 GET_QUEUE_SYNC_VARS(context
, q
);
134 if (serial_task
&& serial_sync
) Release(thr
, pc
, serial_sync
);
135 if (!serial_task
&& concurrent_sync
) Release(thr
, pc
, concurrent_sync
);
137 if (q
) q
= GetTargetQueueFromQueue(q
);
141 static void dispatch_callback_wrap(void *param
) {
142 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap
);
143 tsan_block_context_t
*context
= (tsan_block_context_t
*)param
;
145 dispatch_sync_pre_execute(thr
, pc
, context
);
147 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
148 context
->orig_work(context
->orig_context
);
149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
151 dispatch_sync_post_execute(thr
, pc
, context
);
153 if (context
->free_context_in_callback
) user_free(thr
, pc
, context
);
156 static void invoke_block(void *param
) {
157 dispatch_block_t block
= (dispatch_block_t
)param
;
161 static void invoke_and_release_block(void *param
) {
162 dispatch_block_t block
= (dispatch_block_t
)param
;
164 Block_release(block
);
167 #define DISPATCH_INTERCEPT_B(name, barrier) \
168 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
169 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
170 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
171 dispatch_block_t heap_block = Block_copy(block); \
172 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
173 tsan_block_context_t *new_context = \
174 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
175 new_context->is_barrier_block = barrier; \
176 Release(thr, pc, (uptr)new_context); \
177 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
178 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
182 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
183 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
184 DISPATCH_NOESCAPE dispatch_block_t block) { \
185 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
186 tsan_block_context_t new_context = { \
187 q, block, &invoke_block, false, true, barrier, 0}; \
188 Release(thr, pc, (uptr)&new_context); \
189 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
190 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
191 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
192 Acquire(thr, pc, (uptr)&new_context); \
195 #define DISPATCH_INTERCEPT_F(name, barrier) \
196 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
197 dispatch_function_t work) { \
198 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
199 tsan_block_context_t *new_context = \
200 AllocContext(thr, pc, q, context, work); \
201 new_context->is_barrier_block = barrier; \
202 Release(thr, pc, (uptr)new_context); \
203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
204 REAL(name)(q, new_context, dispatch_callback_wrap); \
205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
208 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
209 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
210 dispatch_function_t work) { \
211 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
212 tsan_block_context_t new_context = { \
213 q, context, work, false, true, barrier, 0}; \
214 Release(thr, pc, (uptr)&new_context); \
215 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
216 REAL(name)(q, &new_context, dispatch_callback_wrap); \
217 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
218 Acquire(thr, pc, (uptr)&new_context); \
221 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
222 // context, which is used to synchronize (we release the context before
223 // submitting, and the callback acquires it before executing the original
225 DISPATCH_INTERCEPT_B(dispatch_async
, false)
226 DISPATCH_INTERCEPT_B(dispatch_barrier_async
, true)
227 DISPATCH_INTERCEPT_F(dispatch_async_f
, false)
228 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f
, true)
229 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync
, false)
230 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync
, true)
231 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f
, false)
232 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f
, true)
234 TSAN_INTERCEPTOR(void, dispatch_after
, dispatch_time_t when
,
235 dispatch_queue_t queue
, dispatch_block_t block
) {
236 SCOPED_TSAN_INTERCEPTOR(dispatch_after
, when
, queue
, block
);
237 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
238 dispatch_block_t heap_block
= Block_copy(block
);
239 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
240 tsan_block_context_t
*new_context
=
241 AllocContext(thr
, pc
, queue
, heap_block
, &invoke_and_release_block
);
242 Release(thr
, pc
, (uptr
)new_context
);
243 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
244 REAL(dispatch_after_f
)(when
, queue
, new_context
, dispatch_callback_wrap
);
245 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
248 TSAN_INTERCEPTOR(void, dispatch_after_f
, dispatch_time_t when
,
249 dispatch_queue_t queue
, void *context
,
250 dispatch_function_t work
) {
251 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f
, when
, queue
, context
, work
);
252 WRAP(dispatch_after
)(when
, queue
, ^(void) {
257 // GCD's dispatch_once implementation has a fast path that contains a racy read
258 // and it's inlined into user's code. Furthermore, this fast path doesn't
259 // establish a proper happens-before relations between the initialization and
260 // code following the call to dispatch_once. We could deal with this in
261 // instrumented code, but there's not much we can do about it in system
262 // libraries. Let's disable the fast path (by never storing the value ~0 to
263 // predicate), so the interceptor is always called, and let's add proper release
264 // and acquire semantics. Since TSan does not see its own atomic stores, the
265 // race on predicate won't be reported - the only accesses to it that TSan sees
266 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
267 // both a macro and a real function, we want to intercept the function, so we
268 // need to undefine the macro.
270 TSAN_INTERCEPTOR(void, dispatch_once
, dispatch_once_t
*predicate
,
271 DISPATCH_NOESCAPE dispatch_block_t block
) {
272 SCOPED_INTERCEPTOR_RAW(dispatch_once
, predicate
, block
);
273 atomic_uint32_t
*a
= reinterpret_cast<atomic_uint32_t
*>(predicate
);
274 u32 v
= atomic_load(a
, memory_order_acquire
);
276 atomic_compare_exchange_strong(a
, &v
, 1, memory_order_relaxed
)) {
277 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
279 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
280 Release(thr
, pc
, (uptr
)a
);
281 atomic_store(a
, 2, memory_order_release
);
284 internal_sched_yield();
285 v
= atomic_load(a
, memory_order_acquire
);
287 Acquire(thr
, pc
, (uptr
)a
);
291 #undef dispatch_once_f
292 TSAN_INTERCEPTOR(void, dispatch_once_f
, dispatch_once_t
*predicate
,
293 void *context
, dispatch_function_t function
) {
294 SCOPED_INTERCEPTOR_RAW(dispatch_once_f
, predicate
, context
, function
);
295 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
296 WRAP(dispatch_once
)(predicate
, ^(void) {
299 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
302 TSAN_INTERCEPTOR(long_t
, dispatch_semaphore_signal
,
303 dispatch_semaphore_t dsema
) {
304 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal
, dsema
);
305 Release(thr
, pc
, (uptr
)dsema
);
306 return REAL(dispatch_semaphore_signal
)(dsema
);
309 TSAN_INTERCEPTOR(long_t
, dispatch_semaphore_wait
, dispatch_semaphore_t dsema
,
310 dispatch_time_t timeout
) {
311 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait
, dsema
, timeout
);
312 long_t result
= REAL(dispatch_semaphore_wait
)(dsema
, timeout
);
313 if (result
== 0) Acquire(thr
, pc
, (uptr
)dsema
);
317 TSAN_INTERCEPTOR(long_t
, dispatch_group_wait
, dispatch_group_t group
,
318 dispatch_time_t timeout
) {
319 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait
, group
, timeout
);
320 long_t result
= REAL(dispatch_group_wait
)(group
, timeout
);
321 if (result
== 0) Acquire(thr
, pc
, (uptr
)group
);
325 TSAN_INTERCEPTOR(void, dispatch_group_leave
, dispatch_group_t group
) {
326 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave
, group
);
327 // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
328 Release(thr
, pc
, (uptr
)group
);
329 REAL(dispatch_group_leave
)(group
);
332 TSAN_INTERCEPTOR(void, dispatch_group_async
, dispatch_group_t group
,
333 dispatch_queue_t queue
, dispatch_block_t block
) {
334 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async
, group
, queue
, block
);
335 dispatch_retain(group
);
336 dispatch_group_enter(group
);
337 __block dispatch_block_t block_copy
= (dispatch_block_t
)_Block_copy(block
);
338 WRAP(dispatch_async
)(queue
, ^(void) {
340 _Block_release(block_copy
);
341 WRAP(dispatch_group_leave
)(group
);
342 dispatch_release(group
);
346 TSAN_INTERCEPTOR(void, dispatch_group_async_f
, dispatch_group_t group
,
347 dispatch_queue_t queue
, void *context
,
348 dispatch_function_t work
) {
349 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f
, group
, queue
, context
, work
);
350 dispatch_retain(group
);
351 dispatch_group_enter(group
);
352 WRAP(dispatch_async
)(queue
, ^(void) {
354 WRAP(dispatch_group_leave
)(group
);
355 dispatch_release(group
);
359 TSAN_INTERCEPTOR(void, dispatch_group_notify
, dispatch_group_t group
,
360 dispatch_queue_t q
, dispatch_block_t block
) {
361 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify
, group
, q
, block
);
363 // To make sure the group is still available in the callback (otherwise
364 // it can be already destroyed). Will be released in the callback.
365 dispatch_retain(group
);
367 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
368 dispatch_block_t heap_block
= Block_copy(^(void) {
370 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback
);
371 // Released when leaving the group (dispatch_group_leave).
372 Acquire(thr
, pc
, (uptr
)group
);
374 dispatch_release(group
);
377 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
378 tsan_block_context_t
*new_context
=
379 AllocContext(thr
, pc
, q
, heap_block
, &invoke_and_release_block
);
380 new_context
->is_barrier_block
= true;
381 Release(thr
, pc
, (uptr
)new_context
);
382 REAL(dispatch_group_notify_f
)(group
, q
, new_context
, dispatch_callback_wrap
);
385 TSAN_INTERCEPTOR(void, dispatch_group_notify_f
, dispatch_group_t group
,
386 dispatch_queue_t q
, void *context
, dispatch_function_t work
) {
387 WRAP(dispatch_group_notify
)(group
, q
, ^(void) { work(context
); });
390 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler
,
391 dispatch_source_t source
, dispatch_block_t handler
) {
392 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler
, source
, handler
);
393 if (handler
== nullptr)
394 return REAL(dispatch_source_set_event_handler
)(source
, nullptr);
395 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
396 __block tsan_block_context_t new_context
= {
397 q
, handler
, &invoke_block
, false, false, false, 0 };
398 dispatch_block_t new_handler
= Block_copy(^(void) {
399 new_context
.orig_context
= handler
; // To explicitly capture "handler".
400 dispatch_callback_wrap(&new_context
);
402 uptr submit_sync
= (uptr
)&new_context
;
403 Release(thr
, pc
, submit_sync
);
404 REAL(dispatch_source_set_event_handler
)(source
, new_handler
);
405 Block_release(new_handler
);
408 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f
,
409 dispatch_source_t source
, dispatch_function_t handler
) {
410 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f
, source
, handler
);
411 if (handler
== nullptr)
412 return REAL(dispatch_source_set_event_handler
)(source
, nullptr);
413 dispatch_block_t block
= ^(void) {
414 handler(dispatch_get_context(source
));
416 WRAP(dispatch_source_set_event_handler
)(source
, block
);
419 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler
,
420 dispatch_source_t source
, dispatch_block_t handler
) {
421 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler
, source
, handler
);
422 if (handler
== nullptr)
423 return REAL(dispatch_source_set_cancel_handler
)(source
, nullptr);
424 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
425 __block tsan_block_context_t new_context
= {
426 q
, handler
, &invoke_block
, false, false, false, 0};
427 dispatch_block_t new_handler
= Block_copy(^(void) {
428 new_context
.orig_context
= handler
; // To explicitly capture "handler".
429 dispatch_callback_wrap(&new_context
);
431 uptr submit_sync
= (uptr
)&new_context
;
432 Release(thr
, pc
, submit_sync
);
433 REAL(dispatch_source_set_cancel_handler
)(source
, new_handler
);
434 Block_release(new_handler
);
437 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f
,
438 dispatch_source_t source
, dispatch_function_t handler
) {
439 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f
, source
,
441 if (handler
== nullptr)
442 return REAL(dispatch_source_set_cancel_handler
)(source
, nullptr);
443 dispatch_block_t block
= ^(void) {
444 handler(dispatch_get_context(source
));
446 WRAP(dispatch_source_set_cancel_handler
)(source
, block
);
449 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler
,
450 dispatch_source_t source
, dispatch_block_t handler
) {
451 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler
, source
,
453 if (handler
== nullptr)
454 return REAL(dispatch_source_set_registration_handler
)(source
, nullptr);
455 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
456 __block tsan_block_context_t new_context
= {
457 q
, handler
, &invoke_block
, false, false, false, 0};
458 dispatch_block_t new_handler
= Block_copy(^(void) {
459 new_context
.orig_context
= handler
; // To explicitly capture "handler".
460 dispatch_callback_wrap(&new_context
);
462 uptr submit_sync
= (uptr
)&new_context
;
463 Release(thr
, pc
, submit_sync
);
464 REAL(dispatch_source_set_registration_handler
)(source
, new_handler
);
465 Block_release(new_handler
);
468 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f
,
469 dispatch_source_t source
, dispatch_function_t handler
) {
470 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f
, source
,
472 if (handler
== nullptr)
473 return REAL(dispatch_source_set_registration_handler
)(source
, nullptr);
474 dispatch_block_t block
= ^(void) {
475 handler(dispatch_get_context(source
));
477 WRAP(dispatch_source_set_registration_handler
)(source
, block
);
480 TSAN_INTERCEPTOR(void, dispatch_apply
, size_t iterations
,
481 dispatch_queue_t queue
,
482 DISPATCH_NOESCAPE
void (^block
)(size_t)) {
483 SCOPED_TSAN_INTERCEPTOR(dispatch_apply
, iterations
, queue
, block
);
485 void *parent_to_child_sync
= nullptr;
486 uptr parent_to_child_sync_uptr
= (uptr
)&parent_to_child_sync
;
487 void *child_to_parent_sync
= nullptr;
488 uptr child_to_parent_sync_uptr
= (uptr
)&child_to_parent_sync
;
490 Release(thr
, pc
, parent_to_child_sync_uptr
);
491 void (^new_block
)(size_t) = ^(size_t iteration
) {
492 SCOPED_INTERCEPTOR_RAW(dispatch_apply
);
493 Acquire(thr
, pc
, parent_to_child_sync_uptr
);
494 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
496 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
497 Release(thr
, pc
, child_to_parent_sync_uptr
);
499 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
500 REAL(dispatch_apply
)(iterations
, queue
, new_block
);
501 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
502 Acquire(thr
, pc
, child_to_parent_sync_uptr
);
505 TSAN_INTERCEPTOR(void, dispatch_apply_f
, size_t iterations
,
506 dispatch_queue_t queue
, void *context
,
507 void (*work
)(void *, size_t)) {
508 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f
, iterations
, queue
, context
, work
);
509 void (^new_block
)(size_t) = ^(size_t iteration
) {
510 work(context
, iteration
);
512 WRAP(dispatch_apply
)(iterations
, queue
, new_block
);
515 DECLARE_REAL_AND_INTERCEPTOR(void, free
, void *ptr
)
516 DECLARE_REAL_AND_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
)
518 TSAN_INTERCEPTOR(dispatch_data_t
, dispatch_data_create
, const void *buffer
,
519 size_t size
, dispatch_queue_t q
, dispatch_block_t destructor
) {
520 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create
, buffer
, size
, q
, destructor
);
521 if ((q
== nullptr) || (destructor
== DISPATCH_DATA_DESTRUCTOR_DEFAULT
))
522 return REAL(dispatch_data_create
)(buffer
, size
, q
, destructor
);
524 if (destructor
== DISPATCH_DATA_DESTRUCTOR_FREE
)
525 destructor
= ^(void) { WRAP(free
)((void *)(uintptr_t)buffer
); };
526 else if (destructor
== DISPATCH_DATA_DESTRUCTOR_MUNMAP
)
527 destructor
= ^(void) { WRAP(munmap
)((void *)(uintptr_t)buffer
, size
); };
529 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
530 dispatch_block_t heap_block
= Block_copy(destructor
);
531 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
532 tsan_block_context_t
*new_context
=
533 AllocContext(thr
, pc
, q
, heap_block
, &invoke_and_release_block
);
534 uptr submit_sync
= (uptr
)new_context
;
535 Release(thr
, pc
, submit_sync
);
536 return REAL(dispatch_data_create
)(buffer
, size
, q
, ^(void) {
537 dispatch_callback_wrap(new_context
);
541 typedef void (^fd_handler_t
)(dispatch_data_t data
, int error
);
542 typedef void (^cleanup_handler_t
)(int error
);
544 TSAN_INTERCEPTOR(void, dispatch_read
, dispatch_fd_t fd
, size_t length
,
545 dispatch_queue_t q
, fd_handler_t h
) {
546 SCOPED_TSAN_INTERCEPTOR(dispatch_read
, fd
, length
, q
, h
);
547 __block tsan_block_context_t new_context
= {
548 q
, nullptr, &invoke_block
, false, false, false, 0};
549 fd_handler_t new_h
= Block_copy(^(dispatch_data_t data
, int error
) {
550 new_context
.orig_context
= ^(void) {
553 dispatch_callback_wrap(&new_context
);
555 uptr submit_sync
= (uptr
)&new_context
;
556 Release(thr
, pc
, submit_sync
);
557 REAL(dispatch_read
)(fd
, length
, q
, new_h
);
558 Block_release(new_h
);
561 TSAN_INTERCEPTOR(void, dispatch_write
, dispatch_fd_t fd
, dispatch_data_t data
,
562 dispatch_queue_t q
, fd_handler_t h
) {
563 SCOPED_TSAN_INTERCEPTOR(dispatch_write
, fd
, data
, q
, h
);
564 __block tsan_block_context_t new_context
= {
565 q
, nullptr, &invoke_block
, false, false, false, 0};
566 fd_handler_t new_h
= Block_copy(^(dispatch_data_t data
, int error
) {
567 new_context
.orig_context
= ^(void) {
570 dispatch_callback_wrap(&new_context
);
572 uptr submit_sync
= (uptr
)&new_context
;
573 Release(thr
, pc
, submit_sync
);
574 REAL(dispatch_write
)(fd
, data
, q
, new_h
);
575 Block_release(new_h
);
578 TSAN_INTERCEPTOR(void, dispatch_io_read
, dispatch_io_t channel
, off_t offset
,
579 size_t length
, dispatch_queue_t q
, dispatch_io_handler_t h
) {
580 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read
, channel
, offset
, length
, q
, h
);
581 __block tsan_block_context_t new_context
= {
582 q
, nullptr, &invoke_block
, false, false, false, 0};
583 dispatch_io_handler_t new_h
=
584 Block_copy(^(bool done
, dispatch_data_t data
, int error
) {
585 new_context
.orig_context
= ^(void) {
586 h(done
, data
, error
);
588 dispatch_callback_wrap(&new_context
);
590 uptr submit_sync
= (uptr
)&new_context
;
591 Release(thr
, pc
, submit_sync
);
592 REAL(dispatch_io_read
)(channel
, offset
, length
, q
, new_h
);
593 Block_release(new_h
);
596 TSAN_INTERCEPTOR(void, dispatch_io_write
, dispatch_io_t channel
, off_t offset
,
597 dispatch_data_t data
, dispatch_queue_t q
,
598 dispatch_io_handler_t h
) {
599 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write
, channel
, offset
, data
, q
, h
);
600 __block tsan_block_context_t new_context
= {
601 q
, nullptr, &invoke_block
, false, false, false, 0};
602 dispatch_io_handler_t new_h
=
603 Block_copy(^(bool done
, dispatch_data_t data
, int error
) {
604 new_context
.orig_context
= ^(void) {
605 h(done
, data
, error
);
607 dispatch_callback_wrap(&new_context
);
609 uptr submit_sync
= (uptr
)&new_context
;
610 Release(thr
, pc
, submit_sync
);
611 REAL(dispatch_io_write
)(channel
, offset
, data
, q
, new_h
);
612 Block_release(new_h
);
615 TSAN_INTERCEPTOR(void, dispatch_io_barrier
, dispatch_io_t channel
,
616 dispatch_block_t barrier
) {
617 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier
, channel
, barrier
);
618 __block tsan_block_context_t new_context
= {
619 nullptr, nullptr, &invoke_block
, false, false, false, 0};
620 new_context
.non_queue_sync_object
= (uptr
)channel
;
621 new_context
.is_barrier_block
= true;
622 dispatch_block_t new_block
= Block_copy(^(void) {
623 new_context
.orig_context
= ^(void) {
626 dispatch_callback_wrap(&new_context
);
628 uptr submit_sync
= (uptr
)&new_context
;
629 Release(thr
, pc
, submit_sync
);
630 REAL(dispatch_io_barrier
)(channel
, new_block
);
631 Block_release(new_block
);
634 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create
, dispatch_io_type_t type
,
635 dispatch_fd_t fd
, dispatch_queue_t q
, cleanup_handler_t h
) {
636 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create
, type
, fd
, q
, h
);
637 __block dispatch_io_t new_channel
= nullptr;
638 __block tsan_block_context_t new_context
= {
639 q
, nullptr, &invoke_block
, false, false, false, 0};
640 cleanup_handler_t new_h
= Block_copy(^(int error
) {
642 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
643 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
645 new_context
.orig_context
= ^(void) {
648 dispatch_callback_wrap(&new_context
);
650 uptr submit_sync
= (uptr
)&new_context
;
651 Release(thr
, pc
, submit_sync
);
652 new_channel
= REAL(dispatch_io_create
)(type
, fd
, q
, new_h
);
653 Block_release(new_h
);
657 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create_with_path
,
658 dispatch_io_type_t type
, const char *path
, int oflag
,
659 mode_t mode
, dispatch_queue_t q
, cleanup_handler_t h
) {
660 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path
, type
, path
, oflag
, mode
,
662 __block dispatch_io_t new_channel
= nullptr;
663 __block tsan_block_context_t new_context
= {
664 q
, nullptr, &invoke_block
, false, false, false, 0};
665 cleanup_handler_t new_h
= Block_copy(^(int error
) {
667 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
668 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
670 new_context
.orig_context
= ^(void) {
673 dispatch_callback_wrap(&new_context
);
675 uptr submit_sync
= (uptr
)&new_context
;
676 Release(thr
, pc
, submit_sync
);
678 REAL(dispatch_io_create_with_path
)(type
, path
, oflag
, mode
, q
, new_h
);
679 Block_release(new_h
);
683 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create_with_io
,
684 dispatch_io_type_t type
, dispatch_io_t io
, dispatch_queue_t q
,
685 cleanup_handler_t h
) {
686 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io
, type
, io
, q
, h
);
687 __block dispatch_io_t new_channel
= nullptr;
688 __block tsan_block_context_t new_context
= {
689 q
, nullptr, &invoke_block
, false, false, false, 0};
690 cleanup_handler_t new_h
= Block_copy(^(int error
) {
692 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
693 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
695 new_context
.orig_context
= ^(void) {
698 dispatch_callback_wrap(&new_context
);
700 uptr submit_sync
= (uptr
)&new_context
;
701 Release(thr
, pc
, submit_sync
);
702 new_channel
= REAL(dispatch_io_create_with_io
)(type
, io
, q
, new_h
);
703 Block_release(new_h
);
707 TSAN_INTERCEPTOR(void, dispatch_io_close
, dispatch_io_t channel
,
708 dispatch_io_close_flags_t flags
) {
709 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close
, channel
, flags
);
710 Release(thr
, pc
, (uptr
)channel
); // Acquire() in dispatch_io_create[_*].
711 return REAL(dispatch_io_close
)(channel
, flags
);
714 // Resuming a suspended queue needs to synchronize with all subsequent
715 // executions of blocks in that queue.
716 TSAN_INTERCEPTOR(void, dispatch_resume
, dispatch_object_t o
) {
717 SCOPED_TSAN_INTERCEPTOR(dispatch_resume
, o
);
718 Release(thr
, pc
, (uptr
)o
); // Synchronizes with the Acquire() on serial_sync
719 // in dispatch_sync_pre_execute
720 return REAL(dispatch_resume
)(o
);
723 } // namespace __tsan
725 #endif // SANITIZER_MAC