1 //===-- tsan_libdispatch_mac.cc -------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Mac-specific libdispatch (GCD) support.
11 //===----------------------------------------------------------------------===//
13 #include "sanitizer_common/sanitizer_platform.h"
16 #include "sanitizer_common/sanitizer_common.h"
17 #include "interception/interception.h"
18 #include "tsan_interceptors.h"
19 #include "tsan_platform.h"
23 #include <dispatch/dispatch.h>
26 typedef long long_t
; // NOLINT
31 dispatch_queue_t queue
;
33 dispatch_function_t orig_work
;
34 bool free_context_in_callback
;
35 bool submitted_synchronously
;
36 bool is_barrier_block
;
37 uptr non_queue_sync_object
;
38 } tsan_block_context_t
;
40 // The offsets of different fields of the dispatch_queue_t structure, exported
41 // by libdispatch.dylib.
42 extern "C" struct dispatch_queue_offsets_s
{
43 const uint16_t dqo_version
;
44 const uint16_t dqo_label
;
45 const uint16_t dqo_label_size
;
46 const uint16_t dqo_flags
;
47 const uint16_t dqo_flags_size
;
48 const uint16_t dqo_serialnum
;
49 const uint16_t dqo_serialnum_size
;
50 const uint16_t dqo_width
;
51 const uint16_t dqo_width_size
;
52 const uint16_t dqo_running
;
53 const uint16_t dqo_running_size
;
54 const uint16_t dqo_suspend_cnt
;
55 const uint16_t dqo_suspend_cnt_size
;
56 const uint16_t dqo_target_queue
;
57 const uint16_t dqo_target_queue_size
;
58 const uint16_t dqo_priority
;
59 const uint16_t dqo_priority_size
;
60 } dispatch_queue_offsets
;
62 static bool IsQueueSerial(dispatch_queue_t q
) {
63 CHECK_EQ(dispatch_queue_offsets
.dqo_width_size
, 2);
64 uptr width
= *(uint16_t *)(((uptr
)q
) + dispatch_queue_offsets
.dqo_width
);
69 static dispatch_queue_t
GetTargetQueueFromSource(dispatch_source_t source
) {
70 CHECK_EQ(dispatch_queue_offsets
.dqo_target_queue_size
, 8);
71 dispatch_queue_t target_queue
=
72 *(dispatch_queue_t
*)(((uptr
)source
) +
73 dispatch_queue_offsets
.dqo_target_queue
);
74 CHECK_NE(target_queue
, 0);
78 static tsan_block_context_t
*AllocContext(ThreadState
*thr
, uptr pc
,
79 dispatch_queue_t queue
,
81 dispatch_function_t orig_work
) {
82 tsan_block_context_t
*new_context
=
83 (tsan_block_context_t
*)user_alloc(thr
, pc
, sizeof(tsan_block_context_t
));
84 new_context
->queue
= queue
;
85 new_context
->orig_context
= orig_context
;
86 new_context
->orig_work
= orig_work
;
87 new_context
->free_context_in_callback
= true;
88 new_context
->submitted_synchronously
= false;
89 new_context
->is_barrier_block
= false;
93 static void dispatch_callback_wrap(void *param
) {
94 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap
);
95 tsan_block_context_t
*context
= (tsan_block_context_t
*)param
;
96 bool is_queue_serial
= context
->queue
&& IsQueueSerial(context
->queue
);
97 uptr sync_ptr
= (uptr
)context
->queue
?: context
->non_queue_sync_object
;
99 uptr serial_sync
= (uptr
)sync_ptr
;
100 uptr concurrent_sync
= ((uptr
)sync_ptr
) + sizeof(uptr
);
101 uptr submit_sync
= (uptr
)context
;
102 bool serial_task
= context
->is_barrier_block
|| is_queue_serial
;
104 Acquire(thr
, pc
, submit_sync
);
105 Acquire(thr
, pc
, serial_sync
);
106 if (serial_task
) Acquire(thr
, pc
, concurrent_sync
);
108 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
109 context
->orig_work(context
->orig_context
);
110 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
112 Release(thr
, pc
, serial_task
? serial_sync
: concurrent_sync
);
113 if (context
->submitted_synchronously
) Release(thr
, pc
, submit_sync
);
115 if (context
->free_context_in_callback
) user_free(thr
, pc
, context
);
118 static void invoke_block(void *param
) {
119 dispatch_block_t block
= (dispatch_block_t
)param
;
123 static void invoke_and_release_block(void *param
) {
124 dispatch_block_t block
= (dispatch_block_t
)param
;
126 Block_release(block
);
129 #define DISPATCH_INTERCEPT_B(name, barrier) \
130 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
131 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
132 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
133 dispatch_block_t heap_block = Block_copy(block); \
134 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
135 tsan_block_context_t *new_context = \
136 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
137 new_context->is_barrier_block = barrier; \
138 Release(thr, pc, (uptr)new_context); \
139 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
140 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
144 #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
145 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
146 SCOPED_TSAN_INTERCEPTOR(name, q, block); \
147 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
148 dispatch_block_t heap_block = Block_copy(block); \
149 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
150 tsan_block_context_t new_context = { \
151 q, heap_block, &invoke_and_release_block, false, true, barrier, 0}; \
152 Release(thr, pc, (uptr)&new_context); \
153 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
154 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
155 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
156 Acquire(thr, pc, (uptr)&new_context); \
159 #define DISPATCH_INTERCEPT_F(name, barrier) \
160 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
161 dispatch_function_t work) { \
162 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
163 tsan_block_context_t *new_context = \
164 AllocContext(thr, pc, q, context, work); \
165 new_context->is_barrier_block = barrier; \
166 Release(thr, pc, (uptr)new_context); \
167 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
168 REAL(name)(q, new_context, dispatch_callback_wrap); \
169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
172 #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
173 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
174 dispatch_function_t work) { \
175 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
176 tsan_block_context_t new_context = { \
177 q, context, work, false, true, barrier, 0}; \
178 Release(thr, pc, (uptr)&new_context); \
179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
180 REAL(name)(q, &new_context, dispatch_callback_wrap); \
181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
182 Acquire(thr, pc, (uptr)&new_context); \
185 // We wrap dispatch_async, dispatch_sync and friends where we allocate a new
186 // context, which is used to synchronize (we release the context before
187 // submitting, and the callback acquires it before executing the original
189 DISPATCH_INTERCEPT_B(dispatch_async
, false)
190 DISPATCH_INTERCEPT_B(dispatch_barrier_async
, true)
191 DISPATCH_INTERCEPT_F(dispatch_async_f
, false)
192 DISPATCH_INTERCEPT_F(dispatch_barrier_async_f
, true)
193 DISPATCH_INTERCEPT_SYNC_B(dispatch_sync
, false)
194 DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_sync
, true)
195 DISPATCH_INTERCEPT_SYNC_F(dispatch_sync_f
, false)
196 DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_sync_f
, true)
198 TSAN_INTERCEPTOR(void, dispatch_after
, dispatch_time_t when
,
199 dispatch_queue_t queue
, dispatch_block_t block
) {
200 SCOPED_TSAN_INTERCEPTOR(dispatch_after
, when
, queue
, block
);
201 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
202 dispatch_block_t heap_block
= Block_copy(block
);
203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
204 tsan_block_context_t
*new_context
=
205 AllocContext(thr
, pc
, queue
, heap_block
, &invoke_and_release_block
);
206 Release(thr
, pc
, (uptr
)new_context
);
207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
208 REAL(dispatch_after_f
)(when
, queue
, new_context
, dispatch_callback_wrap
);
209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
212 TSAN_INTERCEPTOR(void, dispatch_after_f
, dispatch_time_t when
,
213 dispatch_queue_t queue
, void *context
,
214 dispatch_function_t work
) {
215 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f
, when
, queue
, context
, work
);
216 WRAP(dispatch_after
)(when
, queue
, ^(void) {
221 // GCD's dispatch_once implementation has a fast path that contains a racy read
222 // and it's inlined into user's code. Furthermore, this fast path doesn't
223 // establish a proper happens-before relations between the initialization and
224 // code following the call to dispatch_once. We could deal with this in
225 // instrumented code, but there's not much we can do about it in system
226 // libraries. Let's disable the fast path (by never storing the value ~0 to
227 // predicate), so the interceptor is always called, and let's add proper release
228 // and acquire semantics. Since TSan does not see its own atomic stores, the
229 // race on predicate won't be reported - the only accesses to it that TSan sees
230 // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
231 // both a macro and a real function, we want to intercept the function, so we
232 // need to undefine the macro.
234 TSAN_INTERCEPTOR(void, dispatch_once
, dispatch_once_t
*predicate
,
235 dispatch_block_t block
) {
236 SCOPED_INTERCEPTOR_RAW(dispatch_once
, predicate
, block
);
237 atomic_uint32_t
*a
= reinterpret_cast<atomic_uint32_t
*>(predicate
);
238 u32 v
= atomic_load(a
, memory_order_acquire
);
240 atomic_compare_exchange_strong(a
, &v
, 1, memory_order_relaxed
)) {
241 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
243 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
244 Release(thr
, pc
, (uptr
)a
);
245 atomic_store(a
, 2, memory_order_release
);
248 internal_sched_yield();
249 v
= atomic_load(a
, memory_order_acquire
);
251 Acquire(thr
, pc
, (uptr
)a
);
255 #undef dispatch_once_f
256 TSAN_INTERCEPTOR(void, dispatch_once_f
, dispatch_once_t
*predicate
,
257 void *context
, dispatch_function_t function
) {
258 SCOPED_INTERCEPTOR_RAW(dispatch_once_f
, predicate
, context
, function
);
259 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
260 WRAP(dispatch_once
)(predicate
, ^(void) {
263 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
266 TSAN_INTERCEPTOR(long_t
, dispatch_semaphore_signal
,
267 dispatch_semaphore_t dsema
) {
268 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal
, dsema
);
269 Release(thr
, pc
, (uptr
)dsema
);
270 return REAL(dispatch_semaphore_signal
)(dsema
);
273 TSAN_INTERCEPTOR(long_t
, dispatch_semaphore_wait
, dispatch_semaphore_t dsema
,
274 dispatch_time_t timeout
) {
275 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait
, dsema
, timeout
);
276 long_t result
= REAL(dispatch_semaphore_wait
)(dsema
, timeout
);
277 if (result
== 0) Acquire(thr
, pc
, (uptr
)dsema
);
281 TSAN_INTERCEPTOR(long_t
, dispatch_group_wait
, dispatch_group_t group
,
282 dispatch_time_t timeout
) {
283 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait
, group
, timeout
);
284 long_t result
= REAL(dispatch_group_wait
)(group
, timeout
);
285 if (result
== 0) Acquire(thr
, pc
, (uptr
)group
);
289 TSAN_INTERCEPTOR(void, dispatch_group_leave
, dispatch_group_t group
) {
290 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave
, group
);
291 // Acquired in the group noticifaction callback in dispatch_group_notify[_f].
292 Release(thr
, pc
, (uptr
)group
);
293 REAL(dispatch_group_leave
)(group
);
296 TSAN_INTERCEPTOR(void, dispatch_group_async
, dispatch_group_t group
,
297 dispatch_queue_t queue
, dispatch_block_t block
) {
298 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async
, group
, queue
, block
);
299 dispatch_retain(group
);
300 dispatch_group_enter(group
);
301 __block dispatch_block_t block_copy
= (dispatch_block_t
)_Block_copy(block
);
302 WRAP(dispatch_async
)(queue
, ^(void) {
304 _Block_release(block_copy
);
305 WRAP(dispatch_group_leave
)(group
);
306 dispatch_release(group
);
310 TSAN_INTERCEPTOR(void, dispatch_group_async_f
, dispatch_group_t group
,
311 dispatch_queue_t queue
, void *context
,
312 dispatch_function_t work
) {
313 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f
, group
, queue
, context
, work
);
314 dispatch_retain(group
);
315 dispatch_group_enter(group
);
316 WRAP(dispatch_async
)(queue
, ^(void) {
318 WRAP(dispatch_group_leave
)(group
);
319 dispatch_release(group
);
323 TSAN_INTERCEPTOR(void, dispatch_group_notify
, dispatch_group_t group
,
324 dispatch_queue_t q
, dispatch_block_t block
) {
325 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify
, group
, q
, block
);
327 // To make sure the group is still available in the callback (otherwise
328 // it can be already destroyed). Will be released in the callback.
329 dispatch_retain(group
);
331 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
332 dispatch_block_t heap_block
= Block_copy(^(void) {
334 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback
);
335 // Released when leaving the group (dispatch_group_leave).
336 Acquire(thr
, pc
, (uptr
)group
);
338 dispatch_release(group
);
341 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
342 tsan_block_context_t
*new_context
=
343 AllocContext(thr
, pc
, q
, heap_block
, &invoke_and_release_block
);
344 new_context
->is_barrier_block
= true;
345 Release(thr
, pc
, (uptr
)new_context
);
346 REAL(dispatch_group_notify_f
)(group
, q
, new_context
, dispatch_callback_wrap
);
349 TSAN_INTERCEPTOR(void, dispatch_group_notify_f
, dispatch_group_t group
,
350 dispatch_queue_t q
, void *context
, dispatch_function_t work
) {
351 WRAP(dispatch_group_notify
)(group
, q
, ^(void) { work(context
); });
354 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler
,
355 dispatch_source_t source
, dispatch_block_t handler
) {
356 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler
, source
, handler
);
357 if (handler
== nullptr)
358 return REAL(dispatch_source_set_event_handler
)(source
, nullptr);
359 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
360 __block tsan_block_context_t new_context
= {
361 q
, handler
, &invoke_block
, false, false, false, 0 };
362 dispatch_block_t new_handler
= Block_copy(^(void) {
363 new_context
.orig_context
= handler
; // To explicitly capture "handler".
364 dispatch_callback_wrap(&new_context
);
366 uptr submit_sync
= (uptr
)&new_context
;
367 Release(thr
, pc
, submit_sync
);
368 REAL(dispatch_source_set_event_handler
)(source
, new_handler
);
369 Block_release(new_handler
);
372 TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f
,
373 dispatch_source_t source
, dispatch_function_t handler
) {
374 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f
, source
, handler
);
375 if (handler
== nullptr)
376 return REAL(dispatch_source_set_event_handler
)(source
, nullptr);
377 dispatch_block_t block
= ^(void) {
378 handler(dispatch_get_context(source
));
380 WRAP(dispatch_source_set_event_handler
)(source
, block
);
383 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler
,
384 dispatch_source_t source
, dispatch_block_t handler
) {
385 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler
, source
, handler
);
386 if (handler
== nullptr)
387 return REAL(dispatch_source_set_cancel_handler
)(source
, nullptr);
388 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
389 __block tsan_block_context_t new_context
= {
390 q
, handler
, &invoke_block
, false, false, false, 0};
391 dispatch_block_t new_handler
= Block_copy(^(void) {
392 new_context
.orig_context
= handler
; // To explicitly capture "handler".
393 dispatch_callback_wrap(&new_context
);
395 uptr submit_sync
= (uptr
)&new_context
;
396 Release(thr
, pc
, submit_sync
);
397 REAL(dispatch_source_set_cancel_handler
)(source
, new_handler
);
398 Block_release(new_handler
);
401 TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f
,
402 dispatch_source_t source
, dispatch_function_t handler
) {
403 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f
, source
,
405 if (handler
== nullptr)
406 return REAL(dispatch_source_set_cancel_handler
)(source
, nullptr);
407 dispatch_block_t block
= ^(void) {
408 handler(dispatch_get_context(source
));
410 WRAP(dispatch_source_set_cancel_handler
)(source
, block
);
413 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler
,
414 dispatch_source_t source
, dispatch_block_t handler
) {
415 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler
, source
,
417 if (handler
== nullptr)
418 return REAL(dispatch_source_set_registration_handler
)(source
, nullptr);
419 dispatch_queue_t q
= GetTargetQueueFromSource(source
);
420 __block tsan_block_context_t new_context
= {
421 q
, handler
, &invoke_block
, false, false, false, 0};
422 dispatch_block_t new_handler
= Block_copy(^(void) {
423 new_context
.orig_context
= handler
; // To explicitly capture "handler".
424 dispatch_callback_wrap(&new_context
);
426 uptr submit_sync
= (uptr
)&new_context
;
427 Release(thr
, pc
, submit_sync
);
428 REAL(dispatch_source_set_registration_handler
)(source
, new_handler
);
429 Block_release(new_handler
);
432 TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f
,
433 dispatch_source_t source
, dispatch_function_t handler
) {
434 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f
, source
,
436 if (handler
== nullptr)
437 return REAL(dispatch_source_set_registration_handler
)(source
, nullptr);
438 dispatch_block_t block
= ^(void) {
439 handler(dispatch_get_context(source
));
441 WRAP(dispatch_source_set_registration_handler
)(source
, block
);
444 TSAN_INTERCEPTOR(void, dispatch_apply
, size_t iterations
,
445 dispatch_queue_t queue
, void (^block
)(size_t)) {
446 SCOPED_TSAN_INTERCEPTOR(dispatch_apply
, iterations
, queue
, block
);
448 void *parent_to_child_sync
= nullptr;
449 uptr parent_to_child_sync_uptr
= (uptr
)&parent_to_child_sync
;
450 void *child_to_parent_sync
= nullptr;
451 uptr child_to_parent_sync_uptr
= (uptr
)&child_to_parent_sync
;
453 Release(thr
, pc
, parent_to_child_sync_uptr
);
454 void (^new_block
)(size_t) = ^(size_t iteration
) {
455 SCOPED_INTERCEPTOR_RAW(dispatch_apply
);
456 Acquire(thr
, pc
, parent_to_child_sync_uptr
);
457 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
459 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
460 Release(thr
, pc
, child_to_parent_sync_uptr
);
462 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
463 REAL(dispatch_apply
)(iterations
, queue
, new_block
);
464 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
465 Acquire(thr
, pc
, child_to_parent_sync_uptr
);
468 TSAN_INTERCEPTOR(void, dispatch_apply_f
, size_t iterations
,
469 dispatch_queue_t queue
, void *context
,
470 void (*work
)(void *, size_t)) {
471 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f
, iterations
, queue
, context
, work
);
472 void (^new_block
)(size_t) = ^(size_t iteration
) {
473 work(context
, iteration
);
475 WRAP(dispatch_apply
)(iterations
, queue
, new_block
);
478 DECLARE_REAL_AND_INTERCEPTOR(void, free
, void *ptr
)
479 DECLARE_REAL_AND_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
)
481 TSAN_INTERCEPTOR(dispatch_data_t
, dispatch_data_create
, const void *buffer
,
482 size_t size
, dispatch_queue_t q
, dispatch_block_t destructor
) {
483 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create
, buffer
, size
, q
, destructor
);
484 if ((q
== nullptr) || (destructor
== DISPATCH_DATA_DESTRUCTOR_DEFAULT
))
485 return REAL(dispatch_data_create
)(buffer
, size
, q
, destructor
);
487 if (destructor
== DISPATCH_DATA_DESTRUCTOR_FREE
)
488 destructor
= ^(void) { WRAP(free
)((void *)buffer
); };
489 else if (destructor
== DISPATCH_DATA_DESTRUCTOR_MUNMAP
)
490 destructor
= ^(void) { WRAP(munmap
)((void *)buffer
, size
); };
492 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
493 dispatch_block_t heap_block
= Block_copy(destructor
);
494 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
495 tsan_block_context_t
*new_context
=
496 AllocContext(thr
, pc
, q
, heap_block
, &invoke_and_release_block
);
497 uptr submit_sync
= (uptr
)new_context
;
498 Release(thr
, pc
, submit_sync
);
499 return REAL(dispatch_data_create
)(buffer
, size
, q
, ^(void) {
500 dispatch_callback_wrap(new_context
);
504 typedef void (^fd_handler_t
)(dispatch_data_t data
, int error
);
505 typedef void (^cleanup_handler_t
)(int error
);
507 TSAN_INTERCEPTOR(void, dispatch_read
, dispatch_fd_t fd
, size_t length
,
508 dispatch_queue_t q
, fd_handler_t h
) {
509 SCOPED_TSAN_INTERCEPTOR(dispatch_read
, fd
, length
, q
, h
);
510 __block tsan_block_context_t new_context
= {
511 q
, nullptr, &invoke_block
, false, false, false, 0};
512 fd_handler_t new_h
= Block_copy(^(dispatch_data_t data
, int error
) {
513 new_context
.orig_context
= ^(void) {
516 dispatch_callback_wrap(&new_context
);
518 uptr submit_sync
= (uptr
)&new_context
;
519 Release(thr
, pc
, submit_sync
);
520 REAL(dispatch_read
)(fd
, length
, q
, new_h
);
521 Block_release(new_h
);
524 TSAN_INTERCEPTOR(void, dispatch_write
, dispatch_fd_t fd
, dispatch_data_t data
,
525 dispatch_queue_t q
, fd_handler_t h
) {
526 SCOPED_TSAN_INTERCEPTOR(dispatch_write
, fd
, data
, q
, h
);
527 __block tsan_block_context_t new_context
= {
528 q
, nullptr, &invoke_block
, false, false, false, 0};
529 fd_handler_t new_h
= Block_copy(^(dispatch_data_t data
, int error
) {
530 new_context
.orig_context
= ^(void) {
533 dispatch_callback_wrap(&new_context
);
535 uptr submit_sync
= (uptr
)&new_context
;
536 Release(thr
, pc
, submit_sync
);
537 REAL(dispatch_write
)(fd
, data
, q
, new_h
);
538 Block_release(new_h
);
541 TSAN_INTERCEPTOR(void, dispatch_io_read
, dispatch_io_t channel
, off_t offset
,
542 size_t length
, dispatch_queue_t q
, dispatch_io_handler_t h
) {
543 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read
, channel
, offset
, length
, q
, h
);
544 __block tsan_block_context_t new_context
= {
545 q
, nullptr, &invoke_block
, false, false, false, 0};
546 dispatch_io_handler_t new_h
=
547 Block_copy(^(bool done
, dispatch_data_t data
, int error
) {
548 new_context
.orig_context
= ^(void) {
549 h(done
, data
, error
);
551 dispatch_callback_wrap(&new_context
);
553 uptr submit_sync
= (uptr
)&new_context
;
554 Release(thr
, pc
, submit_sync
);
555 REAL(dispatch_io_read
)(channel
, offset
, length
, q
, new_h
);
556 Block_release(new_h
);
559 TSAN_INTERCEPTOR(void, dispatch_io_write
, dispatch_io_t channel
, off_t offset
,
560 dispatch_data_t data
, dispatch_queue_t q
,
561 dispatch_io_handler_t h
) {
562 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write
, channel
, offset
, data
, q
, h
);
563 __block tsan_block_context_t new_context
= {
564 q
, nullptr, &invoke_block
, false, false, false, 0};
565 dispatch_io_handler_t new_h
=
566 Block_copy(^(bool done
, dispatch_data_t data
, int error
) {
567 new_context
.orig_context
= ^(void) {
568 h(done
, data
, error
);
570 dispatch_callback_wrap(&new_context
);
572 uptr submit_sync
= (uptr
)&new_context
;
573 Release(thr
, pc
, submit_sync
);
574 REAL(dispatch_io_write
)(channel
, offset
, data
, q
, new_h
);
575 Block_release(new_h
);
578 TSAN_INTERCEPTOR(void, dispatch_io_barrier
, dispatch_io_t channel
,
579 dispatch_block_t barrier
) {
580 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier
, channel
, barrier
);
581 __block tsan_block_context_t new_context
= {
582 nullptr, nullptr, &invoke_block
, false, false, false, 0};
583 new_context
.non_queue_sync_object
= (uptr
)channel
;
584 new_context
.is_barrier_block
= true;
585 dispatch_block_t new_block
= Block_copy(^(void) {
586 new_context
.orig_context
= ^(void) {
589 dispatch_callback_wrap(&new_context
);
591 uptr submit_sync
= (uptr
)&new_context
;
592 Release(thr
, pc
, submit_sync
);
593 REAL(dispatch_io_barrier
)(channel
, new_block
);
594 Block_release(new_block
);
597 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create
, dispatch_io_type_t type
,
598 dispatch_fd_t fd
, dispatch_queue_t q
, cleanup_handler_t h
) {
599 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create
, type
, fd
, q
, h
);
600 __block dispatch_io_t new_channel
= nullptr;
601 __block tsan_block_context_t new_context
= {
602 q
, nullptr, &invoke_block
, false, false, false, 0};
603 cleanup_handler_t new_h
= Block_copy(^(int error
) {
605 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
606 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
608 new_context
.orig_context
= ^(void) {
611 dispatch_callback_wrap(&new_context
);
613 uptr submit_sync
= (uptr
)&new_context
;
614 Release(thr
, pc
, submit_sync
);
615 new_channel
= REAL(dispatch_io_create
)(type
, fd
, q
, new_h
);
616 Block_release(new_h
);
620 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create_with_path
,
621 dispatch_io_type_t type
, const char *path
, int oflag
,
622 mode_t mode
, dispatch_queue_t q
, cleanup_handler_t h
) {
623 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path
, type
, path
, oflag
, mode
,
625 __block dispatch_io_t new_channel
= nullptr;
626 __block tsan_block_context_t new_context
= {
627 q
, nullptr, &invoke_block
, false, false, false, 0};
628 cleanup_handler_t new_h
= Block_copy(^(int error
) {
630 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
631 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
633 new_context
.orig_context
= ^(void) {
636 dispatch_callback_wrap(&new_context
);
638 uptr submit_sync
= (uptr
)&new_context
;
639 Release(thr
, pc
, submit_sync
);
641 REAL(dispatch_io_create_with_path
)(type
, path
, oflag
, mode
, q
, new_h
);
642 Block_release(new_h
);
646 TSAN_INTERCEPTOR(dispatch_io_t
, dispatch_io_create_with_io
,
647 dispatch_io_type_t type
, dispatch_io_t io
, dispatch_queue_t q
,
648 cleanup_handler_t h
) {
649 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io
, type
, io
, q
, h
);
650 __block dispatch_io_t new_channel
= nullptr;
651 __block tsan_block_context_t new_context
= {
652 q
, nullptr, &invoke_block
, false, false, false, 0};
653 cleanup_handler_t new_h
= Block_copy(^(int error
) {
655 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback
);
656 Acquire(thr
, pc
, (uptr
)new_channel
); // Release() in dispatch_io_close.
658 new_context
.orig_context
= ^(void) {
661 dispatch_callback_wrap(&new_context
);
663 uptr submit_sync
= (uptr
)&new_context
;
664 Release(thr
, pc
, submit_sync
);
665 new_channel
= REAL(dispatch_io_create_with_io
)(type
, io
, q
, new_h
);
666 Block_release(new_h
);
670 TSAN_INTERCEPTOR(void, dispatch_io_close
, dispatch_io_t channel
,
671 dispatch_io_close_flags_t flags
) {
672 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close
, channel
, flags
);
673 Release(thr
, pc
, (uptr
)channel
); // Acquire() in dispatch_io_create[_*].
674 return REAL(dispatch_io_close
)(channel
, flags
);
677 } // namespace __tsan
679 #endif // SANITIZER_MAC