1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Mac-specific interceptors.
11 //===----------------------------------------------------------------------===//
13 #include "sanitizer_common/sanitizer_platform.h"
16 #include "interception/interception.h"
17 #include "tsan_interceptors.h"
18 #include "tsan_interface.h"
19 #include "tsan_interface_ann.h"
21 #include <libkern/OSAtomic.h>
24 typedef long long_t
; // NOLINT
28 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
29 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
30 // actually aliases of each other, and we cannot have different interceptors for
31 // them, because they're actually the same function. Thus, we have to stay
32 // conservative and treat the non-barrier versions as mo_acq_rel.
33 static const morder kMacOrderBarrier
= mo_acq_rel
;
34 static const morder kMacOrderNonBarrier
= mo_acq_rel
;
36 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
37 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
38 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
39 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \
42 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
43 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \
44 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \
45 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \
48 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
49 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
50 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
51 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \
54 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
56 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \
57 SCOPED_TSAN_INTERCEPTOR(f, ptr); \
58 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \
61 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \
62 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
63 kMacOrderNonBarrier) \
64 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
66 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \
67 kMacOrderNonBarrier) \
68 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \
71 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \
72 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \
73 kMacOrderNonBarrier) \
74 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \
76 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
77 kMacOrderNonBarrier) \
78 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \
79 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
81 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd
, fetch_add
,
82 OSATOMIC_INTERCEPTOR_PLUS_X
)
83 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement
, fetch_add
,
84 OSATOMIC_INTERCEPTOR_PLUS_1
)
85 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement
, fetch_sub
,
86 OSATOMIC_INTERCEPTOR_MINUS_1
)
87 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr
, fetch_or
, OSATOMIC_INTERCEPTOR_PLUS_X
,
89 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd
, fetch_and
,
90 OSATOMIC_INTERCEPTOR_PLUS_X
, OSATOMIC_INTERCEPTOR
)
91 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor
, fetch_xor
,
92 OSATOMIC_INTERCEPTOR_PLUS_X
, OSATOMIC_INTERCEPTOR
)
94 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \
95 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \
96 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \
97 return tsan_atomic_f##_compare_exchange_strong( \
98 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
99 kMacOrderNonBarrier, kMacOrderNonBarrier); \
102 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \
104 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \
105 return tsan_atomic_f##_compare_exchange_strong( \
106 (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \
107 kMacOrderBarrier, kMacOrderNonBarrier); \
110 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt
, __tsan_atomic32
, a32
, int)
111 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong
, __tsan_atomic64
, a64
,
113 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr
, __tsan_atomic64
, a64
,
115 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32
, __tsan_atomic32
, a32
,
117 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64
, __tsan_atomic64
, a64
,
120 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \
121 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
122 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \
123 char *byte_ptr = ((char *)ptr) + (n >> 3); \
124 char bit = 0x80u >> (n & 7); \
125 char mask = clear ? ~bit : bit; \
126 char orig_byte = op((a8 *)byte_ptr, mask, mo); \
127 return orig_byte & bit; \
130 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \
131 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
132 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
134 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet
, __tsan_atomic8_fetch_or
, false)
135 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear
, __tsan_atomic8_fetch_and
,
138 TSAN_INTERCEPTOR(void, OSAtomicEnqueue
, OSQueueHead
*list
, void *item
,
140 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue
, list
, item
, offset
);
141 __tsan_release(item
);
142 REAL(OSAtomicEnqueue
)(list
, item
, offset
);
145 TSAN_INTERCEPTOR(void *, OSAtomicDequeue
, OSQueueHead
*list
, size_t offset
) {
146 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue
, list
, offset
);
147 void *item
= REAL(OSAtomicDequeue
)(list
, offset
);
148 if (item
) __tsan_acquire(item
);
152 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
155 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue
, OSFifoQueueHead
*list
, void *item
,
157 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue
, list
, item
, offset
);
158 __tsan_release(item
);
159 REAL(OSAtomicFifoEnqueue
)(list
, item
, offset
);
162 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue
, OSFifoQueueHead
*list
,
164 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue
, list
, offset
);
165 void *item
= REAL(OSAtomicFifoDequeue
)(list
, offset
);
166 if (item
) __tsan_acquire(item
);
172 TSAN_INTERCEPTOR(void, OSSpinLockLock
, volatile OSSpinLock
*lock
) {
173 CHECK(!cur_thread()->is_dead
);
174 if (!cur_thread()->is_inited
) {
175 return REAL(OSSpinLockLock
)(lock
);
177 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock
, lock
);
178 REAL(OSSpinLockLock
)(lock
);
179 Acquire(thr
, pc
, (uptr
)lock
);
182 TSAN_INTERCEPTOR(bool, OSSpinLockTry
, volatile OSSpinLock
*lock
) {
183 CHECK(!cur_thread()->is_dead
);
184 if (!cur_thread()->is_inited
) {
185 return REAL(OSSpinLockTry
)(lock
);
187 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry
, lock
);
188 bool result
= REAL(OSSpinLockTry
)(lock
);
190 Acquire(thr
, pc
, (uptr
)lock
);
194 TSAN_INTERCEPTOR(void, OSSpinLockUnlock
, volatile OSSpinLock
*lock
) {
195 CHECK(!cur_thread()->is_dead
);
196 if (!cur_thread()->is_inited
) {
197 return REAL(OSSpinLockUnlock
)(lock
);
199 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock
, lock
);
200 Release(thr
, pc
, (uptr
)lock
);
201 REAL(OSSpinLockUnlock
)(lock
);
204 TSAN_INTERCEPTOR(void, os_lock_lock
, void *lock
) {
205 CHECK(!cur_thread()->is_dead
);
206 if (!cur_thread()->is_inited
) {
207 return REAL(os_lock_lock
)(lock
);
209 SCOPED_TSAN_INTERCEPTOR(os_lock_lock
, lock
);
210 REAL(os_lock_lock
)(lock
);
211 Acquire(thr
, pc
, (uptr
)lock
);
214 TSAN_INTERCEPTOR(bool, os_lock_trylock
, void *lock
) {
215 CHECK(!cur_thread()->is_dead
);
216 if (!cur_thread()->is_inited
) {
217 return REAL(os_lock_trylock
)(lock
);
219 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock
, lock
);
220 bool result
= REAL(os_lock_trylock
)(lock
);
222 Acquire(thr
, pc
, (uptr
)lock
);
226 TSAN_INTERCEPTOR(void, os_lock_unlock
, void *lock
) {
227 CHECK(!cur_thread()->is_dead
);
228 if (!cur_thread()->is_inited
) {
229 return REAL(os_lock_unlock
)(lock
);
231 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock
, lock
);
232 Release(thr
, pc
, (uptr
)lock
);
233 REAL(os_lock_unlock
)(lock
);
236 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler
,
237 xpc_connection_t connection
, xpc_handler_t handler
) {
238 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler
, connection
,
240 Release(thr
, pc
, (uptr
)connection
);
241 xpc_handler_t new_handler
= ^(xpc_object_t object
) {
243 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler
);
244 Acquire(thr
, pc
, (uptr
)connection
);
248 REAL(xpc_connection_set_event_handler
)(connection
, new_handler
);
251 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier
, xpc_connection_t connection
,
252 dispatch_block_t barrier
) {
253 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier
, connection
, barrier
);
254 Release(thr
, pc
, (uptr
)connection
);
255 dispatch_block_t new_barrier
= ^() {
257 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier
);
258 Acquire(thr
, pc
, (uptr
)connection
);
262 REAL(xpc_connection_send_barrier
)(connection
, new_barrier
);
265 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply
,
266 xpc_connection_t connection
, xpc_object_t message
,
267 dispatch_queue_t replyq
, xpc_handler_t handler
) {
268 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply
, connection
,
269 message
, replyq
, handler
);
270 Release(thr
, pc
, (uptr
)connection
);
271 xpc_handler_t new_handler
= ^(xpc_object_t object
) {
273 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply
);
274 Acquire(thr
, pc
, (uptr
)connection
);
278 REAL(xpc_connection_send_message_with_reply
)
279 (connection
, message
, replyq
, new_handler
);
282 // On macOS, libc++ is always linked dynamically, so intercepting works the
284 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
287 struct fake_shared_weak_count
{
288 volatile a64 shared_owners
;
289 volatile a64 shared_weak_owners
;
290 virtual void _unused_0x0() = 0;
291 virtual void _unused_0x8() = 0;
292 virtual void on_zero_shared() = 0;
293 virtual void _unused_0x18() = 0;
294 virtual void on_zero_shared_weak() = 0;
298 // This adds a libc++ interceptor for:
299 // void __shared_weak_count::__release_shared() _NOEXCEPT;
300 // Shared and weak pointers in C++ maintain reference counts via atomics in
301 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
302 // destructor code. This interceptor re-implements the whole function so that
303 // the mo_acq_rel semantics of the atomic decrement are visible.
305 // Unfortunately, this interceptor cannot simply Acquire/Release some sync
306 // object and call the original function, because it would have a race between
307 // the sync and the destruction of the object. Calling both under a lock will
308 // not work because the destructor can invoke this interceptor again (and even
309 // in a different thread, so recursive locks don't help).
310 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv
,
311 fake_shared_weak_count
*o
) {
312 if (!flags()->shared_ptr_interceptor
)
313 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv
)(o
);
315 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv
,
317 if (__tsan_atomic64_fetch_add(&o
->shared_owners
, -1, mo_release
) == 0) {
318 Acquire(thr
, pc
, (uptr
)&o
->shared_owners
);
320 if (__tsan_atomic64_fetch_add(&o
->shared_weak_owners
, -1, mo_release
) ==
322 Acquire(thr
, pc
, (uptr
)&o
->shared_weak_owners
);
323 o
->on_zero_shared_weak();
329 struct call_once_callback_args
{
330 void (*orig_func
)(void *arg
);
335 void call_once_callback_wrapper(void *arg
) {
336 call_once_callback_args
*new_args
= (call_once_callback_args
*)arg
;
337 new_args
->orig_func(new_args
->orig_arg
);
338 __tsan_release(new_args
->flag
);
342 // This adds a libc++ interceptor for:
343 // void __call_once(volatile unsigned long&, void*, void(*)(void*));
344 // C++11 call_once is implemented via an internal function __call_once which is
345 // inside libc++.dylib, and the atomic release store inside it is thus
346 // TSan-invisible. To avoid false positives, this interceptor wraps the callback
347 // function and performs an explicit Release after the user code has run.
348 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E
, void *flag
,
349 void *arg
, void (*func
)(void *arg
)) {
350 call_once_callback_args new_args
= {func
, arg
, flag
};
351 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E
)(flag
, &new_args
,
352 call_once_callback_wrapper
);
355 } // namespace __tsan
357 #endif // SANITIZER_MAC