libsanitizer/tsan/tsan_interceptors_mac.cc

   1 //===-- tsan_interceptors_mac.cc ------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Mac-specific interceptors.
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "sanitizer_common/sanitizer_platform.h"
  14 #if SANITIZER_MAC
  15
  16 #include "interception/interception.h"
  17 #include "tsan_interceptors.h"
  18 #include "tsan_interface.h"
  19 #include "tsan_interface_ann.h"
  20
  21 #include <libkern/OSAtomic.h>
  22 #include <xpc/xpc.h>
  23
  24 typedef long long_t;  // NOLINT
  25
  26 namespace __tsan {
  27
  28 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,
  29 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are
  30 // actually aliases of each other, and we cannot have different interceptors for
  31 // them, because they're actually the same function.  Thus, we have to stay
  32 // conservative and treat the non-barrier versions as mo_acq_rel.
  33 static const morder kMacOrderBarrier = mo_acq_rel;
  34 static const morder kMacOrderNonBarrier = mo_acq_rel;
  35
  36 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  37   TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                 \
  38     SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                 \
  39     return tsan_atomic_f((volatile tsan_t *)ptr, x, mo);                \
  40   }
  41
  42 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  43   TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) {                        \
  44     SCOPED_TSAN_INTERCEPTOR(f, x, ptr);                                        \
  45     return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x;                   \
  46   }
  47
  48 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \
  49   TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                             \
  50     SCOPED_TSAN_INTERCEPTOR(f, ptr);                                           \
  51     return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1;                   \
  52   }
  53
  54 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \
  55                                      mo)                                    \
  56   TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) {                          \
  57     SCOPED_TSAN_INTERCEPTOR(f, ptr);                                        \
  58     return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1;                \
  59   }
  60
  61 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m)                  \
  62   m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,             \
  63     kMacOrderNonBarrier)                                                       \
  64   m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,    \
  65     kMacOrderBarrier)                                                          \
  66   m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f,             \
  67     kMacOrderNonBarrier)                                                       \
  68   m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f,    \
  69     kMacOrderBarrier)
  70
  71 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig)             \
  72   m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f,            \
  73     kMacOrderNonBarrier)                                                       \
  74   m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f,   \
  75     kMacOrderBarrier)                                                          \
  76   m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \
  77     kMacOrderNonBarrier)                                                       \
  78   m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier,                           \
  79     __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)
  80
  81 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,
  82                                  OSATOMIC_INTERCEPTOR_PLUS_X)
  83 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,
  84                                  OSATOMIC_INTERCEPTOR_PLUS_1)
  85 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,
  86                                  OSATOMIC_INTERCEPTOR_MINUS_1)
  87 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,
  88                               OSATOMIC_INTERCEPTOR)
  89 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,
  90                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
  91 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,
  92                               OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)
  93
  94 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t)              \
  95   TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) {    \
  96     SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr);                  \
  97     return tsan_atomic_f##_compare_exchange_strong(                         \
  98         (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,             \
  99         kMacOrderNonBarrier, kMacOrderNonBarrier);                          \
 100   }                                                                         \
 101                                                                             \
 102   TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value,              \
 103                    t volatile *ptr) {                                       \
 104     SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr);         \
 105     return tsan_atomic_f##_compare_exchange_strong(                         \
 106         (tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value,             \
 107         kMacOrderBarrier, kMacOrderNonBarrier);                             \
 108   }
 109
 110 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)
 111 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,
 112                           long_t)
 113 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,
 114                           void *)
 115 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,
 116                           int32_t)
 117 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,
 118                           int64_t)
 119
 120 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo)          \
 121   TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \
 122     SCOPED_TSAN_INTERCEPTOR(f, n, ptr);                       \
 123     char *byte_ptr = ((char *)ptr) + (n >> 3);                \
 124     char bit = 0x80u >> (n & 7);                              \
 125     char mask = clear ? ~bit : bit;                           \
 126     char orig_byte = op((a8 *)byte_ptr, mask, mo);            \
 127     return orig_byte & bit;                                   \
 128   }
 129
 130 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear)               \
 131   OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \
 132   OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)
 133
 134 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)
 135 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,
 136                             true)
 137
 138 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,
 139                  size_t offset) {
 140   SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);
 141   __tsan_release(item);
 142   REAL(OSAtomicEnqueue)(list, item, offset);
 143 }
 144
 145 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {
 146   SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);
 147   void *item = REAL(OSAtomicDequeue)(list, offset);
 148   if (item) __tsan_acquire(item);
 149   return item;
 150 }
 151
 152 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.
 153 #if !SANITIZER_IOS
 154
 155 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,
 156                  size_t offset) {
 157   SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);
 158   __tsan_release(item);
 159   REAL(OSAtomicFifoEnqueue)(list, item, offset);
 160 }
 161
 162 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,
 163                  size_t offset) {
 164   SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);
 165   void *item = REAL(OSAtomicFifoDequeue)(list, offset);
 166   if (item) __tsan_acquire(item);
 167   return item;
 168 }
 169
 170 #endif
 171
 172 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {
 173   CHECK(!cur_thread()->is_dead);
 174   if (!cur_thread()->is_inited) {
 175     return REAL(OSSpinLockLock)(lock);
 176   }
 177   SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);
 178   REAL(OSSpinLockLock)(lock);
 179   Acquire(thr, pc, (uptr)lock);
 180 }
 181
 182 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {
 183   CHECK(!cur_thread()->is_dead);
 184   if (!cur_thread()->is_inited) {
 185     return REAL(OSSpinLockTry)(lock);
 186   }
 187   SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);
 188   bool result = REAL(OSSpinLockTry)(lock);
 189   if (result)
 190     Acquire(thr, pc, (uptr)lock);
 191   return result;
 192 }
 193
 194 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {
 195   CHECK(!cur_thread()->is_dead);
 196   if (!cur_thread()->is_inited) {
 197     return REAL(OSSpinLockUnlock)(lock);
 198   }
 199   SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);
 200   Release(thr, pc, (uptr)lock);
 201   REAL(OSSpinLockUnlock)(lock);
 202 }
 203
 204 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {
 205   CHECK(!cur_thread()->is_dead);
 206   if (!cur_thread()->is_inited) {
 207     return REAL(os_lock_lock)(lock);
 208   }
 209   SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);
 210   REAL(os_lock_lock)(lock);
 211   Acquire(thr, pc, (uptr)lock);
 212 }
 213
 214 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {
 215   CHECK(!cur_thread()->is_dead);
 216   if (!cur_thread()->is_inited) {
 217     return REAL(os_lock_trylock)(lock);
 218   }
 219   SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);
 220   bool result = REAL(os_lock_trylock)(lock);
 221   if (result)
 222     Acquire(thr, pc, (uptr)lock);
 223   return result;
 224 }
 225
 226 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
 227   CHECK(!cur_thread()->is_dead);
 228   if (!cur_thread()->is_inited) {
 229     return REAL(os_lock_unlock)(lock);
 230   }
 231   SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);
 232   Release(thr, pc, (uptr)lock);
 233   REAL(os_lock_unlock)(lock);
 234 }
 235
 236 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
 237                  xpc_connection_t connection, xpc_handler_t handler) {
 238   SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
 239                           handler);
 240   Release(thr, pc, (uptr)connection);
 241   xpc_handler_t new_handler = ^(xpc_object_t object) {
 242     {
 243       SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);
 244       Acquire(thr, pc, (uptr)connection);
 245     }
 246     handler(object);
 247   };
 248   REAL(xpc_connection_set_event_handler)(connection, new_handler);
 249 }
 250
 251 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,
 252                  dispatch_block_t barrier) {
 253   SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);
 254   Release(thr, pc, (uptr)connection);
 255   dispatch_block_t new_barrier = ^() {
 256     {
 257       SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);
 258       Acquire(thr, pc, (uptr)connection);
 259     }
 260     barrier();
 261   };
 262   REAL(xpc_connection_send_barrier)(connection, new_barrier);
 263 }
 264
 265 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,
 266                  xpc_connection_t connection, xpc_object_t message,
 267                  dispatch_queue_t replyq, xpc_handler_t handler) {
 268   SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,
 269                           message, replyq, handler);
 270   Release(thr, pc, (uptr)connection);
 271   xpc_handler_t new_handler = ^(xpc_object_t object) {
 272     {
 273       SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);
 274       Acquire(thr, pc, (uptr)connection);
 275     }
 276     handler(object);
 277   };
 278   REAL(xpc_connection_send_message_with_reply)
 279   (connection, message, replyq, new_handler);
 280 }
 281
 282 // On macOS, libc++ is always linked dynamically, so intercepting works the
 283 // usual way.
 284 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
 285
 286 namespace {
 287 struct fake_shared_weak_count {
 288   volatile a64 shared_owners;
 289   volatile a64 shared_weak_owners;
 290   virtual void _unused_0x0() = 0;
 291   virtual void _unused_0x8() = 0;
 292   virtual void on_zero_shared() = 0;
 293   virtual void _unused_0x18() = 0;
 294   virtual void on_zero_shared_weak() = 0;
 295 };
 296 }  // namespace
 297
 298 // This adds a libc++ interceptor for:
 299 //     void __shared_weak_count::__release_shared() _NOEXCEPT;
 300 // Shared and weak pointers in C++ maintain reference counts via atomics in
 301 // libc++.dylib, which are TSan-invisible, and this leads to false positives in
 302 // destructor code.  This interceptor re-implements the whole function so that
 303 // the mo_acq_rel semantics of the atomic decrement are visible.
 304 //
 305 // Unfortunately, this interceptor cannot simply Acquire/Release some sync
 306 // object and call the original function, because it would have a race between
 307 // the sync and the destruction of the object.  Calling both under a lock will
 308 // not work because the destructor can invoke this interceptor again (and even
 309 // in a different thread, so recursive locks don't help).
 310 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,
 311                    fake_shared_weak_count *o) {
 312   if (!flags()->shared_ptr_interceptor)
 313     return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);
 314
 315   SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,
 316                           o);
 317   if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {
 318     Acquire(thr, pc, (uptr)&o->shared_owners);
 319     o->on_zero_shared();
 320     if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==
 321         0) {
 322       Acquire(thr, pc, (uptr)&o->shared_weak_owners);
 323       o->on_zero_shared_weak();
 324     }
 325   }
 326 }
 327
 328 namespace {
 329 struct call_once_callback_args {
 330   void (*orig_func)(void *arg);
 331   void *orig_arg;
 332   void *flag;
 333 };
 334
 335 void call_once_callback_wrapper(void *arg) {
 336   call_once_callback_args *new_args = (call_once_callback_args *)arg;
 337   new_args->orig_func(new_args->orig_arg);
 338   __tsan_release(new_args->flag);
 339 }
 340 }  // namespace
 341
 342 // This adds a libc++ interceptor for:
 343 //     void __call_once(volatile unsigned long&, void*, void(*)(void*));
 344 // C++11 call_once is implemented via an internal function __call_once which is
 345 // inside libc++.dylib, and the atomic release store inside it is thus
 346 // TSan-invisible. To avoid false positives, this interceptor wraps the callback
 347 // function and performs an explicit Release after the user code has run.
 348 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,
 349                    void *arg, void (*func)(void *arg)) {
 350   call_once_callback_args new_args = {func, arg, flag};
 351   REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,
 352                                             call_once_callback_wrapper);
 353 }
 354
 355 }  // namespace __tsan
 356
 357 #endif  // SANITIZER_MAC