| //===-- tsan_interceptors_libdispatch.cpp ---------------------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file is a part of ThreadSanitizer (TSan), a race detector. |
| // |
| // Support for intercepting libdispatch (GCD). |
| //===----------------------------------------------------------------------===// |
| |
| #include "sanitizer_common/sanitizer_common.h" |
| #include "interception/interception.h" |
| #include "tsan_interceptors.h" |
| #include "tsan_rtl.h" |
| |
| #include "BlocksRuntime/Block.h" |
| #include "tsan_dispatch_defs.h" |
| |
| #if SANITIZER_MAC |
| # include <Availability.h> |
| #endif |
| |
| namespace __tsan { |
| typedef u16 uint16_t; |
| |
| typedef struct { |
| dispatch_queue_t queue; |
| void *orig_context; |
| dispatch_function_t orig_work; |
| bool free_context_in_callback; |
| bool submitted_synchronously; |
| bool is_barrier_block; |
| uptr non_queue_sync_object; |
| } block_context_t; |
| |
| // The offsets of different fields of the dispatch_queue_t structure, exported |
| // by libdispatch.dylib. |
| extern "C" struct dispatch_queue_offsets_s { |
| const uint16_t dqo_version; |
| const uint16_t dqo_label; |
| const uint16_t dqo_label_size; |
| const uint16_t dqo_flags; |
| const uint16_t dqo_flags_size; |
| const uint16_t dqo_serialnum; |
| const uint16_t dqo_serialnum_size; |
| const uint16_t dqo_width; |
| const uint16_t dqo_width_size; |
| const uint16_t dqo_running; |
| const uint16_t dqo_running_size; |
| const uint16_t dqo_suspend_cnt; |
| const uint16_t dqo_suspend_cnt_size; |
| const uint16_t dqo_target_queue; |
| const uint16_t dqo_target_queue_size; |
| const uint16_t dqo_priority; |
| const uint16_t dqo_priority_size; |
| } dispatch_queue_offsets; |
| |
| static bool IsQueueSerial(dispatch_queue_t q) { |
| CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); |
| uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); |
| CHECK_NE(width, 0); |
| return width == 1; |
| } |
| |
| static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { |
| CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); |
| dispatch_queue_t tq = *( |
| dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); |
| return tq; |
| } |
| |
| static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { |
| dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); |
| CHECK_NE(tq, 0); |
| return tq; |
| } |
| |
| static block_context_t *AllocContext(ThreadState *thr, uptr pc, |
| dispatch_queue_t queue, void *orig_context, |
| dispatch_function_t orig_work) { |
| block_context_t *new_context = |
| (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); |
| new_context->queue = queue; |
| new_context->orig_context = orig_context; |
| new_context->orig_work = orig_work; |
| new_context->free_context_in_callback = true; |
| new_context->submitted_synchronously = false; |
| new_context->is_barrier_block = false; |
| new_context->non_queue_sync_object = 0; |
| return new_context; |
| } |
| |
| #define GET_QUEUE_SYNC_VARS(context, q) \ |
| bool is_queue_serial = q && IsQueueSerial(q); \ |
| uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ |
| uptr serial_sync = (uptr)sync_ptr; \ |
| uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ |
| bool serial_task = context->is_barrier_block || is_queue_serial |
| |
| static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, |
| block_context_t *context) { |
| uptr submit_sync = (uptr)context; |
| Acquire(thr, pc, submit_sync); |
| |
| dispatch_queue_t q = context->queue; |
| do { |
| GET_QUEUE_SYNC_VARS(context, q); |
| if (serial_sync) Acquire(thr, pc, serial_sync); |
| if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); |
| |
| if (q) q = GetTargetQueueFromQueue(q); |
| } while (q); |
| } |
| |
| static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, |
| block_context_t *context) { |
| uptr submit_sync = (uptr)context; |
| if (context->submitted_synchronously) Release(thr, pc, submit_sync); |
| |
| dispatch_queue_t q = context->queue; |
| do { |
| GET_QUEUE_SYNC_VARS(context, q); |
| if (serial_task && serial_sync) Release(thr, pc, serial_sync); |
| if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); |
| |
| if (q) q = GetTargetQueueFromQueue(q); |
| } while (q); |
| } |
| |
| static void dispatch_callback_wrap(void *param) { |
| SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); |
| block_context_t *context = (block_context_t *)param; |
| |
| dispatch_sync_pre_execute(thr, pc, context); |
| |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| context->orig_work(context->orig_context); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| |
| dispatch_sync_post_execute(thr, pc, context); |
| |
| if (context->free_context_in_callback) user_free(thr, pc, context); |
| } |
| |
| static void invoke_block(void *param) { |
| dispatch_block_t block = (dispatch_block_t)param; |
| block(); |
| } |
| |
| static void invoke_and_release_block(void *param) { |
| dispatch_block_t block = (dispatch_block_t)param; |
| block(); |
| Block_release(block); |
| } |
| |
| #define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ |
| TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ |
| SCOPED_TSAN_INTERCEPTOR(name, q, block); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ |
| dispatch_block_t heap_block = Block_copy(block); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ |
| block_context_t *new_context = \ |
| AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ |
| new_context->is_barrier_block = barrier; \ |
| Release(thr, pc, (uptr)new_context); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ |
| REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ |
| } |
| |
| #define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ |
| TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ |
| DISPATCH_NOESCAPE dispatch_block_t block) { \ |
| SCOPED_TSAN_INTERCEPTOR(name, q, block); \ |
| block_context_t new_context = { \ |
| q, block, &invoke_block, false, true, barrier, 0}; \ |
| Release(thr, pc, (uptr)&new_context); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ |
| REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ |
| Acquire(thr, pc, (uptr)&new_context); \ |
| } |
| |
| #define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ |
| TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ |
| dispatch_function_t work) { \ |
| SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ |
| block_context_t *new_context = \ |
| AllocContext(thr, pc, q, context, work); \ |
| new_context->is_barrier_block = barrier; \ |
| Release(thr, pc, (uptr)new_context); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ |
| REAL(name)(q, new_context, dispatch_callback_wrap); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ |
| } |
| |
| #define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ |
| TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ |
| dispatch_function_t work) { \ |
| SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ |
| block_context_t new_context = { \ |
| q, context, work, false, true, barrier, 0}; \ |
| Release(thr, pc, (uptr)&new_context); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ |
| REAL(name)(q, &new_context, dispatch_callback_wrap); \ |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ |
| Acquire(thr, pc, (uptr)&new_context); \ |
| } |
| |
| #define DISPATCH_INTERCEPT(name, barrier) \ |
| DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ |
| DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ |
| DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ |
| DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) |
| |
| // We wrap dispatch_async, dispatch_sync and friends where we allocate a new |
| // context, which is used to synchronize (we release the context before |
| // submitting, and the callback acquires it before executing the original |
| // callback). |
| DISPATCH_INTERCEPT(dispatch, false) |
| DISPATCH_INTERCEPT(dispatch_barrier, true) |
| |
| // dispatch_async_and_wait() and friends were introduced in macOS 10.14. |
| // Linking of these interceptors fails when using an older SDK. |
| #if !SANITIZER_MAC || defined(__MAC_10_14) |
| // macOS 10.14 is greater than our minimal deployment target. To ensure we |
| // generate a weak reference so the TSan dylib continues to work on older |
| // systems, we need to forward declare the intercepted functions as "weak |
| // imports". Note that this file is multi-platform, so we cannot include the |
| // actual header file (#include <dispatch/dispatch.h>). |
| SANITIZER_WEAK_IMPORT void dispatch_async_and_wait( |
| dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); |
| SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f( |
| dispatch_queue_t queue, void *context, dispatch_function_t work); |
| SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait( |
| dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block); |
| SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f( |
| dispatch_queue_t queue, void *context, dispatch_function_t work); |
| |
| DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false) |
| DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false) |
| DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true) |
| DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true) |
| #endif |
| |
| |
| DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, |
| dispatch_queue_t queue, void *context, dispatch_function_t work) |
| |
| TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, |
| dispatch_queue_t queue, dispatch_block_t block) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| dispatch_block_t heap_block = Block_copy(block); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| block_context_t *new_context = |
| AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); |
| Release(thr, pc, (uptr)new_context); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, |
| dispatch_queue_t queue, void *context, |
| dispatch_function_t work) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); |
| WRAP(dispatch_after)(when, queue, ^(void) { |
| work(context); |
| }); |
| } |
| |
| // GCD's dispatch_once implementation has a fast path that contains a racy read |
| // and it's inlined into user's code. Furthermore, this fast path doesn't |
| // establish a proper happens-before relations between the initialization and |
| // code following the call to dispatch_once. We could deal with this in |
| // instrumented code, but there's not much we can do about it in system |
| // libraries. Let's disable the fast path (by never storing the value ~0 to |
| // predicate), so the interceptor is always called, and let's add proper release |
| // and acquire semantics. Since TSan does not see its own atomic stores, the |
| // race on predicate won't be reported - the only accesses to it that TSan sees |
| // are the loads on the fast path. Loads don't race. Secondly, dispatch_once is |
| // both a macro and a real function, we want to intercept the function, so we |
| // need to undefine the macro. |
| #undef dispatch_once |
| TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, |
| DISPATCH_NOESCAPE dispatch_block_t block) { |
| SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); |
| atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); |
| u32 v = atomic_load(a, memory_order_acquire); |
| if (v == 0 && |
| atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| block(); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| Release(thr, pc, (uptr)a); |
| atomic_store(a, 2, memory_order_release); |
| } else { |
| while (v != 2) { |
| internal_sched_yield(); |
| v = atomic_load(a, memory_order_acquire); |
| } |
| Acquire(thr, pc, (uptr)a); |
| } |
| } |
| |
| #undef dispatch_once_f |
| TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, |
| void *context, dispatch_function_t function) { |
| SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| WRAP(dispatch_once)(predicate, ^(void) { |
| function(context); |
| }); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| } |
| |
| TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, |
| dispatch_semaphore_t dsema) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); |
| Release(thr, pc, (uptr)dsema); |
| return REAL(dispatch_semaphore_signal)(dsema); |
| } |
| |
| TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, |
| dispatch_time_t timeout) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); |
| long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); |
| if (result == 0) Acquire(thr, pc, (uptr)dsema); |
| return result; |
| } |
| |
| TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, |
| dispatch_time_t timeout) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); |
| long_t result = REAL(dispatch_group_wait)(group, timeout); |
| if (result == 0) Acquire(thr, pc, (uptr)group); |
| return result; |
| } |
| |
| // Used, but not intercepted. |
| extern "C" void dispatch_group_enter(dispatch_group_t group); |
| |
| TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); |
| // Acquired in the group notification callback in dispatch_group_notify[_f]. |
| Release(thr, pc, (uptr)group); |
| REAL(dispatch_group_leave)(group); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, |
| dispatch_queue_t queue, dispatch_block_t block) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); |
| dispatch_retain(group); |
| dispatch_group_enter(group); |
| __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); |
| WRAP(dispatch_async)(queue, ^(void) { |
| block_copy(); |
| Block_release(block_copy); |
| WRAP(dispatch_group_leave)(group); |
| dispatch_release(group); |
| }); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, |
| dispatch_queue_t queue, void *context, |
| dispatch_function_t work) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); |
| dispatch_retain(group); |
| dispatch_group_enter(group); |
| WRAP(dispatch_async)(queue, ^(void) { |
| work(context); |
| WRAP(dispatch_group_leave)(group); |
| dispatch_release(group); |
| }); |
| } |
| |
| DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, |
| dispatch_queue_t q, void *context, dispatch_function_t work) |
| |
| TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, |
| dispatch_queue_t q, dispatch_block_t block) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); |
| |
| // To make sure the group is still available in the callback (otherwise |
| // it can be already destroyed). Will be released in the callback. |
| dispatch_retain(group); |
| |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| dispatch_block_t heap_block = Block_copy(^(void) { |
| { |
| SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); |
| // Released when leaving the group (dispatch_group_leave). |
| Acquire(thr, pc, (uptr)group); |
| } |
| dispatch_release(group); |
| block(); |
| }); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| block_context_t *new_context = |
| AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); |
| new_context->is_barrier_block = true; |
| Release(thr, pc, (uptr)new_context); |
| REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, |
| dispatch_queue_t q, void *context, dispatch_function_t work) { |
| WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, |
| dispatch_source_t source, dispatch_block_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_event_handler)(source, nullptr); |
| dispatch_queue_t q = GetTargetQueueFromSource(source); |
| __block block_context_t new_context = { |
| q, handler, &invoke_block, false, false, false, 0 }; |
| dispatch_block_t new_handler = Block_copy(^(void) { |
| new_context.orig_context = handler; // To explicitly capture "handler". |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_source_set_event_handler)(source, new_handler); |
| Block_release(new_handler); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, |
| dispatch_source_t source, dispatch_function_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_event_handler)(source, nullptr); |
| dispatch_block_t block = ^(void) { |
| handler(dispatch_get_context(source)); |
| }; |
| WRAP(dispatch_source_set_event_handler)(source, block); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, |
| dispatch_source_t source, dispatch_block_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_cancel_handler)(source, nullptr); |
| dispatch_queue_t q = GetTargetQueueFromSource(source); |
| __block block_context_t new_context = { |
| q, handler, &invoke_block, false, false, false, 0}; |
| dispatch_block_t new_handler = Block_copy(^(void) { |
| new_context.orig_context = handler; // To explicitly capture "handler". |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_source_set_cancel_handler)(source, new_handler); |
| Block_release(new_handler); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, |
| dispatch_source_t source, dispatch_function_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, |
| handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_cancel_handler)(source, nullptr); |
| dispatch_block_t block = ^(void) { |
| handler(dispatch_get_context(source)); |
| }; |
| WRAP(dispatch_source_set_cancel_handler)(source, block); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, |
| dispatch_source_t source, dispatch_block_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, |
| handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_registration_handler)(source, nullptr); |
| dispatch_queue_t q = GetTargetQueueFromSource(source); |
| __block block_context_t new_context = { |
| q, handler, &invoke_block, false, false, false, 0}; |
| dispatch_block_t new_handler = Block_copy(^(void) { |
| new_context.orig_context = handler; // To explicitly capture "handler". |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_source_set_registration_handler)(source, new_handler); |
| Block_release(new_handler); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, |
| dispatch_source_t source, dispatch_function_t handler) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, |
| handler); |
| if (handler == nullptr) |
| return REAL(dispatch_source_set_registration_handler)(source, nullptr); |
| dispatch_block_t block = ^(void) { |
| handler(dispatch_get_context(source)); |
| }; |
| WRAP(dispatch_source_set_registration_handler)(source, block); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, |
| dispatch_queue_t queue, |
| DISPATCH_NOESCAPE void (^block)(size_t)) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); |
| |
| u8 sync1, sync2; |
| uptr parent_to_child_sync = (uptr)&sync1; |
| uptr child_to_parent_sync = (uptr)&sync2; |
| |
| Release(thr, pc, parent_to_child_sync); |
| void (^new_block)(size_t) = ^(size_t iteration) { |
| SCOPED_INTERCEPTOR_RAW(dispatch_apply); |
| Acquire(thr, pc, parent_to_child_sync); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| block(iteration); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| Release(thr, pc, child_to_parent_sync); |
| }; |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| REAL(dispatch_apply)(iterations, queue, new_block); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| Acquire(thr, pc, child_to_parent_sync); |
| } |
| |
| static void invoke_block_iteration(void *param, size_t iteration) { |
| auto block = (void (^)(size_t)) param; |
| block(iteration); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, |
| dispatch_queue_t queue, void *context, |
| void (*work)(void *, size_t)) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); |
| |
| // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch |
| // implements dispatch_apply in terms of dispatch_apply_f. |
| u8 sync1, sync2; |
| uptr parent_to_child_sync = (uptr)&sync1; |
| uptr child_to_parent_sync = (uptr)&sync2; |
| |
| Release(thr, pc, parent_to_child_sync); |
| void (^new_block)(size_t) = ^(size_t iteration) { |
| SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); |
| Acquire(thr, pc, parent_to_child_sync); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| work(context, iteration); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| Release(thr, pc, child_to_parent_sync); |
| }; |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| Acquire(thr, pc, child_to_parent_sync); |
| } |
| |
| DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) |
| DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) |
| |
| TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, |
| size_t size, dispatch_queue_t q, dispatch_block_t destructor) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); |
| if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) |
| return REAL(dispatch_data_create)(buffer, size, q, destructor); |
| |
| if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) |
| destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; |
| else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) |
| destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; |
| |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); |
| dispatch_block_t heap_block = Block_copy(destructor); |
| SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); |
| block_context_t *new_context = |
| AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); |
| uptr submit_sync = (uptr)new_context; |
| Release(thr, pc, submit_sync); |
| return REAL(dispatch_data_create)(buffer, size, q, ^(void) { |
| dispatch_callback_wrap(new_context); |
| }); |
| } |
| |
| typedef void (^fd_handler_t)(dispatch_data_t data, int error); |
| typedef void (^cleanup_handler_t)(int error); |
| |
| TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, |
| dispatch_queue_t q, fd_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { |
| new_context.orig_context = ^(void) { |
| h(data, error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_read)(fd, length, q, new_h); |
| Block_release(new_h); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, |
| dispatch_queue_t q, fd_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { |
| new_context.orig_context = ^(void) { |
| h(data, error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_write)(fd, data, q, new_h); |
| Block_release(new_h); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, |
| size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| dispatch_io_handler_t new_h = |
| Block_copy(^(bool done, dispatch_data_t data, int error) { |
| new_context.orig_context = ^(void) { |
| h(done, data, error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_io_read)(channel, offset, length, q, new_h); |
| Block_release(new_h); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, |
| dispatch_data_t data, dispatch_queue_t q, |
| dispatch_io_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| dispatch_io_handler_t new_h = |
| Block_copy(^(bool done, dispatch_data_t data, int error) { |
| new_context.orig_context = ^(void) { |
| h(done, data, error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_io_write)(channel, offset, data, q, new_h); |
| Block_release(new_h); |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, |
| dispatch_block_t barrier) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); |
| __block block_context_t new_context = { |
| nullptr, nullptr, &invoke_block, false, false, false, 0}; |
| new_context.non_queue_sync_object = (uptr)channel; |
| new_context.is_barrier_block = true; |
| dispatch_block_t new_block = Block_copy(^(void) { |
| new_context.orig_context = ^(void) { |
| barrier(); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| REAL(dispatch_io_barrier)(channel, new_block); |
| Block_release(new_block); |
| } |
| |
| TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, |
| dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); |
| __block dispatch_io_t new_channel = nullptr; |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| cleanup_handler_t new_h = Block_copy(^(int error) { |
| { |
| SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); |
| Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. |
| } |
| new_context.orig_context = ^(void) { |
| h(error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); |
| Block_release(new_h); |
| return new_channel; |
| } |
| |
| TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, |
| dispatch_io_type_t type, const char *path, int oflag, |
| mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, |
| q, h); |
| __block dispatch_io_t new_channel = nullptr; |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| cleanup_handler_t new_h = Block_copy(^(int error) { |
| { |
| SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); |
| Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. |
| } |
| new_context.orig_context = ^(void) { |
| h(error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| new_channel = |
| REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); |
| Block_release(new_h); |
| return new_channel; |
| } |
| |
| TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, |
| dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, |
| cleanup_handler_t h) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); |
| __block dispatch_io_t new_channel = nullptr; |
| __block block_context_t new_context = { |
| q, nullptr, &invoke_block, false, false, false, 0}; |
| cleanup_handler_t new_h = Block_copy(^(int error) { |
| { |
| SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); |
| Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. |
| } |
| new_context.orig_context = ^(void) { |
| h(error); |
| }; |
| dispatch_callback_wrap(&new_context); |
| }); |
| uptr submit_sync = (uptr)&new_context; |
| Release(thr, pc, submit_sync); |
| new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); |
| Block_release(new_h); |
| return new_channel; |
| } |
| |
| TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, |
| dispatch_io_close_flags_t flags) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); |
| Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. |
| return REAL(dispatch_io_close)(channel, flags); |
| } |
| |
| // Resuming a suspended queue needs to synchronize with all subsequent |
| // executions of blocks in that queue. |
| TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { |
| SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); |
| Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync |
| // in dispatch_sync_pre_execute |
| return REAL(dispatch_resume)(o); |
| } |
| |
| void InitializeLibdispatchInterceptors() { |
| INTERCEPT_FUNCTION(dispatch_async); |
| INTERCEPT_FUNCTION(dispatch_async_f); |
| INTERCEPT_FUNCTION(dispatch_sync); |
| INTERCEPT_FUNCTION(dispatch_sync_f); |
| INTERCEPT_FUNCTION(dispatch_barrier_async); |
| INTERCEPT_FUNCTION(dispatch_barrier_async_f); |
| INTERCEPT_FUNCTION(dispatch_barrier_sync); |
| INTERCEPT_FUNCTION(dispatch_barrier_sync_f); |
| INTERCEPT_FUNCTION(dispatch_async_and_wait); |
| INTERCEPT_FUNCTION(dispatch_async_and_wait_f); |
| INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait); |
| INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f); |
| INTERCEPT_FUNCTION(dispatch_after); |
| INTERCEPT_FUNCTION(dispatch_after_f); |
| INTERCEPT_FUNCTION(dispatch_once); |
| INTERCEPT_FUNCTION(dispatch_once_f); |
| INTERCEPT_FUNCTION(dispatch_semaphore_signal); |
| INTERCEPT_FUNCTION(dispatch_semaphore_wait); |
| INTERCEPT_FUNCTION(dispatch_group_wait); |
| INTERCEPT_FUNCTION(dispatch_group_leave); |
| INTERCEPT_FUNCTION(dispatch_group_async); |
| INTERCEPT_FUNCTION(dispatch_group_async_f); |
| INTERCEPT_FUNCTION(dispatch_group_notify); |
| INTERCEPT_FUNCTION(dispatch_group_notify_f); |
| INTERCEPT_FUNCTION(dispatch_source_set_event_handler); |
| INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); |
| INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); |
| INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); |
| INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); |
| INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); |
| INTERCEPT_FUNCTION(dispatch_apply); |
| INTERCEPT_FUNCTION(dispatch_apply_f); |
| INTERCEPT_FUNCTION(dispatch_data_create); |
| INTERCEPT_FUNCTION(dispatch_read); |
| INTERCEPT_FUNCTION(dispatch_write); |
| INTERCEPT_FUNCTION(dispatch_io_read); |
| INTERCEPT_FUNCTION(dispatch_io_write); |
| INTERCEPT_FUNCTION(dispatch_io_barrier); |
| INTERCEPT_FUNCTION(dispatch_io_create); |
| INTERCEPT_FUNCTION(dispatch_io_create_with_path); |
| INTERCEPT_FUNCTION(dispatch_io_create_with_io); |
| INTERCEPT_FUNCTION(dispatch_io_close); |
| INTERCEPT_FUNCTION(dispatch_resume); |
| } |
| |
| } // namespace __tsan |
