Google Git
Sign in
llvm / openmp / 8f53081a5431b87cab41bab1f8f56bd75fcc089b / . / rc3 / runtime / src / z_Windows_NT_util.c
blob: f3ae0a49b0a98a4ec78a1e97af33afe3abe45b50 [file] [log] [blame]
/*
* z_Windows_NT_util.c -- platform specific routines.
*/
//===----------------------------------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//
#include "kmp.h"
#include "kmp_itt.h"
#include "kmp_i18n.h"
#include "kmp_io.h"
#include "kmp_wait_release.h"
/* This code is related to NtQuerySystemInformation() function. This function
is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
number of running threads in the system. */
#include <ntstatus.h>
#include <ntsecapi.h> // UNICODE_STRING
enum SYSTEM_INFORMATION_CLASS {
SystemProcessInformation = 5
}; // SYSTEM_INFORMATION_CLASS
struct CLIENT_ID {
HANDLE UniqueProcess;
HANDLE UniqueThread;
}; // struct CLIENT_ID
enum THREAD_STATE {
StateInitialized,
StateReady,
StateRunning,
StateStandby,
StateTerminated,
StateWait,
StateTransition,
StateUnknown
}; // enum THREAD_STATE
struct VM_COUNTERS {
SIZE_T PeakVirtualSize;
SIZE_T VirtualSize;
ULONG PageFaultCount;
SIZE_T PeakWorkingSetSize;
SIZE_T WorkingSetSize;
SIZE_T QuotaPeakPagedPoolUsage;
SIZE_T QuotaPagedPoolUsage;
SIZE_T QuotaPeakNonPagedPoolUsage;
SIZE_T QuotaNonPagedPoolUsage;
SIZE_T PagefileUsage;
SIZE_T PeakPagefileUsage;
SIZE_T PrivatePageCount;
}; // struct VM_COUNTERS
struct SYSTEM_THREAD {
LARGE_INTEGER KernelTime;
LARGE_INTEGER UserTime;
LARGE_INTEGER CreateTime;
ULONG WaitTime;
LPVOID StartAddress;
CLIENT_ID ClientId;
DWORD Priority;
LONG BasePriority;
ULONG ContextSwitchCount;
THREAD_STATE State;
ULONG WaitReason;
}; // SYSTEM_THREAD
KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
#if KMP_ARCH_X86
KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 52 );
#else
KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State ) == 68 );
#endif
struct SYSTEM_PROCESS_INFORMATION {
ULONG NextEntryOffset;
ULONG NumberOfThreads;
LARGE_INTEGER Reserved[ 3 ];
LARGE_INTEGER CreateTime;
LARGE_INTEGER UserTime;
LARGE_INTEGER KernelTime;
UNICODE_STRING ImageName;
DWORD BasePriority;
HANDLE ProcessId;
HANDLE ParentProcessId;
ULONG HandleCount;
ULONG Reserved2[ 2 ];
VM_COUNTERS VMCounters;
IO_COUNTERS IOCounters;
SYSTEM_THREAD Threads[ 1 ];
}; // SYSTEM_PROCESS_INFORMATION
typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) == 0 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime ) == 32 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName ) == 56 );
#if KMP_ARCH_X86
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 68 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 76 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 88 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 136 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 184 );
#else
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId ) == 80 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount ) == 96 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters ) == 112 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters ) == 208 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads ) == 256 );
#endif
typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
HMODULE ntdll = NULL;
/* End of NtQuerySystemInformation()-related code */
#if KMP_GROUP_AFFINITY
static HMODULE kernel32 = NULL;
#endif /* KMP_GROUP_AFFINITY */
/* ----------------------------------------------------------------------------------- */
/* ----------------------------------------------------------------------------------- */
#if KMP_HANDLE_SIGNALS
typedef void (* sig_func_t )( int );
static sig_func_t __kmp_sighldrs[ NSIG ];
static int __kmp_siginstalled[ NSIG ];
#endif
static HANDLE __kmp_monitor_ev;
static kmp_int64 __kmp_win32_time;
double __kmp_win32_tick;
int __kmp_init_runtime = FALSE;
CRITICAL_SECTION __kmp_win32_section;
void
__kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
{
InitializeCriticalSection( & mx->cs );
#if USE_ITT_BUILD
__kmp_itt_system_object_created( & mx->cs, "Critical Section" );
#endif /* USE_ITT_BUILD */
}
void
__kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
{
DeleteCriticalSection( & mx->cs );
}
void
__kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
{
EnterCriticalSection( & mx->cs );
}
void
__kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
{
LeaveCriticalSection( & mx->cs );
}
void
__kmp_win32_cond_init( kmp_win32_cond_t *cv )
{
cv->waiters_count_ = 0;
cv->wait_generation_count_ = 0;
cv->release_count_ = 0;
/* Initialize the critical section */
__kmp_win32_mutex_init( & cv->waiters_count_lock_ );
/* Create a manual-reset event. */
cv->event_ = CreateEvent( NULL, // no security
TRUE, // manual-reset
FALSE, // non-signaled initially
NULL ); // unnamed
#if USE_ITT_BUILD
__kmp_itt_system_object_created( cv->event_, "Event" );
#endif /* USE_ITT_BUILD */
}
void
__kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
{
__kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
__kmp_free_handle( cv->event_ );
memset( cv, '\0', sizeof( *cv ) );
}
/* TODO associate cv with a team instead of a thread so as to optimize
* the case where we wake up a whole team */
void
__kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load )
{
int my_generation;
int last_waiter;
/* Avoid race conditions */
__kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
/* Increment count of waiters */
cv->waiters_count_++;
/* Store current generation in our activation record. */
my_generation = cv->wait_generation_count_;
__kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
__kmp_win32_mutex_unlock( mx );
for (;;) {
int wait_done;
/* Wait until the event is signaled */
WaitForSingleObject( cv->event_, INFINITE );
__kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
/* Exit the loop when the <cv->event_> is signaled and
* there are still waiting threads from this <wait_generation>
* that haven't been released from this wait yet. */
wait_done = ( cv->release_count_ > 0 ) &&
( cv->wait_generation_count_ != my_generation );
__kmp_win32_mutex_unlock( &cv->waiters_count_lock_);
/* there used to be a semicolon after the if statement,
* it looked like a bug, so i removed it */
if( wait_done )
break;
}
__kmp_win32_mutex_lock( mx );
__kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
cv->waiters_count_--;
cv->release_count_--;
last_waiter = ( cv->release_count_ == 0 );
__kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
if( last_waiter ) {
/* We're the last waiter to be notified, so reset the manual event. */
ResetEvent( cv->event_ );
}
}
void
__kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
{
__kmp_win32_mutex_lock( &cv->waiters_count_lock_ );
if( cv->waiters_count_ > 0 ) {
SetEvent( cv->event_ );
/* Release all the threads in this generation. */
cv->release_count_ = cv->waiters_count_;
/* Start a new generation. */
cv->wait_generation_count_++;
}
__kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
}
void
__kmp_win32_cond_signal( kmp_win32_cond_t *cv )
{
__kmp_win32_cond_broadcast( cv );
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void
__kmp_enable( int new_state )
{
if (__kmp_init_runtime)
LeaveCriticalSection( & __kmp_win32_section );
}
void
__kmp_disable( int *old_state )
{
*old_state = 0;
if (__kmp_init_runtime)
EnterCriticalSection( & __kmp_win32_section );
}
void
__kmp_suspend_initialize( void )
{
/* do nothing */
}
static void
__kmp_suspend_initialize_thread( kmp_info_t *th )
{
if ( ! TCR_4( th->th.th_suspend_init ) ) {
/* this means we haven't initialized the suspension pthread objects for this thread
in this instance of the process */
__kmp_win32_cond_init( &th->th.th_suspend_cv );
__kmp_win32_mutex_init( &th->th.th_suspend_mx );
TCW_4( th->th.th_suspend_init, TRUE );
}
}
void
__kmp_suspend_uninitialize_thread( kmp_info_t *th )
{
if ( TCR_4( th->th.th_suspend_init ) ) {
/* this means we have initialize the suspension pthread objects for this thread
in this instance of the process */
__kmp_win32_cond_destroy( & th->th.th_suspend_cv );
__kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
TCW_4( th->th.th_suspend_init, FALSE );
}
}
/* This routine puts the calling thread to sleep after setting the
* sleep bit for the indicated flag variable to true.
*/
template <class C>
static inline void __kmp_suspend_template( int th_gtid, C *flag )
{
kmp_info_t *th = __kmp_threads[th_gtid];
int status;
typename C::flag_t old_spin;
KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );
__kmp_suspend_initialize_thread( th );
__kmp_win32_mutex_lock( &th->th.th_suspend_mx );
KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
th_gtid, flag->get() ) );
/* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
gets called first?
*/
old_spin = flag->set_sleeping();
KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
th_gtid, flag->get(), *(flag->get()) ) );
if ( flag->done_check_val(old_spin) ) {
old_spin = flag->unset_sleeping();
KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
th_gtid, flag->get()) );
} else {
#ifdef DEBUG_SUSPEND
__kmp_suspend_count++;
#endif
/* Encapsulate in a loop as the documentation states that this may
* "with low probability" return when the condition variable has
* not been signaled or broadcast
*/
int deactivated = FALSE;
TCW_PTR(th->th.th_sleep_loc, (void *)flag);
while ( flag->is_sleeping() ) {
KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
th_gtid ) );
// Mark the thread as no longer active (only in the first iteration of the loop).
if ( ! deactivated ) {
th->th.th_active = FALSE;
if ( th->th.th_active_in_pool ) {
th->th.th_active_in_pool = FALSE;
KMP_TEST_THEN_DEC32(
(kmp_int32 *) &__kmp_thread_pool_active_nth );
KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
}
deactivated = TRUE;
__kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
}
else {
__kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
}
#ifdef KMP_DEBUG
if( flag->is_sleeping() ) {
KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
}
#endif /* KMP_DEBUG */
} // while
// Mark the thread as active again (if it was previous marked as inactive)
if ( deactivated ) {
th->th.th_active = TRUE;
if ( TCR_4(th->th.th_in_pool) ) {
KMP_TEST_THEN_INC32(
(kmp_int32 *) &__kmp_thread_pool_active_nth );
th->th.th_active_in_pool = TRUE;
}
}
}
__kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
}
void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
__kmp_suspend_template(th_gtid, flag);
}
void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
__kmp_suspend_template(th_gtid, flag);
}
void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
__kmp_suspend_template(th_gtid, flag);
}
/* This routine signals the thread specified by target_gtid to wake up
* after setting the sleep bit indicated by the flag argument to FALSE
*/
template <class C>
static inline void __kmp_resume_template( int target_gtid, C *flag )
{
kmp_info_t *th = __kmp_threads[target_gtid];
int status;
#ifdef KMP_DEBUG
int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
#endif
KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
__kmp_suspend_initialize_thread( th );
__kmp_win32_mutex_lock( &th->th.th_suspend_mx );
if (!flag) { // coming from __kmp_null_resume_wrapper
flag = (C *)th->th.th_sleep_loc;
}
// First, check if the flag is null or its type has changed. If so, someone else woke it up.
if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
gtid, target_gtid, NULL ) );
__kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
return;
}
else {
typename C::flag_t old_spin = flag->unset_sleeping();
if ( !flag->is_sleeping_val(old_spin) ) {
KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): "
"%u => %u\n",
gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
__kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
return;
}
}
TCW_PTR(th->th.th_sleep_loc, NULL);
KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
gtid, target_gtid, flag->get() ) );
__kmp_win32_cond_signal( &th->th.th_suspend_cv );
__kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
gtid, target_gtid ) );
}
void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
__kmp_resume_template(target_gtid, flag);
}
void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
__kmp_resume_template(target_gtid, flag);
}
void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
__kmp_resume_template(target_gtid, flag);
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void
__kmp_yield( int cond )
{
if (cond)
Sleep(0);
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void
__kmp_gtid_set_specific( int gtid )
{
if( __kmp_init_gtid ) {
KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
gtid, __kmp_gtid_threadprivate_key ));
if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
KMP_FATAL( TLSSetValueFailed );
} else {
KA_TRACE( 50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
}
}
int
__kmp_gtid_get_specific()
{
int gtid;
if( !__kmp_init_gtid ) {
KA_TRACE( 50, ("__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
return KMP_GTID_SHUTDOWN;
}
gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
if ( gtid == 0 ) {
gtid = KMP_GTID_DNE;
}
else {
gtid--;
}
KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
__kmp_gtid_threadprivate_key, gtid ));
return gtid;
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
#if KMP_GROUP_AFFINITY
//
// Only 1 DWORD in the mask should have any procs set.
// Return the appropriate index, or -1 for an invalid mask.
//
int
__kmp_get_proc_group( kmp_affin_mask_t const *mask )
{
int i;
int group = -1;
for (i = 0; i < __kmp_num_proc_groups; i++) {
#if KMP_USE_HWLOC
// On windows, the long type is always 32 bits
unsigned long first_32_bits = hwloc_bitmap_to_ith_ulong((hwloc_const_bitmap_t)mask, i*2);
unsigned long second_32_bits = hwloc_bitmap_to_ith_ulong((hwloc_const_bitmap_t)mask, i*2+1);
if (first_32_bits == 0 && second_32_bits == 0) {
continue;
}
#else
if (mask[i] == 0) {
continue;
}
#endif
if (group >= 0) {
return -1;
}
group = i;
}
return group;
}
#endif /* KMP_GROUP_AFFINITY */
int
__kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
{
#if KMP_USE_HWLOC
int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
#else
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) {
//
// Check for a valid mask.
//
GROUP_AFFINITY ga;
int group = __kmp_get_proc_group( mask );
if (group < 0) {
if (abort_on_error) {
KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
}
return -1;
}
//
// Transform the bit vector into a GROUP_AFFINITY struct
// and make the system call to set affinity.
//
ga.Group = group;
ga.Mask = mask[group];
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
}
else
# endif /* KMP_GROUP_AFFINITY */
{
if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
}
}
#endif /* KMP_USE_HWLOC */
return 0;
}
int
__kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
{
#if KMP_USE_HWLOC
int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
if (retval >= 0) {
return 0;
}
int error = errno;
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FatalSysError ),
KMP_ERR( error ),
__kmp_msg_null
);
}
return error;
#else /* KMP_USE_HWLOC */
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) {
KMP_CPU_ZERO(mask);
GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);
if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups)
|| (ga.Mask == 0)) {
return -1;
}
mask[ga.Group] = ga.Mask;
}
else
# endif /* KMP_GROUP_AFFINITY */
{
kmp_affin_mask_t newMask, sysMask, retval;
if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
if (! retval) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
return error;
}
newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
if (! newMask) {
DWORD error = GetLastError();
if (abort_on_error) {
__kmp_msg(
kmp_ms_fatal,
KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
KMP_ERR(error),
__kmp_msg_null
);
}
}
*mask = retval;
}
#endif /* KMP_USE_HWLOC */
return 0;
}
void
__kmp_affinity_bind_thread( int proc )
{
#if KMP_USE_HWLOC
kmp_affin_mask_t *mask;
KMP_CPU_ALLOC_ON_STACK(mask);
KMP_CPU_ZERO(mask);
KMP_CPU_SET(proc, mask);
__kmp_set_system_affinity(mask, TRUE);
KMP_CPU_FREE_FROM_STACK(mask);
#else /* KMP_USE_HWLOC */
# if KMP_GROUP_AFFINITY
if (__kmp_num_proc_groups > 1) {
//
// Form the GROUP_AFFINITY struct directly, rather than filling
// out a bit vector and calling __kmp_set_system_affinity().
//
GROUP_AFFINITY ga;
KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
* CHAR_BIT * sizeof(DWORD_PTR))));
ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
DWORD error = GetLastError();
if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
__kmp_msg(
kmp_ms_warning,
KMP_MSG( CantSetThreadAffMask ),
KMP_ERR( error ),
__kmp_msg_null
);
}
}
}
else
# endif /* KMP_GROUP_AFFINITY */
{
kmp_affin_mask_t mask;
KMP_CPU_ZERO(&mask);
KMP_CPU_SET(proc, &mask);
__kmp_set_system_affinity(&mask, TRUE);
}
#endif /* KMP_USE_HWLOC */
}
void
__kmp_affinity_determine_capable( const char *env_var )
{
//
// All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
//
#if KMP_GROUP_AFFINITY
KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(kmp_affin_mask_t));
#else
KMP_AFFINITY_ENABLE(sizeof(kmp_affin_mask_t));
#endif
KA_TRACE( 10, (
"__kmp_affinity_determine_capable: "
"Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n",
__kmp_affin_mask_size
) );
}
double
__kmp_read_cpu_time( void )
{
FILETIME CreationTime, ExitTime, KernelTime, UserTime;
int status;
double cpu_time;
cpu_time = 0;
status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
&ExitTime, &KernelTime, &UserTime );
if (status) {
double sec = 0;
sec += KernelTime.dwHighDateTime;
sec += UserTime.dwHighDateTime;
/* Shift left by 32 bits */
sec *= (double) (1 << 16) * (double) (1 << 16);
sec += KernelTime.dwLowDateTime;
sec += UserTime.dwLowDateTime;
cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
}
return cpu_time;
}
int
__kmp_read_system_info( struct kmp_sys_info *info )
{
info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
info->minflt = 0; /* the number of page faults serviced without any I/O */
info->majflt = 0; /* the number of page faults serviced that required I/O */
info->nswap = 0; /* the number of times a process was "swapped" out of memory */
info->inblock = 0; /* the number of times the file system had to perform input */
info->oublock = 0; /* the number of times the file system had to perform output */
info->nvcsw = 0; /* the number of times a context switch was voluntarily */
info->nivcsw = 0; /* the number of times a context switch was forced */
return 1;
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void
__kmp_runtime_initialize( void )
{
SYSTEM_INFO info;
kmp_str_buf_t path;
UINT path_size;
if ( __kmp_init_runtime ) {
return;
};
#if KMP_DYNAMIC_LIB
/* Pin dynamic library for the lifetime of application */
{
// First, turn off error message boxes
UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS);
HMODULE h;
BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
|GET_MODULE_HANDLE_EX_FLAG_PIN,
(LPCTSTR)&__kmp_serial_initialize, &h);
KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
SetErrorMode (err_mode); // Restore error mode
KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") );
}
#endif
InitializeCriticalSection( & __kmp_win32_section );
#if USE_ITT_BUILD
__kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" );
#endif /* USE_ITT_BUILD */
__kmp_initialize_system_tick();
#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
if ( ! __kmp_cpuinfo.initialized ) {
__kmp_query_cpuid( & __kmp_cpuinfo );
}; // if
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
/* Set up minimum number of threads to switch to TLS gtid */
#if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB
// Windows* OS, static library.
/*
New thread may use stack space previously used by another thread, currently terminated.
On Windows* OS, in case of static linking, we do not know the moment of thread termination,
and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead
threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly
finds gtid (by searching through stack addresses of all known threads) for unregistered
foreign tread.
Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id()
does not search through stacks, but get gtid from TLS immediately.
--ln
*/
__kmp_tls_gtid_min = 0;
#else
__kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
#endif
/* for the static library */
if ( !__kmp_gtid_threadprivate_key ) {
__kmp_gtid_threadprivate_key = TlsAlloc();
if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
KMP_FATAL( TLSOutOfIndexes );
}
}
//
// Load ntdll.dll.
//
/*
Simple
GetModuleHandle( "ntdll.dl" )
is not suitable due to security issue (see
http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
path to the library.
*/
__kmp_str_buf_init( & path );
path_size = GetSystemDirectory( path.str, path.size );
KMP_DEBUG_ASSERT( path_size > 0 );
if ( path_size >= path.size ) {
//
// Buffer is too short. Expand the buffer and try again.
//
__kmp_str_buf_reserve( & path, path_size );
path_size = GetSystemDirectory( path.str, path.size );
KMP_DEBUG_ASSERT( path_size > 0 );
}; // if
if ( path_size > 0 && path_size < path.size ) {
//
// Now we have system directory name in the buffer.
// Append backslash and name of dll to form full path,
//
path.used = path_size;
__kmp_str_buf_print( & path, "\\%s", "ntdll.dll" );
//
// Now load ntdll using full path.
//
ntdll = GetModuleHandle( path.str );
}
KMP_DEBUG_ASSERT( ntdll != NULL );
if ( ntdll != NULL ) {
NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" );
}
KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );
#if KMP_GROUP_AFFINITY
//
// Load kernel32.dll.
// Same caveat - must use full system path name.
//
if ( path_size > 0 && path_size < path.size ) {
//
// Truncate the buffer back to just the system path length,
// discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
//
path.used = path_size;
__kmp_str_buf_print( & path, "\\%s", "kernel32.dll" );
//
// Load kernel32.dll using full path.
//
kernel32 = GetModuleHandle( path.str );
KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) );
//
// Load the function pointers to kernel32.dll routines
// that may or may not exist on this system.
//
if ( kernel32 != NULL ) {
__kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" );
__kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" );
__kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" );
__kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" );
KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) );
KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) );
KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) );
KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) );
KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) );
//
// See if group affinity is supported on this system.
// If so, calculate the #groups and #procs.
//
// Group affinity was introduced with Windows* 7 OS and
// Windows* Server 2008 R2 OS.
//
if ( ( __kmp_GetActiveProcessorCount != NULL )
&& ( __kmp_GetActiveProcessorGroupCount != NULL )
&& ( __kmp_GetThreadGroupAffinity != NULL )
&& ( __kmp_SetThreadGroupAffinity != NULL )
&& ( ( __kmp_num_proc_groups
= __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
//
// Calculate the total number of active OS procs.
//
int i;
KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
__kmp_xproc = 0;
for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
DWORD size = __kmp_GetActiveProcessorCount( i );
__kmp_xproc += size;
KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
}
}
else {
KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
}
}
}
if ( __kmp_num_proc_groups <= 1 ) {
GetSystemInfo( & info );
__kmp_xproc = info.dwNumberOfProcessors;
}
#else
GetSystemInfo( & info );
__kmp_xproc = info.dwNumberOfProcessors;
#endif /* KMP_GROUP_AFFINITY */
//
// If the OS said there were 0 procs, take a guess and use a value of 2.
// This is done for Linux* OS, also. Do we need error / warning?
//
if ( __kmp_xproc <= 0 ) {
__kmp_xproc = 2;
}
KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );
__kmp_str_buf_free( & path );
#if USE_ITT_BUILD
__kmp_itt_initialize();
#endif /* USE_ITT_BUILD */
__kmp_init_runtime = TRUE;
} // __kmp_runtime_initialize
void
__kmp_runtime_destroy( void )
{
if ( ! __kmp_init_runtime ) {
return;
}
#if USE_ITT_BUILD
__kmp_itt_destroy();
#endif /* USE_ITT_BUILD */
/* we can't DeleteCriticalsection( & __kmp_win32_section ); */
/* due to the KX_TRACE() commands */
KA_TRACE( 40, ("__kmp_runtime_destroy\n" ));
if( __kmp_gtid_threadprivate_key ) {
TlsFree( __kmp_gtid_threadprivate_key );
__kmp_gtid_threadprivate_key = 0;
}
__kmp_affinity_uninitialize();
DeleteCriticalSection( & __kmp_win32_section );
ntdll = NULL;
NtQuerySystemInformation = NULL;
#if KMP_ARCH_X86_64
kernel32 = NULL;
__kmp_GetActiveProcessorCount = NULL;
__kmp_GetActiveProcessorGroupCount = NULL;
__kmp_GetThreadGroupAffinity = NULL;
__kmp_SetThreadGroupAffinity = NULL;
#endif // KMP_ARCH_X86_64
__kmp_init_runtime = FALSE;
}
void
__kmp_terminate_thread( int gtid )
{
kmp_info_t *th = __kmp_threads[ gtid ];
if( !th ) return;
KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );
if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
/* It's OK, the thread may have exited already */
}
__kmp_free_handle( th->th.th_info.ds.ds_thread );
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void
__kmp_clear_system_time( void )
{
BOOL status;
LARGE_INTEGER time;
status = QueryPerformanceCounter( & time );
__kmp_win32_time = (kmp_int64) time.QuadPart;
}
void
__kmp_initialize_system_tick( void )
{
{
BOOL status;
LARGE_INTEGER freq;
status = QueryPerformanceFrequency( & freq );
if (! status) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ),
KMP_ERR( error ),
__kmp_msg_null
);
}
else {
__kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
}
}
}
/* Calculate the elapsed wall clock time for the user */
void
__kmp_elapsed( double *t )
{
BOOL status;
LARGE_INTEGER now;
status = QueryPerformanceCounter( & now );
*t = ((double) now.QuadPart) * __kmp_win32_tick;
}
/* Calculate the elapsed wall clock tick for the user */
void
__kmp_elapsed_tick( double *t )
{
*t = __kmp_win32_tick;
}
void
__kmp_read_system_time( double *delta )
{
if (delta != NULL) {
BOOL status;
LARGE_INTEGER now;
status = QueryPerformanceCounter( & now );
*delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
* __kmp_win32_tick;
}
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
void * __stdcall
__kmp_launch_worker( void *arg )
{
volatile void *stack_data;
void *exit_val;
void *padding = 0;
kmp_info_t *this_thr = (kmp_info_t *) arg;
int gtid;
gtid = this_thr->th.th_info.ds.ds_gtid;
__kmp_gtid_set_specific( gtid );
#ifdef KMP_TDATA_GTID
#error "This define causes problems with LoadLibrary() + declspec(thread) " \
"on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
"reference: http://support.microsoft.com/kb/118816"
//__kmp_gtid = gtid;
#endif
#if USE_ITT_BUILD
__kmp_itt_thread_name( gtid );
#endif /* USE_ITT_BUILD */
__kmp_affinity_set_init_mask( gtid, FALSE );
#if KMP_ARCH_X86 || KMP_ARCH_X86_64
//
// Set the FP control regs to be a copy of
// the parallel initialization thread's.
//
__kmp_clear_x87_fpu_status_word();
__kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
__kmp_load_mxcsr( &__kmp_init_mxcsr );
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
if ( __kmp_stkoffset > 0 && gtid > 0 ) {
padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
}
KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
__kmp_check_stack_overlap( this_thr );
}
KMP_MB();
exit_val = __kmp_launch_thread( this_thr );
KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
KMP_MB();
return exit_val;
}
/* The monitor thread controls all of the threads in the complex */
void * __stdcall
__kmp_launch_monitor( void *arg )
{
DWORD wait_status;
kmp_thread_t monitor;
int status;
int interval;
kmp_info_t *this_thr = (kmp_info_t *) arg;
KMP_DEBUG_ASSERT(__kmp_init_monitor);
TCW_4( __kmp_init_monitor, 2 ); // AC: Signal the library that monitor has started
// TODO: hide "2" in enum (like {true,false,started})
this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) );
monitor = GetCurrentThread();
/* set thread priority */
status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
if (! status) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadPriority ),
KMP_ERR( error ),
__kmp_msg_null
);
}
/* register us as monitor */
__kmp_gtid_set_specific( KMP_GTID_MONITOR );
#ifdef KMP_TDATA_GTID
#error "This define causes problems with LoadLibrary() + declspec(thread) " \
"on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
"reference: http://support.microsoft.com/kb/118816"
//__kmp_gtid = KMP_GTID_MONITOR;
#endif
#if USE_ITT_BUILD
__kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore monitor thread.
#endif /* USE_ITT_BUILD */
KMP_MB(); /* Flush all pending memory write invalidates. */
interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */
while (! TCR_4(__kmp_global.g.g_done)) {
/* This thread monitors the state of the system */
KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );
wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );
if (wait_status == WAIT_TIMEOUT) {
TCW_4( __kmp_global.g.g_time.dt.t_value,
TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
}
KMP_MB(); /* Flush all pending memory write invalidates. */
}
KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) );
status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
if (! status) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetThreadPriority ),
KMP_ERR( error ),
__kmp_msg_null
);
}
if (__kmp_global.g.g_abort != 0) {
/* now we need to terminate the worker threads */
/* the value of t_abort is the signal we caught */
int gtid;
KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );
/* terminate the OpenMP worker threads */
/* TODO this is not valid for sibling threads!!
* the uber master might not be 0 anymore.. */
for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
__kmp_terminate_thread( gtid );
__kmp_cleanup();
Sleep( 0 );
KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );
if (__kmp_global.g.g_abort > 0) {
raise( __kmp_global.g.g_abort );
}
}
TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
KMP_MB();
return arg;
}
void
__kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
{
kmp_thread_t handle;
DWORD idThread;
KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );
th->th.th_info.ds.ds_gtid = gtid;
if ( KMP_UBER_GTID(gtid) ) {
int stack_data;
/* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use.
Is it appropriate to just use GetCurrentThread? When should we close this handle? When
unregistering the root?
*/
{
BOOL rc;
rc = DuplicateHandle(
GetCurrentProcess(),
GetCurrentThread(),
GetCurrentProcess(),
&th->th.th_info.ds.ds_thread,
0,
FALSE,
DUPLICATE_SAME_ACCESS
);
KMP_ASSERT( rc );
KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
(LPVOID)th,
th->th.th_info.ds.ds_thread ) );
th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
}
if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
/* we will dynamically update the stack range if gtid_mode == 1 */
TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
__kmp_check_stack_overlap( th );
}
}
else {
KMP_MB(); /* Flush all pending memory write invalidates. */
/* Set stack size for this thread now. */
KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
" bytes\n", stack_size ) );
stack_size += gtid * __kmp_stkoffset;
TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %"
KMP_SIZE_T_SPEC
" bytes, &__kmp_launch_worker = %p, th = %p, "
"&idThread = %p\n",
(SIZE_T) stack_size,
(LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
(LPVOID) th, &idThread ) );
handle = CreateThread( NULL, (SIZE_T) stack_size,
(LPTHREAD_START_ROUTINE) __kmp_launch_worker,
(LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %"
KMP_SIZE_T_SPEC
" bytes, &__kmp_launch_worker = %p, th = %p, "
"idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
(SIZE_T) stack_size,
(LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
(LPVOID) th, idThread, handle ) );
if ( handle == 0 ) {
DWORD error = GetLastError();
__kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null);
} else {
th->th.th_info.ds.ds_thread = handle;
}
KMP_MB(); /* Flush all pending memory write invalidates. */
}
KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
}
int
__kmp_still_running(kmp_info_t *th) {
return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
}
void
__kmp_create_monitor( kmp_info_t *th )
{
kmp_thread_t handle;
DWORD idThread;
int ideal, new_ideal;
if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
// We don't need monitor thread in case of MAX_BLOCKTIME
KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
th->th.th_info.ds.ds_gtid = 0;
TCW_4( __kmp_init_monitor, 2 ); // Signal to stop waiting for monitor creation
return;
}
KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );
KMP_MB(); /* Flush all pending memory write invalidates. */
__kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
if ( __kmp_monitor_ev == NULL ) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantCreateEvent ),
KMP_ERR( error ),
__kmp_msg_null
);
}; // if
#if USE_ITT_BUILD
__kmp_itt_system_object_created( __kmp_monitor_ev, "Event" );
#endif /* USE_ITT_BUILD */
th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
// FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
// to automatically expand stacksize based on CreateThread error code.
if ( __kmp_monitor_stksize == 0 ) {
__kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
}
if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
__kmp_monitor_stksize = __kmp_sys_min_stksize;
}
KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
(int) __kmp_monitor_stksize ) );
TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
(LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
(LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
if (handle == 0) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantCreateThread ),
KMP_ERR( error ),
__kmp_msg_null
);
}
else
th->th.th_info.ds.ds_thread = handle;
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n",
(void *) th->th.th_info.ds.ds_thread ) );
}
/*
Check to see if thread is still alive.
NOTE: The ExitProcess(code) system call causes all threads to Terminate
with a exit_val = code. Because of this we can not rely on
exit_val having any particular value. So this routine may
return STILL_ALIVE in exit_val even after the thread is dead.
*/
int
__kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
{
DWORD rc;
rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
if ( rc == 0 ) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FunctionError, "GetExitCodeThread()" ),
KMP_ERR( error ),
__kmp_msg_null
);
}; // if
return ( *exit_val == STILL_ACTIVE );
}
void
__kmp_exit_thread(
int exit_status
) {
ExitThread( exit_status );
} // __kmp_exit_thread
/*
This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
*/
static void
__kmp_reap_common( kmp_info_t * th )
{
DWORD exit_val;
KMP_MB(); /* Flush all pending memory write invalidates. */
KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );
/*
2006-10-19:
There are two opposite situations:
1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread
function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very
slow".)
2. Windows* OS may kill thread before it resets ds_alive flag.
Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting.
*/
{
// TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to
// cover this usage also.
void * obj = NULL;
register kmp_uint32 spins;
#if USE_ITT_BUILD
KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive );
#endif /* USE_ITT_BUILD */
KMP_INIT_YIELD( spins );
do {
#if USE_ITT_BUILD
KMP_FSYNC_SPIN_PREPARE( obj );
#endif /* USE_ITT_BUILD */
__kmp_is_thread_alive( th, &exit_val );
KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
KMP_YIELD_SPIN( spins );
} while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
#if USE_ITT_BUILD
if ( exit_val == STILL_ACTIVE ) {
KMP_FSYNC_CANCEL( obj );
} else {
KMP_FSYNC_SPIN_ACQUIRED( obj );
}; // if
#endif /* USE_ITT_BUILD */
}
__kmp_free_handle( th->th.th_info.ds.ds_thread );
/*
* NOTE: The ExitProcess(code) system call causes all threads to Terminate
* with a exit_val = code. Because of this we can not rely on
* exit_val having any particular value.
*/
if ( exit_val == STILL_ACTIVE ) {
KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) );
} else if ( (void *) exit_val != (void *) th) {
KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
}; // if
KA_TRACE( 10,
(
"__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n",
th->th.th_info.ds.ds_gtid,
th->th.th_info.ds.ds_thread
)
);
th->th.th_info.ds.ds_thread = 0;
th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
th->th.th_info.ds.ds_thread_id = 0;
KMP_MB(); /* Flush all pending memory write invalidates. */
}
void
__kmp_reap_monitor( kmp_info_t *th )
{
int status;
KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n",
(void *) th->th.th_info.ds.ds_thread ) );
// If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
// If both tid and gtid are 0, it means the monitor did not ever start.
// If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
return;
}; // if
KMP_MB(); /* Flush all pending memory write invalidates. */
status = SetEvent( __kmp_monitor_ev );
if ( status == FALSE ) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantSetEvent ),
KMP_ERR( error ),
__kmp_msg_null
);
}
KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
__kmp_reap_common( th );
__kmp_free_handle( __kmp_monitor_ev );
KMP_MB(); /* Flush all pending memory write invalidates. */
}
void
__kmp_reap_worker( kmp_info_t * th )
{
KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
__kmp_reap_common( th );
}
/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */
#if KMP_HANDLE_SIGNALS
static void
__kmp_team_handler( int signo )
{
if ( __kmp_global.g.g_abort == 0 ) {
// Stage 1 signal handler, let's shut down all of the threads.
if ( __kmp_debug_buf ) {
__kmp_dump_debug_buffer();
}; // if
KMP_MB(); // Flush all pending memory write invalidates.
TCW_4( __kmp_global.g.g_abort, signo );
KMP_MB(); // Flush all pending memory write invalidates.
TCW_4( __kmp_global.g.g_done, TRUE );
KMP_MB(); // Flush all pending memory write invalidates.
}
} // __kmp_team_handler
static
sig_func_t __kmp_signal( int signum, sig_func_t handler ) {
sig_func_t old = signal( signum, handler );
if ( old == SIG_ERR ) {
int error = errno;
__kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null );
}; // if
return old;
}
static void
__kmp_install_one_handler(
int sig,
sig_func_t handler,
int parallel_init
) {
sig_func_t old;
KMP_MB(); /* Flush all pending memory write invalidates. */
KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) );
if ( parallel_init ) {
old = __kmp_signal( sig, handler );
// SIG_DFL on Windows* OS in NULL or 0.
if ( old == __kmp_sighldrs[ sig ] ) {
__kmp_siginstalled[ sig ] = 1;
} else {
// Restore/keep user's handler if one previously installed.
old = __kmp_signal( sig, old );
}; // if
} else {
// Save initial/system signal handlers to see if user handlers installed.
// 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with
// parallel_init == TRUE.
old = __kmp_signal( sig, SIG_DFL );
__kmp_sighldrs[ sig ] = old;
__kmp_signal( sig, old );
}; // if
KMP_MB(); /* Flush all pending memory write invalidates. */
} // __kmp_install_one_handler
static void
__kmp_remove_one_handler( int sig ) {
if ( __kmp_siginstalled[ sig ] ) {
sig_func_t old;
KMP_MB(); // Flush all pending memory write invalidates.
KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) );
old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
if ( old != __kmp_team_handler ) {
KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
old = __kmp_signal( sig, old );
}; // if
__kmp_sighldrs[ sig ] = NULL;
__kmp_siginstalled[ sig ] = 0;
KMP_MB(); // Flush all pending memory write invalidates.
}; // if
} // __kmp_remove_one_handler
void
__kmp_install_signals( int parallel_init )
{
KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) );
if ( ! __kmp_handle_signals ) {
KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
return;
}; // if
__kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
__kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
__kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
__kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
__kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
__kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
} // __kmp_install_signals
void
__kmp_remove_signals( void )
{
int sig;
KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) );
for ( sig = 1; sig < NSIG; ++ sig ) {
__kmp_remove_one_handler( sig );
}; // for sig
} // __kmp_remove_signals
#endif // KMP_HANDLE_SIGNALS
/* Put the thread to sleep for a time period */
void
__kmp_thread_sleep( int millis )
{
DWORD status;
status = SleepEx( (DWORD) millis, FALSE );
if ( status ) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( FunctionError, "SleepEx()" ),
KMP_ERR( error ),
__kmp_msg_null
);
}
}
/* Determine whether the given address is mapped into the current address space. */
int
__kmp_is_address_mapped( void * addr )
{
DWORD status;
MEMORY_BASIC_INFORMATION lpBuffer;
SIZE_T dwLength;
dwLength = sizeof(MEMORY_BASIC_INFORMATION);
status = VirtualQuery( addr, &lpBuffer, dwLength );
return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
(( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
}
kmp_uint64
__kmp_hardware_timestamp(void)
{
kmp_uint64 r = 0;
QueryPerformanceCounter((LARGE_INTEGER*) &r);
return r;
}
/* Free handle and check the error code */
void
__kmp_free_handle( kmp_thread_t tHandle )
{
/* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */
BOOL rc;
rc = CloseHandle( tHandle );
if ( !rc ) {
DWORD error = GetLastError();
__kmp_msg(
kmp_ms_fatal,
KMP_MSG( CantCloseHandle ),
KMP_ERR( error ),
__kmp_msg_null
);
}
}
int
__kmp_get_load_balance( int max ) {
static ULONG glb_buff_size = 100 * 1024;
static int glb_running_threads = 0; /* Saved count of the running threads for the thread balance algortihm */
static double glb_call_time = 0; /* Thread balance algorithm call time */
int running_threads = 0; // Number of running threads in the system.
NTSTATUS status = 0;
ULONG buff_size = 0;
ULONG info_size = 0;
void * buffer = NULL;
PSYSTEM_PROCESS_INFORMATION spi = NULL;
int first_time = 1;
double call_time = 0.0; //start, finish;
__kmp_elapsed( & call_time );
if ( glb_call_time &&
( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
running_threads = glb_running_threads;
goto finish;
}
glb_call_time = call_time;
// Do not spend time on running algorithm if we have a permanent error.
if ( NtQuerySystemInformation == NULL ) {
running_threads = -1;
goto finish;
}; // if
if ( max <= 0 ) {
max = INT_MAX;
}; // if
do {
if ( first_time ) {
buff_size = glb_buff_size;
} else {
buff_size = 2 * buff_size;
}
buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
if ( buffer == NULL ) {
running_threads = -1;
goto finish;
}; // if
status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
first_time = 0;
} while ( status == STATUS_INFO_LENGTH_MISMATCH );
glb_buff_size = buff_size;
#define CHECK( cond ) \
{ \
KMP_DEBUG_ASSERT( cond ); \
if ( ! ( cond ) ) { \
running_threads = -1; \
goto finish; \
} \
}
CHECK( buff_size >= info_size );
spi = PSYSTEM_PROCESS_INFORMATION( buffer );
for ( ; ; ) {
ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
HANDLE pid = spi->ProcessId;
ULONG num = spi->NumberOfThreads;
CHECK( num >= 1 );
size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 );
CHECK( offset + spi_size < info_size ); // Make sure process info record fits the buffer.
if ( spi->NextEntryOffset != 0 ) {
CHECK( spi_size <= spi->NextEntryOffset ); // And do not overlap with the next record.
}; // if
// pid == 0 corresponds to the System Idle Process. It always has running threads
// on all cores. So, we don't consider the running threads of this process.
if ( pid != 0 ) {
for ( int i = 0; i < num; ++ i ) {
THREAD_STATE state = spi->Threads[ i ].State;
// Count threads that have Ready or Running state.
// !!! TODO: Why comment does not match the code???
if ( state == StateRunning ) {
++ running_threads;
// Stop counting running threads if the number is already greater than
// the number of available cores
if ( running_threads >= max ) {
goto finish;
}
} // if
}; // for i
} // if
if ( spi->NextEntryOffset == 0 ) {
break;
}; // if
spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
}; // forever
#undef CHECK
finish: // Clean up and exit.
if ( buffer != NULL ) {
KMP_INTERNAL_FREE( buffer );
}; // if
glb_running_threads = running_threads;
return running_threads;
} //__kmp_get_load_balance()