| /* |
| * kmp_csupport.c -- kfront linkage support for OpenMP. |
| */ |
| |
| |
| //===----------------------------------------------------------------------===// |
| // |
| // The LLVM Compiler Infrastructure |
| // |
| // This file is dual licensed under the MIT and the University of Illinois Open |
| // Source Licenses. See LICENSE.txt for details. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| |
| #include "omp.h" /* extern "C" declarations of user-visible routines */ |
| #include "kmp.h" |
| #include "kmp_i18n.h" |
| #include "kmp_itt.h" |
| #include "kmp_lock.h" |
| #include "kmp_error.h" |
| #include "kmp_stats.h" |
| |
| #if OMPT_SUPPORT |
| #include "ompt-internal.h" |
| #include "ompt-specific.h" |
| #endif |
| |
| #define MAX_MESSAGE 512 |
| |
| /* ------------------------------------------------------------------------ */ |
| /* ------------------------------------------------------------------------ */ |
| |
| /* flags will be used in future, e.g., to implement */ |
| /* openmp_strict library restrictions */ |
| |
| /*! |
| * @ingroup STARTUP_SHUTDOWN |
| * @param loc in source location information |
| * @param flags in for future use (currently ignored) |
| * |
| * Initialize the runtime library. This call is optional; if it is not made then |
| * it will be implicitly called by attempts to use other library functions. |
| * |
| */ |
| void |
| __kmpc_begin(ident_t *loc, kmp_int32 flags) |
| { |
| // By default __kmp_ignore_mppbeg() returns TRUE. |
| if (__kmp_ignore_mppbeg() == FALSE) { |
| __kmp_internal_begin(); |
| |
| KC_TRACE( 10, ("__kmpc_begin: called\n" ) ); |
| } |
| } |
| |
| /*! |
| * @ingroup STARTUP_SHUTDOWN |
| * @param loc source location information |
| * |
| * Shutdown the runtime library. This is also optional, and even if called will not |
| * do anything unless the `KMP_IGNORE_MPPEND` environment variable is set to zero. |
| */ |
| void |
| __kmpc_end(ident_t *loc) |
| { |
| // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end() call no-op. |
| // However, this can be overridden with KMP_IGNORE_MPPEND environment variable. |
| // If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend() returns FALSE and __kmpc_end() |
| // will unregister this root (it can cause library shut down). |
| if (__kmp_ignore_mppend() == FALSE) { |
| KC_TRACE( 10, ("__kmpc_end: called\n" ) ); |
| KA_TRACE( 30, ("__kmpc_end\n" )); |
| |
| __kmp_internal_end_thread( -1 ); |
| } |
| } |
| |
| /*! |
| @ingroup THREAD_STATES |
| @param loc Source location information. |
| @return The global thread index of the active thread. |
| |
| This function can be called in any context. |
| |
| If the runtime has ony been entered at the outermost level from a |
| single (necessarily non-OpenMP<sup>*</sup>) thread, then the thread number is that |
| which would be returned by omp_get_thread_num() in the outermost |
| active parallel construct. (Or zero if there is no active parallel |
| construct, since the master thread is necessarily thread zero). |
| |
| If multiple non-OpenMP threads all enter an OpenMP construct then this |
| will be a unique thread identifier among all the threads created by |
| the OpenMP runtime (but the value cannote be defined in terms of |
| OpenMP thread ids returned by omp_get_thread_num()). |
| |
| */ |
| kmp_int32 |
| __kmpc_global_thread_num(ident_t *loc) |
| { |
| kmp_int32 gtid = __kmp_entry_gtid(); |
| |
| KC_TRACE( 10, ("__kmpc_global_thread_num: T#%d\n", gtid ) ); |
| |
| return gtid; |
| } |
| |
| /*! |
| @ingroup THREAD_STATES |
| @param loc Source location information. |
| @return The number of threads under control of the OpenMP<sup>*</sup> runtime |
| |
| This function can be called in any context. |
| It returns the total number of threads under the control of the OpenMP runtime. That is |
| not a number that can be determined by any OpenMP standard calls, since the library may be |
| called from more than one non-OpenMP thread, and this reflects the total over all such calls. |
| Similarly the runtime maintains underlying threads even when they are not active (since the cost |
| of creating and destroying OS threads is high), this call counts all such threads even if they are not |
| waiting for work. |
| */ |
| kmp_int32 |
| __kmpc_global_num_threads(ident_t *loc) |
| { |
| KC_TRACE( 10, ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) ); |
| |
| return TCR_4(__kmp_nth); |
| } |
| |
| /*! |
| @ingroup THREAD_STATES |
| @param loc Source location information. |
| @return The thread number of the calling thread in the innermost active parallel construct. |
| |
| */ |
| kmp_int32 |
| __kmpc_bound_thread_num(ident_t *loc) |
| { |
| KC_TRACE( 10, ("__kmpc_bound_thread_num: called\n" ) ); |
| return __kmp_tid_from_gtid( __kmp_entry_gtid() ); |
| } |
| |
| /*! |
| @ingroup THREAD_STATES |
| @param loc Source location information. |
| @return The number of threads in the innermost active parallel construct. |
| */ |
| kmp_int32 |
| __kmpc_bound_num_threads(ident_t *loc) |
| { |
| KC_TRACE( 10, ("__kmpc_bound_num_threads: called\n" ) ); |
| |
| return __kmp_entry_thread() -> th.th_team -> t.t_nproc; |
| } |
| |
| /*! |
| * @ingroup DEPRECATED |
| * @param loc location description |
| * |
| * This function need not be called. It always returns TRUE. |
| */ |
| kmp_int32 |
| __kmpc_ok_to_fork(ident_t *loc) |
| { |
| #ifndef KMP_DEBUG |
| |
| return TRUE; |
| |
| #else |
| |
| const char *semi2; |
| const char *semi3; |
| int line_no; |
| |
| if (__kmp_par_range == 0) { |
| return TRUE; |
| } |
| semi2 = loc->psource; |
| if (semi2 == NULL) { |
| return TRUE; |
| } |
| semi2 = strchr(semi2, ';'); |
| if (semi2 == NULL) { |
| return TRUE; |
| } |
| semi2 = strchr(semi2 + 1, ';'); |
| if (semi2 == NULL) { |
| return TRUE; |
| } |
| if (__kmp_par_range_filename[0]) { |
| const char *name = semi2 - 1; |
| while ((name > loc->psource) && (*name != '/') && (*name != ';')) { |
| name--; |
| } |
| if ((*name == '/') || (*name == ';')) { |
| name++; |
| } |
| if (strncmp(__kmp_par_range_filename, name, semi2 - name)) { |
| return __kmp_par_range < 0; |
| } |
| } |
| semi3 = strchr(semi2 + 1, ';'); |
| if (__kmp_par_range_routine[0]) { |
| if ((semi3 != NULL) && (semi3 > semi2) |
| && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) { |
| return __kmp_par_range < 0; |
| } |
| } |
| if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) { |
| if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) { |
| return __kmp_par_range > 0; |
| } |
| return __kmp_par_range < 0; |
| } |
| return TRUE; |
| |
| #endif /* KMP_DEBUG */ |
| |
| } |
| |
| /*! |
| @ingroup THREAD_STATES |
| @param loc Source location information. |
| @return 1 if this thread is executing inside an active parallel region, zero if not. |
| */ |
| kmp_int32 |
| __kmpc_in_parallel( ident_t *loc ) |
| { |
| return __kmp_entry_thread() -> th.th_root -> r.r_active; |
| } |
| |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param global_tid global thread number |
| @param num_threads number of threads requested for this parallel construct |
| |
| Set the number of threads to be used by the next fork spawned by this thread. |
| This call is only required if the parallel construct has a `num_threads` clause. |
| */ |
| void |
| __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads ) |
| { |
| KA_TRACE( 20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n", |
| global_tid, num_threads ) ); |
| |
| __kmp_push_num_threads( loc, global_tid, num_threads ); |
| } |
| |
| void |
| __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid ) |
| { |
| KA_TRACE( 20, ("__kmpc_pop_num_threads: enter\n" ) ); |
| |
| /* the num_threads are automatically popped */ |
| } |
| |
| |
| #if OMP_40_ENABLED |
| |
| void |
| __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind ) |
| { |
| KA_TRACE( 20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", |
| global_tid, proc_bind ) ); |
| |
| __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind ); |
| } |
| |
| #endif /* OMP_40_ENABLED */ |
| |
| |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param argc total number of arguments in the ellipsis |
| @param microtask pointer to callback routine consisting of outlined parallel construct |
| @param ... pointers to shared variables that aren't global |
| |
| Do the actual fork and call the microtask in the relevant number of threads. |
| */ |
| void |
| __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) |
| { |
| int gtid = __kmp_entry_gtid(); |
| |
| #if (KMP_STATS_ENABLED) |
| int inParallel = __kmpc_in_parallel(loc); |
| if (inParallel) |
| { |
| KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL); |
| } |
| else |
| { |
| KMP_COUNT_BLOCK(OMP_PARALLEL); |
| } |
| #endif |
| |
| // maybe to save thr_state is enough here |
| { |
| va_list ap; |
| va_start( ap, microtask ); |
| |
| #if OMPT_SUPPORT |
| int tid = __kmp_tid_from_gtid( gtid ); |
| kmp_info_t *master_th = __kmp_threads[ gtid ]; |
| kmp_team_t *parent_team = master_th->th.th_team; |
| if (ompt_enabled) { |
| parent_team->t.t_implicit_task_taskdata[tid]. |
| ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0); |
| } |
| #endif |
| |
| #if INCLUDE_SSC_MARKS |
| SSC_MARK_FORKING(); |
| #endif |
| __kmp_fork_call( loc, gtid, fork_context_intel, |
| argc, |
| #if OMPT_SUPPORT |
| VOLATILE_CAST(void *) microtask, // "unwrapped" task |
| #endif |
| VOLATILE_CAST(microtask_t) microtask, // "wrapped" task |
| VOLATILE_CAST(launch_t) __kmp_invoke_task_func, |
| /* TODO: revert workaround for Intel(R) 64 tracker #96 */ |
| #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX |
| &ap |
| #else |
| ap |
| #endif |
| ); |
| #if INCLUDE_SSC_MARKS |
| SSC_MARK_JOINING(); |
| #endif |
| __kmp_join_call( loc, gtid |
| #if OMPT_SUPPORT |
| , fork_context_intel |
| #endif |
| ); |
| |
| va_end( ap ); |
| |
| #if OMPT_SUPPORT |
| if (ompt_enabled) { |
| parent_team->t.t_implicit_task_taskdata[tid]. |
| ompt_task_info.frame.reenter_runtime_frame = 0; |
| } |
| #endif |
| } |
| } |
| |
| #if OMP_40_ENABLED |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param global_tid global thread number |
| @param num_teams number of teams requested for the teams construct |
| @param num_threads number of threads per team requested for the teams construct |
| |
| Set the number of teams to be used by the teams construct. |
| This call is only required if the teams construct has a `num_teams` clause |
| or a `thread_limit` clause (or both). |
| */ |
| void |
| __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads ) |
| { |
| KA_TRACE( 20, ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n", |
| global_tid, num_teams, num_threads ) ); |
| |
| __kmp_push_num_teams( loc, global_tid, num_teams, num_threads ); |
| } |
| |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param argc total number of arguments in the ellipsis |
| @param microtask pointer to callback routine consisting of outlined teams construct |
| @param ... pointers to shared variables that aren't global |
| |
| Do the actual fork and call the microtask in the relevant number of threads. |
| */ |
| void |
| __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) |
| { |
| int gtid = __kmp_entry_gtid(); |
| kmp_info_t *this_thr = __kmp_threads[ gtid ]; |
| va_list ap; |
| va_start( ap, microtask ); |
| |
| KMP_COUNT_BLOCK(OMP_TEAMS); |
| |
| // remember teams entry point and nesting level |
| this_thr->th.th_teams_microtask = microtask; |
| this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level; // AC: can be >0 on host |
| |
| #if OMPT_SUPPORT |
| kmp_team_t *parent_team = this_thr->th.th_team; |
| int tid = __kmp_tid_from_gtid( gtid ); |
| if (ompt_enabled) { |
| parent_team->t.t_implicit_task_taskdata[tid]. |
| ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(0); |
| } |
| #endif |
| |
| // check if __kmpc_push_num_teams called, set default number of teams otherwise |
| if ( this_thr->th.th_teams_size.nteams == 0 ) { |
| __kmp_push_num_teams( loc, gtid, 0, 0 ); |
| } |
| KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1); |
| KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1); |
| KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1); |
| |
| __kmp_fork_call( loc, gtid, fork_context_intel, |
| argc, |
| #if OMPT_SUPPORT |
| VOLATILE_CAST(void *) microtask, // "unwrapped" task |
| #endif |
| VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task |
| VOLATILE_CAST(launch_t) __kmp_invoke_teams_master, |
| #if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX |
| &ap |
| #else |
| ap |
| #endif |
| ); |
| __kmp_join_call( loc, gtid |
| #if OMPT_SUPPORT |
| , fork_context_intel |
| #endif |
| ); |
| |
| #if OMPT_SUPPORT |
| if (ompt_enabled) { |
| parent_team->t.t_implicit_task_taskdata[tid]. |
| ompt_task_info.frame.reenter_runtime_frame = NULL; |
| } |
| #endif |
| |
| this_thr->th.th_teams_microtask = NULL; |
| this_thr->th.th_teams_level = 0; |
| *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L; |
| va_end( ap ); |
| } |
| #endif /* OMP_40_ENABLED */ |
| |
| |
| // |
| // I don't think this function should ever have been exported. |
| // The __kmpc_ prefix was misapplied. I'm fairly certain that no generated |
| // openmp code ever called it, but it's been exported from the RTL for so |
| // long that I'm afraid to remove the definition. |
| // |
| int |
| __kmpc_invoke_task_func( int gtid ) |
| { |
| return __kmp_invoke_task_func( gtid ); |
| } |
| |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param global_tid global thread number |
| |
| Enter a serialized parallel construct. This interface is used to handle a |
| conditional parallel region, like this, |
| @code |
| #pragma omp parallel if (condition) |
| @endcode |
| when the condition is false. |
| */ |
| void |
| __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) |
| { |
| __kmp_serialized_parallel(loc, global_tid); /* The implementation is now in kmp_runtime.c so that it can share static functions with |
| * kmp_fork_call since the tasks to be done are similar in each case. |
| */ |
| } |
| |
| /*! |
| @ingroup PARALLEL |
| @param loc source location information |
| @param global_tid global thread number |
| |
| Leave a serialized parallel construct. |
| */ |
| void |
| __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) |
| { |
| kmp_internal_control_t *top; |
| kmp_info_t *this_thr; |
| kmp_team_t *serial_team; |
| |
| KC_TRACE( 10, ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) ); |
| |
| /* skip all this code for autopar serialized loops since it results in |
| unacceptable overhead */ |
| if( loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR ) ) |
| return; |
| |
| // Not autopar code |
| if( ! TCR_4( __kmp_init_parallel ) ) |
| __kmp_parallel_initialize(); |
| |
| this_thr = __kmp_threads[ global_tid ]; |
| serial_team = this_thr->th.th_serial_team; |
| |
| #if OMP_45_ENABLED |
| kmp_task_team_t * task_team = this_thr->th.th_task_team; |
| |
| // we need to wait for the proxy tasks before finishing the thread |
| if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks ) |
| __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL) ); // is an ITT object needed here? |
| #endif |
| |
| KMP_MB(); |
| KMP_DEBUG_ASSERT( serial_team ); |
| KMP_ASSERT( serial_team -> t.t_serialized ); |
| KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team ); |
| KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team ); |
| KMP_DEBUG_ASSERT( serial_team -> t.t_threads ); |
| KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr ); |
| |
| /* If necessary, pop the internal control stack values and replace the team values */ |
| top = serial_team -> t.t_control_stack_top; |
| if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) { |
| copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top ); |
| serial_team -> t.t_control_stack_top = top -> next; |
| __kmp_free(top); |
| } |
| |
| //if( serial_team -> t.t_serialized > 1 ) |
| serial_team -> t.t_level--; |
| |
| /* pop dispatch buffers stack */ |
| KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer); |
| { |
| dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer; |
| serial_team->t.t_dispatch->th_disp_buffer = |
| serial_team->t.t_dispatch->th_disp_buffer->next; |
| __kmp_free( disp_buffer ); |
| } |
| |
| -- serial_team -> t.t_serialized; |
| if ( serial_team -> t.t_serialized == 0 ) { |
| |
| /* return to the parallel section */ |
| |
| #if KMP_ARCH_X86 || KMP_ARCH_X86_64 |
| if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) { |
| __kmp_clear_x87_fpu_status_word(); |
| __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word ); |
| __kmp_load_mxcsr( &serial_team->t.t_mxcsr ); |
| } |
| #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */ |
| |
| this_thr -> th.th_team = serial_team -> t.t_parent; |
| this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid; |
| |
| /* restore values cached in the thread */ |
| this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc; /* JPH */ |
| this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0]; /* JPH */ |
| this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized; |
| |
| /* TODO the below shouldn't need to be adjusted for serialized teams */ |
| this_thr -> th.th_dispatch = & this_thr -> th.th_team -> |
| t.t_dispatch[ serial_team -> t.t_master_tid ]; |
| |
| __kmp_pop_current_task_from_thread( this_thr ); |
| |
| KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 ); |
| this_thr -> th.th_current_task -> td_flags.executing = 1; |
| |
| if ( __kmp_tasking_mode != tskm_immediate_exec ) { |
| // Copy the task team from the new child / old parent team to the thread. |
| this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]; |
| KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n", |
| global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) ); |
| } |
| } else { |
| if ( __kmp_tasking_mode != tskm_immediate_exec ) { |
| KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n", |
| global_tid, serial_team, serial_team -> t.t_serialized ) ); |
| } |
| } |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_parallel( global_tid, NULL ); |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information. |
| |
| Execute <tt>flush</tt>. This is implemented as a full memory fence. (Though |
| depending on the memory ordering convention obeyed by the compiler |
| even that may not be necessary). |
| */ |
| void |
| __kmpc_flush(ident_t *loc) |
| { |
| KC_TRACE( 10, ("__kmpc_flush: called\n" ) ); |
| |
| /* need explicit __mf() here since use volatile instead in library */ |
| KMP_MB(); /* Flush all pending memory write invalidates. */ |
| |
| #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 ) |
| #if KMP_MIC |
| // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used. |
| // We shouldn't need it, though, since the ABI rules require that |
| // * If the compiler generates NGO stores it also generates the fence |
| // * If users hand-code NGO stores they should insert the fence |
| // therefore no incomplete unordered stores should be visible. |
| #else |
| // C74404 |
| // This is to address non-temporal store instructions (sfence needed). |
| // The clflush instruction is addressed either (mfence needed). |
| // Probably the non-temporal load monvtdqa instruction should also be addressed. |
| // mfence is a SSE2 instruction. Do not execute it if CPU is not SSE2. |
| if ( ! __kmp_cpuinfo.initialized ) { |
| __kmp_query_cpuid( & __kmp_cpuinfo ); |
| }; // if |
| if ( ! __kmp_cpuinfo.sse2 ) { |
| // CPU cannot execute SSE2 instructions. |
| } else { |
| #if KMP_COMPILER_ICC |
| _mm_mfence(); |
| #elif KMP_COMPILER_MSVC |
| MemoryBarrier(); |
| #else |
| __sync_synchronize(); |
| #endif // KMP_COMPILER_ICC |
| }; // if |
| #endif // KMP_MIC |
| #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64) |
| // Nothing to see here move along |
| #elif KMP_ARCH_PPC64 |
| // Nothing needed here (we have a real MB above). |
| #if KMP_OS_CNK |
| // The flushing thread needs to yield here; this prevents a |
| // busy-waiting thread from saturating the pipeline. flush is |
| // often used in loops like this: |
| // while (!flag) { |
| // #pragma omp flush(flag) |
| // } |
| // and adding the yield here is good for at least a 10x speedup |
| // when running >2 threads per core (on the NAS LU benchmark). |
| __kmp_yield(TRUE); |
| #endif |
| #else |
| #error Unknown or unsupported architecture |
| #endif |
| |
| } |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid thread id. |
| |
| Execute a barrier. |
| */ |
| void |
| __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) |
| { |
| KMP_COUNT_BLOCK(OMP_BARRIER); |
| KC_TRACE( 10, ("__kmpc_barrier: called T#%d\n", global_tid ) ); |
| |
| if (! TCR_4(__kmp_init_parallel)) |
| __kmp_parallel_initialize(); |
| |
| if ( __kmp_env_consistency_check ) { |
| if ( loc == 0 ) { |
| KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user? |
| }; // if |
| |
| __kmp_check_barrier( global_tid, ct_barrier, loc ); |
| } |
| |
| __kmp_threads[ global_tid ]->th.th_ident = loc; |
| // TODO: explicit barrier_wait_id: |
| // this function is called when 'barrier' directive is present or |
| // implicit barrier at the end of a worksharing construct. |
| // 1) better to add a per-thread barrier counter to a thread data structure |
| // 2) set to 0 when a new team is created |
| // 4) no sync is required |
| |
| __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL ); |
| } |
| |
| /* The BARRIER for a MASTER section is always explicit */ |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param global_tid global thread number . |
| @return 1 if this thread should execute the <tt>master</tt> block, 0 otherwise. |
| */ |
| kmp_int32 |
| __kmpc_master(ident_t *loc, kmp_int32 global_tid) |
| { |
| int status = 0; |
| |
| KC_TRACE( 10, ("__kmpc_master: called T#%d\n", global_tid ) ); |
| |
| if( ! TCR_4( __kmp_init_parallel ) ) |
| __kmp_parallel_initialize(); |
| |
| if( KMP_MASTER_GTID( global_tid )) { |
| KMP_COUNT_BLOCK(OMP_MASTER); |
| KMP_PUSH_PARTITIONED_TIMER(OMP_master); |
| status = 1; |
| } |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (status) { |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_master_begin)) { |
| kmp_info_t *this_thr = __kmp_threads[ global_tid ]; |
| kmp_team_t *team = this_thr -> th.th_team; |
| |
| int tid = __kmp_tid_from_gtid( global_tid ); |
| ompt_callbacks.ompt_callback(ompt_event_master_begin)( |
| team->t.ompt_team_info.parallel_id, |
| team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); |
| } |
| } |
| #endif |
| |
| if ( __kmp_env_consistency_check ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| if (status) |
| __kmp_push_sync( global_tid, ct_master, loc, NULL, 0 ); |
| else |
| __kmp_check_sync( global_tid, ct_master, loc, NULL, 0 ); |
| #else |
| if (status) |
| __kmp_push_sync( global_tid, ct_master, loc, NULL ); |
| else |
| __kmp_check_sync( global_tid, ct_master, loc, NULL ); |
| #endif |
| } |
| |
| return status; |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param global_tid global thread number . |
| |
| Mark the end of a <tt>master</tt> region. This should only be called by the thread |
| that executes the <tt>master</tt> region. |
| */ |
| void |
| __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) |
| { |
| KC_TRACE( 10, ("__kmpc_end_master: called T#%d\n", global_tid ) ); |
| |
| KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid )); |
| KMP_POP_PARTITIONED_TIMER(); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| kmp_info_t *this_thr = __kmp_threads[ global_tid ]; |
| kmp_team_t *team = this_thr -> th.th_team; |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_master_end)) { |
| int tid = __kmp_tid_from_gtid( global_tid ); |
| ompt_callbacks.ompt_callback(ompt_event_master_end)( |
| team->t.ompt_team_info.parallel_id, |
| team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); |
| } |
| #endif |
| |
| if ( __kmp_env_consistency_check ) { |
| if( global_tid < 0 ) |
| KMP_WARNING( ThreadIdentInvalid ); |
| |
| if( KMP_MASTER_GTID( global_tid )) |
| __kmp_pop_sync( global_tid, ct_master, loc ); |
| } |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param gtid global thread number. |
| |
| Start execution of an <tt>ordered</tt> construct. |
| */ |
| void |
| __kmpc_ordered( ident_t * loc, kmp_int32 gtid ) |
| { |
| int cid = 0; |
| kmp_info_t *th; |
| KMP_DEBUG_ASSERT( __kmp_init_serial ); |
| |
| KC_TRACE( 10, ("__kmpc_ordered: called T#%d\n", gtid )); |
| |
| if (! TCR_4(__kmp_init_parallel)) |
| __kmp_parallel_initialize(); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_ordered_prep( gtid ); |
| // TODO: ordered_wait_id |
| #endif /* USE_ITT_BUILD */ |
| |
| th = __kmp_threads[ gtid ]; |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled) { |
| /* OMPT state update */ |
| th->th.ompt_thread_info.wait_id = (uint64_t) loc; |
| th->th.ompt_thread_info.state = ompt_state_wait_ordered; |
| |
| /* OMPT event callback */ |
| if (ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) { |
| ompt_callbacks.ompt_callback(ompt_event_wait_ordered)( |
| th->th.ompt_thread_info.wait_id); |
| } |
| } |
| #endif |
| |
| if ( th -> th.th_dispatch -> th_deo_fcn != 0 ) |
| (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc ); |
| else |
| __kmp_parallel_deo( & gtid, & cid, loc ); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled) { |
| /* OMPT state update */ |
| th->th.ompt_thread_info.state = ompt_state_work_parallel; |
| th->th.ompt_thread_info.wait_id = 0; |
| |
| /* OMPT event callback */ |
| if (ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) { |
| ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)( |
| th->th.ompt_thread_info.wait_id); |
| } |
| } |
| #endif |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_ordered_start( gtid ); |
| #endif /* USE_ITT_BUILD */ |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param gtid global thread number. |
| |
| End execution of an <tt>ordered</tt> construct. |
| */ |
| void |
| __kmpc_end_ordered( ident_t * loc, kmp_int32 gtid ) |
| { |
| int cid = 0; |
| kmp_info_t *th; |
| |
| KC_TRACE( 10, ("__kmpc_end_ordered: called T#%d\n", gtid ) ); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_ordered_end( gtid ); |
| // TODO: ordered_wait_id |
| #endif /* USE_ITT_BUILD */ |
| |
| th = __kmp_threads[ gtid ]; |
| |
| if ( th -> th.th_dispatch -> th_dxo_fcn != 0 ) |
| (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc ); |
| else |
| __kmp_parallel_dxo( & gtid, & cid, loc ); |
| |
| #if OMPT_SUPPORT && OMPT_BLAME |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_release_ordered)) { |
| ompt_callbacks.ompt_callback(ompt_event_release_ordered)( |
| th->th.ompt_thread_info.wait_id); |
| } |
| #endif |
| } |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| static __forceinline void |
| __kmp_init_indirect_csptr(kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid, kmp_indirect_locktag_t tag) |
| { |
| // Pointer to the allocated indirect lock is written to crit, while indexing is ignored. |
| void *idx; |
| kmp_indirect_lock_t **lck; |
| lck = (kmp_indirect_lock_t **)crit; |
| kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag); |
| KMP_I_LOCK_FUNC(ilk, init)(ilk->lock); |
| KMP_SET_I_LOCK_LOCATION(ilk, loc); |
| KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section); |
| KA_TRACE(20, ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag)); |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_creating(ilk->lock, loc); |
| #endif |
| int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk); |
| if (status == 0) { |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_destroyed(ilk->lock); |
| #endif |
| // We don't really need to destroy the unclaimed lock here since it will be cleaned up at program exit. |
| //KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx); |
| } |
| KMP_DEBUG_ASSERT(*lck != NULL); |
| } |
| |
| // Fast-path acquire tas lock |
| #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) { \ |
| kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ |
| if (l->lk.poll != KMP_LOCK_FREE(tas) || \ |
| ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \ |
| kmp_uint32 spins; \ |
| KMP_FSYNC_PREPARE(l); \ |
| KMP_INIT_YIELD(spins); \ |
| if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ |
| KMP_YIELD(TRUE); \ |
| } else { \ |
| KMP_YIELD_SPIN(spins); \ |
| } \ |
| kmp_backoff_t backoff = __kmp_spin_backoff_params; \ |
| while (l->lk.poll != KMP_LOCK_FREE(tas) || \ |
| ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \ |
| __kmp_spin_backoff(&backoff); \ |
| if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \ |
| KMP_YIELD(TRUE); \ |
| } else { \ |
| KMP_YIELD_SPIN(spins); \ |
| } \ |
| } \ |
| } \ |
| KMP_FSYNC_ACQUIRED(l); \ |
| } |
| |
| // Fast-path test tas lock |
| #define KMP_TEST_TAS_LOCK(lock, gtid, rc) { \ |
| kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \ |
| rc = l->lk.poll == KMP_LOCK_FREE(tas) && \ |
| KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas)); \ |
| } |
| |
| // Fast-path release tas lock |
| #define KMP_RELEASE_TAS_LOCK(lock, gtid) { \ |
| TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); \ |
| KMP_MB(); \ |
| } |
| |
| #if KMP_USE_FUTEX |
| |
| # include <unistd.h> |
| # include <sys/syscall.h> |
| # ifndef FUTEX_WAIT |
| # define FUTEX_WAIT 0 |
| # endif |
| # ifndef FUTEX_WAKE |
| # define FUTEX_WAKE 1 |
| # endif |
| |
| // Fast-path acquire futex lock |
| #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) { \ |
| kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ |
| kmp_int32 gtid_code = (gtid+1) << 1; \ |
| KMP_MB(); \ |
| KMP_FSYNC_PREPARE(ftx); \ |
| kmp_int32 poll_val; \ |
| while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \ |
| KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \ |
| kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \ |
| if (!cond) { \ |
| if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | KMP_LOCK_BUSY(1, futex))) { \ |
| continue; \ |
| } \ |
| poll_val |= KMP_LOCK_BUSY(1, futex); \ |
| } \ |
| kmp_int32 rc; \ |
| if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \ |
| continue; \ |
| } \ |
| gtid_code |= 1; \ |
| } \ |
| KMP_FSYNC_ACQUIRED(ftx); \ |
| } |
| |
| // Fast-path test futex lock |
| #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) { \ |
| kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ |
| if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1 << 1, futex))) { \ |
| KMP_FSYNC_ACQUIRED(ftx); \ |
| rc = TRUE; \ |
| } else { \ |
| rc = FALSE; \ |
| } \ |
| } |
| |
| // Fast-path release futex lock |
| #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) { \ |
| kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \ |
| KMP_MB(); \ |
| KMP_FSYNC_RELEASING(ftx); \ |
| kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \ |
| if (KMP_LOCK_STRIP(poll_val) & 1) { \ |
| syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \ |
| } \ |
| KMP_MB(); \ |
| KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \ |
| } |
| |
| #endif // KMP_USE_FUTEX |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| static kmp_user_lock_p |
| __kmp_get_critical_section_ptr( kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid ) |
| { |
| kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit; |
| |
| // |
| // Because of the double-check, the following load |
| // doesn't need to be volatile. |
| // |
| kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp ); |
| |
| if ( lck == NULL ) { |
| void * idx; |
| |
| // Allocate & initialize the lock. |
| // Remember allocated locks in table in order to free them in __kmp_cleanup() |
| lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section ); |
| __kmp_init_user_lock_with_checks( lck ); |
| __kmp_set_user_lock_location( lck, loc ); |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_creating( lck ); |
| // __kmp_itt_critical_creating() should be called *before* the first usage of underlying |
| // lock. It is the only place where we can guarantee it. There are chances the lock will |
| // destroyed with no usage, but it is not a problem, because this is not real event seen |
| // by user but rather setting name for object (lock). See more details in kmp_itt.h. |
| #endif /* USE_ITT_BUILD */ |
| |
| // |
| // Use a cmpxchg instruction to slam the start of the critical |
| // section with the lock pointer. If another thread beat us |
| // to it, deallocate the lock, and use the lock that the other |
| // thread allocated. |
| // |
| int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck ); |
| |
| if ( status == 0 ) { |
| // Deallocate the lock and reload the value. |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_destroyed( lck ); |
| // Let ITT know the lock is destroyed and the same memory location may be reused for |
| // another purpose. |
| #endif /* USE_ITT_BUILD */ |
| __kmp_destroy_user_lock_with_checks( lck ); |
| __kmp_user_lock_free( &idx, gtid, lck ); |
| lck = (kmp_user_lock_p)TCR_PTR( *lck_pp ); |
| KMP_DEBUG_ASSERT( lck != NULL ); |
| } |
| } |
| return lck; |
| } |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param global_tid global thread number . |
| @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or |
| some other suitably unique value. |
| |
| Enter code protected by a `critical` construct. |
| This function blocks until the executing thread can enter the critical section. |
| */ |
| void |
| __kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) |
| { |
| #if KMP_USE_DYNAMIC_LOCK |
| __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none); |
| #else |
| KMP_COUNT_BLOCK(OMP_CRITICAL); |
| KMP_TIME_PARTITIONED_BLOCK(OMP_critical_wait); /* Time spent waiting to enter the critical section */ |
| kmp_user_lock_p lck; |
| |
| KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) ); |
| |
| //TODO: add THR_OVHD_STATE |
| |
| KMP_CHECK_USER_LOCK_INIT(); |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) { |
| lck = (kmp_user_lock_p)crit; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) { |
| lck = (kmp_user_lock_p)crit; |
| } |
| #endif |
| else { // ticket, queuing or drdpa |
| lck = __kmp_get_critical_section_ptr( crit, loc, global_tid ); |
| } |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_critical, loc, lck ); |
| |
| /* since the critical directive binds to all threads, not just |
| * the current team we have to check this even if we are in a |
| * serialized team */ |
| /* also, even if we are the uber thread, we still have to conduct the lock, |
| * as we have to contend with sibling threads */ |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_acquiring( lck ); |
| #endif /* USE_ITT_BUILD */ |
| // Value of 'crit' should be good for using as a critical_id of the critical section directive. |
| __kmp_acquire_user_lock_with_checks( lck, global_tid ); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_acquired( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| KMP_START_EXPLICIT_TIMER(OMP_critical); |
| KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid )); |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| // Converts the given hint to an internal lock implementation |
| static __forceinline kmp_dyna_lockseq_t |
| __kmp_map_hint_to_lock(uintptr_t hint) |
| { |
| #if KMP_USE_TSX |
| # define KMP_TSX_LOCK(seq) lockseq_##seq |
| #else |
| # define KMP_TSX_LOCK(seq) __kmp_user_lock_seq |
| #endif |
| |
| #if KMP_ARCH_X86 || KMP_ARCH_X86_64 |
| # define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm) |
| #else |
| # define KMP_CPUINFO_RTM 0 |
| #endif |
| |
| // Hints that do not require further logic |
| if (hint & kmp_lock_hint_hle) |
| return KMP_TSX_LOCK(hle); |
| if (hint & kmp_lock_hint_rtm) |
| return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq; |
| if (hint & kmp_lock_hint_adaptive) |
| return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq; |
| |
| // Rule out conflicting hints first by returning the default lock |
| if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended)) |
| return __kmp_user_lock_seq; |
| if ((hint & omp_lock_hint_speculative) && (hint & omp_lock_hint_nonspeculative)) |
| return __kmp_user_lock_seq; |
| |
| // Do not even consider speculation when it appears to be contended |
| if (hint & omp_lock_hint_contended) |
| return lockseq_queuing; |
| |
| // Uncontended lock without speculation |
| if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative)) |
| return lockseq_tas; |
| |
| // HLE lock for speculation |
| if (hint & omp_lock_hint_speculative) |
| return KMP_TSX_LOCK(hle); |
| |
| return __kmp_user_lock_seq; |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param global_tid global thread number. |
| @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, |
| or some other suitably unique value. |
| @param hint the lock hint. |
| |
| Enter code protected by a `critical` construct with a hint. The hint value is used to suggest a lock implementation. |
| This function blocks until the executing thread can enter the critical section unless the hint suggests use of |
| speculative execution and the hardware supports it. |
| */ |
| void |
| __kmpc_critical_with_hint( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit, uintptr_t hint ) |
| { |
| KMP_COUNT_BLOCK(OMP_CRITICAL); |
| kmp_user_lock_p lck; |
| |
| KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) ); |
| |
| kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit; |
| // Check if it is initialized. |
| if (*lk == 0) { |
| kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint); |
| if (KMP_IS_D_LOCK(lckseq)) { |
| KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(lckseq)); |
| } else { |
| __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq)); |
| } |
| } |
| // Branch for accessing the actual lock object and set operation. This branching is inevitable since |
| // this lock initialization does not follow the normal dispatch path (lock table is not used). |
| if (KMP_EXTRACT_D_TAG(lk) != 0) { |
| lck = (kmp_user_lock_p)lk; |
| if (__kmp_env_consistency_check) { |
| __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint)); |
| } |
| # if USE_ITT_BUILD |
| __kmp_itt_critical_acquiring(lck); |
| # endif |
| # if KMP_USE_INLINED_TAS |
| if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) { |
| KMP_ACQUIRE_TAS_LOCK(lck, global_tid); |
| } else |
| # elif KMP_USE_INLINED_FUTEX |
| if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) { |
| KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid); |
| } else |
| # endif |
| { |
| KMP_D_LOCK_FUNC(lk, set)(lk, global_tid); |
| } |
| } else { |
| kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk); |
| lck = ilk->lock; |
| if (__kmp_env_consistency_check) { |
| __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint)); |
| } |
| # if USE_ITT_BUILD |
| __kmp_itt_critical_acquiring(lck); |
| # endif |
| KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_acquired( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| KMP_PUSH_PARTITIONED_TIMER(OMP_critical); |
| KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid )); |
| } // __kmpc_critical_with_hint |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param global_tid global thread number . |
| @param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or |
| some other suitably unique value. |
| |
| Leave a critical section, releasing any lock that was held during its execution. |
| */ |
| void |
| __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit) |
| { |
| kmp_user_lock_p lck; |
| |
| KC_TRACE( 10, ("__kmpc_end_critical: called T#%d\n", global_tid )); |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { |
| lck = (kmp_user_lock_p)crit; |
| KMP_ASSERT(lck != NULL); |
| if (__kmp_env_consistency_check) { |
| __kmp_pop_sync(global_tid, ct_critical, loc); |
| } |
| # if USE_ITT_BUILD |
| __kmp_itt_critical_releasing( lck ); |
| # endif |
| # if KMP_USE_INLINED_TAS |
| if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) { |
| KMP_RELEASE_TAS_LOCK(lck, global_tid); |
| } else |
| # elif KMP_USE_INLINED_FUTEX |
| if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) { |
| KMP_RELEASE_FUTEX_LOCK(lck, global_tid); |
| } else |
| # endif |
| { |
| KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid); |
| } |
| } else { |
| kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit)); |
| KMP_ASSERT(ilk != NULL); |
| lck = ilk->lock; |
| if (__kmp_env_consistency_check) { |
| __kmp_pop_sync(global_tid, ct_critical, loc); |
| } |
| # if USE_ITT_BUILD |
| __kmp_itt_critical_releasing( lck ); |
| # endif |
| KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid); |
| } |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) { |
| lck = (kmp_user_lock_p)crit; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) { |
| lck = (kmp_user_lock_p)crit; |
| } |
| #endif |
| else { // ticket, queuing or drdpa |
| lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit)); |
| } |
| |
| KMP_ASSERT(lck != NULL); |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_sync( global_tid, ct_critical, loc ); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_critical_releasing( lck ); |
| #endif /* USE_ITT_BUILD */ |
| // Value of 'crit' should be good for using as a critical_id of the critical section directive. |
| __kmp_release_user_lock_with_checks( lck, global_tid ); |
| |
| #if OMPT_SUPPORT && OMPT_BLAME |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_release_critical)) { |
| ompt_callbacks.ompt_callback(ompt_event_release_critical)( |
| (uint64_t) lck); |
| } |
| #endif |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| KMP_POP_PARTITIONED_TIMER(); |
| KA_TRACE( 15, ("__kmpc_end_critical: done T#%d\n", global_tid )); |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid thread id. |
| @return one if the thread should execute the master block, zero otherwise |
| |
| Start execution of a combined barrier and master. The barrier is executed inside this function. |
| */ |
| kmp_int32 |
| __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) |
| { |
| int status; |
| |
| KC_TRACE( 10, ("__kmpc_barrier_master: called T#%d\n", global_tid ) ); |
| |
| if (! TCR_4(__kmp_init_parallel)) |
| __kmp_parallel_initialize(); |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_check_barrier( global_tid, ct_barrier, loc ); |
| |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL ); |
| |
| return (status != 0) ? 0 : 1; |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid thread id. |
| |
| Complete the execution of a combined barrier and master. This function should |
| only be called at the completion of the <tt>master</tt> code. Other threads will |
| still be waiting at the barrier and this call releases them. |
| */ |
| void |
| __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid) |
| { |
| KC_TRACE( 10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid )); |
| |
| __kmp_end_split_barrier ( bs_plain_barrier, global_tid ); |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid thread id. |
| @return one if the thread should execute the master block, zero otherwise |
| |
| Start execution of a combined barrier and master(nowait) construct. |
| The barrier is executed inside this function. |
| There is no equivalent "end" function, since the |
| */ |
| kmp_int32 |
| __kmpc_barrier_master_nowait( ident_t * loc, kmp_int32 global_tid ) |
| { |
| kmp_int32 ret; |
| |
| KC_TRACE( 10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid )); |
| |
| if (! TCR_4(__kmp_init_parallel)) |
| __kmp_parallel_initialize(); |
| |
| if ( __kmp_env_consistency_check ) { |
| if ( loc == 0 ) { |
| KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user? |
| } |
| __kmp_check_barrier( global_tid, ct_barrier, loc ); |
| } |
| |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL ); |
| |
| ret = __kmpc_master (loc, global_tid); |
| |
| if ( __kmp_env_consistency_check ) { |
| /* there's no __kmpc_end_master called; so the (stats) */ |
| /* actions of __kmpc_end_master are done here */ |
| |
| if ( global_tid < 0 ) { |
| KMP_WARNING( ThreadIdentInvalid ); |
| } |
| if (ret) { |
| /* only one thread should do the pop since only */ |
| /* one did the push (see __kmpc_master()) */ |
| |
| __kmp_pop_sync( global_tid, ct_master, loc ); |
| } |
| } |
| |
| return (ret); |
| } |
| |
| /* The BARRIER for a SINGLE process section is always explicit */ |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information |
| @param global_tid global thread number |
| @return One if this thread should execute the single construct, zero otherwise. |
| |
| Test whether to execute a <tt>single</tt> construct. |
| There are no implicit barriers in the two "single" calls, rather the compiler should |
| introduce an explicit barrier if it is required. |
| */ |
| |
| kmp_int32 |
| __kmpc_single(ident_t *loc, kmp_int32 global_tid) |
| { |
| kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE ); |
| |
| if (rc) { |
| // We are going to execute the single statement, so we should count it. |
| KMP_COUNT_BLOCK(OMP_SINGLE); |
| KMP_PUSH_PARTITIONED_TIMER(OMP_single); |
| } |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| kmp_info_t *this_thr = __kmp_threads[ global_tid ]; |
| kmp_team_t *team = this_thr -> th.th_team; |
| int tid = __kmp_tid_from_gtid( global_tid ); |
| |
| if (ompt_enabled) { |
| if (rc) { |
| if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) { |
| ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)( |
| team->t.ompt_team_info.parallel_id, |
| team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id, |
| team->t.ompt_team_info.microtask); |
| } |
| } else { |
| if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) { |
| ompt_callbacks.ompt_callback(ompt_event_single_others_begin)( |
| team->t.ompt_team_info.parallel_id, |
| team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); |
| } |
| this_thr->th.ompt_thread_info.state = ompt_state_wait_single; |
| } |
| } |
| #endif |
| |
| return rc; |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information |
| @param global_tid global thread number |
| |
| Mark the end of a <tt>single</tt> construct. This function should |
| only be called by the thread that executed the block of code protected |
| by the `single` construct. |
| */ |
| void |
| __kmpc_end_single(ident_t *loc, kmp_int32 global_tid) |
| { |
| __kmp_exit_single( global_tid ); |
| KMP_POP_PARTITIONED_TIMER(); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| kmp_info_t *this_thr = __kmp_threads[ global_tid ]; |
| kmp_team_t *team = this_thr -> th.th_team; |
| int tid = __kmp_tid_from_gtid( global_tid ); |
| |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) { |
| ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)( |
| team->t.ompt_team_info.parallel_id, |
| team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id); |
| } |
| #endif |
| } |
| |
| /*! |
| @ingroup WORK_SHARING |
| @param loc Source location |
| @param global_tid Global thread id |
| |
| Mark the end of a statically scheduled loop. |
| */ |
| void |
| __kmpc_for_static_fini( ident_t *loc, kmp_int32 global_tid ) |
| { |
| KE_TRACE( 10, ("__kmpc_for_static_fini called T#%d\n", global_tid)); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_loop_end)) { |
| ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); |
| ompt_task_info_t *task_info = __ompt_get_taskinfo(0); |
| ompt_callbacks.ompt_callback(ompt_event_loop_end)( |
| team_info->parallel_id, task_info->task_id); |
| } |
| #endif |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_workshare( global_tid, ct_pdo, loc ); |
| } |
| |
| /* |
| * User routines which take C-style arguments (call by value) |
| * different from the Fortran equivalent routines |
| */ |
| |
| void |
| ompc_set_num_threads( int arg ) |
| { |
| // !!!!! TODO: check the per-task binding |
| __kmp_set_num_threads( arg, __kmp_entry_gtid() ); |
| } |
| |
| void |
| ompc_set_dynamic( int flag ) |
| { |
| kmp_info_t *thread; |
| |
| /* For the thread-private implementation of the internal controls */ |
| thread = __kmp_entry_thread(); |
| |
| __kmp_save_internal_controls( thread ); |
| |
| set__dynamic( thread, flag ? TRUE : FALSE ); |
| } |
| |
| void |
| ompc_set_nested( int flag ) |
| { |
| kmp_info_t *thread; |
| |
| /* For the thread-private internal controls implementation */ |
| thread = __kmp_entry_thread(); |
| |
| __kmp_save_internal_controls( thread ); |
| |
| set__nested( thread, flag ? TRUE : FALSE ); |
| } |
| |
| void |
| ompc_set_max_active_levels( int max_active_levels ) |
| { |
| /* TO DO */ |
| /* we want per-task implementation of this internal control */ |
| |
| /* For the per-thread internal controls implementation */ |
| __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels ); |
| } |
| |
| void |
| ompc_set_schedule( omp_sched_t kind, int modifier ) |
| { |
| // !!!!! TODO: check the per-task binding |
| __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier ); |
| } |
| |
| int |
| ompc_get_ancestor_thread_num( int level ) |
| { |
| return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level ); |
| } |
| |
| int |
| ompc_get_team_size( int level ) |
| { |
| return __kmp_get_team_size( __kmp_entry_gtid(), level ); |
| } |
| |
| void |
| kmpc_set_stacksize( int arg ) |
| { |
| // __kmp_aux_set_stacksize initializes the library if needed |
| __kmp_aux_set_stacksize( arg ); |
| } |
| |
| void |
| kmpc_set_stacksize_s( size_t arg ) |
| { |
| // __kmp_aux_set_stacksize initializes the library if needed |
| __kmp_aux_set_stacksize( arg ); |
| } |
| |
| void |
| kmpc_set_blocktime( int arg ) |
| { |
| int gtid, tid; |
| kmp_info_t *thread; |
| |
| gtid = __kmp_entry_gtid(); |
| tid = __kmp_tid_from_gtid(gtid); |
| thread = __kmp_thread_from_gtid(gtid); |
| |
| __kmp_aux_set_blocktime( arg, thread, tid ); |
| } |
| |
| void |
| kmpc_set_library( int arg ) |
| { |
| // __kmp_user_set_library initializes the library if needed |
| __kmp_user_set_library( (enum library_type)arg ); |
| } |
| |
| void |
| kmpc_set_defaults( char const * str ) |
| { |
| // __kmp_aux_set_defaults initializes the library if needed |
| __kmp_aux_set_defaults( str, KMP_STRLEN( str ) ); |
| } |
| |
| void |
| kmpc_set_disp_num_buffers( int arg ) |
| { |
| // ignore after initialization because some teams have already |
| // allocated dispatch buffers |
| if( __kmp_init_serial == 0 && arg > 0 ) |
| __kmp_dispatch_num_buffers = arg; |
| } |
| |
| int |
| kmpc_set_affinity_mask_proc( int proc, void **mask ) |
| { |
| #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED |
| return -1; |
| #else |
| if ( ! TCR_4(__kmp_init_middle) ) { |
| __kmp_middle_initialize(); |
| } |
| return __kmp_aux_set_affinity_mask_proc( proc, mask ); |
| #endif |
| } |
| |
| int |
| kmpc_unset_affinity_mask_proc( int proc, void **mask ) |
| { |
| #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED |
| return -1; |
| #else |
| if ( ! TCR_4(__kmp_init_middle) ) { |
| __kmp_middle_initialize(); |
| } |
| return __kmp_aux_unset_affinity_mask_proc( proc, mask ); |
| #endif |
| } |
| |
| int |
| kmpc_get_affinity_mask_proc( int proc, void **mask ) |
| { |
| #if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED |
| return -1; |
| #else |
| if ( ! TCR_4(__kmp_init_middle) ) { |
| __kmp_middle_initialize(); |
| } |
| return __kmp_aux_get_affinity_mask_proc( proc, mask ); |
| #endif |
| } |
| |
| |
| /* -------------------------------------------------------------------------- */ |
| /*! |
| @ingroup THREADPRIVATE |
| @param loc source location information |
| @param gtid global thread number |
| @param cpy_size size of the cpy_data buffer |
| @param cpy_data pointer to data to be copied |
| @param cpy_func helper function to call for copying data |
| @param didit flag variable: 1=single thread; 0=not single thread |
| |
| __kmpc_copyprivate implements the interface for the private data broadcast needed for |
| the copyprivate clause associated with a single region in an OpenMP<sup>*</sup> program (both C and Fortran). |
| All threads participating in the parallel region call this routine. |
| One of the threads (called the single thread) should have the <tt>didit</tt> variable set to 1 |
| and all other threads should have that variable set to 0. |
| All threads pass a pointer to a data buffer (cpy_data) that they have built. |
| |
| The OpenMP specification forbids the use of nowait on the single region when a copyprivate |
| clause is present. However, @ref __kmpc_copyprivate implements a barrier internally to avoid |
| race conditions, so the code generation for the single region should avoid generating a barrier |
| after the call to @ref __kmpc_copyprivate. |
| |
| The <tt>gtid</tt> parameter is the global thread id for the current thread. |
| The <tt>loc</tt> parameter is a pointer to source location information. |
| |
| Internal implementation: The single thread will first copy its descriptor address (cpy_data) |
| to a team-private location, then the other threads will each call the function pointed to by |
| the parameter cpy_func, which carries out the copy by copying the data using the cpy_data buffer. |
| |
| The cpy_func routine used for the copy and the contents of the data area defined by cpy_data |
| and cpy_size may be built in any fashion that will allow the copy to be done. For instance, |
| the cpy_data buffer can hold the actual data to be copied or it may hold a list of pointers |
| to the data. The cpy_func routine must interpret the cpy_data buffer appropriately. |
| |
| The interface to cpy_func is as follows: |
| @code |
| void cpy_func( void *destination, void *source ) |
| @endcode |
| where void *destination is the cpy_data pointer for the thread being copied to |
| and void *source is the cpy_data pointer for the thread being copied from. |
| */ |
| void |
| __kmpc_copyprivate( ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void*,void*), kmp_int32 didit ) |
| { |
| void **data_ptr; |
| |
| KC_TRACE( 10, ("__kmpc_copyprivate: called T#%d\n", gtid )); |
| |
| KMP_MB(); |
| |
| data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data; |
| |
| if ( __kmp_env_consistency_check ) { |
| if ( loc == 0 ) { |
| KMP_WARNING( ConstructIdentInvalid ); |
| } |
| } |
| |
| /* ToDo: Optimize the following two barriers into some kind of split barrier */ |
| |
| if (didit) *data_ptr = cpy_data; |
| |
| /* This barrier is not a barrier region boundary */ |
| #if USE_ITT_NOTIFY |
| __kmp_threads[gtid]->th.th_ident = loc; |
| #endif |
| __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL ); |
| |
| if (! didit) (*cpy_func)( cpy_data, *data_ptr ); |
| |
| /* Consider next barrier the user-visible barrier for barrier region boundaries */ |
| /* Nesting checks are already handled by the single construct checks */ |
| |
| #if USE_ITT_NOTIFY |
| __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g. tasks can overwrite the location) |
| #endif |
| __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL ); |
| } |
| |
| /* -------------------------------------------------------------------------- */ |
| |
| #define INIT_LOCK __kmp_init_user_lock_with_checks |
| #define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks |
| #define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks |
| #define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed |
| #define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks |
| #define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed |
| #define RELEASE_LOCK __kmp_release_user_lock_with_checks |
| #define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks |
| #define TEST_LOCK __kmp_test_user_lock_with_checks |
| #define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks |
| #define DESTROY_LOCK __kmp_destroy_user_lock_with_checks |
| #define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks |
| |
| |
| /* |
| * TODO: Make check abort messages use location info & pass it |
| * into with_checks routines |
| */ |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| // internal lock initializer |
| static __forceinline void |
| __kmp_init_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq) |
| { |
| if (KMP_IS_D_LOCK(seq)) { |
| KMP_INIT_D_LOCK(lock, seq); |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL); |
| #endif |
| } else { |
| KMP_INIT_I_LOCK(lock, seq); |
| #if USE_ITT_BUILD |
| kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock); |
| __kmp_itt_lock_creating(ilk->lock, loc); |
| #endif |
| } |
| } |
| |
| // internal nest lock initializer |
| static __forceinline void |
| __kmp_init_nest_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq) |
| { |
| #if KMP_USE_TSX |
| // Don't have nested lock implementation for speculative locks |
| if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive) |
| seq = __kmp_user_lock_seq; |
| #endif |
| switch (seq) { |
| case lockseq_tas: |
| seq = lockseq_nested_tas; |
| break; |
| #if KMP_USE_FUTEX |
| case lockseq_futex: |
| seq = lockseq_nested_futex; |
| break; |
| #endif |
| case lockseq_ticket: |
| seq = lockseq_nested_ticket; |
| break; |
| case lockseq_queuing: |
| seq = lockseq_nested_queuing; |
| break; |
| case lockseq_drdpa: |
| seq = lockseq_nested_drdpa; |
| break; |
| default: |
| seq = lockseq_nested_queuing; |
| } |
| KMP_INIT_I_LOCK(lock, seq); |
| #if USE_ITT_BUILD |
| kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock); |
| __kmp_itt_lock_creating(ilk->lock, loc); |
| #endif |
| } |
| |
| /* initialize the lock with a hint */ |
| void |
| __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint) |
| { |
| KMP_DEBUG_ASSERT(__kmp_init_serial); |
| if (__kmp_env_consistency_check && user_lock == NULL) { |
| KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint"); |
| } |
| |
| __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint)); |
| } |
| |
| /* initialize the lock with a hint */ |
| void |
| __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint) |
| { |
| KMP_DEBUG_ASSERT(__kmp_init_serial); |
| if (__kmp_env_consistency_check && user_lock == NULL) { |
| KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint"); |
| } |
| |
| __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint)); |
| } |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| |
| /* initialize the lock */ |
| void |
| __kmpc_init_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| KMP_DEBUG_ASSERT(__kmp_init_serial); |
| if (__kmp_env_consistency_check && user_lock == NULL) { |
| KMP_FATAL(LockIsUninitialized, "omp_init_lock"); |
| } |
| __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq); |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| static char const * const func = "omp_init_lock"; |
| kmp_user_lock_p lck; |
| KMP_DEBUG_ASSERT( __kmp_init_serial ); |
| |
| if ( __kmp_env_consistency_check ) { |
| if ( user_lock == NULL ) { |
| KMP_FATAL( LockIsUninitialized, func ); |
| } |
| } |
| |
| KMP_CHECK_USER_LOCK_INIT(); |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_user_lock_allocate( user_lock, gtid, 0 ); |
| } |
| INIT_LOCK( lck ); |
| __kmp_set_user_lock_location( lck, loc ); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_init_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_init_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_creating( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } // __kmpc_init_lock |
| |
| /* initialize the lock */ |
| void |
| __kmpc_init_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| KMP_DEBUG_ASSERT(__kmp_init_serial); |
| if (__kmp_env_consistency_check && user_lock == NULL) { |
| KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock"); |
| } |
| __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq); |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| static char const * const func = "omp_init_nest_lock"; |
| kmp_user_lock_p lck; |
| KMP_DEBUG_ASSERT( __kmp_init_serial ); |
| |
| if ( __kmp_env_consistency_check ) { |
| if ( user_lock == NULL ) { |
| KMP_FATAL( LockIsUninitialized, func ); |
| } |
| } |
| |
| KMP_CHECK_USER_LOCK_INIT(); |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_user_lock_allocate( user_lock, gtid, 0 ); |
| } |
| |
| INIT_NESTED_LOCK( lck ); |
| __kmp_set_user_lock_location( lck, loc ); |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_creating( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } // __kmpc_init_nest_lock |
| |
| void |
| __kmpc_destroy_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| # if USE_ITT_BUILD |
| kmp_user_lock_p lck; |
| if (KMP_EXTRACT_D_TAG(user_lock) == 0) { |
| lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock; |
| } else { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| __kmp_itt_lock_destroyed(lck); |
| # endif |
| KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock); |
| #else |
| kmp_user_lock_p lck; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_lock" ); |
| } |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_destroy_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_destroy_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_destroyed( lck ); |
| #endif /* USE_ITT_BUILD */ |
| DESTROY_LOCK( lck ); |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| ; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| ; |
| } |
| #endif |
| else { |
| __kmp_user_lock_free( user_lock, gtid, lck ); |
| } |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } // __kmpc_destroy_lock |
| |
| /* destroy the lock */ |
| void |
| __kmpc_destroy_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| # if USE_ITT_BUILD |
| kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock); |
| __kmp_itt_lock_destroyed(ilk->lock); |
| # endif |
| KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock); |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_nest_lock" ); |
| } |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_destroyed( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| DESTROY_NESTED_LOCK( lck ); |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| ; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| ; |
| } |
| #endif |
| else { |
| __kmp_user_lock_free( user_lock, gtid, lck ); |
| } |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } // __kmpc_destroy_nest_lock |
| |
| void |
| __kmpc_set_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| KMP_COUNT_BLOCK(OMP_set_lock); |
| #if KMP_USE_DYNAMIC_LOCK |
| int tag = KMP_EXTRACT_D_TAG(user_lock); |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); // itt function will get to the right lock object. |
| # endif |
| # if KMP_USE_INLINED_TAS |
| if (tag == locktag_tas && !__kmp_env_consistency_check) { |
| KMP_ACQUIRE_TAS_LOCK(user_lock, gtid); |
| } else |
| # elif KMP_USE_INLINED_FUTEX |
| if (tag == locktag_futex && !__kmp_env_consistency_check) { |
| KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid); |
| } else |
| # endif |
| { |
| __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid); |
| } |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); |
| # endif |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_set_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| ACQUIRE_LOCK( lck, gtid ); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquired( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_acquired_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_acquired_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| void |
| __kmpc_set_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); |
| # endif |
| KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid); |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); |
| #endif |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled) { |
| // missing support here: need to know whether acquired first or not |
| } |
| #endif |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| int acquire_status; |
| kmp_user_lock_p lck; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_set_nest_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| ACQUIRE_NESTED_LOCK( lck, gtid, &acquire_status ); |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquired( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| #if OMPT_SUPPORT && OMPT_TRACE |
| if (ompt_enabled) { |
| if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) { |
| if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first)) |
| ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first)((uint64_t) lck); |
| } else { |
| if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next)) |
| ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next)((uint64_t) lck); |
| } |
| } |
| #endif |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| void |
| __kmpc_unset_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ) |
| { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| int tag = KMP_EXTRACT_D_TAG(user_lock); |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); |
| # endif |
| # if KMP_USE_INLINED_TAS |
| if (tag == locktag_tas && !__kmp_env_consistency_check) { |
| KMP_RELEASE_TAS_LOCK(user_lock, gtid); |
| } else |
| # elif KMP_USE_INLINED_FUTEX |
| if (tag == locktag_futex && !__kmp_env_consistency_check) { |
| KMP_RELEASE_FUTEX_LOCK(user_lock, gtid); |
| } else |
| # endif |
| { |
| __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid); |
| } |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| |
| /* Can't use serial interval since not block structured */ |
| /* release the lock */ |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) |
| // "fast" path implemented to fix customer performance issue |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock ); |
| #endif /* USE_ITT_BUILD */ |
| TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0); |
| KMP_MB(); |
| return; |
| #else |
| lck = (kmp_user_lock_p)user_lock; |
| #endif |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_unset_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_releasing( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| RELEASE_LOCK( lck, gtid ); |
| |
| #if OMPT_SUPPORT && OMPT_BLAME |
| if (ompt_enabled && |
| ompt_callbacks.ompt_callback(ompt_event_release_lock)) { |
| ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck); |
| } |
| #endif |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| /* release the lock */ |
| void |
| __kmpc_unset_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ) |
| { |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock); |
| # endif |
| KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid); |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| |
| /* Can't use serial interval since not block structured */ |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) |
| // "fast" path implemented to fix customer performance issue |
| kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock; |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock ); |
| #endif /* USE_ITT_BUILD */ |
| if ( --(tl->lk.depth_locked) == 0 ) { |
| TCW_4(tl->lk.poll, 0); |
| } |
| KMP_MB(); |
| return; |
| #else |
| lck = (kmp_user_lock_p)user_lock; |
| #endif |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_unset_nest_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_releasing( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| int release_status; |
| release_status = RELEASE_NESTED_LOCK( lck, gtid ); |
| #if OMPT_SUPPORT && OMPT_BLAME |
| if (ompt_enabled) { |
| if (release_status == KMP_LOCK_RELEASED) { |
| if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) { |
| ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)( |
| (uint64_t) lck); |
| } |
| } else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) { |
| ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)( |
| (uint64_t) lck); |
| } |
| } |
| #endif |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| /* try to acquire the lock */ |
| int |
| __kmpc_test_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ) |
| { |
| KMP_COUNT_BLOCK(OMP_test_lock); |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| int rc; |
| int tag = KMP_EXTRACT_D_TAG(user_lock); |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); |
| # endif |
| # if KMP_USE_INLINED_TAS |
| if (tag == locktag_tas && !__kmp_env_consistency_check) { |
| KMP_TEST_TAS_LOCK(user_lock, gtid, rc); |
| } else |
| # elif KMP_USE_INLINED_FUTEX |
| if (tag == locktag_futex && !__kmp_env_consistency_check) { |
| KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc); |
| } else |
| # endif |
| { |
| rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid); |
| } |
| if (rc) { |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); |
| # endif |
| return FTN_TRUE; |
| } else { |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock); |
| # endif |
| return FTN_FALSE; |
| } |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| int rc; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) |
| && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_test_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| rc = TEST_LOCK( lck, gtid ); |
| #if USE_ITT_BUILD |
| if ( rc ) { |
| __kmp_itt_lock_acquired( lck ); |
| } else { |
| __kmp_itt_lock_cancelled( lck ); |
| } |
| #endif /* USE_ITT_BUILD */ |
| return ( rc ? FTN_TRUE : FTN_FALSE ); |
| |
| /* Can't use serial interval since not block structured */ |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| /* try to acquire the lock */ |
| int |
| __kmpc_test_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock ) |
| { |
| #if KMP_USE_DYNAMIC_LOCK |
| int rc; |
| # if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); |
| # endif |
| rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid); |
| # if USE_ITT_BUILD |
| if (rc) { |
| __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock); |
| } else { |
| __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock); |
| } |
| # endif |
| return rc; |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| kmp_user_lock_p lck; |
| int rc; |
| |
| if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll ) |
| + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #if KMP_USE_FUTEX |
| else if ( ( __kmp_user_lock_kind == lk_futex ) |
| && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked ) |
| <= OMP_NEST_LOCK_T_SIZE ) ) { |
| lck = (kmp_user_lock_p)user_lock; |
| } |
| #endif |
| else { |
| lck = __kmp_lookup_user_lock( user_lock, "omp_test_nest_lock" ); |
| } |
| |
| #if USE_ITT_BUILD |
| __kmp_itt_lock_acquiring( lck ); |
| #endif /* USE_ITT_BUILD */ |
| |
| rc = TEST_NESTED_LOCK( lck, gtid ); |
| #if USE_ITT_BUILD |
| if ( rc ) { |
| __kmp_itt_lock_acquired( lck ); |
| } else { |
| __kmp_itt_lock_cancelled( lck ); |
| } |
| #endif /* USE_ITT_BUILD */ |
| return rc; |
| |
| /* Can't use serial interval since not block structured */ |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| |
| /*--------------------------------------------------------------------------------------------------------------------*/ |
| |
| /* |
| * Interface to fast scalable reduce methods routines |
| */ |
| |
| // keep the selected method in a thread local structure for cross-function usage: will be used in __kmpc_end_reduce* functions; |
| // another solution: to re-determine the method one more time in __kmpc_end_reduce* functions (new prototype required then) |
| // AT: which solution is better? |
| #define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \ |
| ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) ) |
| |
| #define __KMP_GET_REDUCTION_METHOD(gtid) \ |
| ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) |
| |
| // description of the packed_reduction_method variable: look at the macros in kmp.h |
| |
| |
| // used in a critical section reduce block |
| static __forceinline void |
| __kmp_enter_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) { |
| |
| // this lock was visible to a customer and to the threading profile tool as a serial overhead span |
| // (although it's used for an internal purpose only) |
| // why was it visible in previous implementation? |
| // should we keep it visible in new reduce block? |
| kmp_user_lock_p lck; |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit; |
| // Check if it is initialized. |
| if (*lk == 0) { |
| if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { |
| KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(__kmp_user_lock_seq)); |
| } else { |
| __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(__kmp_user_lock_seq)); |
| } |
| } |
| // Branch for accessing the actual lock object and set operation. This branching is inevitable since |
| // this lock initialization does not follow the normal dispatch path (lock table is not used). |
| if (KMP_EXTRACT_D_TAG(lk) != 0) { |
| lck = (kmp_user_lock_p)lk; |
| KMP_DEBUG_ASSERT(lck != NULL); |
| if (__kmp_env_consistency_check) { |
| __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq); |
| } |
| KMP_D_LOCK_FUNC(lk, set)(lk, global_tid); |
| } else { |
| kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk); |
| lck = ilk->lock; |
| KMP_DEBUG_ASSERT(lck != NULL); |
| if (__kmp_env_consistency_check) { |
| __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq); |
| } |
| KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid); |
| } |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| // We know that the fast reduction code is only emitted by Intel compilers |
| // with 32 byte critical sections. If there isn't enough space, then we |
| // have to use a pointer. |
| if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) { |
| lck = (kmp_user_lock_p)crit; |
| } |
| else { |
| lck = __kmp_get_critical_section_ptr( crit, loc, global_tid ); |
| } |
| KMP_DEBUG_ASSERT( lck != NULL ); |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_critical, loc, lck ); |
| |
| __kmp_acquire_user_lock_with_checks( lck, global_tid ); |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } |
| |
| // used in a critical section reduce block |
| static __forceinline void |
| __kmp_end_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) { |
| |
| kmp_user_lock_p lck; |
| |
| #if KMP_USE_DYNAMIC_LOCK |
| |
| if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) { |
| lck = (kmp_user_lock_p)crit; |
| if (__kmp_env_consistency_check) |
| __kmp_pop_sync(global_tid, ct_critical, loc); |
| KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid); |
| } else { |
| kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit)); |
| if (__kmp_env_consistency_check) |
| __kmp_pop_sync(global_tid, ct_critical, loc); |
| KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid); |
| } |
| |
| #else // KMP_USE_DYNAMIC_LOCK |
| |
| // We know that the fast reduction code is only emitted by Intel compilers with 32 byte critical |
| // sections. If there isn't enough space, then we have to use a pointer. |
| if ( __kmp_base_user_lock_size > 32 ) { |
| lck = *( (kmp_user_lock_p *) crit ); |
| KMP_ASSERT( lck != NULL ); |
| } else { |
| lck = (kmp_user_lock_p) crit; |
| } |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_sync( global_tid, ct_critical, loc ); |
| |
| __kmp_release_user_lock_with_checks( lck, global_tid ); |
| |
| #endif // KMP_USE_DYNAMIC_LOCK |
| } // __kmp_end_critical_section_reduce_block |
| |
| |
| /* 2.a.i. Reduce Block without a terminating barrier */ |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid global thread number |
| @param num_vars number of items (variables) to be reduced |
| @param reduce_size size of data in bytes to be reduced |
| @param reduce_data pointer to data to be reduced |
| @param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data |
| @param lck pointer to the unique lock data structure |
| @result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed |
| |
| The nowait version is used for a reduce clause with the nowait argument. |
| */ |
| kmp_int32 |
| __kmpc_reduce_nowait( |
| ident_t *loc, kmp_int32 global_tid, |
| kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data), |
| kmp_critical_name *lck ) { |
| |
| KMP_COUNT_BLOCK(REDUCE_nowait); |
| int retval = 0; |
| PACKED_REDUCTION_METHOD_T packed_reduction_method; |
| #if OMP_40_ENABLED |
| kmp_team_t *team; |
| kmp_info_t *th; |
| int teams_swapped = 0, task_state; |
| #endif |
| KA_TRACE( 10, ( "__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) ); |
| |
| // why do we need this initialization here at all? |
| // Reduction clause can not be used as a stand-alone directive. |
| |
| // do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed |
| // possible detection of false-positive race by the threadchecker ??? |
| if( ! TCR_4( __kmp_init_parallel ) ) |
| __kmp_parallel_initialize(); |
| |
| // check correctness of reduce block nesting |
| #if KMP_USE_DYNAMIC_LOCK |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 ); |
| #else |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_reduce, loc, NULL ); |
| #endif |
| |
| #if OMP_40_ENABLED |
| th = __kmp_thread_from_gtid(global_tid); |
| if( th->th.th_teams_microtask ) { // AC: check if we are inside the teams construct? |
| team = th->th.th_team; |
| if( team->t.t_level == th->th.th_teams_level ) { |
| // this is reduction at teams construct |
| KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0 |
| // Let's swap teams temporarily for the reduction barrier |
| teams_swapped = 1; |
| th->th.th_info.ds.ds_tid = team->t.t_master_tid; |
| th->th.th_team = team->t.t_parent; |
| th->th.th_team_nproc = th->th.th_team->t.t_nproc; |
| th->th.th_task_team = th->th.th_team->t.t_task_team[0]; |
| task_state = th->th.th_task_state; |
| th->th.th_task_state = 0; |
| } |
| } |
| #endif // OMP_40_ENABLED |
| |
| // packed_reduction_method value will be reused by __kmp_end_reduce* function, the value should be kept in a variable |
| // the variable should be either a construct-specific or thread-specific property, not a team specific property |
| // (a thread can reach the next reduce block on the next construct, reduce method may differ on the next construct) |
| // an ident_t "loc" parameter could be used as a construct-specific property (what if loc == 0?) |
| // (if both construct-specific and team-specific variables were shared, then unness extra syncs should be needed) |
| // a thread-specific variable is better regarding two issues above (next construct and extra syncs) |
| // a thread-specific "th_local.reduction_method" variable is used currently |
| // each thread executes 'determine' and 'set' lines (no need to execute by one thread, to avoid unness extra syncs) |
| |
| packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck ); |
| __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method ); |
| |
| if( packed_reduction_method == critical_reduce_block ) { |
| |
| __kmp_enter_critical_section_reduce_block( loc, global_tid, lck ); |
| retval = 1; |
| |
| } else if( packed_reduction_method == empty_reduce_block ) { |
| |
| // usage: if team size == 1, no synchronization is required ( Intel platforms only ) |
| retval = 1; |
| |
| } else if( packed_reduction_method == atomic_reduce_block ) { |
| |
| retval = 2; |
| |
| // all threads should do this pop here (because __kmpc_end_reduce_nowait() won't be called by the code gen) |
| // (it's not quite good, because the checking block has been closed by this 'pop', |
| // but atomic operation has not been executed yet, will be executed slightly later, literally on next instruction) |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_sync( global_tid, ct_reduce, loc ); |
| |
| } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) { |
| |
| //AT: performance issue: a real barrier here |
| //AT: (if master goes slow, other threads are blocked here waiting for the master to come and release them) |
| //AT: (it's not what a customer might expect specifying NOWAIT clause) |
| //AT: (specifying NOWAIT won't result in improvement of performance, it'll be confusing to a customer) |
| //AT: another implementation of *barrier_gather*nowait() (or some other design) might go faster |
| // and be more in line with sense of NOWAIT |
| //AT: TO DO: do epcc test and compare times |
| |
| // this barrier should be invisible to a customer and to the threading profile tool |
| // (it's neither a terminating barrier nor customer's code, it's used for an internal purpose) |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func ); |
| retval = ( retval != 0 ) ? ( 0 ) : ( 1 ); |
| |
| // all other workers except master should do this pop here |
| // ( none of other workers will get to __kmpc_end_reduce_nowait() ) |
| if ( __kmp_env_consistency_check ) { |
| if( retval == 0 ) { |
| __kmp_pop_sync( global_tid, ct_reduce, loc ); |
| } |
| } |
| |
| } else { |
| |
| // should never reach this block |
| KMP_ASSERT( 0 ); // "unexpected method" |
| |
| } |
| #if OMP_40_ENABLED |
| if( teams_swapped ) { |
| // Restore thread structure |
| th->th.th_info.ds.ds_tid = 0; |
| th->th.th_team = team; |
| th->th.th_team_nproc = team->t.t_nproc; |
| th->th.th_task_team = team->t.t_task_team[task_state]; |
| th->th.th_task_state = task_state; |
| } |
| #endif |
| KA_TRACE( 10, ( "__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) ); |
| |
| return retval; |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid global thread id. |
| @param lck pointer to the unique lock data structure |
| |
| Finish the execution of a reduce nowait. |
| */ |
| void |
| __kmpc_end_reduce_nowait( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) { |
| |
| PACKED_REDUCTION_METHOD_T packed_reduction_method; |
| |
| KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) ); |
| |
| packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid ); |
| |
| if( packed_reduction_method == critical_reduce_block ) { |
| |
| __kmp_end_critical_section_reduce_block( loc, global_tid, lck ); |
| |
| } else if( packed_reduction_method == empty_reduce_block ) { |
| |
| // usage: if team size == 1, no synchronization is required ( on Intel platforms only ) |
| |
| } else if( packed_reduction_method == atomic_reduce_block ) { |
| |
| // neither master nor other workers should get here |
| // (code gen does not generate this call in case 2: atomic reduce block) |
| // actually it's better to remove this elseif at all; |
| // after removal this value will checked by the 'else' and will assert |
| |
| } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) { |
| |
| // only master gets here |
| |
| } else { |
| |
| // should never reach this block |
| KMP_ASSERT( 0 ); // "unexpected method" |
| |
| } |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_sync( global_tid, ct_reduce, loc ); |
| |
| KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) ); |
| |
| return; |
| } |
| |
| /* 2.a.ii. Reduce Block with a terminating barrier */ |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid global thread number |
| @param num_vars number of items (variables) to be reduced |
| @param reduce_size size of data in bytes to be reduced |
| @param reduce_data pointer to data to be reduced |
| @param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data |
| @param lck pointer to the unique lock data structure |
| @result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed |
| |
| A blocking reduce that includes an implicit barrier. |
| */ |
| kmp_int32 |
| __kmpc_reduce( |
| ident_t *loc, kmp_int32 global_tid, |
| kmp_int32 num_vars, size_t reduce_size, void *reduce_data, |
| void (*reduce_func)(void *lhs_data, void *rhs_data), |
| kmp_critical_name *lck ) |
| { |
| KMP_COUNT_BLOCK(REDUCE_wait); |
| int retval = 0; |
| PACKED_REDUCTION_METHOD_T packed_reduction_method; |
| |
| KA_TRACE( 10, ( "__kmpc_reduce() enter: called T#%d\n", global_tid ) ); |
| |
| // why do we need this initialization here at all? |
| // Reduction clause can not be a stand-alone directive. |
| |
| // do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed |
| // possible detection of false-positive race by the threadchecker ??? |
| if( ! TCR_4( __kmp_init_parallel ) ) |
| __kmp_parallel_initialize(); |
| |
| // check correctness of reduce block nesting |
| #if KMP_USE_DYNAMIC_LOCK |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 ); |
| #else |
| if ( __kmp_env_consistency_check ) |
| __kmp_push_sync( global_tid, ct_reduce, loc, NULL ); |
| #endif |
| |
| packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck ); |
| __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method ); |
| |
| if( packed_reduction_method == critical_reduce_block ) { |
| |
| __kmp_enter_critical_section_reduce_block( loc, global_tid, lck ); |
| retval = 1; |
| |
| } else if( packed_reduction_method == empty_reduce_block ) { |
| |
| // usage: if team size == 1, no synchronization is required ( Intel platforms only ) |
| retval = 1; |
| |
| } else if( packed_reduction_method == atomic_reduce_block ) { |
| |
| retval = 2; |
| |
| } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) { |
| |
| //case tree_reduce_block: |
| // this barrier should be visible to a customer and to the threading profile tool |
| // (it's a terminating barrier on constructs if NOWAIT not specified) |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; // needed for correct notification of frames |
| #endif |
| retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func ); |
| retval = ( retval != 0 ) ? ( 0 ) : ( 1 ); |
| |
| // all other workers except master should do this pop here |
| // ( none of other workers except master will enter __kmpc_end_reduce() ) |
| if ( __kmp_env_consistency_check ) { |
| if( retval == 0 ) { // 0: all other workers; 1: master |
| __kmp_pop_sync( global_tid, ct_reduce, loc ); |
| } |
| } |
| |
| } else { |
| |
| // should never reach this block |
| KMP_ASSERT( 0 ); // "unexpected method" |
| |
| } |
| |
| KA_TRACE( 10, ( "__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) ); |
| |
| return retval; |
| } |
| |
| /*! |
| @ingroup SYNCHRONIZATION |
| @param loc source location information |
| @param global_tid global thread id. |
| @param lck pointer to the unique lock data structure |
| |
| Finish the execution of a blocking reduce. |
| The <tt>lck</tt> pointer must be the same as that used in the corresponding start function. |
| */ |
| void |
| __kmpc_end_reduce( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) { |
| |
| PACKED_REDUCTION_METHOD_T packed_reduction_method; |
| |
| KA_TRACE( 10, ( "__kmpc_end_reduce() enter: called T#%d\n", global_tid ) ); |
| |
| packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid ); |
| |
| // this barrier should be visible to a customer and to the threading profile tool |
| // (it's a terminating barrier on constructs if NOWAIT not specified) |
| |
| if( packed_reduction_method == critical_reduce_block ) { |
| |
| __kmp_end_critical_section_reduce_block( loc, global_tid, lck ); |
| |
| // TODO: implicit barrier: should be exposed |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL ); |
| |
| } else if( packed_reduction_method == empty_reduce_block ) { |
| |
| // usage: if team size == 1, no synchronization is required ( Intel platforms only ) |
| |
| // TODO: implicit barrier: should be exposed |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL ); |
| |
| } else if( packed_reduction_method == atomic_reduce_block ) { |
| |
| // TODO: implicit barrier: should be exposed |
| #if USE_ITT_NOTIFY |
| __kmp_threads[global_tid]->th.th_ident = loc; |
| #endif |
| __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL ); |
| |
| } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) { |
| |
| // only master executes here (master releases all other workers) |
| __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid ); |
| |
| } else { |
| |
| // should never reach this block |
| KMP_ASSERT( 0 ); // "unexpected method" |
| |
| } |
| |
| if ( __kmp_env_consistency_check ) |
| __kmp_pop_sync( global_tid, ct_reduce, loc ); |
| |
| KA_TRACE( 10, ( "__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) ); |
| |
| return; |
| } |
| |
| #undef __KMP_GET_REDUCTION_METHOD |
| #undef __KMP_SET_REDUCTION_METHOD |
| |
| /*-- end of interface to fast scalable reduce routines ---------------------------------------------------------------*/ |
| |
| kmp_uint64 |
| __kmpc_get_taskid() { |
| |
| kmp_int32 gtid; |
| kmp_info_t * thread; |
| |
| gtid = __kmp_get_gtid(); |
| if ( gtid < 0 ) { |
| return 0; |
| }; // if |
| thread = __kmp_thread_from_gtid( gtid ); |
| return thread->th.th_current_task->td_task_id; |
| |
| } // __kmpc_get_taskid |
| |
| |
| kmp_uint64 |
| __kmpc_get_parent_taskid() { |
| |
| kmp_int32 gtid; |
| kmp_info_t * thread; |
| kmp_taskdata_t * parent_task; |
| |
| gtid = __kmp_get_gtid(); |
| if ( gtid < 0 ) { |
| return 0; |
| }; // if |
| thread = __kmp_thread_from_gtid( gtid ); |
| parent_task = thread->th.th_current_task->td_parent; |
| return ( parent_task == NULL ? 0 : parent_task->td_task_id ); |
| |
| } // __kmpc_get_parent_taskid |
| |
| void __kmpc_place_threads(int nS, int sO, int nC, int cO, int nT) |
| { |
| if ( ! __kmp_init_serial ) { |
| __kmp_serial_initialize(); |
| } |
| __kmp_place_num_sockets = nS; |
| __kmp_place_socket_offset = sO; |
| __kmp_place_num_cores = nC; |
| __kmp_place_core_offset = cO; |
| __kmp_place_num_threads_per_core = nT; |
| } |
| |
| #if OMP_45_ENABLED |
| /*! |
| @ingroup WORK_SHARING |
| @param loc source location information. |
| @param gtid global thread number. |
| @param num_dims number of associated doacross loops. |
| @param dims info on loops bounds. |
| |
| Initialize doacross loop information. |
| Expect compiler send us inclusive bounds, |
| e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2. |
| */ |
| void |
| __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, struct kmp_dim * dims) |
| { |
| int j, idx; |
| kmp_int64 last, trace_count; |
| kmp_info_t *th = __kmp_threads[gtid]; |
| kmp_team_t *team = th->th.th_team; |
| kmp_uint32 *flags; |
| kmp_disp_t *pr_buf = th->th.th_dispatch; |
| dispatch_shared_info_t *sh_buf; |
| |
| KA_TRACE(20,("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n", |
| gtid, num_dims, !team->t.t_serialized)); |
| KMP_DEBUG_ASSERT(dims != NULL); |
| KMP_DEBUG_ASSERT(num_dims > 0); |
| |
| if( team->t.t_serialized ) { |
| KA_TRACE(20,("__kmpc_doacross_init() exit: serialized team\n")); |
| return; // no dependencies if team is serialized |
| } |
| KMP_DEBUG_ASSERT(team->t.t_nproc > 1); |
| idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for the next loop |
| sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers]; |
| |
| // Save bounds info into allocated private buffer |
| KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL); |
| pr_buf->th_doacross_info = |
| (kmp_int64*)__kmp_thread_malloc(th, sizeof(kmp_int64)*(4 * num_dims + 1)); |
| KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); |
| pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions |
| // Save also address of num_done in order to access it later without knowing the buffer index |
| pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done; |
| pr_buf->th_doacross_info[2] = dims[0].lo; |
| pr_buf->th_doacross_info[3] = dims[0].up; |
| pr_buf->th_doacross_info[4] = dims[0].st; |
| last = 5; |
| for( j = 1; j < num_dims; ++j ) { |
| kmp_int64 range_length; // To keep ranges of all dimensions but the first dims[0] |
| if( dims[j].st == 1 ) { // most common case |
| // AC: should we care of ranges bigger than LLONG_MAX? (not for now) |
| range_length = dims[j].up - dims[j].lo + 1; |
| } else { |
| if( dims[j].st > 0 ) { |
| KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo); |
| range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1; |
| } else { // negative increment |
| KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up); |
| range_length = (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1; |
| } |
| } |
| pr_buf->th_doacross_info[last++] = range_length; |
| pr_buf->th_doacross_info[last++] = dims[j].lo; |
| pr_buf->th_doacross_info[last++] = dims[j].up; |
| pr_buf->th_doacross_info[last++] = dims[j].st; |
| } |
| |
| // Compute total trip count. |
| // Start with range of dims[0] which we don't need to keep in the buffer. |
| if( dims[0].st == 1 ) { // most common case |
| trace_count = dims[0].up - dims[0].lo + 1; |
| } else if( dims[0].st > 0 ) { |
| KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo); |
| trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1; |
| } else { // negative increment |
| KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up); |
| trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1; |
| } |
| for( j = 1; j < num_dims; ++j ) { |
| trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges |
| } |
| KMP_DEBUG_ASSERT(trace_count > 0); |
| |
| // Check if shared buffer is not occupied by other loop (idx - __kmp_dispatch_num_buffers) |
| if( idx != sh_buf->doacross_buf_idx ) { |
| // Shared buffer is occupied, wait for it to be free |
| __kmp_wait_yield_4( (kmp_uint32*)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL ); |
| } |
| // Check if we are the first thread. After the CAS the first thread gets 0, |
| // others get 1 if initialization is in progress, allocated pointer otherwise. |
| flags = (kmp_uint32*)KMP_COMPARE_AND_STORE_RET64( |
| (kmp_int64*)&sh_buf->doacross_flags,NULL,(kmp_int64)1); |
| if( flags == NULL ) { |
| // we are the first thread, allocate the array of flags |
| kmp_int64 size = trace_count / 8 + 8; // in bytes, use single bit per iteration |
| sh_buf->doacross_flags = (kmp_uint32*)__kmp_thread_calloc(th, size, 1); |
| } else if( (kmp_int64)flags == 1 ) { |
| // initialization is still in progress, need to wait |
| while( (volatile kmp_int64)sh_buf->doacross_flags == 1 ) { |
| KMP_YIELD(TRUE); |
| } |
| } |
| KMP_DEBUG_ASSERT((kmp_int64)sh_buf->doacross_flags > 1); // check value of pointer |
| pr_buf->th_doacross_flags = sh_buf->doacross_flags; // save private copy in order to not |
| // touch shared buffer on each iteration |
| KA_TRACE(20,("__kmpc_doacross_init() exit: T#%d\n", gtid)); |
| } |
| |
| void |
| __kmpc_doacross_wait(ident_t *loc, int gtid, long long *vec) |
| { |
| kmp_int32 shft, num_dims, i; |
| kmp_uint32 flag; |
| kmp_int64 iter_number; // iteration number of "collapsed" loop nest |
| kmp_info_t *th = __kmp_threads[gtid]; |
| kmp_team_t *team = th->th.th_team; |
| kmp_disp_t *pr_buf; |
| kmp_int64 lo, up, st; |
| |
| KA_TRACE(20,("__kmpc_doacross_wait() enter: called T#%d\n", gtid)); |
| if( team->t.t_serialized ) { |
| KA_TRACE(20,("__kmpc_doacross_wait() exit: serialized team\n")); |
| return; // no dependencies if team is serialized |
| } |
| |
| // calculate sequential iteration number and check out-of-bounds condition |
| pr_buf = th->th.th_dispatch; |
| KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); |
| num_dims = pr_buf->th_doacross_info[0]; |
| lo = pr_buf->th_doacross_info[2]; |
| up = pr_buf->th_doacross_info[3]; |
| st = pr_buf->th_doacross_info[4]; |
| if( st == 1 ) { // most common case |
| if( vec[0] < lo || vec[0] > up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[0], lo, up)); |
| return; |
| } |
| iter_number = vec[0] - lo; |
| } else if( st > 0 ) { |
| if( vec[0] < lo || vec[0] > up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[0], lo, up)); |
| return; |
| } |
| iter_number = (kmp_uint64)(vec[0] - lo) / st; |
| } else { // negative increment |
| if( vec[0] > lo || vec[0] < up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[0], lo, up)); |
| return; |
| } |
| iter_number = (kmp_uint64)(lo - vec[0]) / (-st); |
| } |
| for( i = 1; i < num_dims; ++i ) { |
| kmp_int64 iter, ln; |
| kmp_int32 j = i * 4; |
| ln = pr_buf->th_doacross_info[j + 1]; |
| lo = pr_buf->th_doacross_info[j + 2]; |
| up = pr_buf->th_doacross_info[j + 3]; |
| st = pr_buf->th_doacross_info[j + 4]; |
| if( st == 1 ) { |
| if( vec[i] < lo || vec[i] > up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[i], lo, up)); |
| return; |
| } |
| iter = vec[i] - lo; |
| } else if( st > 0 ) { |
| if( vec[i] < lo || vec[i] > up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[i], lo, up)); |
| return; |
| } |
| iter = (kmp_uint64)(vec[i] - lo) / st; |
| } else { // st < 0 |
| if( vec[i] > lo || vec[i] < up ) { |
| KA_TRACE(20,( |
| "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n", |
| gtid, vec[i], lo, up)); |
| return; |
| } |
| iter = (kmp_uint64)(lo - vec[i]) / (-st); |
| } |
| iter_number = iter + ln * iter_number; |
| } |
| shft = iter_number % 32; // use 32-bit granularity |
| iter_number >>= 5; // divided by 32 |
| flag = 1 << shft; |
| while( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 ) { |
| KMP_YIELD(TRUE); |
| } |
| KA_TRACE(20,("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n", |
| gtid, (iter_number<<5)+shft)); |
| } |
| |
| void |
| __kmpc_doacross_post(ident_t *loc, int gtid, long long *vec) |
| { |
| kmp_int32 shft, num_dims, i; |
| kmp_uint32 flag; |
| kmp_int64 iter_number; // iteration number of "collapsed" loop nest |
| kmp_info_t *th = __kmp_threads[gtid]; |
| kmp_team_t *team = th->th.th_team; |
| kmp_disp_t *pr_buf; |
| kmp_int64 lo, st; |
| |
| KA_TRACE(20,("__kmpc_doacross_post() enter: called T#%d\n", gtid)); |
| if( team->t.t_serialized ) { |
| KA_TRACE(20,("__kmpc_doacross_post() exit: serialized team\n")); |
| return; // no dependencies if team is serialized |
| } |
| |
| // calculate sequential iteration number (same as in "wait" but no out-of-bounds checks) |
| pr_buf = th->th.th_dispatch; |
| KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL); |
| num_dims = pr_buf->th_doacross_info[0]; |
| lo = pr_buf->th_doacross_info[2]; |
| st = pr_buf->th_doacross_info[4]; |
| if( st == 1 ) { // most common case |
| iter_number = vec[0] - lo; |
| } else if( st > 0 ) { |
| iter_number = (kmp_uint64)(vec[0] - lo) / st; |
| } else { // negative increment |
| iter_number = (kmp_uint64)(lo - vec[0]) / (-st); |
| } |
| for( i = 1; i < num_dims; ++i ) { |
| kmp_int64 iter, ln; |
| kmp_int32 j = i * 4; |
| ln = pr_buf->th_doacross_info[j + 1]; |
| lo = pr_buf->th_doacross_info[j + 2]; |
| st = pr_buf->th_doacross_info[j + 4]; |
| if( st == 1 ) { |
| iter = vec[i] - lo; |
| } else if( st > 0 ) { |
| iter = (kmp_uint64)(vec[i] - lo) / st; |
| } else { // st < 0 |
| iter = (kmp_uint64)(lo - vec[i]) / (-st); |
| } |
| iter_number = iter + ln * iter_number; |
| } |
| shft = iter_number % 32; // use 32-bit granularity |
| iter_number >>= 5; // divided by 32 |
| flag = 1 << shft; |
| if( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 ) |
| KMP_TEST_THEN_OR32( (kmp_int32*)&pr_buf->th_doacross_flags[iter_number], (kmp_int32)flag ); |
| KA_TRACE(20,("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", |
| gtid, (iter_number<<5)+shft)); |
| } |
| |
| void |
| __kmpc_doacross_fini(ident_t *loc, int gtid) |
| { |
| kmp_int64 num_done; |
| kmp_info_t *th = __kmp_threads[gtid]; |
| kmp_team_t *team = th->th.th_team; |
| kmp_disp_t *pr_buf = th->th.th_dispatch; |
| |
| KA_TRACE(20,("__kmpc_doacross_fini() enter: called T#%d\n", gtid)); |
| if( team->t.t_serialized ) { |
| KA_TRACE(20,("__kmpc_doacross_fini() exit: serialized team %p\n", team)); |
| return; // nothing to do |
| } |
| num_done = KMP_TEST_THEN_INC64((kmp_int64*)pr_buf->th_doacross_info[1]) + 1; |
| if( num_done == th->th.th_team_nproc ) { |
| // we are the last thread, need to free shared resources |
| int idx = pr_buf->th_doacross_buf_idx - 1; |
| dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers]; |
| KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done); |
| KMP_DEBUG_ASSERT(num_done == (kmp_int64)sh_buf->doacross_num_done); |
| KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx); |
| __kmp_thread_free(th, (void*)sh_buf->doacross_flags); |
| sh_buf->doacross_flags = NULL; |
| sh_buf->doacross_num_done = 0; |
| sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use |
| } |
| // free private resources (need to keep buffer index forever) |
| __kmp_thread_free(th, (void*)pr_buf->th_doacross_info); |
| pr_buf->th_doacross_info = NULL; |
| KA_TRACE(20,("__kmpc_doacross_fini() exit: T#%d\n", gtid)); |
| } |
| #endif |
| |
| // end of file // |
| |