| /* | |
| * kmp_threadprivate.cpp -- OpenMP threadprivate support library | |
| */ | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "kmp.h" | |
| #include "kmp_i18n.h" | |
| #include "kmp_itt.h" | |
| #define USE_CHECKS_COMMON | |
| #define KMP_INLINE_SUBR 1 | |
| void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr, | |
| void *data_addr, size_t pc_size); | |
| struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr, | |
| void *data_addr, | |
| size_t pc_size); | |
| struct shared_table __kmp_threadprivate_d_table; | |
| static | |
| #ifdef KMP_INLINE_SUBR | |
| __forceinline | |
| #endif | |
| struct private_common * | |
| __kmp_threadprivate_find_task_common(struct common_table *tbl, int gtid, | |
| void *pc_addr) | |
| { | |
| struct private_common *tn; | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, called with " | |
| "address %p\n", | |
| gtid, pc_addr)); | |
| dump_list(); | |
| #endif | |
| for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) { | |
| if (tn->gbl_addr == pc_addr) { | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, found " | |
| "node %p on list\n", | |
| gtid, pc_addr)); | |
| #endif | |
| return tn; | |
| } | |
| } | |
| return 0; | |
| } | |
| static | |
| #ifdef KMP_INLINE_SUBR | |
| __forceinline | |
| #endif | |
| struct shared_common * | |
| __kmp_find_shared_task_common(struct shared_table *tbl, int gtid, | |
| void *pc_addr) { | |
| struct shared_common *tn; | |
| for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) { | |
| if (tn->gbl_addr == pc_addr) { | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| KC_TRACE( | |
| 10, | |
| ("__kmp_find_shared_task_common: thread#%d, found node %p on list\n", | |
| gtid, pc_addr)); | |
| #endif | |
| return tn; | |
| } | |
| } | |
| return 0; | |
| } | |
| // Create a template for the data initialized storage. Either the template is | |
| // NULL indicating zero fill, or the template is a copy of the original data. | |
| static struct private_data *__kmp_init_common_data(void *pc_addr, | |
| size_t pc_size) { | |
| struct private_data *d; | |
| size_t i; | |
| char *p; | |
| d = (struct private_data *)__kmp_allocate(sizeof(struct private_data)); | |
| /* | |
| d->data = 0; // AC: commented out because __kmp_allocate zeroes the | |
| memory | |
| d->next = 0; | |
| */ | |
| d->size = pc_size; | |
| d->more = 1; | |
| p = (char *)pc_addr; | |
| for (i = pc_size; i > 0; --i) { | |
| if (*p++ != '\0') { | |
| d->data = __kmp_allocate(pc_size); | |
| KMP_MEMCPY(d->data, pc_addr, pc_size); | |
| break; | |
| } | |
| } | |
| return d; | |
| } | |
| // Initialize the data area from the template. | |
| static void __kmp_copy_common_data(void *pc_addr, struct private_data *d) { | |
| char *addr = (char *)pc_addr; | |
| for (size_t offset = 0; d != 0; d = d->next) { | |
| for (int i = d->more; i > 0; --i) { | |
| if (d->data == 0) | |
| memset(&addr[offset], '\0', d->size); | |
| else | |
| KMP_MEMCPY(&addr[offset], d->data, d->size); | |
| offset += d->size; | |
| } | |
| } | |
| } | |
| /* we are called from __kmp_serial_initialize() with __kmp_initz_lock held. */ | |
| void __kmp_common_initialize(void) { | |
| if (!TCR_4(__kmp_init_common)) { | |
| int q; | |
| #ifdef KMP_DEBUG | |
| int gtid; | |
| #endif | |
| __kmp_threadpriv_cache_list = NULL; | |
| #ifdef KMP_DEBUG | |
| /* verify the uber masters were initialized */ | |
| for (gtid = 0; gtid < __kmp_threads_capacity; gtid++) | |
| if (__kmp_root[gtid]) { | |
| KMP_DEBUG_ASSERT(__kmp_root[gtid]->r.r_uber_thread); | |
| for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) | |
| KMP_DEBUG_ASSERT( | |
| !__kmp_root[gtid]->r.r_uber_thread->th.th_pri_common->data[q]); | |
| /* __kmp_root[ gitd ]-> r.r_uber_thread -> | |
| * th.th_pri_common -> data[ q ] = 0;*/ | |
| } | |
| #endif /* KMP_DEBUG */ | |
| for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) | |
| __kmp_threadprivate_d_table.data[q] = 0; | |
| TCW_4(__kmp_init_common, TRUE); | |
| } | |
| } | |
| /* Call all destructors for threadprivate data belonging to all threads. | |
| Currently unused! */ | |
| void __kmp_common_destroy(void) { | |
| if (TCR_4(__kmp_init_common)) { | |
| int q; | |
| TCW_4(__kmp_init_common, FALSE); | |
| for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) { | |
| int gtid; | |
| struct private_common *tn; | |
| struct shared_common *d_tn; | |
| /* C++ destructors need to be called once per thread before exiting. | |
| Don't call destructors for primary thread though unless we used copy | |
| constructor */ | |
| for (d_tn = __kmp_threadprivate_d_table.data[q]; d_tn; | |
| d_tn = d_tn->next) { | |
| if (d_tn->is_vec) { | |
| if (d_tn->dt.dtorv != 0) { | |
| for (gtid = 0; gtid < __kmp_all_nth; ++gtid) { | |
| if (__kmp_threads[gtid]) { | |
| if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) | |
| : (!KMP_UBER_GTID(gtid))) { | |
| tn = __kmp_threadprivate_find_task_common( | |
| __kmp_threads[gtid]->th.th_pri_common, gtid, | |
| d_tn->gbl_addr); | |
| if (tn) { | |
| (*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len); | |
| } | |
| } | |
| } | |
| } | |
| if (d_tn->obj_init != 0) { | |
| (*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len); | |
| } | |
| } | |
| } else { | |
| if (d_tn->dt.dtor != 0) { | |
| for (gtid = 0; gtid < __kmp_all_nth; ++gtid) { | |
| if (__kmp_threads[gtid]) { | |
| if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) | |
| : (!KMP_UBER_GTID(gtid))) { | |
| tn = __kmp_threadprivate_find_task_common( | |
| __kmp_threads[gtid]->th.th_pri_common, gtid, | |
| d_tn->gbl_addr); | |
| if (tn) { | |
| (*d_tn->dt.dtor)(tn->par_addr); | |
| } | |
| } | |
| } | |
| } | |
| if (d_tn->obj_init != 0) { | |
| (*d_tn->dt.dtor)(d_tn->obj_init); | |
| } | |
| } | |
| } | |
| } | |
| __kmp_threadprivate_d_table.data[q] = 0; | |
| } | |
| } | |
| } | |
| /* Call all destructors for threadprivate data belonging to this thread */ | |
| void __kmp_common_destroy_gtid(int gtid) { | |
| struct private_common *tn; | |
| struct shared_common *d_tn; | |
| if (!TCR_4(__kmp_init_gtid)) { | |
| // This is possible when one of multiple roots initiates early library | |
| // termination in a sequential region while other teams are active, and its | |
| // child threads are about to end. | |
| return; | |
| } | |
| KC_TRACE(10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid)); | |
| if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) : (!KMP_UBER_GTID(gtid))) { | |
| if (TCR_4(__kmp_init_common)) { | |
| /* Cannot do this here since not all threads have destroyed their data */ | |
| /* TCW_4(__kmp_init_common, FALSE); */ | |
| for (tn = __kmp_threads[gtid]->th.th_pri_head; tn; tn = tn->link) { | |
| d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid, | |
| tn->gbl_addr); | |
| if (d_tn == NULL) | |
| continue; | |
| if (d_tn->is_vec) { | |
| if (d_tn->dt.dtorv != 0) { | |
| (void)(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len); | |
| if (d_tn->obj_init != 0) { | |
| (void)(*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len); | |
| } | |
| } | |
| } else { | |
| if (d_tn->dt.dtor != 0) { | |
| (void)(*d_tn->dt.dtor)(tn->par_addr); | |
| if (d_tn->obj_init != 0) { | |
| (void)(*d_tn->dt.dtor)(d_tn->obj_init); | |
| } | |
| } | |
| } | |
| } | |
| KC_TRACE(30, ("__kmp_common_destroy_gtid: T#%d threadprivate destructors " | |
| "complete\n", | |
| gtid)); | |
| } | |
| } | |
| } | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| static void dump_list(void) { | |
| int p, q; | |
| for (p = 0; p < __kmp_all_nth; ++p) { | |
| if (!__kmp_threads[p]) | |
| continue; | |
| for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) { | |
| if (__kmp_threads[p]->th.th_pri_common->data[q]) { | |
| struct private_common *tn; | |
| KC_TRACE(10, ("\tdump_list: gtid:%d addresses\n", p)); | |
| for (tn = __kmp_threads[p]->th.th_pri_common->data[q]; tn; | |
| tn = tn->next) { | |
| KC_TRACE(10, | |
| ("\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n", | |
| tn->gbl_addr, tn->par_addr)); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| #endif /* KMP_TASK_COMMON_DEBUG */ | |
| // NOTE: this routine is to be called only from the serial part of the program. | |
| void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr, | |
| void *data_addr, size_t pc_size) { | |
| struct shared_common **lnk_tn, *d_tn; | |
| KMP_DEBUG_ASSERT(__kmp_threads[gtid] && | |
| __kmp_threads[gtid]->th.th_root->r.r_active == 0); | |
| d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid, | |
| pc_addr); | |
| if (d_tn == 0) { | |
| d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); | |
| d_tn->gbl_addr = pc_addr; | |
| d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size); | |
| /* | |
| d_tn->obj_init = 0; // AC: commented out because __kmp_allocate | |
| zeroes the memory | |
| d_tn->ct.ctor = 0; | |
| d_tn->cct.cctor = 0;; | |
| d_tn->dt.dtor = 0; | |
| d_tn->is_vec = FALSE; | |
| d_tn->vec_len = 0L; | |
| */ | |
| d_tn->cmn_size = pc_size; | |
| __kmp_acquire_lock(&__kmp_global_lock, gtid); | |
| lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]); | |
| d_tn->next = *lnk_tn; | |
| *lnk_tn = d_tn; | |
| __kmp_release_lock(&__kmp_global_lock, gtid); | |
| } | |
| } | |
| struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr, | |
| void *data_addr, | |
| size_t pc_size) { | |
| struct private_common *tn, **tt; | |
| struct shared_common *d_tn; | |
| /* +++++++++ START OF CRITICAL SECTION +++++++++ */ | |
| __kmp_acquire_lock(&__kmp_global_lock, gtid); | |
| tn = (struct private_common *)__kmp_allocate(sizeof(struct private_common)); | |
| tn->gbl_addr = pc_addr; | |
| d_tn = __kmp_find_shared_task_common( | |
| &__kmp_threadprivate_d_table, gtid, | |
| pc_addr); /* Only the MASTER data table exists. */ | |
| if (d_tn != 0) { | |
| /* This threadprivate variable has already been seen. */ | |
| if (d_tn->pod_init == 0 && d_tn->obj_init == 0) { | |
| d_tn->cmn_size = pc_size; | |
| if (d_tn->is_vec) { | |
| if (d_tn->ct.ctorv != 0) { | |
| /* Construct from scratch so no prototype exists */ | |
| d_tn->obj_init = 0; | |
| } else if (d_tn->cct.cctorv != 0) { | |
| /* Now data initialize the prototype since it was previously | |
| * registered */ | |
| d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size); | |
| (void)(*d_tn->cct.cctorv)(d_tn->obj_init, pc_addr, d_tn->vec_len); | |
| } else { | |
| d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size); | |
| } | |
| } else { | |
| if (d_tn->ct.ctor != 0) { | |
| /* Construct from scratch so no prototype exists */ | |
| d_tn->obj_init = 0; | |
| } else if (d_tn->cct.cctor != 0) { | |
| /* Now data initialize the prototype since it was previously | |
| registered */ | |
| d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size); | |
| (void)(*d_tn->cct.cctor)(d_tn->obj_init, pc_addr); | |
| } else { | |
| d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size); | |
| } | |
| } | |
| } | |
| } else { | |
| struct shared_common **lnk_tn; | |
| d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); | |
| d_tn->gbl_addr = pc_addr; | |
| d_tn->cmn_size = pc_size; | |
| d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size); | |
| /* | |
| d_tn->obj_init = 0; // AC: commented out because __kmp_allocate | |
| zeroes the memory | |
| d_tn->ct.ctor = 0; | |
| d_tn->cct.cctor = 0; | |
| d_tn->dt.dtor = 0; | |
| d_tn->is_vec = FALSE; | |
| d_tn->vec_len = 0L; | |
| */ | |
| lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]); | |
| d_tn->next = *lnk_tn; | |
| *lnk_tn = d_tn; | |
| } | |
| tn->cmn_size = d_tn->cmn_size; | |
| if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) { | |
| tn->par_addr = (void *)pc_addr; | |
| } else { | |
| tn->par_addr = (void *)__kmp_allocate(tn->cmn_size); | |
| } | |
| __kmp_release_lock(&__kmp_global_lock, gtid); | |
| /* +++++++++ END OF CRITICAL SECTION +++++++++ */ | |
| #ifdef USE_CHECKS_COMMON | |
| if (pc_size > d_tn->cmn_size) { | |
| KC_TRACE( | |
| 10, ("__kmp_threadprivate_insert: THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC | |
| " ,%" KMP_UINTPTR_SPEC ")\n", | |
| pc_addr, pc_size, d_tn->cmn_size)); | |
| KMP_FATAL(TPCommonBlocksInconsist); | |
| } | |
| #endif /* USE_CHECKS_COMMON */ | |
| tt = &(__kmp_threads[gtid]->th.th_pri_common->data[KMP_HASH(pc_addr)]); | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| if (*tt != 0) { | |
| KC_TRACE( | |
| 10, | |
| ("__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n", | |
| gtid, pc_addr)); | |
| } | |
| #endif | |
| tn->next = *tt; | |
| *tt = tn; | |
| #ifdef KMP_TASK_COMMON_DEBUG | |
| KC_TRACE(10, | |
| ("__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n", | |
| gtid, pc_addr)); | |
| dump_list(); | |
| #endif | |
| /* Link the node into a simple list */ | |
| tn->link = __kmp_threads[gtid]->th.th_pri_head; | |
| __kmp_threads[gtid]->th.th_pri_head = tn; | |
| if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) | |
| return tn; | |
| /* if C++ object with copy constructor, use it; | |
| * else if C++ object with constructor, use it for the non-primary thread | |
| copies only; | |
| * else use pod_init and memcpy | |
| * | |
| * C++ constructors need to be called once for each non-primary thread on | |
| * allocate | |
| * C++ copy constructors need to be called once for each thread on allocate */ | |
| /* C++ object with constructors/destructors; don't call constructors for | |
| primary thread though */ | |
| if (d_tn->is_vec) { | |
| if (d_tn->ct.ctorv != 0) { | |
| (void)(*d_tn->ct.ctorv)(tn->par_addr, d_tn->vec_len); | |
| } else if (d_tn->cct.cctorv != 0) { | |
| (void)(*d_tn->cct.cctorv)(tn->par_addr, d_tn->obj_init, d_tn->vec_len); | |
| } else if (tn->par_addr != tn->gbl_addr) { | |
| __kmp_copy_common_data(tn->par_addr, d_tn->pod_init); | |
| } | |
| } else { | |
| if (d_tn->ct.ctor != 0) { | |
| (void)(*d_tn->ct.ctor)(tn->par_addr); | |
| } else if (d_tn->cct.cctor != 0) { | |
| (void)(*d_tn->cct.cctor)(tn->par_addr, d_tn->obj_init); | |
| } else if (tn->par_addr != tn->gbl_addr) { | |
| __kmp_copy_common_data(tn->par_addr, d_tn->pod_init); | |
| } | |
| } | |
| /* !BUILD_OPENMP_C | |
| if (tn->par_addr != tn->gbl_addr) | |
| __kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); */ | |
| return tn; | |
| } | |
| /* ------------------------------------------------------------------------ */ | |
| /* We are currently parallel, and we know the thread id. */ | |
| /* ------------------------------------------------------------------------ */ | |
| /*! | |
| @ingroup THREADPRIVATE | |
| @param loc source location information | |
| @param data pointer to data being privatized | |
| @param ctor pointer to constructor function for data | |
| @param cctor pointer to copy constructor function for data | |
| @param dtor pointer to destructor function for data | |
| Register constructors and destructors for thread private data. | |
| This function is called when executing in parallel, when we know the thread id. | |
| */ | |
| void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor, | |
| kmpc_cctor cctor, kmpc_dtor dtor) { | |
| struct shared_common *d_tn, **lnk_tn; | |
| KC_TRACE(10, ("__kmpc_threadprivate_register: called\n")); | |
| #ifdef USE_CHECKS_COMMON | |
| /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ | |
| KMP_ASSERT(cctor == 0); | |
| #endif /* USE_CHECKS_COMMON */ | |
| /* Only the global data table exists. */ | |
| d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, -1, data); | |
| if (d_tn == 0) { | |
| d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); | |
| d_tn->gbl_addr = data; | |
| d_tn->ct.ctor = ctor; | |
| d_tn->cct.cctor = cctor; | |
| d_tn->dt.dtor = dtor; | |
| /* | |
| d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate | |
| zeroes the memory | |
| d_tn->vec_len = 0L; | |
| d_tn->obj_init = 0; | |
| d_tn->pod_init = 0; | |
| */ | |
| lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]); | |
| d_tn->next = *lnk_tn; | |
| *lnk_tn = d_tn; | |
| } | |
| } | |
| void *__kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data, | |
| size_t size) { | |
| void *ret; | |
| struct private_common *tn; | |
| KC_TRACE(10, ("__kmpc_threadprivate: T#%d called\n", global_tid)); | |
| #ifdef USE_CHECKS_COMMON | |
| if (!__kmp_init_serial) | |
| KMP_FATAL(RTLNotInitialized); | |
| #endif /* USE_CHECKS_COMMON */ | |
| if (!__kmp_threads[global_tid]->th.th_root->r.r_active && !__kmp_foreign_tp) { | |
| /* The parallel address will NEVER overlap with the data_address */ | |
| /* dkp: 3rd arg to kmp_threadprivate_insert_private_data() is the | |
| * data_address; use data_address = data */ | |
| KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting private data\n", | |
| global_tid)); | |
| kmp_threadprivate_insert_private_data(global_tid, data, data, size); | |
| ret = data; | |
| } else { | |
| KC_TRACE( | |
| 50, | |
| ("__kmpc_threadprivate: T#%d try to find private data at address %p\n", | |
| global_tid, data)); | |
| tn = __kmp_threadprivate_find_task_common( | |
| __kmp_threads[global_tid]->th.th_pri_common, global_tid, data); | |
| if (tn) { | |
| KC_TRACE(20, ("__kmpc_threadprivate: T#%d found data\n", global_tid)); | |
| #ifdef USE_CHECKS_COMMON | |
| if ((size_t)size > tn->cmn_size) { | |
| KC_TRACE(10, ("THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC | |
| " ,%" KMP_UINTPTR_SPEC ")\n", | |
| data, size, tn->cmn_size)); | |
| KMP_FATAL(TPCommonBlocksInconsist); | |
| } | |
| #endif /* USE_CHECKS_COMMON */ | |
| } else { | |
| /* The parallel address will NEVER overlap with the data_address */ | |
| /* dkp: 3rd arg to kmp_threadprivate_insert() is the data_address; use | |
| * data_address = data */ | |
| KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting data\n", global_tid)); | |
| tn = kmp_threadprivate_insert(global_tid, data, data, size); | |
| } | |
| ret = tn->par_addr; | |
| } | |
| KC_TRACE(10, ("__kmpc_threadprivate: T#%d exiting; return value = %p\n", | |
| global_tid, ret)); | |
| return ret; | |
| } | |
| static kmp_cached_addr_t *__kmp_find_cache(void *data) { | |
| kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; | |
| while (ptr && ptr->data != data) | |
| ptr = ptr->next; | |
| return ptr; | |
| } | |
| /*! | |
| @ingroup THREADPRIVATE | |
| @param loc source location information | |
| @param global_tid global thread number | |
| @param data pointer to data to privatize | |
| @param size size of data to privatize | |
| @param cache pointer to cache | |
| @return pointer to private storage | |
| Allocate private storage for threadprivate data. | |
| */ | |
| void * | |
| __kmpc_threadprivate_cached(ident_t *loc, | |
| kmp_int32 global_tid, // gtid. | |
| void *data, // Pointer to original global variable. | |
| size_t size, // Size of original global variable. | |
| void ***cache) { | |
| KC_TRACE(10, ("__kmpc_threadprivate_cached: T#%d called with cache: %p, " | |
| "address: %p, size: %" KMP_SIZE_T_SPEC "\n", | |
| global_tid, *cache, data, size)); | |
| if (TCR_PTR(*cache) == 0) { | |
| __kmp_acquire_lock(&__kmp_global_lock, global_tid); | |
| if (TCR_PTR(*cache) == 0) { | |
| __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock); | |
| // Compiler often passes in NULL cache, even if it's already been created | |
| void **my_cache; | |
| kmp_cached_addr_t *tp_cache_addr; | |
| // Look for an existing cache | |
| tp_cache_addr = __kmp_find_cache(data); | |
| if (!tp_cache_addr) { // Cache was never created; do it now | |
| __kmp_tp_cached = 1; | |
| KMP_ITT_IGNORE(my_cache = (void **)__kmp_allocate( | |
| sizeof(void *) * __kmp_tp_capacity + | |
| sizeof(kmp_cached_addr_t));); | |
| // No need to zero the allocated memory; __kmp_allocate does that. | |
| KC_TRACE(50, ("__kmpc_threadprivate_cached: T#%d allocated cache at " | |
| "address %p\n", | |
| global_tid, my_cache)); | |
| /* TODO: free all this memory in __kmp_common_destroy using | |
| * __kmp_threadpriv_cache_list */ | |
| /* Add address of mycache to linked list for cleanup later */ | |
| tp_cache_addr = (kmp_cached_addr_t *)&my_cache[__kmp_tp_capacity]; | |
| tp_cache_addr->addr = my_cache; | |
| tp_cache_addr->data = data; | |
| tp_cache_addr->compiler_cache = cache; | |
| tp_cache_addr->next = __kmp_threadpriv_cache_list; | |
| __kmp_threadpriv_cache_list = tp_cache_addr; | |
| } else { // A cache was already created; use it | |
| my_cache = tp_cache_addr->addr; | |
| tp_cache_addr->compiler_cache = cache; | |
| } | |
| KMP_MB(); | |
| TCW_PTR(*cache, my_cache); | |
| __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock); | |
| KMP_MB(); | |
| } | |
| __kmp_release_lock(&__kmp_global_lock, global_tid); | |
| } | |
| void *ret; | |
| if ((ret = TCR_PTR((*cache)[global_tid])) == 0) { | |
| ret = __kmpc_threadprivate(loc, global_tid, data, (size_t)size); | |
| TCW_PTR((*cache)[global_tid], ret); | |
| } | |
| KC_TRACE(10, | |
| ("__kmpc_threadprivate_cached: T#%d exiting; return value = %p\n", | |
| global_tid, ret)); | |
| return ret; | |
| } | |
| // This function should only be called when both __kmp_tp_cached_lock and | |
| // kmp_forkjoin_lock are held. | |
| void __kmp_threadprivate_resize_cache(int newCapacity) { | |
| KC_TRACE(10, ("__kmp_threadprivate_resize_cache: called with size: %d\n", | |
| newCapacity)); | |
| kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; | |
| while (ptr) { | |
| if (ptr->data) { // this location has an active cache; resize it | |
| void **my_cache; | |
| KMP_ITT_IGNORE(my_cache = | |
| (void **)__kmp_allocate(sizeof(void *) * newCapacity + | |
| sizeof(kmp_cached_addr_t));); | |
| // No need to zero the allocated memory; __kmp_allocate does that. | |
| KC_TRACE(50, ("__kmp_threadprivate_resize_cache: allocated cache at %p\n", | |
| my_cache)); | |
| // Now copy old cache into new cache | |
| void **old_cache = ptr->addr; | |
| for (int i = 0; i < __kmp_tp_capacity; ++i) { | |
| my_cache[i] = old_cache[i]; | |
| } | |
| // Add address of new my_cache to linked list for cleanup later | |
| kmp_cached_addr_t *tp_cache_addr; | |
| tp_cache_addr = (kmp_cached_addr_t *)&my_cache[newCapacity]; | |
| tp_cache_addr->addr = my_cache; | |
| tp_cache_addr->data = ptr->data; | |
| tp_cache_addr->compiler_cache = ptr->compiler_cache; | |
| tp_cache_addr->next = __kmp_threadpriv_cache_list; | |
| __kmp_threadpriv_cache_list = tp_cache_addr; | |
| // Copy new cache to compiler's location: We can copy directly | |
| // to (*compiler_cache) if compiler guarantees it will keep | |
| // using the same location for the cache. This is not yet true | |
| // for some compilers, in which case we have to check if | |
| // compiler_cache is still pointing at old cache, and if so, we | |
| // can point it at the new cache with an atomic compare&swap | |
| // operation. (Old method will always work, but we should shift | |
| // to new method (commented line below) when Intel and Clang | |
| // compilers use new method.) | |
| (void)KMP_COMPARE_AND_STORE_PTR(tp_cache_addr->compiler_cache, old_cache, | |
| my_cache); | |
| // TCW_PTR(*(tp_cache_addr->compiler_cache), my_cache); | |
| // If the store doesn't happen here, the compiler's old behavior will | |
| // inevitably call __kmpc_threadprivate_cache with a new location for the | |
| // cache, and that function will store the resized cache there at that | |
| // point. | |
| // Nullify old cache's data pointer so we skip it next time | |
| ptr->data = NULL; | |
| } | |
| ptr = ptr->next; | |
| } | |
| // After all caches are resized, update __kmp_tp_capacity to the new size | |
| *(volatile int *)&__kmp_tp_capacity = newCapacity; | |
| } | |
| /*! | |
| @ingroup THREADPRIVATE | |
| @param loc source location information | |
| @param data pointer to data being privatized | |
| @param ctor pointer to constructor function for data | |
| @param cctor pointer to copy constructor function for data | |
| @param dtor pointer to destructor function for data | |
| @param vector_length length of the vector (bytes or elements?) | |
| Register vector constructors and destructors for thread private data. | |
| */ | |
| void __kmpc_threadprivate_register_vec(ident_t *loc, void *data, | |
| kmpc_ctor_vec ctor, kmpc_cctor_vec cctor, | |
| kmpc_dtor_vec dtor, | |
| size_t vector_length) { | |
| struct shared_common *d_tn, **lnk_tn; | |
| KC_TRACE(10, ("__kmpc_threadprivate_register_vec: called\n")); | |
| #ifdef USE_CHECKS_COMMON | |
| /* copy constructor must be zero for current code gen (Nov 2002 - jph) */ | |
| KMP_ASSERT(cctor == 0); | |
| #endif /* USE_CHECKS_COMMON */ | |
| d_tn = __kmp_find_shared_task_common( | |
| &__kmp_threadprivate_d_table, -1, | |
| data); /* Only the global data table exists. */ | |
| if (d_tn == 0) { | |
| d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common)); | |
| d_tn->gbl_addr = data; | |
| d_tn->ct.ctorv = ctor; | |
| d_tn->cct.cctorv = cctor; | |
| d_tn->dt.dtorv = dtor; | |
| d_tn->is_vec = TRUE; | |
| d_tn->vec_len = (size_t)vector_length; | |
| // d_tn->obj_init = 0; // AC: __kmp_allocate zeroes the memory | |
| // d_tn->pod_init = 0; | |
| lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]); | |
| d_tn->next = *lnk_tn; | |
| *lnk_tn = d_tn; | |
| } | |
| } | |
| void __kmp_cleanup_threadprivate_caches() { | |
| kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list; | |
| while (ptr) { | |
| void **cache = ptr->addr; | |
| __kmp_threadpriv_cache_list = ptr->next; | |
| if (*ptr->compiler_cache) | |
| *ptr->compiler_cache = NULL; | |
| ptr->compiler_cache = NULL; | |
| ptr->data = NULL; | |
| ptr->addr = NULL; | |
| ptr->next = NULL; | |
| // Threadprivate data pointed at by cache entries are destroyed at end of | |
| // __kmp_launch_thread with __kmp_common_destroy_gtid. | |
| __kmp_free(cache); // implicitly frees ptr too | |
| ptr = __kmp_threadpriv_cache_list; | |
| } | |
| } |