blob: 38d5139212be529df451f6f32ea0745c26e91f74 [file] [log] [blame]
//===----------------------------------------------------------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//
// Forward declaration as the following 2 functions are declared as friend in offload_engine.h
// CLANG does not like static to been after friend declaration.
static void __offload_init_library_once(void);
static void __offload_fini_library(void);
#include "offload_host.h"
#ifdef MYO_SUPPORT
#include "offload_myo_host.h"
#endif
#include <malloc.h>
#ifndef TARGET_WINNT
#include <alloca.h>
#include <elf.h>
#endif // TARGET_WINNT
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <algorithm>
#include <bitset>
#if defined(HOST_WINNT)
#define PATH_SEPARATOR ";"
#else
#define PATH_SEPARATOR ":"
#endif
#define GET_OFFLOAD_NUMBER(timer_data) \
timer_data? timer_data->offload_number : 0
#ifdef TARGET_WINNT
// Small subset of ELF declarations for Windows which is needed to compile
// this file. ELF header is used to understand what binary type is contained
// in the target image - shared library or executable.
typedef uint16_t Elf64_Half;
typedef uint32_t Elf64_Word;
typedef uint64_t Elf64_Addr;
typedef uint64_t Elf64_Off;
#define EI_NIDENT 16
#define ET_EXEC 2
#define ET_DYN 3
typedef struct
{
unsigned char e_ident[EI_NIDENT];
Elf64_Half e_type;
Elf64_Half e_machine;
Elf64_Word e_version;
Elf64_Addr e_entry;
Elf64_Off e_phoff;
Elf64_Off e_shoff;
Elf64_Word e_flags;
Elf64_Half e_ehsize;
Elf64_Half e_phentsize;
Elf64_Half e_phnum;
Elf64_Half e_shentsize;
Elf64_Half e_shnum;
Elf64_Half e_shstrndx;
} Elf64_Ehdr;
#endif // TARGET_WINNT
// Host console and file logging
const char *prefix;
int console_enabled = 0;
int offload_number = 0;
static const char *htrace_envname = "H_TRACE";
static const char *offload_report_envname = "OFFLOAD_REPORT";
static char *timer_envname = "H_TIME";
// Trace information
static const char* vardesc_direction_as_string[] = {
"NOCOPY",
"IN",
"OUT",
"INOUT"
};
static const char* vardesc_type_as_string[] = {
"unknown",
"data",
"data_ptr",
"func_ptr",
"void_ptr",
"string_ptr",
"dv",
"dv_data",
"dv_data_slice",
"dv_ptr",
"dv_ptr_data",
"dv_ptr_data_slice",
"cean_var",
"cean_var_ptr",
"c_data_ptr_array",
"c_func_ptr_array",
"c_void_ptr_array",
"c_string_ptr_array"
};
Engine* mic_engines = 0;
uint32_t mic_engines_total = 0;
pthread_key_t mic_thread_key;
MicEnvVar mic_env_vars;
uint64_t cpu_frequency = 0;
// MIC_STACKSIZE
uint32_t mic_stack_size = 12 * 1024 * 1024;
// MIC_BUFFERSIZE
uint64_t mic_buffer_size = 0;
// MIC_LD_LIBRARY_PATH
char* mic_library_path = 0;
// MIC_PROXY_IO
bool mic_proxy_io = true;
// MIC_PROXY_FS_ROOT
char* mic_proxy_fs_root = 0;
// Threshold for creating buffers with large pages. Buffer is created
// with large pages hint if its size exceeds the threshold value.
// By default large pages are disabled right now (by setting default
// value for threshold to MAX) due to HSD 4114629.
uint64_t __offload_use_2mb_buffers = 0xffffffffffffffffULL;
static const char *mic_use_2mb_buffers_envname =
"MIC_USE_2MB_BUFFERS";
static uint64_t __offload_use_async_buffer_write = 2 * 1024 * 1024;
static const char *mic_use_async_buffer_write_envname =
"MIC_USE_ASYNC_BUFFER_WRITE";
static uint64_t __offload_use_async_buffer_read = 2 * 1024 * 1024;
static const char *mic_use_async_buffer_read_envname =
"MIC_USE_ASYNC_BUFFER_READ";
// device initialization type
OffloadInitType __offload_init_type = c_init_on_offload_all;
static const char *offload_init_envname = "OFFLOAD_INIT";
// active wait
static bool __offload_active_wait = true;
static const char *offload_active_wait_envname = "OFFLOAD_ACTIVE_WAIT";
// OMP_DEFAULT_DEVICE
int __omp_device_num = 0;
static const char *omp_device_num_envname = "OMP_DEFAULT_DEVICE";
// The list of pending target libraries
static bool __target_libs;
static TargetImageList __target_libs_list;
static mutex_t __target_libs_lock;
static mutex_t stack_alloc_lock;
// Target executable
TargetImage* __target_exe;
static char * offload_get_src_base(void * ptr, uint8_t type)
{
char *base;
if (VAR_TYPE_IS_PTR(type)) {
base = *static_cast<char**>(ptr);
}
else if (VAR_TYPE_IS_SCALAR(type)) {
base = static_cast<char*>(ptr);
}
else if (VAR_TYPE_IS_DV_DATA_SLICE(type) || VAR_TYPE_IS_DV_DATA(type)) {
ArrDesc *dvp;
if (VAR_TYPE_IS_DV_DATA_SLICE(type)) {
const arr_desc *ap = static_cast<const arr_desc*>(ptr);
dvp = (type == c_dv_data_slice) ?
reinterpret_cast<ArrDesc*>(ap->base) :
*reinterpret_cast<ArrDesc**>(ap->base);
}
else {
dvp = (type == c_dv_data) ?
static_cast<ArrDesc*>(ptr) :
*static_cast<ArrDesc**>(ptr);
}
base = reinterpret_cast<char*>(dvp->Base);
}
else {
base = NULL;
}
return base;
}
void OffloadDescriptor::report_coi_error(error_types msg, COIRESULT res)
{
// special case for the 'process died' error
if (res == COI_PROCESS_DIED) {
m_device.fini_process(true);
}
else {
switch (msg) {
case c_buf_create:
if (res == COI_OUT_OF_MEMORY) {
msg = c_buf_create_out_of_mem;
}
/* fallthru */
case c_buf_create_from_mem:
case c_buf_get_address:
case c_pipeline_create:
case c_pipeline_run_func:
LIBOFFLOAD_ERROR(msg, m_device.get_logical_index(), res);
break;
case c_buf_read:
case c_buf_write:
case c_buf_copy:
case c_buf_map:
case c_buf_unmap:
case c_buf_destroy:
case c_buf_set_state:
LIBOFFLOAD_ERROR(msg, res);
break;
default:
break;
}
}
exit(1);
}
_Offload_result OffloadDescriptor::translate_coi_error(COIRESULT res) const
{
switch (res) {
case COI_SUCCESS:
return OFFLOAD_SUCCESS;
case COI_PROCESS_DIED:
return OFFLOAD_PROCESS_DIED;
case COI_OUT_OF_MEMORY:
return OFFLOAD_OUT_OF_MEMORY;
default:
return OFFLOAD_ERROR;
}
}
bool OffloadDescriptor::alloc_ptr_data(
PtrData* &ptr_data,
void *base,
int64_t disp,
int64_t size,
int64_t alloc_disp,
int align
)
{
// total length of base
int64_t length = disp + size;
bool is_new;
OFFLOAD_TRACE(3, "Creating association for data: addr %p, length %lld\n",
base, length);
// add new entry
ptr_data = m_device.insert_ptr_data(base, length, is_new);
if (is_new) {
OFFLOAD_TRACE(3, "Added new association\n");
if (length > 0) {
OffloadTimer timer(get_timer_data(), c_offload_host_alloc_buffers);
COIRESULT res;
// align should be a power of 2
if (align > 0 && (align & (align - 1)) == 0) {
// offset within mic_buffer. Can do offset optimization
// only when source address alignment satisfies requested
// alignment on the target (cq172736).
if ((reinterpret_cast<intptr_t>(base) & (align - 1)) == 0) {
ptr_data->mic_offset = reinterpret_cast<intptr_t>(base) & 4095;
}
}
// buffer size and flags
uint64_t buffer_size = length + ptr_data->mic_offset;
uint32_t buffer_flags = 0;
// create buffer with large pages if data length exceeds
// large page threshold
if (length >= __offload_use_2mb_buffers) {
buffer_flags = COI_OPTIMIZE_HUGE_PAGE_SIZE;
}
// create CPU buffer
OFFLOAD_DEBUG_TRACE_1(3,
GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_create_buf_host,
"Creating buffer from source memory %p, "
"length %lld\n", base, length);
// result is not checked because we can continue without cpu
// buffer. In this case we will use COIBufferRead/Write instead
// of COIBufferCopy.
COI::BufferCreateFromMemory(length,
COI_BUFFER_NORMAL,
0,
base,
1,
&m_device.get_process(),
&ptr_data->cpu_buf);
OFFLOAD_DEBUG_TRACE_1(3,
GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_create_buf_mic,
"Creating buffer for sink: size %lld, offset %d, "
"flags =0x%x\n", buffer_size - alloc_disp,
ptr_data->mic_offset, buffer_flags);
// create MIC buffer
res = COI::BufferCreate(buffer_size - alloc_disp,
COI_BUFFER_NORMAL,
buffer_flags,
0,
1,
&m_device.get_process(),
&ptr_data->mic_buf);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_create, res);
}
ptr_data->alloc_ptr_data_lock.unlock();
return false;
}
// make buffer valid on the device.
res = COI::BufferSetState(ptr_data->mic_buf,
m_device.get_process(),
COI_BUFFER_VALID,
COI_BUFFER_NO_MOVE,
0, 0, 0);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_set_state, res);
}
ptr_data->alloc_ptr_data_lock.unlock();
return false;
}
res = COI::BufferSetState(ptr_data->mic_buf,
COI_PROCESS_SOURCE,
COI_BUFFER_INVALID,
COI_BUFFER_NO_MOVE,
0, 0, 0);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_set_state, res);
}
ptr_data->alloc_ptr_data_lock.unlock();
return false;
}
}
ptr_data->alloc_disp = alloc_disp;
ptr_data->alloc_ptr_data_lock.unlock();
}
else {
mutex_locker_t locker(ptr_data->alloc_ptr_data_lock);
OFFLOAD_TRACE(3, "Found existing association: addr %p, length %lld, "
"is_static %d\n",
ptr_data->cpu_addr.start(), ptr_data->cpu_addr.length(),
ptr_data->is_static);
// This is not a new entry. Make sure that provided address range fits
// into existing one.
MemRange addr_range(base, length - ptr_data->alloc_disp);
if (!ptr_data->cpu_addr.contains(addr_range)) {
LIBOFFLOAD_ERROR(c_bad_ptr_mem_range);
exit(1);
}
// if the entry is associated with static data it may not have buffers
// created because they are created on demand.
if (ptr_data->is_static && !init_static_ptr_data(ptr_data)) {
return false;
}
}
return true;
}
bool OffloadDescriptor::find_ptr_data(
PtrData* &ptr_data,
void *base,
int64_t disp,
int64_t size,
bool report_error
)
{
// total length of base
int64_t length = disp + size;
OFFLOAD_TRACE(3, "Looking for association for data: addr %p, "
"length %lld\n", base, length);
// find existing association in pointer table
ptr_data = m_device.find_ptr_data(base);
if (ptr_data == 0) {
if (report_error) {
LIBOFFLOAD_ERROR(c_no_ptr_data, base);
exit(1);
}
OFFLOAD_TRACE(3, "Association does not exist\n");
return true;
}
OFFLOAD_TRACE(3, "Found association: base %p, length %lld, is_static %d\n",
ptr_data->cpu_addr.start(), ptr_data->cpu_addr.length(),
ptr_data->is_static);
// make sure that provided address range fits into existing one
MemRange addr_range(base, length);
if (!ptr_data->cpu_addr.contains(addr_range)) {
if (report_error) {
LIBOFFLOAD_ERROR(c_bad_ptr_mem_range);
exit(1);
}
OFFLOAD_TRACE(3, "Existing association partially overlaps with "
"data address range\n");
ptr_data = 0;
return true;
}
// if the entry is associated with static data it may not have buffers
// created because they are created on demand.
if (ptr_data->is_static && !init_static_ptr_data(ptr_data)) {
return false;
}
return true;
}
bool OffloadDescriptor::init_static_ptr_data(PtrData *ptr_data)
{
OffloadTimer timer(get_timer_data(), c_offload_host_alloc_buffers);
if (ptr_data->cpu_buf == 0) {
OFFLOAD_TRACE(3, "Creating buffer from source memory %llx\n",
ptr_data->cpu_addr.start());
COIRESULT res = COI::BufferCreateFromMemory(
ptr_data->cpu_addr.length(),
COI_BUFFER_NORMAL,
0,
const_cast<void*>(ptr_data->cpu_addr.start()),
1, &m_device.get_process(),
&ptr_data->cpu_buf);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_create_from_mem, res);
}
}
if (ptr_data->mic_buf == 0) {
OFFLOAD_TRACE(3, "Creating buffer from sink memory %llx\n",
ptr_data->mic_addr);
COIRESULT res = COI::BufferCreateFromMemory(
ptr_data->cpu_addr.length(),
COI_BUFFER_NORMAL,
COI_SINK_MEMORY,
reinterpret_cast<void*>(ptr_data->mic_addr),
1, &m_device.get_process(),
&ptr_data->mic_buf);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_create_from_mem, res);
}
}
return true;
}
bool OffloadDescriptor::init_mic_address(PtrData *ptr_data)
{
if (ptr_data->mic_buf != 0 && ptr_data->mic_addr == 0) {
COIRESULT res = COI::BufferGetSinkAddress(ptr_data->mic_buf,
&ptr_data->mic_addr);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_get_address, res);
}
return false;
}
}
return true;
}
bool OffloadDescriptor::nullify_target_stack(
COIBUFFER targ_buf,
uint64_t size
)
{
char * ptr = (char*)malloc(size);
COIRESULT res;
memset(ptr, 0, size);
res = COI::BufferWrite(
targ_buf,
0,
ptr,
size,
COI_COPY_UNSPECIFIED,
0, 0, 0);
free(ptr);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_write, res);
}
return true;
}
bool OffloadDescriptor::offload_stack_memory_manager(
const void * stack_begin,
int routine_id,
int buf_size,
int align,
bool *is_new)
{
mutex_locker_t locker(stack_alloc_lock);
PersistData * new_el;
PersistDataList::iterator it_begin = m_device.m_persist_list.begin();
PersistDataList::iterator it_end;
int erase = 0;
*is_new = false;
for (PersistDataList::iterator it = m_device.m_persist_list.begin();
it != m_device.m_persist_list.end(); it++) {
PersistData cur_el = *it;
if (stack_begin > it->stack_cpu_addr) {
// this stack data must be destroyed
m_destroy_stack.push_front(cur_el.stack_ptr_data);
it_end = it;
erase++;
}
else if (stack_begin == it->stack_cpu_addr) {
if (routine_id != it-> routine_id) {
// this stack data must be destroyed
m_destroy_stack.push_front(cur_el.stack_ptr_data);
it_end = it;
erase++;
break;
}
else {
// stack data is reused
m_stack_ptr_data = it->stack_ptr_data;
if (erase > 0) {
// all obsolete stack sections must be erased from the list
m_device.m_persist_list.erase(it_begin, ++it_end);
m_in_datalen +=
erase * sizeof(new_el->stack_ptr_data->mic_addr);
}
OFFLOAD_TRACE(3, "Reuse of stack buffer with addr %p\n",
m_stack_ptr_data->mic_addr);
return true;
}
}
else if (stack_begin < it->stack_cpu_addr) {
break;
}
}
if (erase > 0) {
// all obsolete stack sections must be erased from the list
m_device.m_persist_list.erase(it_begin, ++it_end);
m_in_datalen += erase * sizeof(new_el->stack_ptr_data->mic_addr);
}
// new stack table is created
new_el = new PersistData(stack_begin, routine_id, buf_size);
// create MIC buffer
COIRESULT res;
uint32_t buffer_flags = 0;
// create buffer with large pages if data length exceeds
// large page threshold
if (buf_size >= __offload_use_2mb_buffers) {
buffer_flags = COI_OPTIMIZE_HUGE_PAGE_SIZE;
}
res = COI::BufferCreate(buf_size,
COI_BUFFER_NORMAL,
buffer_flags,
0,
1,
&m_device.get_process(),
&new_el->stack_ptr_data->mic_buf);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_create, res);
}
return false;
}
// make buffer valid on the device.
res = COI::BufferSetState(new_el->stack_ptr_data->mic_buf,
m_device.get_process(),
COI_BUFFER_VALID,
COI_BUFFER_NO_MOVE,
0, 0, 0);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_set_state, res);
}
return false;
}
res = COI::BufferSetState(new_el->stack_ptr_data->mic_buf,
COI_PROCESS_SOURCE,
COI_BUFFER_INVALID,
COI_BUFFER_NO_MOVE,
0, 0, 0);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
}
else if (m_is_mandatory) {
report_coi_error(c_buf_set_state, res);
}
return false;
}
// persistence algorithm requires target stack initialy to be nullified
if (!nullify_target_stack(new_el->stack_ptr_data->mic_buf, buf_size)) {
return false;
}
m_stack_ptr_data = new_el->stack_ptr_data;
init_mic_address(m_stack_ptr_data);
OFFLOAD_TRACE(3, "Allocating stack buffer with addr %p\n",
m_stack_ptr_data->mic_addr);
m_device.m_persist_list.push_front(*new_el);
init_mic_address(new_el->stack_ptr_data);
*is_new = true;
return true;
}
bool OffloadDescriptor::setup_descriptors(
VarDesc *vars,
VarDesc2 *vars2,
int vars_total,
int entry_id,
const void *stack_addr
)
{
COIRESULT res;
OffloadTimer timer(get_timer_data(), c_offload_host_setup_buffers);
// make a copy of variable descriptors
m_vars_total = vars_total;
if (vars_total > 0) {
m_vars = (VarDesc*) malloc(m_vars_total * sizeof(VarDesc));
memcpy(m_vars, vars, m_vars_total * sizeof(VarDesc));
m_vars_extra = (VarExtra*) malloc(m_vars_total * sizeof(VarExtra));
}
// dependencies
m_in_deps = (COIEVENT*) malloc(sizeof(COIEVENT) * (m_vars_total + 1));
if (m_vars_total > 0) {
m_out_deps = (COIEVENT*) malloc(sizeof(COIEVENT) * m_vars_total);
}
// copyin/copyout data length
m_in_datalen = 0;
m_out_datalen = 0;
// First pass over variable descriptors
// - Calculate size of the input and output non-pointer data
// - Allocate buffers for input and output pointers
for (int i = 0; i < m_vars_total; i++) {
void* alloc_base = NULL;
int64_t alloc_disp = 0;
int64_t alloc_size;
bool src_is_for_mic = (m_vars[i].direction.out ||
m_vars[i].into == NULL);
const char *var_sname = "";
if (vars2 != NULL && i < vars_total) {
if (vars2[i].sname != NULL) {
var_sname = vars2[i].sname;
}
}
OFFLOAD_TRACE(2, " VarDesc %d, var=%s, %s, %s\n",
i, var_sname,
vardesc_direction_as_string[m_vars[i].direction.bits],
vardesc_type_as_string[m_vars[i].type.src]);
if (vars2 != NULL && i < vars_total && vars2[i].dname != NULL) {
OFFLOAD_TRACE(2, " into=%s, %s\n", vars2[i].dname,
vardesc_type_as_string[m_vars[i].type.dst]);
}
OFFLOAD_TRACE(2,
" type_src=%d, type_dstn=%d, direction=%d, "
"alloc_if=%d, free_if=%d, align=%d, mic_offset=%d, flags=0x%x, "
"offset=%lld, size=%lld, count/disp=%lld, ptr=%p, into=%p\n",
m_vars[i].type.src,
m_vars[i].type.dst,
m_vars[i].direction.bits,
m_vars[i].alloc_if,
m_vars[i].free_if,
m_vars[i].align,
m_vars[i].mic_offset,
m_vars[i].flags.bits,
m_vars[i].offset,
m_vars[i].size,
m_vars[i].count,
m_vars[i].ptr,
m_vars[i].into);
if (m_vars[i].alloc != NULL) {
// array descriptor
const arr_desc *ap =
static_cast<const arr_desc*>(m_vars[i].alloc);
// debug dump
__arr_desc_dump(" ", "ALLOC", ap, 0);
__arr_data_offset_and_length(ap, alloc_disp, alloc_size);
alloc_base = reinterpret_cast<void*>(ap->base);
}
m_vars_extra[i].cpu_disp = 0;
m_vars_extra[i].cpu_offset = 0;
m_vars_extra[i].src_data = 0;
m_vars_extra[i].read_rng_src = 0;
m_vars_extra[i].read_rng_dst = 0;
// flag is_arr_ptr_el is 1 only for var_descs generated
// for c_data_ptr_array type
if (i < vars_total) {
m_vars_extra[i].is_arr_ptr_el = 0;
}
switch (m_vars[i].type.src) {
case c_data_ptr_array:
{
const arr_desc *ap;
const VarDesc3 *vd3 =
static_cast<const VarDesc3*>(m_vars[i].ptr);
int flags = vd3->array_fields;
OFFLOAD_TRACE(2,
" pointer array flags = %04x\n", flags);
OFFLOAD_TRACE(2,
" pointer array type is %s\n",
vardesc_type_as_string[flags & 0x3f]);
ap = static_cast<const arr_desc*>(vd3->ptr_array);
__arr_desc_dump(" ", "ptr array", ap, 0);
if (m_vars[i].into) {
ap = static_cast<const arr_desc*>(m_vars[i].into);
__arr_desc_dump(
" ", "into array", ap, 0);
}
if ((flags & (1<<flag_align_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->align_array);
__arr_desc_dump(
" ", "align array", ap, 0);
}
if ((flags & (1<<flag_alloc_if_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->alloc_if_array);
__arr_desc_dump(
" ", "alloc_if array", ap, 0);
}
if ((flags & (1<<flag_free_if_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->free_if_array);
__arr_desc_dump(
" ", "free_if array", ap, 0);
}
if ((flags & (1<<flag_extent_start_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->extent_start);
__arr_desc_dump(
" ", "extent_start array", ap, 0);
} else if ((flags &
(1<<flag_extent_start_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" extent_start scalar = %d\n",
(int64_t)vd3->extent_start);
}
if ((flags & (1<<flag_extent_elements_is_array)) != 0) {
ap = static_cast<const arr_desc*>
(vd3->extent_elements);
__arr_desc_dump(
" ", "extent_elements array", ap, 0);
} else if ((flags &
(1<<flag_extent_elements_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" extent_elements scalar = %d\n",
(int64_t)vd3->extent_elements);
}
if ((flags & (1<<flag_into_start_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->into_start);
__arr_desc_dump(
" ", "into_start array", ap, 0);
} else if ((flags &
(1<<flag_into_start_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" into_start scalar = %d\n",
(int64_t)vd3->into_start);
}
if ((flags & (1<<flag_into_elements_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->into_elements);
__arr_desc_dump(
" ", "into_elements array", ap, 0);
} else if ((flags &
(1<<flag_into_elements_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" into_elements scalar = %d\n",
(int64_t)vd3->into_elements);
}
if ((flags & (1<<flag_alloc_start_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->alloc_start);
__arr_desc_dump(
" ", "alloc_start array", ap, 0);
} else if ((flags &
(1<<flag_alloc_start_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" alloc_start scalar = %d\n",
(int64_t)vd3->alloc_start);
}
if ((flags & (1<<flag_alloc_elements_is_array)) != 0) {
ap = static_cast<const arr_desc*>(vd3->alloc_elements);
__arr_desc_dump(
" ", "alloc_elements array", ap, 0);
} else if ((flags &
(1<<flag_alloc_elements_is_scalar)) != 0) {
OFFLOAD_TRACE(2,
" alloc_elements scalar = %d\n",
(int64_t)vd3->alloc_elements);
}
}
if (!gen_var_descs_for_pointer_array(i)) {
return false;
}
break;
case c_data:
case c_void_ptr:
case c_cean_var:
// In all uses later
// VarDesc.size will have the length of the data to be
// transferred
// VarDesc.disp will have an offset from base
if (m_vars[i].type.src == c_cean_var) {
// array descriptor
const arr_desc *ap =
static_cast<const arr_desc*>(m_vars[i].ptr);
// debug dump
__arr_desc_dump("", "IN/OUT", ap, 0);
// offset and length are derived from the array descriptor
__arr_data_offset_and_length(ap, m_vars[i].disp,
m_vars[i].size);
if (!is_arr_desc_contiguous(ap)) {
m_vars[i].flags.is_noncont_src = 1;
m_vars_extra[i].read_rng_src =
init_read_ranges_arr_desc(ap);
}
// all necessary information about length and offset is
// transferred in var descriptor. There is no need to send
// array descriptor to the target side.
m_vars[i].ptr = reinterpret_cast<void*>(ap->base);
}
else {
m_vars[i].size *= m_vars[i].count;
m_vars[i].disp = 0;
}
if (m_vars[i].direction.bits) {
// make sure that transfer size > 0
if (m_vars[i].size <= 0) {
LIBOFFLOAD_ERROR(c_zero_or_neg_transfer_size);
exit(1);
}
if (m_vars[i].flags.is_static) {
PtrData *ptr_data;
// find data associated with variable
if (!find_ptr_data(ptr_data,
m_vars[i].ptr,
m_vars[i].disp,
m_vars[i].size,
false)) {
return false;
}
if (ptr_data != 0) {
// offset to base from the beginning of the buffer
// memory
m_vars[i].offset =
(char*) m_vars[i].ptr -
(char*) ptr_data->cpu_addr.start();
}
else {
m_vars[i].flags.is_static = false;
if (m_vars[i].into == NULL) {
m_vars[i].flags.is_static_dstn = false;
}
}
m_vars_extra[i].src_data = ptr_data;
}
if (m_is_openmp) {
if (m_vars[i].flags.is_static) {
// Static data is transferred only by omp target
// update construct which passes zeros for
// alloc_if and free_if.
if (m_vars[i].alloc_if || m_vars[i].free_if) {
m_vars[i].direction.bits = c_parameter_nocopy;
}
}
else {
AutoData *auto_data;
if (m_vars[i].alloc_if) {
auto_data = m_device.insert_auto_data(
m_vars[i].ptr, m_vars[i].size);
auto_data->add_reference();
}
else {
// TODO: what should be done if var is not in
// the table?
auto_data = m_device.find_auto_data(
m_vars[i].ptr);
}
// For automatic variables data is transferred
// only if alloc_if == 0 && free_if == 0
// or reference count is 1
if ((m_vars[i].alloc_if || m_vars[i].free_if) &&
auto_data != 0 &&
auto_data->get_reference() != 1) {
m_vars[i].direction.bits = c_parameter_nocopy;
}
// save data for later use
m_vars_extra[i].auto_data = auto_data;
}
}
if (m_vars[i].direction.in &&
!m_vars[i].flags.is_static) {
m_in_datalen += m_vars[i].size;
// for non-static target destination defined as CEAN
// expression we pass to target its size and dist
if (m_vars[i].into == NULL &&
m_vars[i].type.src == c_cean_var) {
m_in_datalen += 2 * sizeof(uint64_t);
}
m_need_runfunction = true;
}
if (m_vars[i].direction.out &&
!m_vars[i].flags.is_static) {
m_out_datalen += m_vars[i].size;
m_need_runfunction = true;
}
}
break;
case c_dv:
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
ArrDesc *dvp = static_cast<ArrDesc*>(m_vars[i].ptr);
// debug dump
__dv_desc_dump("IN/OUT", dvp);
// send dope vector contents excluding base
m_in_datalen += m_vars[i].size - sizeof(uint64_t);
m_need_runfunction = true;
}
break;
case c_string_ptr:
if ((m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) &&
m_vars[i].size == 0) {
m_vars[i].size = 1;
m_vars[i].count =
strlen(*static_cast<char**>(m_vars[i].ptr)) + 1;
}
/* fallthru */
case c_data_ptr:
if (m_vars[i].flags.is_stack_buf &&
!m_vars[i].direction.bits &&
m_vars[i].alloc_if) {
// this var_desc is for stack buffer
bool is_new;
if (!offload_stack_memory_manager(
stack_addr, entry_id,
m_vars[i].count, m_vars[i].align, &is_new)) {
return false;
}
if (is_new) {
m_compute_buffers.push_back(
m_stack_ptr_data->mic_buf);
m_device.m_persist_list.front().cpu_stack_addr =
static_cast<char*>(m_vars[i].ptr);
}
else {
m_vars[i].flags.sink_addr = 1;
m_in_datalen += sizeof(m_stack_ptr_data->mic_addr);
}
m_vars[i].size = m_destroy_stack.size();
m_vars_extra[i].src_data = m_stack_ptr_data;
// need to add reference for buffer
m_need_runfunction = true;
break;
}
/* fallthru */
case c_cean_var_ptr:
case c_dv_ptr:
if (m_vars[i].type.src == c_cean_var_ptr) {
// array descriptor
const arr_desc *ap =
static_cast<const arr_desc*>(m_vars[i].ptr);
// debug dump
__arr_desc_dump("", "IN/OUT", ap, 1);
// offset and length are derived from the array descriptor
__arr_data_offset_and_length(ap, m_vars[i].disp,
m_vars[i].size);
if (!is_arr_desc_contiguous(ap)) {
m_vars[i].flags.is_noncont_src = 1;
m_vars_extra[i].read_rng_src =
init_read_ranges_arr_desc(ap);
}
// all necessary information about length and offset is
// transferred in var descriptor. There is no need to send
// array descriptor to the target side.
m_vars[i].ptr = reinterpret_cast<void*>(ap->base);
}
else if (m_vars[i].type.src == c_dv_ptr) {
// need to send DV to the device unless it is 'nocopy'
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
ArrDesc *dvp = *static_cast<ArrDesc**>(m_vars[i].ptr);
// debug dump
__dv_desc_dump("IN/OUT", dvp);
m_vars[i].direction.bits = c_parameter_in;
}
// no displacement
m_vars[i].disp = 0;
}
else {
// c_data_ptr or c_string_ptr
m_vars[i].size *= m_vars[i].count;
m_vars[i].disp = 0;
}
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
PtrData *ptr_data;
// check that buffer length >= 0
if (m_vars[i].alloc_if &&
m_vars[i].disp + m_vars[i].size < 0) {
LIBOFFLOAD_ERROR(c_zero_or_neg_ptr_len);
exit(1);
}
// base address
void *base = *static_cast<void**>(m_vars[i].ptr);
// allocate buffer if we have no INTO and don't need
// allocation for the ptr at target
if (src_is_for_mic) {
if (m_vars[i].flags.is_stack_buf) {
// for stack persistent objects ptr data is created
// by var_desc with number 0.
// Its ptr_data is stored at m_stack_ptr_data
ptr_data = m_stack_ptr_data;
m_vars[i].flags.sink_addr = 1;
}
else if (m_vars[i].alloc_if) {
// add new entry
if (!alloc_ptr_data(
ptr_data,
base,
(alloc_base != NULL) ?
alloc_disp : m_vars[i].disp,
(alloc_base != NULL) ?
alloc_size : m_vars[i].size,
alloc_disp,
(alloc_base != NULL) ?
0 : m_vars[i].align)) {
return false;
}
if (ptr_data->add_reference() == 0 &&
ptr_data->mic_buf != 0) {
// add buffer to the list of buffers that
// are passed to dispatch call
m_compute_buffers.push_back(
ptr_data->mic_buf);
}
else {
// will send buffer address to device
m_vars[i].flags.sink_addr = 1;
}
if (!ptr_data->is_static) {
// need to add reference for buffer
m_need_runfunction = true;
}
}
else {
bool error_if_not_found = true;
if (m_is_openmp) {
// For omp target update variable is ignored
// if it does not exist.
if (!m_vars[i].alloc_if &&
!m_vars[i].free_if) {
error_if_not_found = false;
}
}
// use existing association from pointer table
if (!find_ptr_data(ptr_data,
base,
m_vars[i].disp,
m_vars[i].size,
error_if_not_found)) {
return false;
}
if (m_is_openmp) {
// make var nocopy if it does not exist
if (ptr_data == 0) {
m_vars[i].direction.bits =
c_parameter_nocopy;
}
}
if (ptr_data != 0) {
m_vars[i].flags.sink_addr = 1;
}
}
if (ptr_data != 0) {
if (m_is_openmp) {
// data is transferred only if
// alloc_if == 0 && free_if == 0
// or reference count is 1
if ((m_vars[i].alloc_if ||
m_vars[i].free_if) &&
ptr_data->get_reference() != 1) {
m_vars[i].direction.bits =
c_parameter_nocopy;
}
}
if (ptr_data->alloc_disp != 0) {
m_vars[i].flags.alloc_disp = 1;
m_in_datalen += sizeof(alloc_disp);
}
if (m_vars[i].flags.sink_addr) {
// get buffers's address on the sink
if (!init_mic_address(ptr_data)) {
return false;
}
m_in_datalen += sizeof(ptr_data->mic_addr);
}
if (!ptr_data->is_static && m_vars[i].free_if) {
// need to decrement buffer reference on target
m_need_runfunction = true;
}
// offset to base from the beginning of the buffer
// memory
m_vars[i].offset = (char*) base -
(char*) ptr_data->cpu_addr.start();
// copy other pointer properties to var descriptor
m_vars[i].mic_offset = ptr_data->mic_offset;
m_vars[i].flags.is_static = ptr_data->is_static;
}
}
else {
if (!find_ptr_data(ptr_data,
base,
m_vars[i].disp,
m_vars[i].size,
false)) {
return false;
}
if (ptr_data) {
m_vars[i].offset =
(char*) base -
(char*) ptr_data->cpu_addr.start();
}
}
// save pointer data
m_vars_extra[i].src_data = ptr_data;
}
break;
case c_func_ptr:
if (m_vars[i].direction.in) {
m_in_datalen += __offload_funcs.max_name_length();
}
if (m_vars[i].direction.out) {
m_out_datalen += __offload_funcs.max_name_length();
}
m_need_runfunction = true;
break;
case c_dv_data:
case c_dv_ptr_data:
case c_dv_data_slice:
case c_dv_ptr_data_slice:
ArrDesc *dvp;
if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) {
const arr_desc *ap;
ap = static_cast<const arr_desc*>(m_vars[i].ptr);
dvp = (m_vars[i].type.src == c_dv_data_slice) ?
reinterpret_cast<ArrDesc*>(ap->base) :
*reinterpret_cast<ArrDesc**>(ap->base);
}
else {
dvp = (m_vars[i].type.src == c_dv_data) ?
static_cast<ArrDesc*>(m_vars[i].ptr) :
*static_cast<ArrDesc**>(m_vars[i].ptr);
}
// if allocatable dope vector isn't allocated don't
// transfer its data
if (!__dv_is_allocated(dvp)) {
m_vars[i].direction.bits = c_parameter_nocopy;
m_vars[i].alloc_if = 0;
m_vars[i].free_if = 0;
}
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
const arr_desc *ap;
if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) {
ap = static_cast<const arr_desc*>(m_vars[i].ptr);
// debug dump
__arr_desc_dump("", "IN/OUT", ap, 0);
}
if (!__dv_is_contiguous(dvp)) {
m_vars[i].flags.is_noncont_src = 1;
m_vars_extra[i].read_rng_src =
init_read_ranges_dv(dvp);
}
// size and displacement
if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.src)) {
// offset and length are derived from the
// array descriptor
__arr_data_offset_and_length(ap,
m_vars[i].disp,
m_vars[i].size);
if (m_vars[i].direction.bits) {
if (!is_arr_desc_contiguous(ap)) {
if (m_vars[i].flags.is_noncont_src) {
LIBOFFLOAD_ERROR(c_slice_of_noncont_array);
return false;
}
m_vars[i].flags.is_noncont_src = 1;
m_vars_extra[i].read_rng_src =
init_read_ranges_arr_desc(ap);
}
}
}
else {
if (m_vars[i].flags.has_length) {
m_vars[i].size =
__dv_data_length(dvp, m_vars[i].count);
}
else {
m_vars[i].size = __dv_data_length(dvp);
}
m_vars[i].disp = 0;
}
// check that length >= 0
if (m_vars[i].alloc_if &&
(m_vars[i].disp + m_vars[i].size < 0)) {
LIBOFFLOAD_ERROR(c_zero_or_neg_ptr_len);
exit(1);
}
// base address
void *base = reinterpret_cast<void*>(dvp->Base);
PtrData *ptr_data;
// allocate buffer if we have no INTO and don't need
// allocation for the ptr at target
if (src_is_for_mic) {
if (m_vars[i].alloc_if) {
// add new entry
if (!alloc_ptr_data(
ptr_data,
base,
(alloc_base != NULL) ?
alloc_disp : m_vars[i].disp,
(alloc_base != NULL) ?
alloc_size : m_vars[i].size,
alloc_disp,
(alloc_base != NULL) ?
0 : m_vars[i].align)) {
return false;
}
if (ptr_data->add_reference() == 0 &&
ptr_data->mic_buf != 0) {
// add buffer to the list of buffers
// that are passed to dispatch call
m_compute_buffers.push_back(
ptr_data->mic_buf);
}
else {
// will send buffer address to device
m_vars[i].flags.sink_addr = 1;
}
if (!ptr_data->is_static) {
// need to add reference for buffer
m_need_runfunction = true;
}
}
else {
bool error_if_not_found = true;
if (m_is_openmp) {
// For omp target update variable is ignored
// if it does not exist.
if (!m_vars[i].alloc_if &&
!m_vars[i].free_if) {
error_if_not_found = false;
}
}
// use existing association from pointer table
if (!find_ptr_data(ptr_data,
base,
m_vars[i].disp,
m_vars[i].size,
error_if_not_found)) {
return false;
}
if (m_is_openmp) {
// make var nocopy if it does not exist
if (ptr_data == 0) {
m_vars[i].direction.bits =
c_parameter_nocopy;
}
}
if (ptr_data != 0) {
// need to update base in dope vector on device
m_vars[i].flags.sink_addr = 1;
}
}
if (ptr_data != 0) {
if (m_is_openmp) {
// data is transferred only if
// alloc_if == 0 && free_if == 0
// or reference count is 1
if ((m_vars[i].alloc_if ||
m_vars[i].free_if) &&
ptr_data->get_reference() != 1) {
m_vars[i].direction.bits =
c_parameter_nocopy;
}
}
if (ptr_data->alloc_disp != 0) {
m_vars[i].flags.alloc_disp = 1;
m_in_datalen += sizeof(alloc_disp);
}
if (m_vars[i].flags.sink_addr) {
// get buffers's address on the sink
if (!init_mic_address(ptr_data)) {
return false;
}
m_in_datalen += sizeof(ptr_data->mic_addr);
}
if (!ptr_data->is_static && m_vars[i].free_if) {
// need to decrement buffer reference on target
m_need_runfunction = true;
}
// offset to base from the beginning of the buffer
// memory
m_vars[i].offset =
(char*) base -
(char*) ptr_data->cpu_addr.start();
// copy other pointer properties to var descriptor
m_vars[i].mic_offset = ptr_data->mic_offset;
m_vars[i].flags.is_static = ptr_data->is_static;
}
}
else { // !src_is_for_mic
if (!find_ptr_data(ptr_data,
base,
m_vars[i].disp,
m_vars[i].size,
false)) {
return false;
}
m_vars[i].offset = !ptr_data ? 0 :
(char*) base -
(char*) ptr_data->cpu_addr.start();
}
// save pointer data
m_vars_extra[i].src_data = ptr_data;
}
break;
default:
LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars[i].type.src);
LIBOFFLOAD_ABORT;
}
if (m_vars[i].type.src == c_data_ptr_array) {
continue;
}
if (src_is_for_mic && m_vars[i].flags.is_stack_buf) {
m_vars[i].offset = static_cast<char*>(m_vars[i].ptr) -
m_device.m_persist_list.front().cpu_stack_addr;
}
// if source is used at CPU save its offset and disp
if (m_vars[i].into == NULL || m_vars[i].direction.in) {
m_vars_extra[i].cpu_offset = m_vars[i].offset;
m_vars_extra[i].cpu_disp = m_vars[i].disp;
}
// If "into" is define we need to do the similar work for it
if (!m_vars[i].into) {
continue;
}
int64_t into_disp =0, into_offset = 0;
switch (m_vars[i].type.dst) {
case c_data_ptr_array:
break;
case c_data:
case c_void_ptr:
case c_cean_var: {
int64_t size = m_vars[i].size;
if (m_vars[i].type.dst == c_cean_var) {
// array descriptor
const arr_desc *ap =
static_cast<const arr_desc*>(m_vars[i].into);
// debug dump
__arr_desc_dump(" ", "INTO", ap, 0);
// offset and length are derived from the array descriptor
__arr_data_offset_and_length(ap, into_disp, size);
if (!is_arr_desc_contiguous(ap)) {
m_vars[i].flags.is_noncont_dst = 1;
m_vars_extra[i].read_rng_dst =
init_read_ranges_arr_desc(ap);
if (!cean_ranges_match(
m_vars_extra[i].read_rng_src,
m_vars_extra[i].read_rng_dst)) {
LIBOFFLOAD_ERROR(c_ranges_dont_match);
exit(1);
}
}
m_vars[i].into = reinterpret_cast<void*>(ap->base);
}
int64_t size_src = m_vars_extra[i].read_rng_src ?
cean_get_transf_size(m_vars_extra[i].read_rng_src) :
m_vars[i].size;
int64_t size_dst = m_vars_extra[i].read_rng_dst ?
cean_get_transf_size(m_vars_extra[i].read_rng_dst) :
size;
// It's supposed that "into" size must be not less
// than src size
if (size_src > size_dst) {
LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes,
size_src, size_dst);
exit(1);
}
if (m_vars[i].direction.bits) {
if (m_vars[i].flags.is_static_dstn) {
PtrData *ptr_data;
// find data associated with variable
if (!find_ptr_data(ptr_data, m_vars[i].into,
into_disp, size, false)) {
return false;
}
if (ptr_data != 0) {
// offset to base from the beginning of the buffer
// memory
into_offset =
(char*) m_vars[i].into -
(char*) ptr_data->cpu_addr.start();
}
else {
m_vars[i].flags.is_static_dstn = false;
}
m_vars_extra[i].dst_data = ptr_data;
}
}
if (m_vars[i].direction.in &&
!m_vars[i].flags.is_static_dstn) {
m_in_datalen += m_vars[i].size;
// for non-static target destination defined as CEAN
// expression we pass to target its size and dist
if (m_vars[i].type.dst == c_cean_var) {
m_in_datalen += 2 * sizeof(uint64_t);
}
m_need_runfunction = true;
}
break;
}
case c_dv:
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
ArrDesc *dvp = static_cast<ArrDesc*>(m_vars[i].into);
// debug dump
__dv_desc_dump("INTO", dvp);
// send dope vector contents excluding base
m_in_datalen += m_vars[i].size - sizeof(uint64_t);
m_need_runfunction = true;
}
break;
case c_string_ptr:
case c_data_ptr:
case c_cean_var_ptr:
case c_dv_ptr: {
int64_t size = m_vars[i].size;
if (m_vars[i].type.dst == c_cean_var_ptr) {
// array descriptor
const arr_desc *ap =
static_cast<const arr_desc*>(m_vars[i].into);
// debug dump
__arr_desc_dump(" ", "INTO", ap, 1);
// offset and length are derived from the array descriptor
__arr_data_offset_and_length(ap, into_disp, size);
if (!is_arr_desc_contiguous(ap)) {
m_vars[i].flags.is_noncont_src = 1;
m_vars_extra[i].read_rng_dst =
init_read_ranges_arr_desc(ap);
if (!cean_ranges_match(
m_vars_extra[i].read_rng_src,
m_vars_extra[i].read_rng_dst)) {
LIBOFFLOAD_ERROR(c_ranges_dont_match);
}
}
m_vars[i].into = reinterpret_cast<char**>(ap->base);
}
else if (m_vars[i].type.dst == c_dv_ptr) {
// need to send DV to the device unless it is 'nocopy'
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
ArrDesc *dvp = *static_cast<ArrDesc**>(m_vars[i].into);
// debug dump
__dv_desc_dump("INTO", dvp);
m_vars[i].direction.bits = c_parameter_in;
}
}
int64_t size_src = m_vars_extra[i].read_rng_src ?
cean_get_transf_size(m_vars_extra[i].read_rng_src) :
m_vars[i].size;
int64_t size_dst = m_vars_extra[i].read_rng_dst ?
cean_get_transf_size(m_vars_extra[i].read_rng_dst) :
size;
// It's supposed that "into" size must be not less than
// src size
if (size_src > size_dst) {
LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes,
size_src, size_dst);
exit(1);
}
if (m_vars[i].direction.bits) {
PtrData *ptr_data;
// base address
void *base = *static_cast<void**>(m_vars[i].into);
if (m_vars[i].direction.in) {
// allocate buffer
if (m_vars[i].flags.is_stack_buf) {
// for stack persistent objects ptr data is created
// by var_desc with number 0.
// Its ptr_data is stored at m_stack_ptr_data
ptr_data = m_stack_ptr_data;
m_vars[i].flags.sink_addr = 1;
}
else if (m_vars[i].alloc_if) {
// add new entry
if (!alloc_ptr_data(
ptr_data,
base,
(alloc_base != NULL) ?
alloc_disp : into_disp,
(alloc_base != NULL) ?
alloc_size : size,
alloc_disp,
(alloc_base != NULL) ?
0 : m_vars[i].align)) {
return false;
}
if (ptr_data->add_reference() == 0 &&
ptr_data->mic_buf != 0) {
// add buffer to the list of buffers that
// are passed to dispatch call
m_compute_buffers.push_back(
ptr_data->mic_buf);
}
else {
// will send buffer address to device
m_vars[i].flags.sink_addr = 1;
}
if (!ptr_data->is_static) {
// need to add reference for buffer
m_need_runfunction = true;
}
}
else {
// use existing association from pointer table
if (!find_ptr_data(ptr_data, base, into_disp, size)) {
return false;
}
m_vars[i].flags.sink_addr = 1;
}
if (ptr_data->alloc_disp != 0) {
m_vars[i].flags.alloc_disp = 1;
m_in_datalen += sizeof(alloc_disp);
}
if (m_vars[i].flags.sink_addr) {
// get buffers's address on the sink
if (!init_mic_address(ptr_data)) {
return false;
}
m_in_datalen += sizeof(ptr_data->mic_addr);
}
if (!ptr_data->is_static && m_vars[i].free_if) {
// need to decrement buffer reference on target
m_need_runfunction = true;
}
// copy other pointer properties to var descriptor
m_vars[i].mic_offset = ptr_data->mic_offset;
m_vars[i].flags.is_static_dstn = ptr_data->is_static;
}
else {
if (!find_ptr_data(ptr_data,
base,
into_disp,
m_vars[i].size,
false)) {
return false;
}
}
if (ptr_data) {
into_offset = ptr_data ?
(char*) base -
(char*) ptr_data->cpu_addr.start() :
0;
}
// save pointer data
m_vars_extra[i].dst_data = ptr_data;
}
break;
}
case c_func_ptr:
break;
case c_dv_data:
case c_dv_ptr_data:
case c_dv_data_slice:
case c_dv_ptr_data_slice:
if (m_vars[i].direction.bits ||
m_vars[i].alloc_if ||
m_vars[i].free_if) {
const arr_desc *ap;
ArrDesc *dvp;
PtrData *ptr_data;
int64_t disp;
int64_t size;
if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst)) {
ap = static_cast<const arr_desc*>(m_vars[i].into);
// debug dump
__arr_desc_dump(" ", "INTO", ap, 0);
dvp = (m_vars[i].type.dst == c_dv_data_slice) ?
reinterpret_cast<ArrDesc*>(ap->base) :
*reinterpret_cast<ArrDesc**>(ap->base);
}
else {
dvp = (m_vars[i].type.dst == c_dv_data) ?
static_cast<ArrDesc*>(m_vars[i].into) :
*static_cast<ArrDesc**>(m_vars[i].into);
}
if (!__dv_is_contiguous(dvp)) {
m_vars[i].flags.is_noncont_dst = 1;
m_vars_extra[i].read_rng_dst =
init_read_ranges_dv(dvp);
}
// size and displacement
if (VAR_TYPE_IS_DV_DATA_SLICE(m_vars[i].type.dst)) {
// offset and length are derived from the array
// descriptor
__arr_data_offset_and_length(ap, into_disp, size);
if (m_vars[i].direction.bits) {
if (!is_arr_desc_contiguous(ap)) {
if (m_vars[i].flags.is_noncont_dst) {
LIBOFFLOAD_ERROR(c_slice_of_noncont_array);
return false;
}
m_vars[i].flags.is_noncont_dst = 1;
m_vars_extra[i].read_rng_dst =
init_read_ranges_arr_desc(ap);
if (!cean_ranges_match(
m_vars_extra[i].read_rng_src,
m_vars_extra[i].read_rng_dst)) {
LIBOFFLOAD_ERROR(c_ranges_dont_match);
}
}
}
}
else {
if (m_vars[i].flags.has_length) {
size = __dv_data_length(dvp, m_vars[i].count);
}
else {
size = __dv_data_length(dvp);
}
disp = 0;
}
int64_t size_src =
m_vars_extra[i].read_rng_src ?
cean_get_transf_size(m_vars_extra[i].read_rng_src) :
m_vars[i].size;
int64_t size_dst =
m_vars_extra[i].read_rng_dst ?
cean_get_transf_size(m_vars_extra[i].read_rng_dst) :
size;
// It's supposed that "into" size must be not less
// than src size
if (size_src > size_dst) {
LIBOFFLOAD_ERROR(c_different_src_and_dstn_sizes,
size_src, size_dst);
exit(1);
}
// base address
void *base = reinterpret_cast<void*>(dvp->Base);
// allocate buffer
if (m_vars[i].direction.in) {
if (m_vars[i].alloc_if) {
// add new entry
if (!alloc_ptr_data(
ptr_data,
base,
(alloc_base != NULL) ?
alloc_disp : into_disp,
(alloc_base != NULL) ?
alloc_size : size,
alloc_disp,
(alloc_base != NULL) ?
0 : m_vars[i].align)) {
return false;
}
if (ptr_data->add_reference() == 0 &&
ptr_data->mic_buf !=0) {
// add buffer to the list of buffers
// that are passed to dispatch call
m_compute_buffers.push_back(
ptr_data->mic_buf);
}
else {
// will send buffer address to device
m_vars[i].flags.sink_addr = 1;
}
if (!ptr_data->is_static) {
// need to add reference for buffer
m_need_runfunction = true;
}
}
else {
// use existing association from pointer table
if (!find_ptr_data(ptr_data, base, into_disp, size)) {
return false;
}
// need to update base in dope vector on device
m_vars[i].flags.sink_addr = 1;
}
if (ptr_data->alloc_disp != 0) {
m_vars[i].flags.alloc_disp = 1;
m_in_datalen += sizeof(alloc_disp);
}
if (m_vars[i].flags.sink_addr) {
// get buffers's address on the sink
if (!init_mic_address(ptr_data)) {
return false;
}
m_in_datalen += sizeof(ptr_data->mic_addr);
}
if (!ptr_data->is_static && m_vars[i].free_if) {
// need to decrement buffer reference on target
m_need_runfunction = true;
}
// offset to base from the beginning of the buffer
// memory
into_offset =
(char*) base - (char*) ptr_data->cpu_addr.start();
// copy other pointer properties to var descriptor
m_vars[i].mic_offset = ptr_data->mic_offset;
m_vars[i].flags.is_static_dstn = ptr_data->is_static;
}
else { // src_is_for_mic
if (!find_ptr_data(ptr_data,
base,
into_disp,
size,
false)) {
return false;
}
into_offset = !ptr_data ?
0 :
(char*) base - (char*) ptr_data->cpu_addr.start();
}
// save pointer data
m_vars_extra[i].dst_data = ptr_data;
}
break;
default:
LIBOFFLOAD_ERROR(c_unknown_var_type, m_vars[i].type.src);
LIBOFFLOAD_ABORT;
}
// if into is used at CPU save its offset and disp
if (m_vars[i].direction.out) {
m_vars_extra[i].cpu_offset = into_offset;
m_vars_extra[i].cpu_disp = into_disp;
}
else {
if (m_vars[i].flags.is_stack_buf) {
into_offset = static_cast<char*>(m_vars[i].into) -
m_device.m_persist_list.front().cpu_stack_addr;
}
m_vars[i].offset = into_offset;
m_vars[i].disp = into_disp;
}
}
return true;
}
bool OffloadDescriptor::setup_misc_data(const char *name)
{
OffloadTimer timer(get_timer_data(), c_offload_host_setup_misc_data);
// we can skip run functon call together with wait if offloaded
// region is empty and there is no user defined non-pointer IN/OUT data
if (m_need_runfunction) {
// variable descriptors are sent as input data
m_in_datalen += m_vars_total * sizeof(VarDesc);
// timer data is sent as a part of the output data
m_out_datalen += OFFLOAD_TIMER_DATALEN();
// max from input data and output data length
uint64_t data_len = m_in_datalen > m_out_datalen ? m_in_datalen :
m_out_datalen;
// Misc data has the following layout
// <Function Descriptor>
// <Function Name>
// <In/Out Data> (optional)
//
// We can transfer copyin/copyout data in misc/return data which can
// be passed to run function call if its size does not exceed
// COI_PIPELINE_MAX_IN_MISC_DATA_LEN. Otherwise we have to allocate
// buffer for it.
m_func_desc_size = sizeof(FunctionDescriptor) + strlen(name) + 1;
m_func_desc_size = (m_func_desc_size + 7) & ~7;
int misc_data_offset = 0;
int misc_data_size = 0;
if (data_len > 0) {
if (m_func_desc_size +
m_in_datalen <= COI_PIPELINE_MAX_IN_MISC_DATA_LEN &&
m_out_datalen <= COI_PIPELINE_MAX_IN_MISC_DATA_LEN) {
// use misc/return data for copyin/copyout
misc_data_offset = m_func_desc_size;
misc_data_size = data_len;
}
else {
OffloadTimer timer_buf(get_timer_data(),
c_offload_host_alloc_data_buffer);
// send/receive data using buffer
COIRESULT res = COI::BufferCreate(data_len,
COI_BUFFER_NORMAL,
0, 0,
1, &m_device.get_process(),
&m_inout_buf);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_create, res);
}
m_compute_buffers.push_back(m_inout_buf);
m_destroy_buffers.push_back(m_inout_buf);
}
}
// initialize function descriptor
m_func_desc = (FunctionDescriptor*) malloc(m_func_desc_size +
misc_data_size);
m_func_desc->console_enabled = console_enabled;
m_func_desc->timer_enabled =
timer_enabled || (offload_report_level && offload_report_enabled);
m_func_desc->offload_report_level = offload_report_level;
m_func_desc->offload_number = GET_OFFLOAD_NUMBER(get_timer_data());
m_func_desc->in_datalen = m_in_datalen;
m_func_desc->out_datalen = m_out_datalen;
m_func_desc->vars_num = m_vars_total;
m_func_desc->data_offset = misc_data_offset;
// append entry name
strcpy(m_func_desc->data, name);
}
return true;
}
bool OffloadDescriptor::wait_dependencies(
const void **waits,
int num_waits
)
{
OffloadTimer timer(get_timer_data(), c_offload_host_wait_deps);
bool ret = true;
for (int i = 0; i < num_waits; i++) {
OffloadDescriptor *task = m_device.find_signal(waits[i], true);
if (task == 0) {
LIBOFFLOAD_ERROR(c_offload1, m_device.get_logical_index(),
waits[i]);
LIBOFFLOAD_ABORT;
}
if (!task->offload_finish()) {
ret = false;
}
task->cleanup();
delete task;
}
return ret;
}
bool OffloadDescriptor::offload(
const char *name,
bool is_empty,
VarDesc *vars,
VarDesc2 *vars2,
int vars_total,
const void **waits,
int num_waits,
const void **signal,
int entry_id,
const void *stack_addr
)
{
if (signal == 0) {
OFFLOAD_DEBUG_TRACE_1(1,
GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_init_func,
"Offload function %s, is_empty=%d, #varDescs=%d, "
"#waits=%d, signal=none\n",
name, is_empty, vars_total, num_waits);
OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_sent_pointer_data,
"#Wait : %d \n", num_waits);
OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_signal,
"none %d\n", 0);
}
else {
OFFLOAD_DEBUG_TRACE_1(1,
GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_init_func,
"Offload function %s, is_empty=%d, #varDescs=%d, "
"#waits=%d, signal=%p\n",
name, is_empty, vars_total, num_waits,
*signal);
OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_signal,
"%d\n", signal);
}
OFFLOAD_REPORT(3, GET_OFFLOAD_NUMBER(get_timer_data()),
c_offload_wait,
"#Wait : %d %p\n", num_waits, waits);
if (m_status != 0) {
m_status->result = OFFLOAD_SUCCESS;
m_status->device_number = m_device.get_logical_index();
}
m_need_runfunction = !is_empty;
// wait for dependencies to finish
if (!wait_dependencies(waits, num_waits)) {
cleanup();
return false;
}
// setup buffers
if (!setup_descriptors(vars, vars2, vars_total, entry_id, stack_addr)) {
cleanup();
return false;
}
// initiate send for pointers. Want to do it as early as possible.
if (!send_pointer_data(signal != 0)) {
cleanup();
return false;
}
// setup misc data for run function
if (!setup_misc_data(name)) {
cleanup();
return false;
}
// gather copyin data into buffer
if (!gather_copyin_data()) {
cleanup();
return false;
}
// Start the computation
if (!compute()) {
cleanup();
return false;
}
// initiate receive for pointers
if (!receive_pointer_data(signal != 0)) {
cleanup();
return false;
}
// if there is a signal save descriptor for the later use.
if (signal != 0) {
m_device.add_signal(*signal, this);
return true;
}
// wait for the offload to finish.
if (!offload_finish()) {
cleanup();
return false;
}
cleanup();
return true;
}
bool OffloadDescriptor::offload_finish()
{
COIRESULT res;
// wait for compute dependencies to become signaled
if (m_in_deps_total > 0) {
OffloadTimer timer(get_timer_data(), c_offload_host_wait_compute);
if (__offload_active_wait) {
// keep CPU busy
do {
res = COI::EventWait(m_in_deps_total, m_in_deps, 0, 1, 0, 0);
}
while (res == COI_TIME_OUT_REACHED);
}
else {
res = COI::EventWait(m_in_deps_total, m_in_deps, -1, 1, 0, 0);
}
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_event_wait, res);
}
}
// scatter copyout data received from target
if (!scatter_copyout_data()) {
return false;
}
// wait for receive dependencies to become signaled
if (m_out_deps_total > 0) {
OffloadTimer timer(get_timer_data(), c_offload_host_wait_buffers_reads);
if (__offload_active_wait) {
// keep CPU busy
do {
res = COI::EventWait(m_out_deps_total, m_out_deps, 0, 1, 0, 0);
}
while (res == COI_TIME_OUT_REACHED);
}
else {
res = COI::EventWait(m_out_deps_total, m_out_deps, -1, 1, 0, 0);
}
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_event_wait, res);
}
}
// destroy buffers
{
OffloadTimer timer(get_timer_data(), c_offload_host_destroy_buffers);
for (BufferList::const_iterator it = m_destroy_buffers.begin();
it != m_destroy_buffers.end(); it++) {
res = COI::BufferDestroy(*it);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_destroy, res);
}
}
}
return true;
}
void OffloadDescriptor::cleanup()
{
// release device in orsl
ORSL::release(m_device.get_logical_index());
OFFLOAD_TIMER_STOP(get_timer_data(), c_offload_host_total_offload);
// report stuff
Offload_Report_Epilog(get_timer_data());
}
bool OffloadDescriptor::is_signaled()
{
bool signaled = true;
COIRESULT res;
// check compute and receive dependencies
if (m_in_deps_total > 0) {
res = COI::EventWait(m_in_deps_total, m_in_deps, 0, 1, 0, 0);
signaled = signaled && (res == COI_SUCCESS);
}
if (m_out_deps_total > 0) {
res = COI::EventWait(m_out_deps_total, m_out_deps, 0, 1, 0, 0);
signaled = signaled && (res == COI_SUCCESS);
}
return signaled;
}
// Send pointer data if source or destination or both of them are
// noncontiguous. There is guarantee that length of destination enough for
// transferred data.
bool OffloadDescriptor::send_noncontiguous_pointer_data(
int i,
PtrData* src_data,
PtrData* dst_data,
COIEVENT *event
)
{
int64_t offset_src, offset_dst;
int64_t length_src, length_dst;
int64_t length_src_cur, length_dst_cur;
int64_t send_size, data_sent = 0;
COIRESULT res;
bool dst_is_empty = true;
bool src_is_empty = true;
// Set length_src and length_dst
length_src = (m_vars_extra[i].read_rng_src) ?
m_vars_extra[i].read_rng_src->range_size : m_vars[i].size;
length_dst = !m_vars[i].into ? length_src :
(m_vars_extra[i].read_rng_dst) ?
m_vars_extra[i].read_rng_dst->range_size : m_vars[i].size;
send_size = (length_src < length_dst) ? length_src : length_dst;
// consequently get contiguous ranges,
// define corresponded destination offset and send data
do {
if (src_is_empty) {
if (m_vars_extra[i].read_rng_src) {
if (!get_next_range(m_vars_extra[i].read_rng_src,
&offset_src)) {
// source ranges are over - nothing to send
break;
}
}
else if (data_sent == 0) {
offset_src = m_vars_extra[i].cpu_disp;
}
else {
break;
}
length_src_cur = length_src;
}
else {
// if source is contiguous or its contiguous range is greater
// than destination one
offset_src += send_size;
}
length_src_cur -= send_size;
src_is_empty = length_src_cur == 0;
if (dst_is_empty) {
if (m_vars[i].into) {
if (m_vars_extra[i].read_rng_dst) {
if (!get_next_range(m_vars_extra[i].read_rng_dst,
&offset_dst)) {
// destination ranges are over
LIBOFFLOAD_ERROR(c_destination_is_over);
return false;
}
}
// into is contiguous.
else {
offset_dst = m_vars[i].disp;
}
length_dst_cur = length_dst;
}
// same as source
else {
offset_dst = offset_src;
length_dst_cur = length_src;
}
}
else {
// if destination is contiguous or its contiguous range is greater
// than source one
offset_dst += send_size;
}
length_dst_cur -= send_size;
dst_is_empty = length_dst_cur == 0;
if (src_data != 0 && src_data->cpu_buf != 0) {
res = COI::BufferCopy(
dst_data->mic_buf,
src_data->cpu_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + offset_dst,
m_vars_extra[i].cpu_offset + offset_src,
send_size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_copy, res);
}
}
else {
char *base = offload_get_src_base(m_vars[i].ptr,
m_vars[i].type.src);
res = COI::BufferWrite(
dst_data->mic_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + offset_dst,
base + offset_src,
send_size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_write, res);
}
}
data_sent += length_src;
}
while (true);
return true;
}
bool OffloadDescriptor::send_pointer_data(bool is_async)
{
OffloadTimer timer(get_timer_data(), c_offload_host_send_pointers);
uint64_t ptr_sent = 0;
COIRESULT res;
// Initiate send for pointer data
for (int i = 0; i < m_vars_total; i++) {
switch (m_vars[i].type.dst) {
case c_data_ptr_array:
break;
case c_data:
case c_void_ptr:
case c_cean_var:
if (m_vars[i].direction.in &&
m_vars[i].flags.is_static_dstn) {
COIEVENT *event =
(is_async ||
m_vars[i].size >= __offload_use_async_buffer_write) ?
&m_in_deps[m_in_deps_total++] : 0;
PtrData* dst_data = m_vars[i].into ?
m_vars_extra[i].dst_data :
m_vars_extra[i].src_data;
PtrData* src_data =
VAR_TYPE_IS_PTR(m_vars[i].type.src) ||
VAR_TYPE_IS_SCALAR(m_vars[i].type.src) &&
m_vars[i].flags.is_static ?
m_vars_extra[i].src_data : 0;
if (m_vars[i].flags.is_noncont_src ||
m_vars[i].flags.is_noncont_dst) {
if (!send_noncontiguous_pointer_data(
i, src_data, dst_data, event)) {
return false;
}
}
else if (src_data != 0 && src_data->cpu_buf != 0) {
res = COI::BufferCopy(
dst_data->mic_buf,
src_data->cpu_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + m_vars[i].disp,
m_vars_extra[i].cpu_offset +
m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_copy, res);
}
}
else {
char *base = offload_get_src_base(m_vars[i].ptr,
m_vars[i].type.src);
res = COI::BufferWrite(
dst_data->mic_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + m_vars[i].disp,
base + m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_write, res);
}
}
ptr_sent += m_vars[i].size;
}
break;
case c_string_ptr:
case c_data_ptr:
case c_cean_var_ptr:
case c_dv_ptr:
if (m_vars[i].direction.in && m_vars[i].size > 0) {
COIEVENT *event =
(is_async ||
m_vars[i].size >= __offload_use_async_buffer_write) ?
&m_in_deps[m_in_deps_total++] : 0;
PtrData* dst_data = m_vars[i].into ?
m_vars_extra[i].dst_data :
m_vars_extra[i].src_data;
PtrData* src_data =
VAR_TYPE_IS_PTR(m_vars[i].type.src) ||
VAR_TYPE_IS_SCALAR(m_vars[i].type.src) &&
m_vars[i].flags.is_static ?
m_vars_extra[i].src_data : 0;
if (m_vars[i].flags.is_noncont_src ||
m_vars[i].flags.is_noncont_dst) {
send_noncontiguous_pointer_data(
i, src_data, dst_data, event);
}
else if (src_data != 0 && src_data->cpu_buf != 0) {
res = COI::BufferCopy(
dst_data->mic_buf,
src_data->cpu_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + m_vars[i].disp,
m_vars_extra[i].cpu_offset +
m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_copy, res);
}
}
else {
char *base = offload_get_src_base(m_vars[i].ptr,
m_vars[i].type.src);
res = COI::BufferWrite(
dst_data->mic_buf,
m_vars[i].mic_offset - dst_data->alloc_disp +
m_vars[i].offset + m_vars[i].disp,
base + m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_write, res);
}
}
ptr_sent += m_vars[i].size;
}
break;
case c_dv_data:
case c_dv_ptr_data:
if (m_vars[i].direction.in &&
m_vars[i].size > 0) {
PtrData *ptr_data = m_vars[i].into ?
m_vars_extra[i].dst_data :
m_vars_extra[i].src_data;
PtrData* src_data = m_vars_extra[i].src_data;
COIEVENT *event =
(is_async ||
m_vars[i].size >= __offload_use_async_buffer_write) ?
&m_in_deps[m_in_deps_total++] : 0;
if (m_vars[i].flags.is_noncont_src ||
m_vars[i].flags.is_noncont_dst) {
send_noncontiguous_pointer_data(
i, src_data, ptr_data, event);
}
else if (src_data && src_data->cpu_buf != 0) {
res = COI::BufferCopy(
ptr_data->mic_buf,
src_data->cpu_buf,
m_vars[i].offset + ptr_data->mic_offset -
ptr_data->alloc_disp +
m_vars[i].disp,
m_vars_extra[i].cpu_offset +
m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_copy, res);
}
}
else {
char *base = offload_get_src_base(m_vars[i].ptr,
m_vars[i].type.src);
res = COI::BufferWrite(
ptr_data->mic_buf,
ptr_data->mic_offset - ptr_data->alloc_disp +
m_vars[i].offset + m_vars[i].disp,
base + m_vars_extra[i].cpu_disp,
m_vars[i].size,
COI_COPY_UNSPECIFIED,
0, 0,
event);
if (res != COI_SUCCESS) {
if (m_status != 0) {
m_status->result = translate_coi_error(res);
return false;
}
report_coi_error(c_buf_write, res);
}
}
ptr_sent += m_vars[i].size;
}
break;