libomptarget/src/api.cpp - llvm-project/openmp - Git at Google

 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Implementation of OpenMP API interface functions.
 //
 //===----------------------------------------------------------------------===//

 #include "device.h"
 #include "omptarget.h"
 #include "private.h"
 #include "rtl.h"

 #include <climits>
 #include <cstdlib>
 #include <cstring>

 EXTERN int omp_get_num_devices(void) {
   TIMESCOPE();
   PM->RTLsMtx.lock();
   size_t DevicesSize = PM->Devices.size();
   PM->RTLsMtx.unlock();

   DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);

   return DevicesSize;
 }

 EXTERN int omp_get_device_num(void) {
   TIMESCOPE();
   int HostDevice = omp_get_initial_device();

   DP("Call to omp_get_device_num returning %d\n", HostDevice);

   return HostDevice;
 }

 EXTERN int omp_get_initial_device(void) {
   TIMESCOPE();
   int HostDevice = omp_get_num_devices();
   DP("Call to omp_get_initial_device returning %d\n", HostDevice);
   return HostDevice;
 }

 EXTERN void *omp_target_alloc(size_t Size, int DeviceNum) {
   return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);
 }

 EXTERN void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum) {
   return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEVICE, __func__);
 }

 EXTERN void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum) {
   return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_HOST, __func__);
 }

 EXTERN void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum) {
   return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_SHARED, __func__);
 }

 EXTERN void *llvm_omp_target_dynamic_shared_alloc() { return nullptr; }
 EXTERN void *llvm_omp_get_dynamic_shared() { return nullptr; }

 EXTERN void omp_target_free(void *DevicePtr, int DeviceNum) {
   TIMESCOPE();
   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
      DeviceNum, DPxPTR(DevicePtr));

   if (!DevicePtr) {
     DP("Call to omp_target_free with NULL ptr\n");
     return;
   }

   if (DeviceNum == omp_get_initial_device()) {
     free(DevicePtr);
     DP("omp_target_free deallocated host ptr\n");
     return;
   }

   if (!deviceIsReady(DeviceNum)) {
     DP("omp_target_free returns, nothing to do\n");
     return;
   }

   PM->Devices[DeviceNum]->deleteData(DevicePtr);
   DP("omp_target_free deallocated device ptr\n");
 }

 EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
   TIMESCOPE();
   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
      DeviceNum, DPxPTR(Ptr));

   if (!Ptr) {
     DP("Call to omp_target_is_present with NULL ptr, returning false\n");
     return false;
   }

   if (DeviceNum == omp_get_initial_device()) {
     DP("Call to omp_target_is_present on host, returning true\n");
     return true;
   }

   PM->RTLsMtx.lock();
   size_t DevicesSize = PM->Devices.size();
   PM->RTLsMtx.unlock();
   if (DevicesSize <= (size_t)DeviceNum) {
     DP("Call to omp_target_is_present with invalid device ID, returning "
        "false\n");
     return false;
   }

   DeviceTy &Device = *PM->Devices[DeviceNum];
   bool IsLast; // not used
   bool IsHostPtr;
   // omp_target_is_present tests whether a host pointer refers to storage that
   // is mapped to a given device. However, due to the lack of the storage size,
   // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
   // lengh array) is mapped instead of the referred storage.
   TargetPointerResultTy TPR =
       Device.getTgtPtrBegin(const_cast<void *>(Ptr), 1, IsLast,
                             /*UpdateRefCount=*/false,
                             /*UseHoldRefCount=*/false, IsHostPtr);
   int Rc = (TPR.TargetPointer != NULL);
   // Under unified memory the host pointer can be returned by the
   // getTgtPtrBegin() function which means that there is no device
   // corresponding point for ptr. This function should return false
   // in that situation.
   if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
     Rc = !IsHostPtr;
   DP("Call to omp_target_is_present returns %d\n", Rc);
   return Rc;
 }

 EXTERN int omp_target_memcpy(void *Dst, const void *Src, size_t Length,
                              size_t DstOffset, size_t SrcOffset, int DstDevice,
                              int SrcDevice) {
   TIMESCOPE();
   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
      "src offset %zu, length %zu\n",
      DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,
      Length);

   if (!Dst || !Src || Length <= 0) {
     if (Length == 0) {
       DP("Call to omp_target_memcpy with zero length, nothing to do\n");
       return OFFLOAD_SUCCESS;
     }

     REPORT("Call to omp_target_memcpy with invalid arguments\n");
     return OFFLOAD_FAIL;
   }

   if (SrcDevice != omp_get_initial_device() && !deviceIsReady(SrcDevice)) {
     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
     return OFFLOAD_FAIL;
   }

   if (DstDevice != omp_get_initial_device() && !deviceIsReady(DstDevice)) {
     REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
     return OFFLOAD_FAIL;
   }

   int Rc = OFFLOAD_SUCCESS;
   void *SrcAddr = (char *)const_cast<void *>(Src) + SrcOffset;
   void *DstAddr = (char *)Dst + DstOffset;

   if (SrcDevice == omp_get_initial_device() &&
       DstDevice == omp_get_initial_device()) {
     DP("copy from host to host\n");
     const void *P = memcpy(DstAddr, SrcAddr, Length);
     if (P == NULL)
       Rc = OFFLOAD_FAIL;
   } else if (SrcDevice == omp_get_initial_device()) {
     DP("copy from host to device\n");
     DeviceTy &DstDev = *PM->Devices[DstDevice];
     AsyncInfoTy AsyncInfo(DstDev);
     Rc = DstDev.submitData(DstAddr, SrcAddr, Length, AsyncInfo);
   } else if (DstDevice == omp_get_initial_device()) {
     DP("copy from device to host\n");
     DeviceTy &SrcDev = *PM->Devices[SrcDevice];
     AsyncInfoTy AsyncInfo(SrcDev);
     Rc = SrcDev.retrieveData(DstAddr, SrcAddr, Length, AsyncInfo);
   } else {
     DP("copy from device to device\n");
     DeviceTy &SrcDev = *PM->Devices[SrcDevice];
     DeviceTy &DstDev = *PM->Devices[DstDevice];
     // First try to use D2D memcpy which is more efficient. If fails, fall back
     // to unefficient way.
     if (SrcDev.isDataExchangable(DstDev)) {
       AsyncInfoTy AsyncInfo(SrcDev);
       Rc = SrcDev.dataExchange(SrcAddr, DstDev, DstAddr, Length, AsyncInfo);
       if (Rc == OFFLOAD_SUCCESS)
         return OFFLOAD_SUCCESS;
     }

     void *Buffer = malloc(Length);
     {
       AsyncInfoTy AsyncInfo(SrcDev);
       Rc = SrcDev.retrieveData(Buffer, SrcAddr, Length, AsyncInfo);
     }
     if (Rc == OFFLOAD_SUCCESS) {
       AsyncInfoTy AsyncInfo(SrcDev);
       Rc = DstDev.submitData(DstAddr, Buffer, Length, AsyncInfo);
     }
     free(Buffer);
   }

   DP("omp_target_memcpy returns %d\n", Rc);
   return Rc;
 }

 EXTERN int
 omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,
                        int NumDims, const size_t *Volume,
                        const size_t *DstOffsets, const size_t *SrcOffsets,
                        const size_t *DstDimensions, const size_t *SrcDimensions,
                        int DstDevice, int SrcDevice) {
   TIMESCOPE();
   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
      "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
      "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
      "volume " DPxMOD ", element size %zu, num_dims %d\n",
      DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),
      DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),
      DPxPTR(Volume), ElementSize, NumDims);

   if (!(Dst || Src)) {
     DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
        INT_MAX);
     return INT_MAX;
   }

   if (!Dst || !Src || ElementSize < 1 || NumDims < 1 || !Volume ||
       !DstOffsets || !SrcOffsets || !DstDimensions || !SrcDimensions) {
     REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
     return OFFLOAD_FAIL;
   }

   int Rc;
   if (NumDims == 1) {
     Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],
                            ElementSize * DstOffsets[0],
                            ElementSize * SrcOffsets[0], DstDevice, SrcDevice);
   } else {
     size_t DstSliceSize = ElementSize;
     size_t SrcSliceSize = ElementSize;
     for (int I = 1; I < NumDims; ++I) {
       DstSliceSize *= DstDimensions[I];
       SrcSliceSize *= SrcDimensions[I];
     }

     size_t DstOff = DstOffsets[0] * DstSliceSize;
     size_t SrcOff = SrcOffsets[0] * SrcSliceSize;
     for (size_t I = 0; I < Volume[0]; ++I) {
       Rc = omp_target_memcpy_rect(
           (char *)Dst + DstOff + DstSliceSize * I,
           (char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I,
           ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,
           DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);

       if (Rc) {
         DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
         return Rc;
       }
     }
   }

   DP("omp_target_memcpy_rect returns %d\n", Rc);
   return Rc;
 }

 EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
                                     size_t Size, size_t DeviceOffset,
                                     int DeviceNum) {
   TIMESCOPE();
   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
      "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
      DPxPTR(HostPtr), DPxPTR(DevicePtr), Size, DeviceOffset, DeviceNum);

   if (!HostPtr || !DevicePtr || Size <= 0) {
     REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
     return OFFLOAD_FAIL;
   }

   if (DeviceNum == omp_get_initial_device()) {
     REPORT("omp_target_associate_ptr: no association possible on the host\n");
     return OFFLOAD_FAIL;
   }

   if (!deviceIsReady(DeviceNum)) {
     REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
     return OFFLOAD_FAIL;
   }

   DeviceTy &Device = *PM->Devices[DeviceNum];
   void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
   int Rc = Device.associatePtr(const_cast<void *>(HostPtr),
                                const_cast<void *>(DeviceAddr), Size);
   DP("omp_target_associate_ptr returns %d\n", Rc);
   return Rc;
 }

 EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
   TIMESCOPE();
   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
      "device_num %d\n",
      DPxPTR(HostPtr), DeviceNum);

   if (!HostPtr) {
     REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
     return OFFLOAD_FAIL;
   }

   if (DeviceNum == omp_get_initial_device()) {
     REPORT(
         "omp_target_disassociate_ptr: no association possible on the host\n");
     return OFFLOAD_FAIL;
   }

   if (!deviceIsReady(DeviceNum)) {
     REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
     return OFFLOAD_FAIL;
   }

   DeviceTy &Device = *PM->Devices[DeviceNum];
   int Rc = Device.disassociatePtr(const_cast<void *>(HostPtr));
   DP("omp_target_disassociate_ptr returns %d\n", Rc);
   return Rc;
 }
	//===----------- api.cpp - Target independent OpenMP target RTL -----------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// Implementation of OpenMP API interface functions.
	//
	//===----------------------------------------------------------------------===//

	#include "device.h"
	#include "omptarget.h"
	#include "private.h"
	#include "rtl.h"

	#include <climits>
	#include <cstdlib>
	#include <cstring>

	EXTERN int omp_get_num_devices(void) {
	TIMESCOPE();
	PM->RTLsMtx.lock();
	size_t DevicesSize = PM->Devices.size();
	PM->RTLsMtx.unlock();

	DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);

	return DevicesSize;
	}

	EXTERN int omp_get_device_num(void) {
	TIMESCOPE();
	int HostDevice = omp_get_initial_device();

	DP("Call to omp_get_device_num returning %d\n", HostDevice);

	return HostDevice;
	}

	EXTERN int omp_get_initial_device(void) {
	TIMESCOPE();
	int HostDevice = omp_get_num_devices();
	DP("Call to omp_get_initial_device returning %d\n", HostDevice);
	return HostDevice;
	}

	EXTERN void *omp_target_alloc(size_t Size, int DeviceNum) {
	return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);
	}

	EXTERN void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum) {
	return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEVICE, __func__);
	}

	EXTERN void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum) {
	return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_HOST, __func__);
	}

	EXTERN void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum) {
	return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_SHARED, __func__);
	}

	EXTERN void *llvm_omp_target_dynamic_shared_alloc() { return nullptr; }
	EXTERN void *llvm_omp_get_dynamic_shared() { return nullptr; }

	EXTERN void omp_target_free(void *DevicePtr, int DeviceNum) {
	TIMESCOPE();
	DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
	DeviceNum, DPxPTR(DevicePtr));

	if (!DevicePtr) {
	DP("Call to omp_target_free with NULL ptr\n");
	return;
	}

	if (DeviceNum == omp_get_initial_device()) {
	free(DevicePtr);
	DP("omp_target_free deallocated host ptr\n");
	return;
	}

	if (!deviceIsReady(DeviceNum)) {
	DP("omp_target_free returns, nothing to do\n");
	return;
	}

	PM->Devices[DeviceNum]->deleteData(DevicePtr);
	DP("omp_target_free deallocated device ptr\n");
	}

	EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
	TIMESCOPE();
	DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
	DeviceNum, DPxPTR(Ptr));

	if (!Ptr) {
	DP("Call to omp_target_is_present with NULL ptr, returning false\n");
	return false;
	}

	if (DeviceNum == omp_get_initial_device()) {
	DP("Call to omp_target_is_present on host, returning true\n");
	return true;
	}

	PM->RTLsMtx.lock();
	size_t DevicesSize = PM->Devices.size();
	PM->RTLsMtx.unlock();
	if (DevicesSize <= (size_t)DeviceNum) {
	DP("Call to omp_target_is_present with invalid device ID, returning "
	"false\n");
	return false;
	}

	DeviceTy &Device = *PM->Devices[DeviceNum];
	bool IsLast; // not used
	bool IsHostPtr;
	// omp_target_is_present tests whether a host pointer refers to storage that
	// is mapped to a given device. However, due to the lack of the storage size,
	// only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
	// lengh array) is mapped instead of the referred storage.
	TargetPointerResultTy TPR =
	Device.getTgtPtrBegin(const_cast<void *>(Ptr), 1, IsLast,
	/UpdateRefCount=/false,
	/UseHoldRefCount=/false, IsHostPtr);
	int Rc = (TPR.TargetPointer != NULL);
	// Under unified memory the host pointer can be returned by the
	// getTgtPtrBegin() function which means that there is no device
	// corresponding point for ptr. This function should return false
	// in that situation.
	if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
	Rc = !IsHostPtr;
	DP("Call to omp_target_is_present returns %d\n", Rc);
	return Rc;
	}

	EXTERN int omp_target_memcpy(void Dst, const void Src, size_t Length,
	size_t DstOffset, size_t SrcOffset, int DstDevice,
	int SrcDevice) {
	TIMESCOPE();
	DP("Call to omp_target_memcpy, dst device %d, src device %d, "
	"dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
	"src offset %zu, length %zu\n",
	DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,
	Length);

	if (!Dst \|\| !Src \|\| Length <= 0) {
	if (Length == 0) {
	DP("Call to omp_target_memcpy with zero length, nothing to do\n");
	return OFFLOAD_SUCCESS;
	}

	REPORT("Call to omp_target_memcpy with invalid arguments\n");
	return OFFLOAD_FAIL;
	}

	if (SrcDevice != omp_get_initial_device() && !deviceIsReady(SrcDevice)) {
	REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
	return OFFLOAD_FAIL;
	}

	if (DstDevice != omp_get_initial_device() && !deviceIsReady(DstDevice)) {
	REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
	return OFFLOAD_FAIL;
	}

	int Rc = OFFLOAD_SUCCESS;
	void SrcAddr = (char )const_cast<void *>(Src) + SrcOffset;
	void DstAddr = (char )Dst + DstOffset;

	if (SrcDevice == omp_get_initial_device() &&
	DstDevice == omp_get_initial_device()) {
	DP("copy from host to host\n");
	const void *P = memcpy(DstAddr, SrcAddr, Length);
	if (P == NULL)
	Rc = OFFLOAD_FAIL;
	} else if (SrcDevice == omp_get_initial_device()) {
	DP("copy from host to device\n");
	DeviceTy &DstDev = *PM->Devices[DstDevice];
	AsyncInfoTy AsyncInfo(DstDev);
	Rc = DstDev.submitData(DstAddr, SrcAddr, Length, AsyncInfo);
	} else if (DstDevice == omp_get_initial_device()) {
	DP("copy from device to host\n");
	DeviceTy &SrcDev = *PM->Devices[SrcDevice];
	AsyncInfoTy AsyncInfo(SrcDev);
	Rc = SrcDev.retrieveData(DstAddr, SrcAddr, Length, AsyncInfo);
	} else {
	DP("copy from device to device\n");
	DeviceTy &SrcDev = *PM->Devices[SrcDevice];
	DeviceTy &DstDev = *PM->Devices[DstDevice];
	// First try to use D2D memcpy which is more efficient. If fails, fall back
	// to unefficient way.
	if (SrcDev.isDataExchangable(DstDev)) {
	AsyncInfoTy AsyncInfo(SrcDev);
	Rc = SrcDev.dataExchange(SrcAddr, DstDev, DstAddr, Length, AsyncInfo);
	if (Rc == OFFLOAD_SUCCESS)
	return OFFLOAD_SUCCESS;
	}

	void *Buffer = malloc(Length);
	{
	AsyncInfoTy AsyncInfo(SrcDev);
	Rc = SrcDev.retrieveData(Buffer, SrcAddr, Length, AsyncInfo);
	}
	if (Rc == OFFLOAD_SUCCESS) {
	AsyncInfoTy AsyncInfo(SrcDev);
	Rc = DstDev.submitData(DstAddr, Buffer, Length, AsyncInfo);
	}
	free(Buffer);
	}

	DP("omp_target_memcpy returns %d\n", Rc);
	return Rc;
	}

	EXTERN int
	omp_target_memcpy_rect(void Dst, const void Src, size_t ElementSize,
	int NumDims, const size_t *Volume,
	const size_t DstOffsets, const size_t SrcOffsets,
	const size_t DstDimensions, const size_t SrcDimensions,
	int DstDevice, int SrcDevice) {
	TIMESCOPE();
	DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
	"dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
	"src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
	"volume " DPxMOD ", element size %zu, num_dims %d\n",
	DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),
	DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),
	DPxPTR(Volume), ElementSize, NumDims);

	if (!(Dst \|\| Src)) {
	DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
	INT_MAX);
	return INT_MAX;
	}

	if (!Dst \|\| !Src \|\| ElementSize < 1 \|\| NumDims < 1 \|\| !Volume \|\|
	!DstOffsets \|\| !SrcOffsets \|\| !DstDimensions \|\| !SrcDimensions) {
	REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
	return OFFLOAD_FAIL;
	}

	int Rc;
	if (NumDims == 1) {
	Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],
	ElementSize * DstOffsets[0],
	ElementSize * SrcOffsets[0], DstDevice, SrcDevice);
	} else {
	size_t DstSliceSize = ElementSize;
	size_t SrcSliceSize = ElementSize;
	for (int I = 1; I < NumDims; ++I) {
	DstSliceSize *= DstDimensions[I];
	SrcSliceSize *= SrcDimensions[I];
	}

	size_t DstOff = DstOffsets[0] * DstSliceSize;
	size_t SrcOff = SrcOffsets[0] * SrcSliceSize;
	for (size_t I = 0; I < Volume[0]; ++I) {
	Rc = omp_target_memcpy_rect(
	(char )Dst + DstOff + DstSliceSize I,
	(char )const_cast<void >(Src) + SrcOff + SrcSliceSize * I,
	ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,
	DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);

	if (Rc) {
	DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
	return Rc;
	}
	}
	}

	DP("omp_target_memcpy_rect returns %d\n", Rc);
	return Rc;
	}

	EXTERN int omp_target_associate_ptr(const void HostPtr, const void DevicePtr,
	size_t Size, size_t DeviceOffset,
	int DeviceNum) {
	TIMESCOPE();
	DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
	"device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
	DPxPTR(HostPtr), DPxPTR(DevicePtr), Size, DeviceOffset, DeviceNum);

	if (!HostPtr \|\| !DevicePtr \|\| Size <= 0) {
	REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
	return OFFLOAD_FAIL;
	}

	if (DeviceNum == omp_get_initial_device()) {
	REPORT("omp_target_associate_ptr: no association possible on the host\n");
	return OFFLOAD_FAIL;
	}

	if (!deviceIsReady(DeviceNum)) {
	REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
	return OFFLOAD_FAIL;
	}

	DeviceTy &Device = *PM->Devices[DeviceNum];
	void DeviceAddr = (void )((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
	int Rc = Device.associatePtr(const_cast<void *>(HostPtr),
	const_cast<void *>(DeviceAddr), Size);
	DP("omp_target_associate_ptr returns %d\n", Rc);
	return Rc;
	}

	EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
	TIMESCOPE();
	DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
	"device_num %d\n",
	DPxPTR(HostPtr), DeviceNum);

	if (!HostPtr) {
	REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
	return OFFLOAD_FAIL;
	}

	if (DeviceNum == omp_get_initial_device()) {
	REPORT(
	"omp_target_disassociate_ptr: no association possible on the host\n");
	return OFFLOAD_FAIL;
	}

	if (!deviceIsReady(DeviceNum)) {
	REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
	return OFFLOAD_FAIL;
	}

	DeviceTy &Device = *PM->Devices[DeviceNum];
	int Rc = Device.disassociatePtr(const_cast<void *>(HostPtr));
	DP("omp_target_disassociate_ptr returns %d\n", Rc);
	return Rc;
	}