| //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This provides a class for OpenMP runtime code generation. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "CGOpenMPRuntime.h" | |
| #include "ABIInfoImpl.h" | |
| #include "CGCXXABI.h" | |
| #include "CGCleanup.h" | |
| #include "CGDebugInfo.h" | |
| #include "CGRecordLayout.h" | |
| #include "CodeGenFunction.h" | |
| #include "TargetInfo.h" | |
| #include "clang/AST/APValue.h" | |
| #include "clang/AST/Attr.h" | |
| #include "clang/AST/Decl.h" | |
| #include "clang/AST/OpenMPClause.h" | |
| #include "clang/AST/StmtOpenMP.h" | |
| #include "clang/AST/StmtVisitor.h" | |
| #include "clang/Basic/DiagnosticFrontend.h" | |
| #include "clang/Basic/OpenMPKinds.h" | |
| #include "clang/Basic/SourceManager.h" | |
| #include "clang/CodeGen/ConstantInitBuilder.h" | |
| #include "clang/CodeGenUtils/RecordLayoutUtils.h" | |
| #include "llvm/ADT/ArrayRef.h" | |
| #include "llvm/ADT/STLExtras.h" | |
| #include "llvm/ADT/SmallSet.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| #include "llvm/ADT/StringExtras.h" | |
| #include "llvm/Analysis/ValueTracking.h" | |
| #include "llvm/Bitcode/BitcodeReader.h" | |
| #include "llvm/IR/Constants.h" | |
| #include "llvm/IR/DerivedTypes.h" | |
| #include "llvm/IR/GlobalValue.h" | |
| #include "llvm/IR/InstrTypes.h" | |
| #include "llvm/IR/Value.h" | |
| #include "llvm/Support/AtomicOrdering.h" | |
| #include "llvm/Support/VirtualFileSystem.h" | |
| #include "llvm/Support/raw_ostream.h" | |
| #include <cassert> | |
| #include <cstdint> | |
| #include <numeric> | |
| #include <optional> | |
| using namespace clang; | |
| using namespace CodeGen; | |
| using namespace llvm::omp; | |
| namespace { | |
| /// Base class for handling code generation inside OpenMP regions. | |
| class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { | |
| public: | |
| /// Kinds of OpenMP regions used in codegen. | |
| enum CGOpenMPRegionKind { | |
| /// Region with outlined function for standalone 'parallel' | |
| /// directive. | |
| ParallelOutlinedRegion, | |
| /// Region with outlined function for standalone 'task' directive. | |
| TaskOutlinedRegion, | |
| /// Region for constructs that do not require function outlining, | |
| /// like 'for', 'sections', 'atomic' etc. directives. | |
| InlinedRegion, | |
| /// Region with outlined function for standalone 'target' directive. | |
| TargetRegion, | |
| }; | |
| CGOpenMPRegionInfo(const CapturedStmt &CS, | |
| const CGOpenMPRegionKind RegionKind, | |
| const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, | |
| bool HasCancel) | |
| : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind), | |
| CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {} | |
| CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind, | |
| const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, | |
| bool HasCancel) | |
| : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen), | |
| Kind(Kind), HasCancel(HasCancel) {} | |
| /// Get a variable or parameter for storing global thread id | |
| /// inside OpenMP construct. | |
| virtual const VarDecl *getThreadIDVariable() const = 0; | |
| /// Emit the captured statement body. | |
| void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; | |
| /// Get an LValue for the current ThreadID variable. | |
| /// \return LValue for thread id variable. This LValue always has type int32*. | |
| virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF); | |
| virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {} | |
| CGOpenMPRegionKind getRegionKind() const { return RegionKind; } | |
| OpenMPDirectiveKind getDirectiveKind() const { return Kind; } | |
| bool hasCancel() const { return HasCancel; } | |
| static bool classof(const CGCapturedStmtInfo *Info) { | |
| return Info->getKind() == CR_OpenMP; | |
| } | |
| ~CGOpenMPRegionInfo() override = default; | |
| protected: | |
| CGOpenMPRegionKind RegionKind; | |
| RegionCodeGenTy CodeGen; | |
| OpenMPDirectiveKind Kind; | |
| bool HasCancel; | |
| }; | |
| /// API for captured statement code generation in OpenMP constructs. | |
| class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo { | |
| public: | |
| CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, | |
| const RegionCodeGenTy &CodeGen, | |
| OpenMPDirectiveKind Kind, bool HasCancel, | |
| StringRef HelperName) | |
| : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind, | |
| HasCancel), | |
| ThreadIDVar(ThreadIDVar), HelperName(HelperName) { | |
| assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); | |
| } | |
| /// Get a variable or parameter for storing global thread id | |
| /// inside OpenMP construct. | |
| const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } | |
| /// Get the name of the capture helper. | |
| StringRef getHelperName() const override { return HelperName; } | |
| static bool classof(const CGCapturedStmtInfo *Info) { | |
| return CGOpenMPRegionInfo::classof(Info) && | |
| cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == | |
| ParallelOutlinedRegion; | |
| } | |
| private: | |
| /// A variable or parameter storing global thread id for OpenMP | |
| /// constructs. | |
| const VarDecl *ThreadIDVar; | |
| StringRef HelperName; | |
| }; | |
| /// API for captured statement code generation in OpenMP constructs. | |
| class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo { | |
| public: | |
| class UntiedTaskActionTy final : public PrePostActionTy { | |
| bool Untied; | |
| const VarDecl *PartIDVar; | |
| const RegionCodeGenTy UntiedCodeGen; | |
| llvm::SwitchInst *UntiedSwitch = nullptr; | |
| public: | |
| UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar, | |
| const RegionCodeGenTy &UntiedCodeGen) | |
| : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {} | |
| void Enter(CodeGenFunction &CGF) override { | |
| if (Untied) { | |
| // Emit task switching point. | |
| LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(PartIDVar), | |
| PartIDVar->getType()->castAs<PointerType>()); | |
| llvm::Value *Res = | |
| CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); | |
| llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); | |
| UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); | |
| CGF.EmitBlock(DoneBB); | |
| CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); | |
| CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); | |
| UntiedSwitch->addCase(CGF.Builder.getInt32(0), | |
| CGF.Builder.GetInsertBlock()); | |
| emitUntiedSwitch(CGF); | |
| } | |
| } | |
| void emitUntiedSwitch(CodeGenFunction &CGF) const { | |
| if (Untied) { | |
| LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(PartIDVar), | |
| PartIDVar->getType()->castAs<PointerType>()); | |
| CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), | |
| PartIdLVal); | |
| UntiedCodeGen(CGF); | |
| CodeGenFunction::JumpDest CurPoint = | |
| CGF.getJumpDestInCurrentScope(".untied.next."); | |
| CGF.EmitBranch(CGF.ReturnBlock.getBlock()); | |
| CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); | |
| UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), | |
| CGF.Builder.GetInsertBlock()); | |
| CGF.EmitBranchThroughCleanup(CurPoint); | |
| CGF.EmitBlock(CurPoint.getBlock()); | |
| } | |
| } | |
| unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); } | |
| }; | |
| CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS, | |
| const VarDecl *ThreadIDVar, | |
| const RegionCodeGenTy &CodeGen, | |
| OpenMPDirectiveKind Kind, bool HasCancel, | |
| const UntiedTaskActionTy &Action) | |
| : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel), | |
| ThreadIDVar(ThreadIDVar), Action(Action) { | |
| assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); | |
| } | |
| /// Get a variable or parameter for storing global thread id | |
| /// inside OpenMP construct. | |
| const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } | |
| /// Get an LValue for the current ThreadID variable. | |
| LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override; | |
| /// Get the name of the capture helper. | |
| StringRef getHelperName() const override { return ".omp_outlined."; } | |
| void emitUntiedSwitch(CodeGenFunction &CGF) override { | |
| Action.emitUntiedSwitch(CGF); | |
| } | |
| static bool classof(const CGCapturedStmtInfo *Info) { | |
| return CGOpenMPRegionInfo::classof(Info) && | |
| cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == | |
| TaskOutlinedRegion; | |
| } | |
| private: | |
| /// A variable or parameter storing global thread id for OpenMP | |
| /// constructs. | |
| const VarDecl *ThreadIDVar; | |
| /// Action for emitting code for untied tasks. | |
| const UntiedTaskActionTy &Action; | |
| }; | |
| /// API for inlined captured statement code generation in OpenMP | |
| /// constructs. | |
| class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { | |
| public: | |
| CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI, | |
| const RegionCodeGenTy &CodeGen, | |
| OpenMPDirectiveKind Kind, bool HasCancel) | |
| : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel), | |
| OldCSI(OldCSI), | |
| OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {} | |
| // Retrieve the value of the context parameter. | |
| llvm::Value *getContextValue() const override { | |
| if (OuterRegionInfo) | |
| return OuterRegionInfo->getContextValue(); | |
| llvm_unreachable("No context value for inlined OpenMP region"); | |
| } | |
| void setContextValue(llvm::Value *V) override { | |
| if (OuterRegionInfo) { | |
| OuterRegionInfo->setContextValue(V); | |
| return; | |
| } | |
| llvm_unreachable("No context value for inlined OpenMP region"); | |
| } | |
| /// Lookup the captured field decl for a variable. | |
| const FieldDecl *lookup(const VarDecl *VD) const override { | |
| if (OuterRegionInfo) | |
| return OuterRegionInfo->lookup(VD); | |
| // If there is no outer outlined region,no need to lookup in a list of | |
| // captured variables, we can use the original one. | |
| return nullptr; | |
| } | |
| FieldDecl *getThisFieldDecl() const override { | |
| if (OuterRegionInfo) | |
| return OuterRegionInfo->getThisFieldDecl(); | |
| return nullptr; | |
| } | |
| /// Get a variable or parameter for storing global thread id | |
| /// inside OpenMP construct. | |
| const VarDecl *getThreadIDVariable() const override { | |
| if (OuterRegionInfo) | |
| return OuterRegionInfo->getThreadIDVariable(); | |
| return nullptr; | |
| } | |
| /// Get an LValue for the current ThreadID variable. | |
| LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { | |
| if (OuterRegionInfo) | |
| return OuterRegionInfo->getThreadIDVariableLValue(CGF); | |
| llvm_unreachable("No LValue for inlined OpenMP construct"); | |
| } | |
| /// Get the name of the capture helper. | |
| StringRef getHelperName() const override { | |
| if (auto *OuterRegionInfo = getOldCSI()) | |
| return OuterRegionInfo->getHelperName(); | |
| llvm_unreachable("No helper name for inlined OpenMP construct"); | |
| } | |
| void emitUntiedSwitch(CodeGenFunction &CGF) override { | |
| if (OuterRegionInfo) | |
| OuterRegionInfo->emitUntiedSwitch(CGF); | |
| } | |
| CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; } | |
| static bool classof(const CGCapturedStmtInfo *Info) { | |
| return CGOpenMPRegionInfo::classof(Info) && | |
| cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion; | |
| } | |
| ~CGOpenMPInlinedRegionInfo() override = default; | |
| private: | |
| /// CodeGen info about outer OpenMP region. | |
| CodeGenFunction::CGCapturedStmtInfo *OldCSI; | |
| CGOpenMPRegionInfo *OuterRegionInfo; | |
| }; | |
| /// API for captured statement code generation in OpenMP target | |
| /// constructs. For this captures, implicit parameters are used instead of the | |
| /// captured fields. The name of the target region has to be unique in a given | |
| /// application so it is provided by the client, because only the client has | |
| /// the information to generate that. | |
| class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo { | |
| public: | |
| CGOpenMPTargetRegionInfo(const CapturedStmt &CS, | |
| const RegionCodeGenTy &CodeGen, StringRef HelperName) | |
| : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target, | |
| /*HasCancel=*/false), | |
| HelperName(HelperName) {} | |
| /// This is unused for target regions because each starts executing | |
| /// with a single thread. | |
| const VarDecl *getThreadIDVariable() const override { return nullptr; } | |
| /// Get the name of the capture helper. | |
| StringRef getHelperName() const override { return HelperName; } | |
| static bool classof(const CGCapturedStmtInfo *Info) { | |
| return CGOpenMPRegionInfo::classof(Info) && | |
| cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion; | |
| } | |
| private: | |
| StringRef HelperName; | |
| }; | |
| static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) { | |
| llvm_unreachable("No codegen for expressions"); | |
| } | |
| /// API for generation of expressions captured in a innermost OpenMP | |
| /// region. | |
| class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo { | |
| public: | |
| CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) | |
| : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, | |
| OMPD_unknown, | |
| /*HasCancel=*/false), | |
| PrivScope(CGF) { | |
| // Make sure the globals captured in the provided statement are local by | |
| // using the privatization logic. We assume the same variable is not | |
| // captured more than once. | |
| for (const auto &C : CS.captures()) { | |
| if (!C.capturesVariable() && !C.capturesVariableByCopy()) | |
| continue; | |
| const VarDecl *VD = C.getCapturedVar(); | |
| if (VD->isLocalVarDeclOrParm()) | |
| continue; | |
| DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), | |
| /*RefersToEnclosingVariableOrCapture=*/false, | |
| VD->getType().getNonReferenceType(), VK_LValue, | |
| C.getLocation()); | |
| PrivScope.addPrivate(VD, CGF.EmitLValue(&DRE).getAddress()); | |
| } | |
| (void)PrivScope.Privatize(); | |
| } | |
| /// Lookup the captured field decl for a variable. | |
| const FieldDecl *lookup(const VarDecl *VD) const override { | |
| if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD)) | |
| return FD; | |
| return nullptr; | |
| } | |
| /// Emit the captured statement body. | |
| void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { | |
| llvm_unreachable("No body for expressions"); | |
| } | |
| /// Get a variable or parameter for storing global thread id | |
| /// inside OpenMP construct. | |
| const VarDecl *getThreadIDVariable() const override { | |
| llvm_unreachable("No thread id for expressions"); | |
| } | |
| /// Get the name of the capture helper. | |
| StringRef getHelperName() const override { | |
| llvm_unreachable("No helper name for expressions"); | |
| } | |
| static bool classof(const CGCapturedStmtInfo *Info) { return false; } | |
| private: | |
| /// Private scope to capture global variables. | |
| CodeGenFunction::OMPPrivateScope PrivScope; | |
| }; | |
| /// RAII for emitting code of OpenMP constructs. | |
| class InlinedOpenMPRegionRAII { | |
| CodeGenFunction &CGF; | |
| llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields; | |
| FieldDecl *LambdaThisCaptureField = nullptr; | |
| const CodeGen::CGBlockInfo *BlockInfo = nullptr; | |
| bool NoInheritance = false; | |
| public: | |
| /// Constructs region for combined constructs. | |
| /// \param CodeGen Code generation sequence for combined directives. Includes | |
| /// a list of functions used for code generation of implicitly inlined | |
| /// regions. | |
| InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, | |
| OpenMPDirectiveKind Kind, bool HasCancel, | |
| bool NoInheritance = true) | |
| : CGF(CGF), NoInheritance(NoInheritance) { | |
| // Start emission for the construct. | |
| CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( | |
| CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); | |
| if (NoInheritance) { | |
| std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); | |
| LambdaThisCaptureField = CGF.LambdaThisCaptureField; | |
| CGF.LambdaThisCaptureField = nullptr; | |
| BlockInfo = CGF.BlockInfo; | |
| CGF.BlockInfo = nullptr; | |
| } | |
| } | |
| ~InlinedOpenMPRegionRAII() { | |
| // Restore original CapturedStmtInfo only if we're done with code emission. | |
| auto *OldCSI = | |
| cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); | |
| delete CGF.CapturedStmtInfo; | |
| CGF.CapturedStmtInfo = OldCSI; | |
| if (NoInheritance) { | |
| std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); | |
| CGF.LambdaThisCaptureField = LambdaThisCaptureField; | |
| CGF.BlockInfo = BlockInfo; | |
| } | |
| } | |
| }; | |
| /// Values for bit flags used in the ident_t to describe the fields. | |
| /// All enumeric elements are named and described in accordance with the code | |
| /// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h | |
| enum OpenMPLocationFlags : unsigned { | |
| /// Use trampoline for internal microtask. | |
| OMP_IDENT_IMD = 0x01, | |
| /// Use c-style ident structure. | |
| OMP_IDENT_KMPC = 0x02, | |
| /// Atomic reduction option for kmpc_reduce. | |
| OMP_ATOMIC_REDUCE = 0x10, | |
| /// Explicit 'barrier' directive. | |
| OMP_IDENT_BARRIER_EXPL = 0x20, | |
| /// Implicit barrier in code. | |
| OMP_IDENT_BARRIER_IMPL = 0x40, | |
| /// Implicit barrier in 'for' directive. | |
| OMP_IDENT_BARRIER_IMPL_FOR = 0x40, | |
| /// Implicit barrier in 'sections' directive. | |
| OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0, | |
| /// Implicit barrier in 'single' directive. | |
| OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140, | |
| /// Call of __kmp_for_static_init for static loop. | |
| OMP_IDENT_WORK_LOOP = 0x200, | |
| /// Call of __kmp_for_static_init for sections. | |
| OMP_IDENT_WORK_SECTIONS = 0x400, | |
| /// Call of __kmp_for_static_init for distribute. | |
| OMP_IDENT_WORK_DISTRIBUTE = 0x800, | |
| LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE) | |
| }; | |
| /// Describes ident structure that describes a source location. | |
| /// All descriptions are taken from | |
| /// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h | |
| /// Original structure: | |
| /// typedef struct ident { | |
| /// kmp_int32 reserved_1; /**< might be used in Fortran; | |
| /// see above */ | |
| /// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; | |
| /// KMP_IDENT_KMPC identifies this union | |
| /// member */ | |
| /// kmp_int32 reserved_2; /**< not really used in Fortran any more; | |
| /// see above */ | |
| ///#if USE_ITT_BUILD | |
| /// /* but currently used for storing | |
| /// region-specific ITT */ | |
| /// /* contextual information. */ | |
| ///#endif /* USE_ITT_BUILD */ | |
| /// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for | |
| /// C++ */ | |
| /// char const *psource; /**< String describing the source location. | |
| /// The string is composed of semi-colon separated | |
| // fields which describe the source file, | |
| /// the function and a pair of line numbers that | |
| /// delimit the construct. | |
| /// */ | |
| /// } ident_t; | |
| enum IdentFieldIndex { | |
| /// might be used in Fortran | |
| IdentField_Reserved_1, | |
| /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member. | |
| IdentField_Flags, | |
| /// Not really used in Fortran any more | |
| IdentField_Reserved_2, | |
| /// Source[4] in Fortran, do not use for C++ | |
| IdentField_Reserved_3, | |
| /// String describing the source location. The string is composed of | |
| /// semi-colon separated fields which describe the source file, the function | |
| /// and a pair of line numbers that delimit the construct. | |
| IdentField_PSource | |
| }; | |
| /// Schedule types for 'omp for' loops (these enumerators are taken from | |
| /// the enum sched_type in kmp.h). | |
| enum OpenMPSchedType { | |
| /// Lower bound for default (unordered) versions. | |
| OMP_sch_lower = 32, | |
| OMP_sch_static_chunked = 33, | |
| OMP_sch_static = 34, | |
| OMP_sch_dynamic_chunked = 35, | |
| OMP_sch_guided_chunked = 36, | |
| OMP_sch_runtime = 37, | |
| OMP_sch_auto = 38, | |
| /// static with chunk adjustment (e.g., simd) | |
| OMP_sch_static_balanced_chunked = 45, | |
| /// Lower bound for 'ordered' versions. | |
| OMP_ord_lower = 64, | |
| OMP_ord_static_chunked = 65, | |
| OMP_ord_static = 66, | |
| OMP_ord_dynamic_chunked = 67, | |
| OMP_ord_guided_chunked = 68, | |
| OMP_ord_runtime = 69, | |
| OMP_ord_auto = 70, | |
| OMP_sch_default = OMP_sch_static, | |
| /// dist_schedule types | |
| OMP_dist_sch_static_chunked = 91, | |
| OMP_dist_sch_static = 92, | |
| /// Fused distribute+for static schedule (entityId = team*nthreads + tid, | |
| /// num_entities = nteams*nthreads). One for_static_init call, no | |
| /// surrounding distribute_static_init. Matches | |
| /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL | |
| /// (openmp/device/include/DeviceTypes.h). | |
| OMP_dist_sch_static_chunked_sch_static_chunkone = 93, | |
| /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. | |
| /// Set if the monotonic schedule modifier was present. | |
| OMP_sch_modifier_monotonic = (1 << 29), | |
| /// Set if the nonmonotonic schedule modifier was present. | |
| OMP_sch_modifier_nonmonotonic = (1 << 30), | |
| }; | |
| /// A basic class for pre|post-action for advanced codegen sequence for OpenMP | |
| /// region. | |
| class CleanupTy final : public EHScopeStack::Cleanup { | |
| PrePostActionTy *Action; | |
| public: | |
| explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {} | |
| void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| Action->Exit(CGF); | |
| } | |
| }; | |
| } // anonymous namespace | |
| void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| if (PrePostAction) { | |
| CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); | |
| Callback(CodeGen, CGF, *PrePostAction); | |
| } else { | |
| PrePostActionTy Action; | |
| Callback(CodeGen, CGF, Action); | |
| } | |
| } | |
| /// Check if the combiner is a call to UDR combiner and if it is so return the | |
| /// UDR decl used for reduction. | |
| static const OMPDeclareReductionDecl * | |
| getReductionInit(const Expr *ReductionOp) { | |
| if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) | |
| if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee())) | |
| if (const auto *DRE = | |
| dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts())) | |
| if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) | |
| return DRD; | |
| return nullptr; | |
| } | |
| static void emitInitWithReductionInitializer(CodeGenFunction &CGF, | |
| const OMPDeclareReductionDecl *DRD, | |
| const Expr *InitOp, | |
| Address Private, Address Original, | |
| QualType Ty) { | |
| if (DRD->getInitializer()) { | |
| std::pair<llvm::Function *, llvm::Function *> Reduction = | |
| CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); | |
| const auto *CE = cast<CallExpr>(InitOp); | |
| const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee()); | |
| const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts(); | |
| const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts(); | |
| const auto *LHSDRE = | |
| cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr()); | |
| const auto *RHSDRE = | |
| cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr()); | |
| CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |
| PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), Private); | |
| PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original); | |
| (void)PrivateScope.Privatize(); | |
| RValue Func = RValue::get(Reduction.second); | |
| CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); | |
| CGF.EmitIgnoredExpr(InitOp); | |
| } else { | |
| llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); | |
| std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); | |
| auto *GV = new llvm::GlobalVariable( | |
| CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, | |
| llvm::GlobalValue::PrivateLinkage, Init, Name); | |
| LValue LV = CGF.MakeNaturalAlignRawAddrLValue(GV, Ty); | |
| RValue InitRVal; | |
| switch (CGF.getEvaluationKind(Ty)) { | |
| case TEK_Scalar: | |
| InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); | |
| break; | |
| case TEK_Complex: | |
| InitRVal = | |
| RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); | |
| break; | |
| case TEK_Aggregate: { | |
| OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV); | |
| CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), | |
| /*IsInitializer=*/false); | |
| return; | |
| } | |
| } | |
| OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); | |
| CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), | |
| /*IsInitializer=*/false); | |
| } | |
| } | |
| /// Emit initialization of arrays of complex types. | |
| /// \param DestAddr Address of the array. | |
| /// \param Type Type of array. | |
| /// \param Init Initial expression of array. | |
| /// \param SrcAddr Address of the original array, required only if the declare | |
| /// reduction has an initializer. | |
| static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, | |
| QualType Type, bool EmitDeclareReductionInit, | |
| const Expr *Init, | |
| const OMPDeclareReductionDecl *DRD, | |
| Address SrcAddr = Address::invalid()) { | |
| // Perform element-by-element initialization. | |
| QualType ElementTy; | |
| // Only the initializer of a declare reduction reads the original item. | |
| bool UseSrc = DRD && DRD->getInitializer(); | |
| assert((!UseSrc || SrcAddr.isValid()) && | |
| "Declare reduction initializer requires the original item address."); | |
| // Drill down to the base element type on both arrays. | |
| const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); | |
| llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); | |
| if (UseSrc) | |
| SrcAddr = SrcAddr.withElementType(DestAddr.getElementType()); | |
| llvm::Value *SrcBegin = nullptr; | |
| if (UseSrc) | |
| SrcBegin = SrcAddr.emitRawPointer(CGF); | |
| llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF); | |
| // Cast from pointer to array type to pointer to single element. | |
| llvm::Value *DestEnd = | |
| CGF.Builder.CreateGEP(DestAddr.getElementType(), DestBegin, NumElements); | |
| // The basic structure here is a while-do loop. | |
| llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); | |
| llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); | |
| llvm::Value *IsEmpty = | |
| CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); | |
| CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); | |
| // Enter the loop body, making that address the current address. | |
| llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); | |
| CGF.EmitBlock(BodyBB); | |
| CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); | |
| llvm::PHINode *SrcElementPHI = nullptr; | |
| Address SrcElementCurrent = Address::invalid(); | |
| if (UseSrc) { | |
| SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, | |
| "omp.arraycpy.srcElementPast"); | |
| SrcElementPHI->addIncoming(SrcBegin, EntryBB); | |
| SrcElementCurrent = | |
| Address(SrcElementPHI, SrcAddr.getElementType(), | |
| SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |
| } | |
| llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( | |
| DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); | |
| DestElementPHI->addIncoming(DestBegin, EntryBB); | |
| Address DestElementCurrent = | |
| Address(DestElementPHI, DestAddr.getElementType(), | |
| DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |
| // Emit copy. | |
| { | |
| CodeGenFunction::RunCleanupsScope InitScope(CGF); | |
| if (EmitDeclareReductionInit) { | |
| emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, | |
| SrcElementCurrent, ElementTy); | |
| } else | |
| CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), | |
| /*IsInitializer=*/false); | |
| } | |
| if (UseSrc) { | |
| // Shift the address forward by one element. | |
| llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( | |
| SrcAddr.getElementType(), SrcElementPHI, /*Idx0=*/1, | |
| "omp.arraycpy.dest.element"); | |
| SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); | |
| } | |
| // Shift the address forward by one element. | |
| llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( | |
| DestAddr.getElementType(), DestElementPHI, /*Idx0=*/1, | |
| "omp.arraycpy.dest.element"); | |
| // Check whether we've reached the end. | |
| llvm::Value *Done = | |
| CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); | |
| CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); | |
| DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); | |
| // Done. | |
| CGF.EmitBlock(DoneBB, /*IsFinished=*/true); | |
| } | |
| LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { | |
| return CGF.EmitOMPSharedLValue(E); | |
| } | |
| LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, | |
| const Expr *E) { | |
| if (const auto *OASE = dyn_cast<ArraySectionExpr>(E)) | |
| return CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false); | |
| return LValue(); | |
| } | |
| void ReductionCodeGen::emitAggregateInitialization( | |
| CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, | |
| const OMPDeclareReductionDecl *DRD) { | |
| // Emit VarDecl with copy init for arrays. | |
| // Get the address of the original variable captured in current | |
| // captured region. | |
| const auto *PrivateVD = | |
| cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); | |
| bool EmitDeclareReductionInit = | |
| DRD && (DRD->getInitializer() || !PrivateVD->hasInit()); | |
| EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), | |
| EmitDeclareReductionInit, | |
| EmitDeclareReductionInit ? ClausesData[N].ReductionOp | |
| : PrivateVD->getInit(), | |
| DRD, SharedAddr); | |
| } | |
| ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds, | |
| ArrayRef<const Expr *> Origs, | |
| ArrayRef<const Expr *> Privates, | |
| ArrayRef<const Expr *> ReductionOps) { | |
| ClausesData.reserve(Shareds.size()); | |
| SharedAddresses.reserve(Shareds.size()); | |
| Sizes.reserve(Shareds.size()); | |
| BaseDecls.reserve(Shareds.size()); | |
| const auto *IOrig = Origs.begin(); | |
| const auto *IPriv = Privates.begin(); | |
| const auto *IRed = ReductionOps.begin(); | |
| for (const Expr *Ref : Shareds) { | |
| ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed); | |
| std::advance(IOrig, 1); | |
| std::advance(IPriv, 1); | |
| std::advance(IRed, 1); | |
| } | |
| } | |
| void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) { | |
| assert(SharedAddresses.size() == N && OrigAddresses.size() == N && | |
| "Number of generated lvalues must be exactly N."); | |
| LValue First = emitSharedLValue(CGF, ClausesData[N].Shared); | |
| LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Shared); | |
| SharedAddresses.emplace_back(First, Second); | |
| if (ClausesData[N].Shared == ClausesData[N].Ref) { | |
| OrigAddresses.emplace_back(First, Second); | |
| } else { | |
| LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); | |
| LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); | |
| OrigAddresses.emplace_back(First, Second); | |
| } | |
| } | |
| void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { | |
| QualType PrivateType = getPrivateType(N); | |
| bool AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref); | |
| if (!PrivateType->isVariablyModifiedType()) { | |
| Sizes.emplace_back( | |
| CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()), | |
| nullptr); | |
| return; | |
| } | |
| llvm::Value *Size; | |
| llvm::Value *SizeInChars; | |
| auto *ElemType = OrigAddresses[N].first.getAddress().getElementType(); | |
| auto *ElemSizeOf = llvm::ConstantInt::get( | |
| CGF.SizeTy, CGF.CGM.getDataLayout().getTypeAllocSize(ElemType)); | |
| if (AsArraySection) { | |
| SizeInChars = | |
| CGF.Builder.CreatePtrDiff(OrigAddresses[N].second.getPointer(CGF), | |
| OrigAddresses[N].first.getPointer(CGF)); | |
| SizeInChars = CGF.Builder.CreateNUWAdd(SizeInChars, ElemSizeOf); | |
| } else { | |
| SizeInChars = | |
| CGF.getTypeSize(OrigAddresses[N].first.getType().getNonReferenceType()); | |
| } | |
| Size = ElemSizeOf->isOne() | |
| ? SizeInChars | |
| : CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); | |
| Sizes.emplace_back(SizeInChars, Size); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap( | |
| CGF, | |
| cast<OpaqueValueExpr>( | |
| CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), | |
| RValue::get(Size)); | |
| CGF.EmitVariablyModifiedType(PrivateType); | |
| } | |
| void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, | |
| llvm::Value *Size) { | |
| QualType PrivateType = getPrivateType(N); | |
| if (!PrivateType->isVariablyModifiedType()) { | |
| assert(!Size && !Sizes[N].second && | |
| "Size should be nullptr for non-variably modified reduction " | |
| "items."); | |
| return; | |
| } | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap( | |
| CGF, | |
| cast<OpaqueValueExpr>( | |
| CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), | |
| RValue::get(Size)); | |
| CGF.EmitVariablyModifiedType(PrivateType); | |
| } | |
| void ReductionCodeGen::emitInitialization( | |
| CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr, | |
| llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) { | |
| assert(SharedAddresses.size() > N && "No variable was generated"); | |
| const auto *PrivateVD = | |
| cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); | |
| const OMPDeclareReductionDecl *DRD = | |
| getReductionInit(ClausesData[N].ReductionOp); | |
| if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { | |
| if (DRD && DRD->getInitializer()) | |
| (void)DefaultInit(CGF); | |
| emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD); | |
| } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { | |
| (void)DefaultInit(CGF); | |
| QualType SharedType = SharedAddresses[N].first.getType(); | |
| emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, | |
| PrivateAddr, SharedAddr, SharedType); | |
| } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && | |
| !CGF.isTrivialInitializer(PrivateVD->getInit())) { | |
| CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, | |
| PrivateVD->getType().getQualifiers(), | |
| /*IsInitializer=*/false); | |
| } | |
| } | |
| bool ReductionCodeGen::needCleanups(unsigned N) { | |
| QualType PrivateType = getPrivateType(N); | |
| QualType::DestructionKind DTorKind = PrivateType.isDestructedType(); | |
| return DTorKind != QualType::DK_none; | |
| } | |
| void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, | |
| Address PrivateAddr) { | |
| QualType PrivateType = getPrivateType(N); | |
| QualType::DestructionKind DTorKind = PrivateType.isDestructedType(); | |
| if (needCleanups(N)) { | |
| PrivateAddr = | |
| PrivateAddr.withElementType(CGF.ConvertTypeForMem(PrivateType)); | |
| CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); | |
| } | |
| } | |
| static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, | |
| LValue BaseLV) { | |
| BaseTy = BaseTy.getNonReferenceType(); | |
| while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && | |
| !CGF.getContext().hasSameType(BaseTy, ElTy)) { | |
| if (const auto *PtrTy = BaseTy->getAs<PointerType>()) { | |
| BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(), PtrTy); | |
| } else { | |
| LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(), BaseTy); | |
| BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); | |
| } | |
| BaseTy = BaseTy->getPointeeType(); | |
| } | |
| return CGF.MakeAddrLValue( | |
| BaseLV.getAddress().withElementType(CGF.ConvertTypeForMem(ElTy)), | |
| BaseLV.getType(), BaseLV.getBaseInfo(), | |
| CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); | |
| } | |
| static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, | |
| Address OriginalBaseAddress, llvm::Value *Addr) { | |
| RawAddress Tmp = RawAddress::invalid(); | |
| Address TopTmp = Address::invalid(); | |
| Address MostTopTmp = Address::invalid(); | |
| BaseTy = BaseTy.getNonReferenceType(); | |
| while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && | |
| !CGF.getContext().hasSameType(BaseTy, ElTy)) { | |
| Tmp = CGF.CreateMemTempWithoutCast(BaseTy); | |
| if (TopTmp.isValid()) | |
| CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); | |
| else | |
| MostTopTmp = Tmp; | |
| TopTmp = Tmp; | |
| BaseTy = BaseTy->getPointeeType(); | |
| } | |
| if (Tmp.isValid()) { | |
| Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Addr, Tmp.getElementType()); | |
| CGF.Builder.CreateStore(Addr, Tmp); | |
| return MostTopTmp; | |
| } | |
| Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Addr, OriginalBaseAddress.getType()); | |
| return OriginalBaseAddress.withPointer(Addr, NotKnownNonNull); | |
| } | |
| static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) { | |
| const VarDecl *OrigVD = nullptr; | |
| if (const auto *OASE = dyn_cast<ArraySectionExpr>(Ref)) { | |
| const Expr *Base = OASE->getBase()->IgnoreParenImpCasts(); | |
| while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Base)) | |
| Base = TempOASE->getBase()->IgnoreParenImpCasts(); | |
| while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) | |
| Base = TempASE->getBase()->IgnoreParenImpCasts(); | |
| DE = cast<DeclRefExpr>(Base); | |
| OrigVD = cast<VarDecl>(DE->getDecl()); | |
| } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) { | |
| const Expr *Base = ASE->getBase()->IgnoreParenImpCasts(); | |
| while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) | |
| Base = TempASE->getBase()->IgnoreParenImpCasts(); | |
| DE = cast<DeclRefExpr>(Base); | |
| OrigVD = cast<VarDecl>(DE->getDecl()); | |
| } | |
| return OrigVD; | |
| } | |
| Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, | |
| Address PrivateAddr) { | |
| const DeclRefExpr *DE; | |
| if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) { | |
| BaseDecls.emplace_back(OrigVD); | |
| LValue OriginalBaseLValue = CGF.EmitLValue(DE); | |
| LValue BaseLValue = | |
| loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), | |
| OriginalBaseLValue); | |
| Address SharedAddr = SharedAddresses[N].first.getAddress(); | |
| llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( | |
| SharedAddr.getElementType(), BaseLValue.getPointer(CGF), | |
| SharedAddr.emitRawPointer(CGF)); | |
| llvm::Value *PrivatePointer = | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| PrivateAddr.emitRawPointer(CGF), SharedAddr.getType()); | |
| llvm::Value *Ptr = CGF.Builder.CreateGEP( | |
| SharedAddr.getElementType(), PrivatePointer, Adjustment); | |
| return castToBase(CGF, OrigVD->getType(), | |
| SharedAddresses[N].first.getType(), | |
| OriginalBaseLValue.getAddress(), Ptr); | |
| } | |
| BaseDecls.emplace_back( | |
| cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl())); | |
| return PrivateAddr; | |
| } | |
| bool ReductionCodeGen::usesReductionInitializer(unsigned N) const { | |
| const OMPDeclareReductionDecl *DRD = | |
| getReductionInit(ClausesData[N].ReductionOp); | |
| return DRD && DRD->getInitializer(); | |
| } | |
| LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { | |
| return CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(getThreadIDVariable()), | |
| getThreadIDVariable()->getType()->castAs<PointerType>()); | |
| } | |
| void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // 1.2.2 OpenMP Language Terminology | |
| // Structured block - An executable statement with a single entry at the | |
| // top and a single exit at the bottom. | |
| // The point of exit cannot be a branch out of the structured block. | |
| // longjmp() and throw() must not violate the entry/exit criteria. | |
| CGF.EHStack.pushTerminate(); | |
| if (S) | |
| CGF.incrementProfileCounter(S); | |
| CodeGen(CGF); | |
| CGF.EHStack.popTerminate(); | |
| } | |
| LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( | |
| CodeGenFunction &CGF) { | |
| return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), | |
| getThreadIDVariable()->getType(), | |
| AlignmentSource::Decl); | |
| } | |
| static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC, | |
| QualType FieldTy) { | |
| auto *Field = FieldDecl::Create( | |
| C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy, | |
| C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()), | |
| /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit); | |
| Field->setAccess(AS_public); | |
| DC->addDecl(Field); | |
| return Field; | |
| } | |
| CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) | |
| : CGM(CGM), OMPBuilder(CGM.getModule()) { | |
| KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); | |
| llvm::OpenMPIRBuilderConfig Config( | |
| CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(), | |
| CGM.getLangOpts().OpenMPOffloadMandatory, | |
| /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false, | |
| hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false); | |
| Config.setDefaultTargetAS( | |
| CGM.getContext().getTargetInfo().getTargetAddressSpace(LangAS::Default)); | |
| Config.setRuntimeCC(CGM.getRuntimeCC()); | |
| OMPBuilder.setConfig(Config); | |
| OMPBuilder.initialize(); | |
| OMPBuilder.loadOffloadInfoMetadata(*CGM.getFileSystem(), | |
| CGM.getLangOpts().OpenMPIsTargetDevice | |
| ? CGM.getLangOpts().OMPHostIRFile | |
| : StringRef{}); | |
| // The user forces the compiler to behave as if omp requires | |
| // unified_shared_memory was given. | |
| if (CGM.getLangOpts().OpenMPForceUSM) { | |
| HasRequiresUnifiedSharedMemory = true; | |
| OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true); | |
| } | |
| } | |
| void CGOpenMPRuntime::clear() { | |
| InternalVars.clear(); | |
| // Clean non-target variable declarations possibly used only in debug info. | |
| for (const auto &Data : EmittedNonTargetVariables) { | |
| if (!Data.getValue().pointsToAliveValue()) | |
| continue; | |
| auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue()); | |
| if (!GV) | |
| continue; | |
| if (!GV->isDeclaration() || GV->getNumUses() > 0) | |
| continue; | |
| GV->eraseFromParent(); | |
| } | |
| } | |
| std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const { | |
| return OMPBuilder.createPlatformSpecificName(Parts); | |
| } | |
| static llvm::Function * | |
| emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, | |
| const Expr *CombinerInitializer, const VarDecl *In, | |
| const VarDecl *Out, bool IsCombiner) { | |
| // void .omp_combiner.(Ty *in, Ty *out); | |
| ASTContext &C = CGM.getContext(); | |
| QualType PtrTy = C.getPointerType(Ty).withRestrict(); | |
| auto *OmpOutParm = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Out->getLocation(), | |
| /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other); | |
| auto *OmpInParm = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, In->getLocation(), | |
| /*Id=*/nullptr, PtrTy, ImplicitParamKind::Other); | |
| FunctionArgList Args{OmpOutParm, OmpInParm}; | |
| const CGFunctionInfo &FnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName( | |
| {IsCombiner ? "omp_combiner" : "omp_initializer", ""}); | |
| auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, | |
| Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| if (CGM.getCodeGenOpts().OptimizationLevel != 0) { | |
| Fn->removeFnAttr(llvm::Attribute::NoInline); | |
| Fn->removeFnAttr(llvm::Attribute::OptimizeNone); | |
| Fn->addFnAttr(llvm::Attribute::AlwaysInline); | |
| } | |
| CodeGenFunction CGF(CGM); | |
| // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions. | |
| // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions. | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), | |
| Out->getLocation()); | |
| CodeGenFunction::OMPPrivateScope Scope(CGF); | |
| Address AddrIn = CGF.GetAddrOfLocalVar(OmpInParm); | |
| Scope.addPrivate( | |
| In, CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) | |
| .getAddress()); | |
| Address AddrOut = CGF.GetAddrOfLocalVar(OmpOutParm); | |
| Scope.addPrivate( | |
| Out, CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) | |
| .getAddress()); | |
| (void)Scope.Privatize(); | |
| if (!IsCombiner && Out->hasInit() && | |
| !CGF.isTrivialInitializer(Out->getInit())) { | |
| CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), | |
| Out->getType().getQualifiers(), | |
| /*IsInitializer=*/true); | |
| } | |
| if (CombinerInitializer) | |
| CGF.EmitIgnoredExpr(CombinerInitializer); | |
| Scope.ForceCleanup(); | |
| CGF.FinishFunction(); | |
| return Fn; | |
| } | |
| void CGOpenMPRuntime::emitUserDefinedReduction( | |
| CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { | |
| if (UDRMap.count(D) > 0) | |
| return; | |
| llvm::Function *Combiner = emitCombinerOrInitializer( | |
| CGM, D->getType(), D->getCombiner(), | |
| cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()), | |
| cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()), | |
| /*IsCombiner=*/true); | |
| llvm::Function *Initializer = nullptr; | |
| if (const Expr *Init = D->getInitializer()) { | |
| Initializer = emitCombinerOrInitializer( | |
| CGM, D->getType(), | |
| D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init | |
| : nullptr, | |
| cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()), | |
| cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()), | |
| /*IsCombiner=*/false); | |
| } | |
| UDRMap.try_emplace(D, Combiner, Initializer); | |
| if (CGF) | |
| FunctionUDRMap[CGF->CurFn].push_back(D); | |
| } | |
| std::pair<llvm::Function *, llvm::Function *> | |
| CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) { | |
| auto I = UDRMap.find(D); | |
| if (I != UDRMap.end()) | |
| return I->second; | |
| emitUserDefinedReduction(/*CGF=*/nullptr, D); | |
| return UDRMap.lookup(D); | |
| } | |
| namespace { | |
| // Temporary RAII solution to perform a push/pop stack event on the OpenMP IR | |
| // Builder if one is present. | |
| struct PushAndPopStackRAII { | |
| PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF, | |
| bool HasCancel, llvm::omp::Directive Kind) | |
| : OMPBuilder(OMPBuilder) { | |
| if (!OMPBuilder) | |
| return; | |
| // The following callback is the crucial part of clangs cleanup process. | |
| // | |
| // NOTE: | |
| // Once the OpenMPIRBuilder is used to create parallel regions (and | |
| // similar), the cancellation destination (Dest below) is determined via | |
| // IP. That means if we have variables to finalize we split the block at IP, | |
| // use the new block (=BB) as destination to build a JumpDest (via | |
| // getJumpDestInCurrentScope(BB)) which then is fed to | |
| // EmitBranchThroughCleanup. Furthermore, there will not be the need | |
| // to push & pop an FinalizationInfo object. | |
| // The FiniCB will still be needed but at the point where the | |
| // OpenMPIRBuilder is asked to construct a parallel (or similar) construct. | |
| auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) { | |
| assert(IP == IP.getNodeParent()->end() && | |
| "Clang CG should cause non-terminated block!"); | |
| CGBuilderTy::InsertPointGuard IPG(CGF.Builder); | |
| CGF.Builder.restoreIP(IP); | |
| CodeGenFunction::JumpDest Dest = | |
| CGF.getOMPCancelDestination(OMPD_parallel); | |
| CGF.EmitBranchThroughCleanup(Dest); | |
| return llvm::Error::success(); | |
| }; | |
| // TODO: Remove this once we emit parallel regions through the | |
| // OpenMPIRBuilder as it can do this setup internally. | |
| llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel}); | |
| OMPBuilder->pushFinalizationCB(std::move(FI)); | |
| } | |
| ~PushAndPopStackRAII() { | |
| if (OMPBuilder) | |
| OMPBuilder->popFinalizationCB(); | |
| } | |
| llvm::OpenMPIRBuilder *OMPBuilder; | |
| }; | |
| } // namespace | |
| static llvm::Function *emitParallelOrTeamsOutlinedFunction( | |
| CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, | |
| const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, | |
| const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) { | |
| assert(ThreadIDVar->getType()->isPointerType() && | |
| "thread id variable must be of type kmp_int32 *"); | |
| CodeGenFunction CGF(CGM, true); | |
| bool HasCancel = false; | |
| if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D)) | |
| HasCancel = OPD->hasCancel(); | |
| else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(&D)) | |
| HasCancel = OPD->hasCancel(); | |
| else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D)) | |
| HasCancel = OPSD->hasCancel(); | |
| else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D)) | |
| HasCancel = OPFD->hasCancel(); | |
| else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D)) | |
| HasCancel = OPFD->hasCancel(); | |
| else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D)) | |
| HasCancel = OPFD->hasCancel(); | |
| else if (const auto *OPFD = | |
| dyn_cast<OMPTeamsDistributeParallelForDirective>(&D)) | |
| HasCancel = OPFD->hasCancel(); | |
| else if (const auto *OPFD = | |
| dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D)) | |
| HasCancel = OPFD->hasCancel(); | |
| // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new | |
| // parallel region to make cancellation barriers work properly. | |
| llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); | |
| PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind); | |
| CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind, | |
| HasCancel, OutlinedHelperName); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| return CGF.GenerateOpenMPCapturedStmtFunction(*CS, D); | |
| } | |
| std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const { | |
| std::string Suffix = getName({"omp_outlined"}); | |
| return (Name + Suffix).str(); | |
| } | |
| std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const { | |
| return getOutlinedHelperName(CGF.CurFn->getName()); | |
| } | |
| std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const { | |
| std::string Suffix = getName({"omp", "reduction", "reduction_func"}); | |
| return (Name + Suffix).str(); | |
| } | |
| llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, | |
| const RegionCodeGenTy &CodeGen) { | |
| const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel); | |
| return emitParallelOrTeamsOutlinedFunction( | |
| CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF), | |
| CodeGen); | |
| } | |
| llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, | |
| const RegionCodeGenTy &CodeGen) { | |
| const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams); | |
| llvm::Function *OutlinedFn = emitParallelOrTeamsOutlinedFunction( | |
| CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(CGF), | |
| CodeGen); | |
| // A teams body is called once per team and is not handed back to the runtime | |
| // as a callback, so unlike a parallel body it cannot be re-entered while a | |
| // call to it is live. | |
| OutlinedFn->setDoesNotRecurse(); | |
| return OutlinedFn; | |
| } | |
| llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction( | |
| const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, | |
| const VarDecl *PartIDVar, const VarDecl *TaskTVar, | |
| OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, | |
| bool Tied, unsigned &NumberOfParts) { | |
| auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, | |
| PrePostActionTy &) { | |
| llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); | |
| llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); | |
| llvm::Value *TaskArgs[] = { | |
| UpLoc, ThreadID, | |
| CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), | |
| TaskTVar->getType()->castAs<PointerType>()) | |
| .getPointer(CGF)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_task), | |
| TaskArgs); | |
| }; | |
| CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar, | |
| UntiedCodeGen); | |
| CodeGen.setAction(Action); | |
| assert(!ThreadIDVar->getType()->isPointerType() && | |
| "thread id variable must be of type kmp_int32 for tasks"); | |
| const OpenMPDirectiveKind Region = | |
| isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop | |
| : OMPD_task; | |
| const CapturedStmt *CS = D.getCapturedStmt(Region); | |
| bool HasCancel = false; | |
| if (const auto *TD = dyn_cast<OMPTaskDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| else if (const auto *TD = dyn_cast<OMPMaskedTaskLoopDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| else if (const auto *TD = dyn_cast<OMPParallelMaskedTaskLoopDirective>(&D)) | |
| HasCancel = TD->hasCancel(); | |
| CodeGenFunction CGF(CGM, true); | |
| CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, | |
| InnermostKind, HasCancel, Action); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); | |
| if (!Tied) | |
| NumberOfParts = Action.getNumberOfParts(); | |
| return Res; | |
| } | |
| void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, | |
| bool AtCurrentPoint) { | |
| auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn]; | |
| assert(!Elem.ServiceInsertPt && "Insert point is set already."); | |
| llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); | |
| if (AtCurrentPoint) { | |
| Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt", | |
| CGF.Builder.GetInsertBlock()); | |
| } else { | |
| Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); | |
| Elem.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt->getIterator()); | |
| } | |
| } | |
| void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { | |
| auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn]; | |
| if (Elem.ServiceInsertPt) { | |
| llvm::Instruction *Ptr = Elem.ServiceInsertPt; | |
| Elem.ServiceInsertPt = nullptr; | |
| Ptr->eraseFromParent(); | |
| } | |
| } | |
| static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| SmallString<128> &Buffer) { | |
| llvm::raw_svector_ostream OS(Buffer); | |
| // Build debug location | |
| PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); | |
| OS << ";"; | |
| if (auto *DbgInfo = CGF.getDebugInfo()) | |
| OS << DbgInfo->remapDIPath(PLoc.getFilename()); | |
| else | |
| OS << PLoc.getFilename(); | |
| OS << ";"; | |
| if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) | |
| OS << FD->getQualifiedNameAsString(); | |
| OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;"; | |
| return OS.str(); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| unsigned Flags, bool EmitLoc) { | |
| uint32_t SrcLocStrSize; | |
| llvm::Constant *SrcLocStr; | |
| if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() == | |
| llvm::codegenoptions::NoDebugInfo) || | |
| Loc.isInvalid()) { | |
| SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize); | |
| } else { | |
| std::string FunctionName; | |
| std::string FileName; | |
| if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) | |
| FunctionName = FD->getQualifiedNameAsString(); | |
| PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); | |
| if (auto *DbgInfo = CGF.getDebugInfo()) | |
| FileName = DbgInfo->remapDIPath(PLoc.getFilename()); | |
| else | |
| FileName = PLoc.getFilename(); | |
| unsigned Line = PLoc.getLine(); | |
| unsigned Column = PLoc.getColumn(); | |
| SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line, | |
| Column, SrcLocStrSize); | |
| } | |
| unsigned Reserved2Flags = getDefaultLocationReserved2Flags(); | |
| return OMPBuilder.getOrCreateIdent( | |
| SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags); | |
| } | |
| llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| assert(CGF.CurFn && "No function in current CodeGenFunction."); | |
| // If the OpenMPIRBuilder is used we need to use it for all thread id calls as | |
| // the clang invariants used below might be broken. | |
| if (CGM.getLangOpts().OpenMPIRBuilder) { | |
| SmallString<128> Buffer; | |
| OMPBuilder.updateToLocation(CGF.Builder); | |
| uint32_t SrcLocStrSize; | |
| auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr( | |
| getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize); | |
| return OMPBuilder.getOrCreateThreadID( | |
| OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize)); | |
| } | |
| llvm::Value *ThreadID = nullptr; | |
| // Check whether we've already cached a load of the thread id in this | |
| // function. | |
| auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); | |
| if (I != OpenMPLocThreadIDMap.end()) { | |
| ThreadID = I->second.ThreadID; | |
| if (ThreadID != nullptr) | |
| return ThreadID; | |
| } | |
| // If exceptions are enabled, do not use parameter to avoid possible crash. | |
| if (auto *OMPRegionInfo = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { | |
| if (OMPRegionInfo->getThreadIDVariable()) { | |
| // Check if this an outlined function with thread id passed as argument. | |
| LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); | |
| llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); | |
| if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || | |
| !CGF.getLangOpts().CXXExceptions || | |
| CGF.Builder.GetInsertBlock() == TopBlock || | |
| !isa<llvm::Instruction>(LVal.getPointer(CGF)) || | |
| cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == | |
| TopBlock || | |
| cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == | |
| CGF.Builder.GetInsertBlock()) { | |
| ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); | |
| // If value loaded in entry block, cache it and use it everywhere in | |
| // function. | |
| if (CGF.Builder.GetInsertBlock() == TopBlock) | |
| OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID; | |
| return ThreadID; | |
| } | |
| } | |
| } | |
| // This is not an outlined function region - need to call __kmpc_int32 | |
| // kmpc_global_thread_num(ident_t *loc). | |
| // Generate thread id value and cache this value for use across the | |
| // function. | |
| auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn]; | |
| if (!Elem.ServiceInsertPt) | |
| setLocThreadIdInsertPt(CGF); | |
| CGBuilderTy::InsertPointGuard IPG(CGF.Builder); | |
| CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt); | |
| auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); | |
| llvm::CallInst *Call = CGF.Builder.CreateCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), | |
| OMPRTL___kmpc_global_thread_num), | |
| emitUpdateLocation(CGF, Loc)); | |
| Call->setCallingConv(CGF.getRuntimeCC()); | |
| Elem.ThreadID = Call; | |
| return Call; | |
| } | |
| void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { | |
| assert(CGF.CurFn && "No function in current CodeGenFunction."); | |
| if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { | |
| clearLocThreadIdInsertPt(CGF); | |
| OpenMPLocThreadIDMap.erase(CGF.CurFn); | |
| } | |
| if (auto I = FunctionUDRMap.find(CGF.CurFn); I != FunctionUDRMap.end()) { | |
| for (const auto *D : I->second) | |
| UDRMap.erase(D); | |
| FunctionUDRMap.erase(I); | |
| } | |
| if (auto I = FunctionUDMMap.find(CGF.CurFn); I != FunctionUDMMap.end()) { | |
| for (const auto *D : I->second) | |
| UDMMap.erase(D); | |
| FunctionUDMMap.erase(I); | |
| } | |
| LastprivateConditionalToTypes.erase(CGF.CurFn); | |
| FunctionToUntiedTaskStackMap.erase(CGF.CurFn); | |
| } | |
| llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() { | |
| return OMPBuilder.IdentPtr; | |
| } | |
| static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind | |
| convertDeviceClause(const VarDecl *VD) { | |
| std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = | |
| OMPDeclareTargetDeclAttr::getDeviceType(VD); | |
| if (!DevTy) | |
| return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone; | |
| switch ((int)*DevTy) { // Avoid -Wcovered-switch-default | |
| case OMPDeclareTargetDeclAttr::DT_Host: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost; | |
| break; | |
| case OMPDeclareTargetDeclAttr::DT_NoHost: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost; | |
| break; | |
| case OMPDeclareTargetDeclAttr::DT_Any: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny; | |
| break; | |
| default: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone; | |
| break; | |
| } | |
| } | |
| static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind | |
| convertCaptureClause(const VarDecl *VD) { | |
| std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType = | |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); | |
| if (!MapType) | |
| return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone; | |
| switch ((int)*MapType) { // Avoid -Wcovered-switch-default | |
| case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo; | |
| break; | |
| case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter; | |
| case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink; | |
| break; | |
| case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local: | |
| // MT_Local variables don't need offload entry (device-local). | |
| llvm_unreachable("MT_Local should not reach convertCaptureClause"); | |
| break; | |
| default: | |
| return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone; | |
| break; | |
| } | |
| } | |
| static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc( | |
| CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder, | |
| SourceLocation BeginLoc, llvm::StringRef ParentName = "") { | |
| auto FileInfoCallBack = [&]() { | |
| SourceManager &SM = CGM.getContext().getSourceManager(); | |
| PresumedLoc PLoc = SM.getPresumedLoc(BeginLoc); | |
| if (!CGM.getFileSystem()->exists(PLoc.getFilename())) | |
| PLoc = SM.getPresumedLoc(BeginLoc, /*UseLineDirectives=*/false); | |
| return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine()); | |
| }; | |
| return OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, | |
| *CGM.getFileSystem(), ParentName); | |
| } | |
| ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { | |
| auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; | |
| auto LinkageForVariable = [&VD, this]() { | |
| return CGM.getLLVMLinkageVarDefinition(VD); | |
| }; | |
| std::vector<llvm::GlobalVariable *> GeneratedRefs; | |
| llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem( | |
| CGM.getContext().getPointerType(VD->getType())); | |
| llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar( | |
| convertCaptureClause(VD), convertDeviceClause(VD), | |
| VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly, | |
| VD->isExternallyVisible(), | |
| getEntryInfoFromPresumedLoc(CGM, OMPBuilder, | |
| VD->getCanonicalDecl()->getBeginLoc()), | |
| CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd, | |
| CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, AddrOfGlobal, | |
| LinkageForVariable); | |
| if (!addr) | |
| return ConstantAddress::invalid(); | |
| return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(VD)); | |
| } | |
| llvm::Constant * | |
| CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { | |
| assert(!CGM.getLangOpts().OpenMPUseTLS || | |
| !CGM.getContext().getTargetInfo().isTLSSupported()); | |
| // Lookup the entry, lazily creating it if necessary. | |
| std::string Suffix = getName({"cache", ""}); | |
| return OMPBuilder.getOrCreateInternalVariable( | |
| CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix).str()); | |
| } | |
| Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, | |
| const VarDecl *VD, | |
| Address VDAddr, | |
| SourceLocation Loc) { | |
| if (CGM.getLangOpts().OpenMPUseTLS && | |
| CGM.getContext().getTargetInfo().isTLSSupported()) | |
| return VDAddr; | |
| llvm::Type *VarTy = VDAddr.getElementType(); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.Int8PtrTy), | |
| CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), | |
| getOrCreateThreadPrivateCache(VD)}; | |
| return Address( | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_threadprivate_cached), | |
| Args), | |
| CGF.Int8Ty, VDAddr.getAlignment()); | |
| } | |
| void CGOpenMPRuntime::emitThreadPrivateVarInit( | |
| CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, | |
| llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { | |
| // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime | |
| // library. | |
| llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_global_thread_num), | |
| OMPLoc); | |
| // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) | |
| // to register constructor/destructor for variable. | |
| llvm::Value *Args[] = { | |
| OMPLoc, | |
| CGF.Builder.CreatePointerCast(VDAddr.emitRawPointer(CGF), CGM.VoidPtrTy), | |
| Ctor, CopyCtor, Dtor}; | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_threadprivate_register), | |
| Args); | |
| } | |
| llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( | |
| const VarDecl *VD, Address VDAddr, SourceLocation Loc, | |
| bool PerformInit, CodeGenFunction *CGF) { | |
| if (CGM.getLangOpts().OpenMPUseTLS && | |
| CGM.getContext().getTargetInfo().isTLSSupported()) | |
| return nullptr; | |
| VD = VD->getDefinition(CGM.getContext()); | |
| if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) { | |
| QualType ASTTy = VD->getType(); | |
| llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr; | |
| const Expr *Init = VD->getAnyInitializer(); | |
| if (CGM.getLangOpts().CPlusPlus && PerformInit) { | |
| // Generate function that re-emits the declaration's initializer into the | |
| // threadprivate copy of the variable VD | |
| CodeGenFunction CtorCGF(CGM); | |
| auto *Dst = ImplicitParamDecl::Create( | |
| CGM.getContext(), /*DC=*/nullptr, Loc, | |
| /*Id=*/nullptr, CGM.getContext().VoidPtrTy, ImplicitParamKind::Other); | |
| FunctionArgList Args{Dst}; | |
| const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration( | |
| CGM.getContext().VoidPtrTy, Args); | |
| llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); | |
| std::string Name = getName({"__kmpc_global_ctor_", ""}); | |
| llvm::Function *Fn = | |
| CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc); | |
| CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI, | |
| Args, Loc, Loc); | |
| llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar( | |
| CtorCGF.GetAddrOfLocalVar(Dst), /*Volatile=*/false, | |
| CGM.getContext().VoidPtrTy, Dst->getLocation()); | |
| Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(ASTTy), | |
| VDAddr.getAlignment()); | |
| CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(), | |
| /*IsInitializer=*/true); | |
| ArgVal = CtorCGF.EmitLoadOfScalar( | |
| CtorCGF.GetAddrOfLocalVar(Dst), /*Volatile=*/false, | |
| CGM.getContext().VoidPtrTy, Dst->getLocation()); | |
| CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue); | |
| CtorCGF.FinishFunction(); | |
| Ctor = Fn; | |
| } | |
| if (VD->getType().isDestructedType() != QualType::DK_none) { | |
| // Generate function that emits destructor call for the threadprivate copy | |
| // of the variable VD | |
| CodeGenFunction DtorCGF(CGM); | |
| auto *Dst = ImplicitParamDecl::Create( | |
| CGM.getContext(), /*DC=*/nullptr, Loc, | |
| /*Id=*/nullptr, CGM.getContext().VoidPtrTy, ImplicitParamKind::Other); | |
| FunctionArgList Args{Dst}; | |
| const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration( | |
| CGM.getContext().VoidTy, Args); | |
| llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); | |
| std::string Name = getName({"__kmpc_global_dtor_", ""}); | |
| llvm::Function *Fn = | |
| CGM.CreateGlobalInitOrCleanUpFunction(FTy, Name, FI, Loc); | |
| auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF); | |
| DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args, | |
| Loc, Loc); | |
| // Create a scope with an artificial location for the body of this function. | |
| auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF); | |
| llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar( | |
| DtorCGF.GetAddrOfLocalVar(Dst), | |
| /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst->getLocation()); | |
| DtorCGF.emitDestroy( | |
| Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), ASTTy, | |
| DtorCGF.getDestroyer(ASTTy.isDestructedType()), | |
| DtorCGF.needsEHCleanup(ASTTy.isDestructedType())); | |
| DtorCGF.FinishFunction(); | |
| Dtor = Fn; | |
| } | |
| // Do not emit init function if it is not required. | |
| if (!Ctor && !Dtor) | |
| return nullptr; | |
| // Copying constructor for the threadprivate variable. | |
| // Must be NULL - reserved by runtime, but currently it requires that this | |
| // parameter is always NULL. Otherwise it fires assertion. | |
| CopyCtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy); | |
| if (Ctor == nullptr) { | |
| Ctor = llvm::Constant::getNullValue(CGM.DefaultPtrTy); | |
| } | |
| if (Dtor == nullptr) { | |
| Dtor = llvm::Constant::getNullValue(CGM.DefaultPtrTy); | |
| } | |
| if (!CGF) { | |
| auto *InitFunctionTy = | |
| llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false); | |
| std::string Name = getName({"__omp_threadprivate_init_", ""}); | |
| llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction( | |
| InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction()); | |
| CodeGenFunction InitCGF(CGM); | |
| FunctionArgList ArgList; | |
| InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, | |
| CGM.getTypes().arrangeNullaryFunction(), ArgList, | |
| Loc, Loc); | |
| emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); | |
| InitCGF.FinishFunction(); | |
| return InitFunction; | |
| } | |
| emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); | |
| } | |
| return nullptr; | |
| } | |
| void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD, | |
| llvm::GlobalValue *GV) { | |
| std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr = | |
| OMPDeclareTargetDeclAttr::getActiveAttr(FD); | |
| // We only need to handle active 'indirect' declare target functions. | |
| if (!ActiveAttr || !(*ActiveAttr)->getIndirect()) | |
| return; | |
| // Get a mangled name to store the new device global in. | |
| llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc( | |
| CGM, OMPBuilder, FD->getCanonicalDecl()->getBeginLoc(), FD->getName()); | |
| SmallString<128> Name; | |
| OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo); | |
| // We need to generate a new global to hold the address of the indirectly | |
| // called device function. Doing this allows us to keep the visibility and | |
| // linkage of the associated function unchanged while allowing the runtime to | |
| // access its value. | |
| llvm::GlobalValue *Addr = GV; | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| llvm::PointerType *FnPtrTy = llvm::PointerType::get( | |
| CGM.getLLVMContext(), | |
| CGM.getModule().getDataLayout().getProgramAddressSpace()); | |
| Addr = new llvm::GlobalVariable( | |
| CGM.getModule(), FnPtrTy, | |
| /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name, | |
| nullptr, llvm::GlobalValue::NotThreadLocal, | |
| CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace()); | |
| Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility); | |
| } | |
| // Register the indirect Vtable: | |
| // This is similar to OMPTargetGlobalVarEntryIndirect, except that the | |
| // size field refers to the size of memory pointed to, not the size of | |
| // the pointer symbol itself (which is implicitly the size of a pointer). | |
| OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo( | |
| Name, Addr, CGM.GetTargetTypeStoreSize(CGM.VoidPtrTy).getQuantity(), | |
| llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect, | |
| llvm::GlobalValue::WeakODRLinkage); | |
| } | |
| void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable, | |
| const VarDecl *VD) { | |
| // TODO: add logic to avoid duplicate vtable registrations per | |
| // translation unit; though for external linkage, this should no | |
| // longer be an issue - or at least we can avoid the issue by | |
| // checking for an existing offloading entry. But, perhaps the | |
| // better approach is to defer emission of the vtables and offload | |
| // entries until later (by tracking a list of items that need to be | |
| // emitted). | |
| llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); | |
| // Generate a new externally visible global to point to the | |
| // internally visible vtable. Doing this allows us to keep the | |
| // visibility and linkage of the associated vtable unchanged while | |
| // allowing the runtime to access its value. The externally | |
| // visible global var needs to be emitted with a unique mangled | |
| // name that won't conflict with similarly named (internal) | |
| // vtables in other translation units. | |
| // Register vtable with source location of dynamic object in map | |
| // clause. | |
| llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc( | |
| CGM, OMPBuilder, VD->getCanonicalDecl()->getBeginLoc(), | |
| VTable->getName()); | |
| llvm::GlobalVariable *Addr = VTable; | |
| SmallString<128> AddrName; | |
| OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(AddrName, EntryInfo); | |
| AddrName.append("addr"); | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| Addr = new llvm::GlobalVariable( | |
| CGM.getModule(), VTable->getType(), | |
| /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable, | |
| AddrName, | |
| /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal, | |
| CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace()); | |
| Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility); | |
| } | |
| OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo( | |
| AddrName, VTable, | |
| CGM.getDataLayout().getTypeAllocSize(VTable->getInitializer()->getType()), | |
| llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable, | |
| llvm::GlobalValue::WeakODRLinkage); | |
| } | |
| void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM, | |
| CXXRecordDecl *CXXRecord, | |
| const VarDecl *VD) { | |
| // Register C++ VTable to OpenMP Offload Entry if it's a new | |
| // CXXRecordDecl. | |
| if (CXXRecord && CXXRecord->hasDefinition() && CXXRecord->isDynamicClass() && | |
| !CGM.getOpenMPRuntime().VTableDeclMap.contains(CXXRecord)) { | |
| auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(CXXRecord, VD); | |
| if (Res.second) { | |
| CGM.EmitVTable(CXXRecord); | |
| CodeGenVTables VTables = CGM.getVTables(); | |
| llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(CXXRecord); | |
| assert(VTablesAddr && "Expected non-null VTable address"); | |
| // Must set VTables to weak since we're emitting them in multiple TUs now | |
| if (VTablesAddr->hasExternalLinkage()) | |
| VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage); | |
| CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTablesAddr, VD); | |
| // Emit VTable for all the fields containing dynamic CXXRecord | |
| for (const FieldDecl *Field : CXXRecord->fields()) { | |
| if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl()) | |
| emitAndRegisterVTable(CGM, RecordDecl, VD); | |
| } | |
| // Emit VTable for all dynamic parent class | |
| for (CXXBaseSpecifier &Base : CXXRecord->bases()) { | |
| if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl()) | |
| emitAndRegisterVTable(CGM, BaseDecl, VD); | |
| } | |
| } | |
| } | |
| } | |
| void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) { | |
| // Register VTable by scanning through the map clause of OpenMP target region. | |
| // Get CXXRecordDecl and VarDecl from Expr. | |
| auto GetVTableDecl = [](const Expr *E) { | |
| QualType VDTy = E->getType(); | |
| CXXRecordDecl *CXXRecord = nullptr; | |
| if (const auto *RefType = VDTy->getAs<LValueReferenceType>()) | |
| VDTy = RefType->getPointeeType(); | |
| if (VDTy->isPointerType()) | |
| CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl(); | |
| else | |
| CXXRecord = VDTy->getAsCXXRecordDecl(); | |
| const VarDecl *VD = nullptr; | |
| if (auto *DRE = dyn_cast<DeclRefExpr>(E)) { | |
| // Handle BindingDecls by redirecting to their DecompositionDecl. | |
| if (auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) | |
| VD = cast<VarDecl>(BD->getDecomposedDecl()); | |
| else | |
| VD = cast<VarDecl>(DRE->getDecl()); | |
| } else if (auto *MRE = dyn_cast<MemberExpr>(E)) { | |
| if (auto *BaseDRE = dyn_cast<DeclRefExpr>(MRE->getBase())) { | |
| if (auto *BaseVD = dyn_cast<VarDecl>(BaseDRE->getDecl())) | |
| VD = BaseVD; | |
| } | |
| } | |
| return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD); | |
| }; | |
| // Collect VTable from OpenMP map clause. | |
| for (const auto *C : D.getClausesOfKind<OMPMapClause>()) { | |
| for (const auto *E : C->varlist()) { | |
| auto DeclPair = GetVTableDecl(E); | |
| // Ensure VD is not null | |
| if (DeclPair.second) | |
| emitAndRegisterVTable(CGM, DeclPair.first, DeclPair.second); | |
| } | |
| } | |
| } | |
| Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, | |
| QualType VarType, | |
| StringRef Name) { | |
| std::string Suffix = getName({"artificial", ""}); | |
| llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); | |
| llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable( | |
| VarLVType, Twine(Name).concat(Suffix).str()); | |
| if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS && | |
| CGM.getTarget().isTLSSupported()) { | |
| GAddr->setThreadLocal(/*Val=*/true); | |
| return Address(GAddr, GAddr->getValueType(), | |
| CGM.getContext().getTypeAlignInChars(VarType)); | |
| } | |
| std::string CacheSuffix = getName({"cache", ""}); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, SourceLocation()), | |
| getThreadID(CGF, SourceLocation()), | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), | |
| CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, | |
| /*isSigned=*/false), | |
| OMPBuilder.getOrCreateInternalVariable( | |
| CGM.VoidPtrPtrTy, | |
| Twine(Name).concat(Suffix).concat(CacheSuffix).str())}; | |
| return Address( | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_threadprivate_cached), | |
| Args), | |
| CGF.Builder.getPtrTy(0)), | |
| VarLVType, CGM.getContext().getTypeAlignInChars(VarType)); | |
| } | |
| void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, | |
| const RegionCodeGenTy &ThenGen, | |
| const RegionCodeGenTy &ElseGen) { | |
| CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); | |
| // If the condition constant folds and can be elided, try to avoid emitting | |
| // the condition and the dead arm of the if/else. | |
| bool CondConstant; | |
| if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { | |
| if (CondConstant) | |
| ThenGen(CGF); | |
| else | |
| ElseGen(CGF); | |
| return; | |
| } | |
| // Otherwise, the condition did not fold, or we couldn't elide it. Just | |
| // emit the conditional branch. | |
| llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); | |
| llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); | |
| llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); | |
| CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); | |
| // Emit the 'then' code. | |
| CGF.EmitBlock(ThenBlock); | |
| ThenGen(CGF); | |
| CGF.EmitBranch(ContBlock); | |
| // Emit the 'else' code if present. | |
| // There is no need to emit line number for unconditional branch. | |
| (void)ApplyDebugLocation::CreateEmpty(CGF); | |
| CGF.EmitBlock(ElseBlock); | |
| ElseGen(CGF); | |
| // There is no need to emit line number for unconditional branch. | |
| (void)ApplyDebugLocation::CreateEmpty(CGF); | |
| CGF.EmitBranch(ContBlock); | |
| // Emit the continuation block for code after the if. | |
| CGF.EmitBlock(ContBlock, /*IsFinished=*/true); | |
| } | |
| void CGOpenMPRuntime::emitParallelCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, | |
| ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond, | |
| llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier, | |
| OpenMPSeverityClauseKind Severity, const Expr *Message) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // The caller waits for the parallel region to finish, so its captured stack | |
| // variables remain alive until the region completes. Global variables also | |
| // remain alive. Record this lifetime guarantee on the outlined function's | |
| // parameters, even if other code has access to the same storage. | |
| // The storage may still be modified or accessed through other pointers. | |
| // For a captured pointer variable, this guarantees the lifetime of the | |
| // variable itself, not the object it points to. | |
| if (!CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| assert(OutlinedFn->arg_size() == CapturedVars.size() + 2 && | |
| "Expected tid and bound tid before the captured arguments"); | |
| for (auto [I, Captured] : llvm::enumerate(CapturedVars)) { | |
| if (Captured->getType()->isPointerTy() && | |
| llvm::isa<llvm::AllocaInst, llvm::GlobalVariable>( | |
| llvm::getUnderlyingObject(Captured))) | |
| OutlinedFn->addParamAttr(I + 2, llvm::Attribute::NoFreeObj); | |
| } | |
| } | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); | |
| auto &M = CGM.getModule(); | |
| auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc, | |
| this](CodeGenFunction &CGF, PrePostActionTy &) { | |
| // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn); | |
| llvm::Value *Args[] = { | |
| RTLoc, | |
| CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars | |
| OutlinedFn}; | |
| llvm::SmallVector<llvm::Value *, 16> RealArgs; | |
| RealArgs.append(std::begin(Args), std::end(Args)); | |
| RealArgs.append(CapturedVars.begin(), CapturedVars.end()); | |
| llvm::FunctionCallee RTLFn = | |
| OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call); | |
| CGF.EmitRuntimeCall(RTLFn, RealArgs); | |
| }; | |
| auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc, | |
| this](CodeGenFunction &CGF, PrePostActionTy &) { | |
| CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); | |
| llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); | |
| // Build calls: | |
| // __kmpc_serialized_parallel(&Loc, GTid); | |
| llvm::Value *Args[] = {RTLoc, ThreadID}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_serialized_parallel), | |
| Args); | |
| // OutlinedFn(>id, &zero_bound, CapturedStruct); | |
| Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); | |
| RawAddress ZeroAddrBound = | |
| CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, | |
| /*Name=*/".bound.zero.addr"); | |
| CGF.Builder.CreateStore(CGF.Builder.getInt32(/*C*/ 0), ZeroAddrBound); | |
| llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; | |
| // ThreadId for serialized parallels is 0. | |
| OutlinedFnArgs.push_back(ThreadIDAddr.emitRawPointer(CGF)); | |
| OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); | |
| OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); | |
| // Ensure we do not inline the function. This is trivially true for the ones | |
| // passed to __kmpc_fork_call but the ones called in serialized regions | |
| // could be inlined. This is not a perfect but it is closer to the invariant | |
| // we want, namely, every data environment starts with a new function. | |
| // TODO: We should pass the if condition to the runtime function and do the | |
| // handling there. Much cleaner code. | |
| OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline); | |
| OutlinedFn->addFnAttr(llvm::Attribute::NoInline); | |
| RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); | |
| // __kmpc_end_serialized_parallel(&Loc, GTid); | |
| llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_end_serialized_parallel), | |
| EndArgs); | |
| }; | |
| if (IfCond) { | |
| emitIfClause(CGF, IfCond, ThenGen, ElseGen); | |
| } else { | |
| RegionCodeGenTy ThenRCG(ThenGen); | |
| ThenRCG(CGF); | |
| } | |
| } | |
| // If we're inside an (outlined) parallel region, use the region info's | |
| // thread-ID variable (it is passed in a first argument of the outlined function | |
| // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in | |
| // regular serial code region, get thread ID by calling kmp_int32 | |
| // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and | |
| // return the address of that temp. | |
| Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| if (auto *OMPRegionInfo = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) | |
| if (OMPRegionInfo->getThreadIDVariable()) | |
| return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(); | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| QualType Int32Ty = | |
| CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); | |
| Address ThreadIDTemp = | |
| CGF.CreateMemTempWithoutCast(Int32Ty, /*Name*/ ".threadid_temp."); | |
| CGF.EmitStoreOfScalar(ThreadID, | |
| CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); | |
| return ThreadIDTemp; | |
| } | |
| llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { | |
| std::string Prefix = Twine("gomp_critical_user_", CriticalName).str(); | |
| std::string Name = getName({Prefix, "var"}); | |
| llvm::GlobalVariable *GV = | |
| OMPBuilder.getOrCreateInternalVariable(KmpCriticalNameTy, Name); | |
| CGM.setDSOLocal(GV); | |
| return GV; | |
| } | |
| namespace { | |
| /// Common pre(post)-action for different OpenMP constructs. | |
| class CommonActionTy final : public PrePostActionTy { | |
| llvm::FunctionCallee EnterCallee; | |
| ArrayRef<llvm::Value *> EnterArgs; | |
| llvm::FunctionCallee ExitCallee; | |
| ArrayRef<llvm::Value *> ExitArgs; | |
| bool Conditional; | |
| llvm::BasicBlock *ContBlock = nullptr; | |
| public: | |
| CommonActionTy(llvm::FunctionCallee EnterCallee, | |
| ArrayRef<llvm::Value *> EnterArgs, | |
| llvm::FunctionCallee ExitCallee, | |
| ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false) | |
| : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee), | |
| ExitArgs(ExitArgs), Conditional(Conditional) {} | |
| void Enter(CodeGenFunction &CGF) override { | |
| llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); | |
| if (Conditional) { | |
| llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); | |
| auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); | |
| ContBlock = CGF.createBasicBlock("omp_if.end"); | |
| // Generate the branch (If-stmt) | |
| CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); | |
| CGF.EmitBlock(ThenBlock); | |
| } | |
| } | |
| void Done(CodeGenFunction &CGF) { | |
| // Emit the rest of blocks/branches | |
| CGF.EmitBranch(ContBlock); | |
| CGF.EmitBlock(ContBlock, true); | |
| } | |
| void Exit(CodeGenFunction &CGF) override { | |
| CGF.EmitRuntimeCall(ExitCallee, ExitArgs); | |
| } | |
| }; | |
| } // anonymous namespace | |
| void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, | |
| StringRef CriticalName, | |
| const RegionCodeGenTy &CriticalOpGen, | |
| SourceLocation Loc, const Expr *Hint) { | |
| // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]); | |
| // CriticalOpGen(); | |
| // __kmpc_end_critical(ident_t *, gtid, Lock); | |
| // Prepare arguments and build a call to __kmpc_critical | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), | |
| Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical); | |
| llvm::Value *LockVar = getCriticalRegionLock(CriticalName); | |
| unsigned LockVarArgIdx = 2; | |
| if (cast<llvm::GlobalVariable>(LockVar)->getAddressSpace() != | |
| RuntimeFcn.getFunctionType() | |
| ->getParamType(LockVarArgIdx) | |
| ->getPointerAddressSpace()) | |
| LockVar = CGF.Builder.CreateAddrSpaceCast( | |
| LockVar, RuntimeFcn.getFunctionType()->getParamType(LockVarArgIdx)); | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| LockVar}; | |
| llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), | |
| std::end(Args)); | |
| if (Hint) { | |
| EnterArgs.push_back(CGF.Builder.CreateIntCast( | |
| CGF.EmitScalarExpr(Hint), CGM.Int32Ty, /*isSigned=*/false)); | |
| } | |
| CommonActionTy Action(RuntimeFcn, EnterArgs, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_critical), | |
| Args); | |
| CriticalOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); | |
| } | |
| void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &MasterOpGen, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // if(__kmpc_master(ident_t *, gtid)) { | |
| // MasterOpGen(); | |
| // __kmpc_end_master(ident_t *, gtid); | |
| // } | |
| // Prepare arguments and build a call to __kmpc_master | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_master), | |
| Args, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_master), | |
| Args, | |
| /*Conditional=*/true); | |
| MasterOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_master, MasterOpGen); | |
| Action.Done(CGF); | |
| } | |
| void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &MaskedOpGen, | |
| SourceLocation Loc, const Expr *Filter) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // if(__kmpc_masked(ident_t *, gtid, filter)) { | |
| // MaskedOpGen(); | |
| // __kmpc_end_masked(iden_t *, gtid); | |
| // } | |
| // Prepare arguments and build a call to __kmpc_masked | |
| llvm::Value *FilterVal = Filter | |
| ? CGF.EmitScalarExpr(Filter, CGF.Int32Ty) | |
| : llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/0); | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| FilterVal}; | |
| llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc), | |
| getThreadID(CGF, Loc)}; | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_masked), | |
| Args, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_masked), | |
| ArgsEnd, | |
| /*Conditional=*/true); | |
| MaskedOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_masked, MaskedOpGen); | |
| Action.Done(CGF); | |
| } | |
| void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { | |
| OMPBuilder.createTaskyield(CGF.Builder); | |
| } else { | |
| // Build call __kmpc_omp_taskyield(loc, thread_id, 0); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_taskyield), | |
| Args); | |
| } | |
| if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) | |
| Region->emitUntiedSwitch(CGF); | |
| } | |
| void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &TaskgroupOpGen, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // __kmpc_taskgroup(ident_t *, gtid); | |
| // TaskgroupOpGen(); | |
| // __kmpc_end_taskgroup(ident_t *, gtid); | |
| // Prepare arguments and build a call to __kmpc_taskgroup | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_taskgroup), | |
| Args, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_taskgroup), | |
| Args); | |
| TaskgroupOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); | |
| } | |
| /// Given an array of pointers to variables, project the address of a | |
| /// given variable. | |
| static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, | |
| unsigned Index, const VarDecl *Var) { | |
| // Pull out the pointer to the variable. | |
| Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); | |
| llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); | |
| llvm::Type *ElemTy = CGF.ConvertTypeForMem(Var->getType()); | |
| return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(Var)); | |
| } | |
| static llvm::Value *emitCopyprivateCopyFunction( | |
| CodeGenModule &CGM, llvm::Type *ArgsElemType, | |
| ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, | |
| ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps, | |
| SourceLocation Loc) { | |
| ASTContext &C = CGM.getContext(); | |
| // void copy_func(void *LHSArg, void *RHSArg); | |
| auto *LHSArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| auto *RHSArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| FunctionArgList Args{LHSArg, RHSArg}; | |
| const auto &CGFI = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| std::string Name = | |
| CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"}); | |
| auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI), | |
| llvm::GlobalValue::InternalLinkage, Name, | |
| &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| Fn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); | |
| // Dest = (void*[n])(LHSArg); | |
| // Src = (void*[n])(RHSArg); | |
| Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(LHSArg)), | |
| CGF.Builder.getPtrTy(0)), | |
| ArgsElemType, CGF.getPointerAlign()); | |
| Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(RHSArg)), | |
| CGF.Builder.getPtrTy(0)), | |
| ArgsElemType, CGF.getPointerAlign()); | |
| // *(Type0*)Dst[0] = *(Type0*)Src[0]; | |
| // *(Type1*)Dst[1] = *(Type1*)Src[1]; | |
| // ... | |
| // *(Typen*)Dst[n] = *(Typen*)Src[n]; | |
| for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) { | |
| const auto *DestVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()); | |
| Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); | |
| const auto *SrcVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()); | |
| Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); | |
| const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl(); | |
| QualType Type = VD->getType(); | |
| CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); | |
| } | |
| CGF.FinishFunction(); | |
| return Fn; | |
| } | |
| void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &SingleOpGen, | |
| SourceLocation Loc, | |
| ArrayRef<const Expr *> CopyprivateVars, | |
| ArrayRef<const Expr *> SrcExprs, | |
| ArrayRef<const Expr *> DstExprs, | |
| ArrayRef<const Expr *> AssignmentOps) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| assert(CopyprivateVars.size() == SrcExprs.size() && | |
| CopyprivateVars.size() == DstExprs.size() && | |
| CopyprivateVars.size() == AssignmentOps.size()); | |
| ASTContext &C = CGM.getContext(); | |
| // int32 did_it = 0; | |
| // if(__kmpc_single(ident_t *, gtid)) { | |
| // SingleOpGen(); | |
| // __kmpc_end_single(ident_t *, gtid); | |
| // did_it = 1; | |
| // } | |
| // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, | |
| // <copy_func>, did_it); | |
| Address DidIt = Address::invalid(); | |
| if (!CopyprivateVars.empty()) { | |
| // int32 did_it = 0; | |
| QualType KmpInt32Ty = | |
| C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); | |
| DidIt = CGF.CreateMemTempWithoutCast(KmpInt32Ty, ".omp.copyprivate.did_it"); | |
| CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); | |
| } | |
| // Prepare arguments and build a call to __kmpc_single | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_single), | |
| Args, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_single), | |
| Args, | |
| /*Conditional=*/true); | |
| SingleOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_single, SingleOpGen); | |
| if (DidIt.isValid()) { | |
| // did_it = 1; | |
| CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); | |
| } | |
| Action.Done(CGF); | |
| // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, | |
| // <copy_func>, did_it); | |
| if (DidIt.isValid()) { | |
| llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size()); | |
| QualType CopyprivateArrayTy = C.getConstantArrayType( | |
| C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, | |
| /*IndexTypeQuals=*/0); | |
| // Create a list of all private variables for copyprivate. | |
| Address CopyprivateList = CGF.CreateMemTempWithoutCast( | |
| CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); | |
| for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) { | |
| Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); | |
| CGF.Builder.CreateStore( | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), | |
| CGF.VoidPtrTy), | |
| Elem); | |
| } | |
| // Build function that copies private values from single region to all other | |
| // threads in the corresponding parallel region. | |
| llvm::Value *CpyFn = emitCopyprivateCopyFunction( | |
| CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy), CopyprivateVars, | |
| SrcExprs, DstExprs, AssignmentOps, Loc); | |
| llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); | |
| Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CopyprivateList, CGF.VoidPtrTy, CGF.Int8Ty); | |
| llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), // ident_t *<loc> | |
| getThreadID(CGF, Loc), // i32 <gtid> | |
| BufSize, // size_t <buf_size> | |
| CL.emitRawPointer(CGF), // void *<copyprivate list> | |
| CpyFn, // void (*) (void *, void *) <copy_func> | |
| DidItVal // i32 did_it | |
| }; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_copyprivate), | |
| Args); | |
| } | |
| } | |
| void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &OrderedOpGen, | |
| SourceLocation Loc, bool IsThreads) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // __kmpc_ordered(ident_t *, gtid); | |
| // OrderedOpGen(); | |
| // __kmpc_end_ordered(ident_t *, gtid); | |
| // Prepare arguments and build a call to __kmpc_ordered | |
| if (IsThreads) { | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_ordered), | |
| Args, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_ordered), | |
| Args); | |
| OrderedOpGen.setAction(Action); | |
| emitInlinedDirective(CGF, OMPD_ordered_blockassoc, OrderedOpGen); | |
| return; | |
| } | |
| emitInlinedDirective(CGF, OMPD_ordered_blockassoc, OrderedOpGen); | |
| } | |
| unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) { | |
| unsigned Flags; | |
| if (Kind == OMPD_for) | |
| Flags = OMP_IDENT_BARRIER_IMPL_FOR; | |
| else if (Kind == OMPD_sections) | |
| Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS; | |
| else if (Kind == OMPD_single) | |
| Flags = OMP_IDENT_BARRIER_IMPL_SINGLE; | |
| else if (Kind == OMPD_barrier) | |
| Flags = OMP_IDENT_BARRIER_EXPL; | |
| else | |
| Flags = OMP_IDENT_BARRIER_IMPL; | |
| return Flags; | |
| } | |
| void CGOpenMPRuntime::getDefaultScheduleAndChunk( | |
| CodeGenFunction &CGF, const OMPLoopDirective &S, | |
| OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const { | |
| // Check if the loop directive is actually a doacross loop directive. In this | |
| // case choose static, 1 schedule. | |
| if (llvm::any_of( | |
| S.getClausesOfKind<OMPOrderedClause>(), | |
| [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) { | |
| ScheduleKind = OMPC_SCHEDULE_static; | |
| // Chunk size is 1 in this case. | |
| llvm::APInt ChunkSize(32, 1); | |
| ChunkExpr = IntegerLiteral::Create( | |
| CGF.getContext(), ChunkSize, | |
| CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), | |
| SourceLocation()); | |
| } | |
| } | |
| void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| OpenMPDirectiveKind Kind, bool EmitChecks, | |
| bool ForceSimpleCall) { | |
| // Check if we should use the OMPBuilder | |
| auto *OMPRegionInfo = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo); | |
| if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { | |
| llvm::OpenMPIRBuilder::InsertPointTy AfterIP = | |
| cantFail(OMPBuilder.createBarrier(CGF.Builder, Kind, ForceSimpleCall, | |
| EmitChecks)); | |
| CGF.Builder.restoreIP(AfterIP); | |
| return; | |
| } | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Build call __kmpc_cancel_barrier(loc, thread_id); | |
| // Build call __kmpc_barrier(loc, thread_id); | |
| unsigned Flags = getDefaultFlagsForBarriers(Kind); | |
| // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc, | |
| // thread_id); | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), | |
| getThreadID(CGF, Loc)}; | |
| if (OMPRegionInfo) { | |
| if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) { | |
| llvm::Value *Result = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), | |
| OMPRTL___kmpc_cancel_barrier), | |
| Args); | |
| if (EmitChecks) { | |
| // if (__kmpc_cancel_barrier()) { | |
| // exit from construct; | |
| // } | |
| llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); | |
| llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); | |
| llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); | |
| CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); | |
| CGF.EmitBlock(ExitBB); | |
| // exit from construct; | |
| CodeGenFunction::JumpDest CancelDestination = | |
| CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); | |
| CGF.EmitBranchThroughCleanup(CancelDestination); | |
| CGF.EmitBlock(ContBB, /*IsFinished=*/true); | |
| } | |
| return; | |
| } | |
| } | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_barrier), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| Expr *ME, bool IsFatal) { | |
| llvm::Value *MVL = ME ? CGF.EmitScalarExpr(ME) | |
| : llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| // Build call void __kmpc_error(ident_t *loc, int severity, const char | |
| // *message) | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/true), | |
| llvm::ConstantInt::get(CGM.Int32Ty, IsFatal ? 2 : 1), | |
| CGF.Builder.CreatePointerCast(MVL, CGM.Int8PtrTy)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_error), | |
| Args); | |
| } | |
| /// Map the OpenMP loop schedule to the runtime enumeration. | |
| static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, | |
| bool Chunked, bool Ordered) { | |
| switch (ScheduleKind) { | |
| case OMPC_SCHEDULE_static: | |
| return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked) | |
| : (Ordered ? OMP_ord_static : OMP_sch_static); | |
| case OMPC_SCHEDULE_dynamic: | |
| return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked; | |
| case OMPC_SCHEDULE_guided: | |
| return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked; | |
| case OMPC_SCHEDULE_runtime: | |
| return Ordered ? OMP_ord_runtime : OMP_sch_runtime; | |
| case OMPC_SCHEDULE_auto: | |
| return Ordered ? OMP_ord_auto : OMP_sch_auto; | |
| case OMPC_SCHEDULE_unknown: | |
| assert(!Chunked && "chunk was specified but schedule kind not known"); | |
| return Ordered ? OMP_ord_static : OMP_sch_static; | |
| } | |
| llvm_unreachable("Unexpected runtime schedule"); | |
| } | |
| /// Map the OpenMP distribute schedule to the runtime enumeration. | |
| static OpenMPSchedType | |
| getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) { | |
| // only static is allowed for dist_schedule | |
| return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static; | |
| } | |
| bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, | |
| bool Chunked) const { | |
| OpenMPSchedType Schedule = | |
| getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); | |
| return Schedule == OMP_sch_static; | |
| } | |
| bool CGOpenMPRuntime::isStaticNonchunked( | |
| OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { | |
| OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked); | |
| return Schedule == OMP_dist_sch_static; | |
| } | |
| bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, | |
| bool Chunked) const { | |
| OpenMPSchedType Schedule = | |
| getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); | |
| return Schedule == OMP_sch_static_chunked; | |
| } | |
| bool CGOpenMPRuntime::isStaticChunked( | |
| OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { | |
| OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked); | |
| return Schedule == OMP_dist_sch_static_chunked; | |
| } | |
| bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { | |
| OpenMPSchedType Schedule = | |
| getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false); | |
| assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here"); | |
| return Schedule != OMP_sch_static; | |
| } | |
| static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, | |
| OpenMPScheduleClauseModifier M1, | |
| OpenMPScheduleClauseModifier M2) { | |
| int Modifier = 0; | |
| switch (M1) { | |
| case OMPC_SCHEDULE_MODIFIER_monotonic: | |
| Modifier = OMP_sch_modifier_monotonic; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_nonmonotonic: | |
| Modifier = OMP_sch_modifier_nonmonotonic; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_simd: | |
| if (Schedule == OMP_sch_static_chunked) | |
| Schedule = OMP_sch_static_balanced_chunked; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_last: | |
| case OMPC_SCHEDULE_MODIFIER_unknown: | |
| break; | |
| } | |
| switch (M2) { | |
| case OMPC_SCHEDULE_MODIFIER_monotonic: | |
| Modifier = OMP_sch_modifier_monotonic; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_nonmonotonic: | |
| Modifier = OMP_sch_modifier_nonmonotonic; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_simd: | |
| if (Schedule == OMP_sch_static_chunked) | |
| Schedule = OMP_sch_static_balanced_chunked; | |
| break; | |
| case OMPC_SCHEDULE_MODIFIER_last: | |
| case OMPC_SCHEDULE_MODIFIER_unknown: | |
| break; | |
| } | |
| // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription. | |
| // If the static schedule kind is specified or if the ordered clause is | |
| // specified, and if the nonmonotonic modifier is not specified, the effect is | |
| // as if the monotonic modifier is specified. Otherwise, unless the monotonic | |
| // modifier is specified, the effect is as if the nonmonotonic modifier is | |
| // specified. | |
| if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) { | |
| if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static || | |
| Schedule == OMP_sch_static_balanced_chunked || | |
| Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static || | |
| Schedule == OMP_dist_sch_static_chunked || | |
| Schedule == OMP_dist_sch_static || | |
| Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone)) | |
| Modifier = OMP_sch_modifier_nonmonotonic; | |
| } | |
| return Schedule | Modifier; | |
| } | |
| void CGOpenMPRuntime::emitForDispatchInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, | |
| bool Ordered, const DispatchRTInput &DispatchValues) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| OpenMPSchedType Schedule = getRuntimeSchedule( | |
| ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered); | |
| assert(Ordered || | |
| (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && | |
| Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && | |
| Schedule != OMP_sch_static_balanced_chunked)); | |
| // Call __kmpc_dispatch_init( | |
| // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, | |
| // kmp_int[32|64] lower, kmp_int[32|64] upper, | |
| // kmp_int[32|64] stride, kmp_int[32|64] chunk); | |
| // If the Chunk was not specified in the clause - use default value 1. | |
| llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk | |
| : CGF.Builder.getIntN(IVSize, 1); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), | |
| getThreadID(CGF, Loc), | |
| CGF.Builder.getInt32(addMonoNonMonoModifier( | |
| CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type | |
| DispatchValues.LB, // Lower | |
| DispatchValues.UB, // Upper | |
| CGF.Builder.getIntN(IVSize, 1), // Stride | |
| Chunk // Chunk | |
| }; | |
| CGF.EmitRuntimeCall(OMPBuilder.createDispatchInitFunction(IVSize, IVSigned), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid); | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.createDispatchDeinitFunction(), Args); | |
| } | |
| static void emitForStaticInitCall( | |
| CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, | |
| llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, | |
| OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, | |
| const CGOpenMPRuntime::StaticRTInput &Values) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| assert(!Values.Ordered); | |
| assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || | |
| Schedule == OMP_sch_static_balanced_chunked || | |
| Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || | |
| Schedule == OMP_dist_sch_static || | |
| Schedule == OMP_dist_sch_static_chunked || | |
| Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone); | |
| // Call __kmpc_for_static_init( | |
| // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, | |
| // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, | |
| // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, | |
| // kmp_int[32|64] incr, kmp_int[32|64] chunk); | |
| llvm::Value *Chunk = Values.Chunk; | |
| if (Chunk == nullptr) { | |
| assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static || | |
| Schedule == OMP_dist_sch_static) && | |
| "expected static non-chunked schedule"); | |
| // If the Chunk was not specified in the clause - use default value 1. | |
| Chunk = CGF.Builder.getIntN(Values.IVSize, 1); | |
| } else { | |
| assert((Schedule == OMP_sch_static_chunked || | |
| Schedule == OMP_sch_static_balanced_chunked || | |
| Schedule == OMP_ord_static_chunked || | |
| Schedule == OMP_dist_sch_static_chunked || | |
| Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) && | |
| "expected static chunked schedule"); | |
| } | |
| llvm::Value *Args[] = { | |
| UpdateLocation, | |
| ThreadId, | |
| CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, | |
| M2)), // Schedule type | |
| Values.IL.emitRawPointer(CGF), // &isLastIter | |
| Values.LB.emitRawPointer(CGF), // &LB | |
| Values.UB.emitRawPointer(CGF), // &UB | |
| Values.ST.emitRawPointer(CGF), // &Stride | |
| CGF.Builder.getIntN(Values.IVSize, 1), // Incr | |
| Chunk // Chunk | |
| }; | |
| CGF.EmitRuntimeCall(ForStaticInitFunction, Args); | |
| } | |
| void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| OpenMPDirectiveKind DKind, | |
| const OpenMPScheduleTy &ScheduleKind, | |
| const StaticRTInput &Values) { | |
| OpenMPSchedType ScheduleNum = | |
| ScheduleKind.UseFusedDistChunkSchedule | |
| ? OMP_dist_sch_static_chunked_sch_static_chunkone | |
| : getRuntimeSchedule(ScheduleKind.Schedule, Values.Chunk != nullptr, | |
| Values.Ordered); | |
| assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) && | |
| "Expected loop-based or sections-based directive."); | |
| llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, | |
| isOpenMPLoopDirective(DKind) | |
| ? OMP_IDENT_WORK_LOOP | |
| : OMP_IDENT_WORK_SECTIONS); | |
| llvm::Value *ThreadId = getThreadID(CGF, Loc); | |
| llvm::FunctionCallee StaticInitFunction = | |
| OMPBuilder.createForStaticInitFunction(Values.IVSize, Values.IVSigned, | |
| false); | |
| auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); | |
| emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, | |
| ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values); | |
| } | |
| void CGOpenMPRuntime::emitDistributeStaticInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| OpenMPDistScheduleClauseKind SchedKind, | |
| const CGOpenMPRuntime::StaticRTInput &Values) { | |
| OpenMPSchedType ScheduleNum = | |
| getRuntimeSchedule(SchedKind, Values.Chunk != nullptr); | |
| llvm::Value *UpdatedLocation = | |
| emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); | |
| llvm::Value *ThreadId = getThreadID(CGF, Loc); | |
| llvm::FunctionCallee StaticInitFunction; | |
| bool isGPUDistribute = | |
| CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU(); | |
| StaticInitFunction = OMPBuilder.createForStaticInitFunction( | |
| Values.IVSize, Values.IVSigned, isGPUDistribute); | |
| emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, | |
| ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown, | |
| OMPC_SCHEDULE_MODIFIER_unknown, Values); | |
| } | |
| void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| OpenMPDirectiveKind DKind) { | |
| assert((DKind == OMPD_distribute || DKind == OMPD_for || | |
| DKind == OMPD_sections) && | |
| "Expected distribute, for, or sections directive kind"); | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc, | |
| isOpenMPDistributeDirective(DKind) || | |
| (DKind == OMPD_target_teams_loop) | |
| ? OMP_IDENT_WORK_DISTRIBUTE | |
| : isOpenMPLoopDirective(DKind) | |
| ? OMP_IDENT_WORK_LOOP | |
| : OMP_IDENT_WORK_SECTIONS), | |
| getThreadID(CGF, Loc)}; | |
| auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); | |
| if (isOpenMPDistributeDirective(DKind) && | |
| CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU()) | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_distribute_static_fini), | |
| Args); | |
| else | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_for_static_fini), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| unsigned IVSize, | |
| bool IVSigned) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid); | |
| llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned), | |
| Args); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, | |
| SourceLocation Loc, unsigned IVSize, | |
| bool IVSigned, Address IL, | |
| Address LB, Address UB, | |
| Address ST) { | |
| // Call __kmpc_dispatch_next( | |
| // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, | |
| // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, | |
| // kmp_int[32|64] *p_stride); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| IL.emitRawPointer(CGF), // &isLastIter | |
| LB.emitRawPointer(CGF), // &Lower | |
| UB.emitRawPointer(CGF), // &Upper | |
| ST.emitRawPointer(CGF) // &Stride | |
| }; | |
| llvm::Value *Call = CGF.EmitRuntimeCall( | |
| OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), Args); | |
| return CGF.EmitScalarConversion( | |
| Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), | |
| CGF.getContext().BoolTy, Loc); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF, | |
| const Expr *Message, | |
| SourceLocation Loc) { | |
| if (!Message) | |
| return llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| return CGF.EmitScalarExpr(Message); | |
| } | |
| llvm::Value * | |
| CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity, | |
| SourceLocation Loc) { | |
| // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is | |
| // as if sev-level is fatal." | |
| return llvm::ConstantInt::get(CGM.Int32Ty, | |
| Severity == OMPC_SEVERITY_warning ? 1 : 2); | |
| } | |
| void CGOpenMPRuntime::emitNumThreadsClause( | |
| CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, | |
| OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity, | |
| SourceLocation SeverityLoc, const Expr *Message, | |
| SourceLocation MessageLoc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| llvm::SmallVector<llvm::Value *, 4> Args( | |
| {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}); | |
| // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) | |
| // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity, | |
| // messsage) if strict modifier is used. | |
| RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads; | |
| if (Modifier == OMPC_NUMTHREADS_strict) { | |
| FnID = OMPRTL___kmpc_push_num_threads_strict; | |
| Args.push_back(emitSeverityClause(Severity, SeverityLoc)); | |
| Args.push_back(emitMessageClause(CGF, Message, MessageLoc)); | |
| } | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args); | |
| } | |
| void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, | |
| ProcBindKind ProcBind, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value."); | |
| // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind) | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| llvm::ConstantInt::get(CGM.IntTy, unsigned(ProcBind), /*isSigned=*/true)}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_push_proc_bind), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, | |
| SourceLocation Loc, llvm::AtomicOrdering AO) { | |
| if (CGF.CGM.getLangOpts().OpenMPIRBuilder) { | |
| OMPBuilder.createFlush(CGF.Builder); | |
| } else { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Build call void __kmpc_flush(ident_t *loc) | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_flush), | |
| emitUpdateLocation(CGF, Loc)); | |
| } | |
| } | |
| namespace { | |
| /// Indexes of fields for type kmp_task_t. | |
| enum KmpTaskTFields { | |
| /// List of shared variables. | |
| KmpTaskTShareds, | |
| /// Task routine. | |
| KmpTaskTRoutine, | |
| /// Partition id for the untied tasks. | |
| KmpTaskTPartId, | |
| /// Function with call of destructors for private variables. | |
| Data1, | |
| /// Task priority. | |
| Data2, | |
| /// (Taskloops only) Lower bound. | |
| KmpTaskTLowerBound, | |
| /// (Taskloops only) Upper bound. | |
| KmpTaskTUpperBound, | |
| /// (Taskloops only) Stride. | |
| KmpTaskTStride, | |
| /// (Taskloops only) Is last iteration flag. | |
| KmpTaskTLastIter, | |
| /// (Taskloops only) Reduction data. | |
| KmpTaskTReductions, | |
| }; | |
| } // anonymous namespace | |
| void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() { | |
| // If we are in simd mode or there are no entries, we don't need to do | |
| // anything. | |
| if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty()) | |
| return; | |
| llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn = | |
| [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind, | |
| const llvm::TargetRegionEntryInfo &EntryInfo) -> void { | |
| SourceLocation Loc; | |
| if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) { | |
| for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(), | |
| E = CGM.getContext().getSourceManager().fileinfo_end(); | |
| I != E; ++I) { | |
| if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID && | |
| I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) { | |
| Loc = CGM.getContext().getSourceManager().translateFileLineCol( | |
| I->getFirst(), EntryInfo.Line, 1); | |
| break; | |
| } | |
| } | |
| } | |
| switch (Kind) { | |
| case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: { | |
| CGM.getDiags().Report(Loc, | |
| diag::err_target_region_offloading_entry_incorrect) | |
| << EntryInfo.ParentName; | |
| } break; | |
| case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: { | |
| CGM.getDiags().Report( | |
| Loc, diag::err_target_var_offloading_entry_incorrect_with_parent) | |
| << EntryInfo.ParentName; | |
| } break; | |
| case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: { | |
| CGM.getDiags().Report(diag::err_target_var_offloading_entry_incorrect); | |
| } break; | |
| case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: { | |
| unsigned DiagID = CGM.getDiags().getCustomDiagID( | |
| DiagnosticsEngine::Error, "Offloading entry for indirect declare " | |
| "target variable is incorrect: the " | |
| "address is invalid."); | |
| CGM.getDiags().Report(DiagID); | |
| } break; | |
| } | |
| }; | |
| OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFn); | |
| } | |
| void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { | |
| if (!KmpRoutineEntryPtrTy) { | |
| // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type. | |
| ASTContext &C = CGM.getContext(); | |
| QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy}; | |
| FunctionProtoType::ExtProtoInfo EPI; | |
| KmpRoutineEntryPtrQTy = C.getPointerType( | |
| C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI)); | |
| KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy); | |
| } | |
| } | |
| namespace { | |
| struct PrivateHelpersTy { | |
| PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original, | |
| const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit) | |
| : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy), | |
| PrivateElemInit(PrivateElemInit) {} | |
| PrivateHelpersTy(const VarDecl *Original) : Original(Original) {} | |
| const Expr *OriginalRef = nullptr; | |
| const VarDecl *Original = nullptr; | |
| const VarDecl *PrivateCopy = nullptr; | |
| const VarDecl *PrivateElemInit = nullptr; | |
| bool isLocalPrivate() const { | |
| return !OriginalRef && !PrivateCopy && !PrivateElemInit; | |
| } | |
| }; | |
| typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy; | |
| } // anonymous namespace | |
| /// For BindingDecls, returns the DecomposedDecl as the original VarDecl. | |
| /// For regular VarDecls, returns the VarDecl itself. | |
| static const VarDecl *getOriginalVarDecl(const ValueDecl *Decl) { | |
| if (const auto *BD = dyn_cast<BindingDecl>(Decl)) | |
| return cast<VarDecl>(BD->getDecomposedDecl()); | |
| return cast<VarDecl>(Decl); | |
| } | |
| static bool isAllocatableDecl(const VarDecl *VD) { | |
| const VarDecl *CVD = VD->getCanonicalDecl(); | |
| if (!CVD->hasAttr<OMPAllocateDeclAttr>()) | |
| return false; | |
| const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>(); | |
| // Use the default allocation. | |
| return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc && | |
| !AA->getAllocator()); | |
| } | |
| static RecordDecl * | |
| createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) { | |
| if (!Privates.empty()) { | |
| ASTContext &C = CGM.getContext(); | |
| // Build struct .kmp_privates_t. { | |
| // /* private vars */ | |
| // }; | |
| RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t"); | |
| RD->startDefinition(); | |
| for (const auto &Pair : Privates) { | |
| const VarDecl *VD = Pair.second.Original; | |
| const VarDecl *PrivateCopy = Pair.second.PrivateCopy; | |
| // For BindingDecls, use PrivateCopy type (binding's actual type). | |
| // For regular variables, use Original type to preserve qualifiers. | |
| // Check OriginalRef to detect BindingDecls since Original may be the | |
| // DecompositionDecl. | |
| bool IsBinding = | |
| Pair.second.OriginalRef && | |
| isa<BindingDecl>( | |
| cast<DeclRefExpr>(Pair.second.OriginalRef)->getDecl()); | |
| QualType Type = IsBinding ? PrivateCopy->getType().getNonReferenceType() | |
| : VD->getType().getNonReferenceType(); | |
| // If the private variable is a local variable with lvalue ref type, | |
| // allocate the pointer instead of the pointee type. | |
| if (Pair.second.isLocalPrivate()) { | |
| if (VD->getType()->isLValueReferenceType()) | |
| Type = C.getPointerType(Type); | |
| if (isAllocatableDecl(VD)) | |
| Type = C.getPointerType(Type); | |
| } | |
| FieldDecl *FD = addFieldToRecordDecl(C, RD, Type); | |
| if (VD->hasAttrs()) { | |
| for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()), | |
| E(VD->getAttrs().end()); | |
| I != E; ++I) | |
| FD->addAttr(*I); | |
| } | |
| } | |
| RD->completeDefinition(); | |
| return RD; | |
| } | |
| return nullptr; | |
| } | |
| static RecordDecl * | |
| createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, | |
| QualType KmpInt32Ty, | |
| QualType KmpRoutineEntryPointerQTy) { | |
| ASTContext &C = CGM.getContext(); | |
| // Build struct kmp_task_t { | |
| // void * shareds; | |
| // kmp_routine_entry_t routine; | |
| // kmp_int32 part_id; | |
| // kmp_cmplrdata_t data1; | |
| // kmp_cmplrdata_t data2; | |
| // For taskloops additional fields: | |
| // kmp_uint64 lb; | |
| // kmp_uint64 ub; | |
| // kmp_int64 st; | |
| // kmp_int32 liter; | |
| // void * reductions; | |
| // }; | |
| RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TagTypeKind::Union); | |
| UD->startDefinition(); | |
| addFieldToRecordDecl(C, UD, KmpInt32Ty); | |
| addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy); | |
| UD->completeDefinition(); | |
| CanQualType KmpCmplrdataTy = C.getCanonicalTagType(UD); | |
| RecordDecl *RD = C.buildImplicitRecord("kmp_task_t"); | |
| RD->startDefinition(); | |
| addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); | |
| addFieldToRecordDecl(C, RD, KmpInt32Ty); | |
| addFieldToRecordDecl(C, RD, KmpCmplrdataTy); | |
| addFieldToRecordDecl(C, RD, KmpCmplrdataTy); | |
| if (isOpenMPTaskLoopDirective(Kind)) { | |
| QualType KmpUInt64Ty = | |
| CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0); | |
| QualType KmpInt64Ty = | |
| CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); | |
| addFieldToRecordDecl(C, RD, KmpUInt64Ty); | |
| addFieldToRecordDecl(C, RD, KmpUInt64Ty); | |
| addFieldToRecordDecl(C, RD, KmpInt64Ty); | |
| addFieldToRecordDecl(C, RD, KmpInt32Ty); | |
| addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| } | |
| RD->completeDefinition(); | |
| return RD; | |
| } | |
| static RecordDecl * | |
| createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, | |
| ArrayRef<PrivateDataTy> Privates) { | |
| ASTContext &C = CGM.getContext(); | |
| // Build struct kmp_task_t_with_privates { | |
| // kmp_task_t task_data; | |
| // .kmp_privates_t. privates; | |
| // }; | |
| RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates"); | |
| RD->startDefinition(); | |
| addFieldToRecordDecl(C, RD, KmpTaskTQTy); | |
| if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) | |
| addFieldToRecordDecl(C, RD, C.getCanonicalTagType(PrivateRD)); | |
| RD->completeDefinition(); | |
| return RD; | |
| } | |
| /// Emit a proxy function which accepts kmp_task_t as the second | |
| /// argument. | |
| /// \code | |
| /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { | |
| /// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt, | |
| /// For taskloops: | |
| /// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter, | |
| /// tt->reductions, tt->shareds); | |
| /// return 0; | |
| /// } | |
| /// \endcode | |
| static llvm::Function * | |
| emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, | |
| OpenMPDirectiveKind Kind, QualType KmpInt32Ty, | |
| QualType KmpTaskTWithPrivatesPtrQTy, | |
| QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, | |
| QualType SharedsPtrTy, llvm::Function *TaskFunction, | |
| llvm::Value *TaskPrivatesMap) { | |
| ASTContext &C = CGM.getContext(); | |
| auto *GtidArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| KmpInt32Ty, ImplicitParamKind::Other); | |
| auto *TaskTypeArg = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| KmpTaskTWithPrivatesPtrQTy.withRestrict(), ImplicitParamKind::Other); | |
| FunctionArgList Args{GtidArg, TaskTypeArg}; | |
| const auto &TaskEntryFnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args); | |
| llvm::FunctionType *TaskEntryTy = | |
| CGM.getTypes().GetFunctionType(TaskEntryFnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""}); | |
| auto *TaskEntry = llvm::Function::Create( | |
| TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| TaskEntry->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| TaskEntry->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, | |
| Loc, Loc); | |
| // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map, | |
| // tt, | |
| // For taskloops: | |
| // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter, | |
| // tt->task_data.shareds); | |
| llvm::Value *GtidParam = CGF.EmitLoadOfScalar( | |
| CGF.GetAddrOfLocalVar(GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); | |
| LValue TDBase = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(TaskTypeArg), | |
| KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); | |
| const auto *KmpTaskTWithPrivatesQTyRD = | |
| KmpTaskTWithPrivatesQTy->castAsRecordDecl(); | |
| LValue Base = | |
| CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl(); | |
| auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); | |
| LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); | |
| llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); | |
| auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds); | |
| LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); | |
| llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.EmitLoadOfScalar(SharedsLVal, Loc), | |
| CGF.ConvertTypeForMem(SharedsPtrTy)); | |
| auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1); | |
| llvm::Value *PrivatesParam; | |
| if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) { | |
| LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); | |
| PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); | |
| } else { | |
| PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| } | |
| llvm::Value *CommonArgs[] = { | |
| GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap, | |
| CGF.Builder | |
| .CreatePointerBitCastOrAddrSpaceCast(TDBase.getAddress(), | |
| CGF.VoidPtrTy, CGF.Int8Ty) | |
| .emitRawPointer(CGF)}; | |
| SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs), | |
| std::end(CommonArgs)); | |
| if (isOpenMPTaskLoopDirective(Kind)) { | |
| auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound); | |
| LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); | |
| llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); | |
| auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound); | |
| LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); | |
| llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); | |
| auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride); | |
| LValue StLVal = CGF.EmitLValueForField(Base, *StFI); | |
| llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); | |
| auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter); | |
| LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); | |
| llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); | |
| auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions); | |
| LValue RLVal = CGF.EmitLValueForField(Base, *RFI); | |
| llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); | |
| CallArgs.push_back(LBParam); | |
| CallArgs.push_back(UBParam); | |
| CallArgs.push_back(StParam); | |
| CallArgs.push_back(LIParam); | |
| CallArgs.push_back(RParam); | |
| } | |
| CallArgs.push_back(SharedsParam); | |
| CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, | |
| CallArgs); | |
| CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), | |
| CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); | |
| CGF.FinishFunction(); | |
| return TaskEntry; | |
| } | |
| static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, | |
| SourceLocation Loc, | |
| QualType KmpInt32Ty, | |
| QualType KmpTaskTWithPrivatesPtrQTy, | |
| QualType KmpTaskTWithPrivatesQTy) { | |
| ASTContext &C = CGM.getContext(); | |
| auto *GtidArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| KmpInt32Ty, ImplicitParamKind::Other); | |
| auto *TaskTypeArg = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| KmpTaskTWithPrivatesPtrQTy.withRestrict(), ImplicitParamKind::Other); | |
| FunctionArgList Args{GtidArg, TaskTypeArg}; | |
| const auto &DestructorFnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args); | |
| llvm::FunctionType *DestructorFnTy = | |
| CGM.getTypes().GetFunctionType(DestructorFnInfo); | |
| std::string Name = | |
| CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""}); | |
| auto *DestructorFn = | |
| llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage, | |
| Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn, | |
| DestructorFnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| DestructorFn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| DestructorFn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, | |
| Args, Loc, Loc); | |
| LValue Base = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(TaskTypeArg), | |
| KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); | |
| const auto *KmpTaskTWithPrivatesQTyRD = | |
| KmpTaskTWithPrivatesQTy->castAsRecordDecl(); | |
| auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| Base = CGF.EmitLValueForField(Base, *FI); | |
| for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) { | |
| if (QualType::DestructionKind DtorKind = | |
| Field->getType().isDestructedType()) { | |
| LValue FieldLValue = CGF.EmitLValueForField(Base, Field); | |
| CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType()); | |
| } | |
| } | |
| CGF.FinishFunction(); | |
| return DestructorFn; | |
| } | |
| /// Emit a privates mapping function for correct handling of private and | |
| /// firstprivate variables. | |
| /// \code | |
| /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1> | |
| /// **noalias priv1,..., <tyn> **noalias privn) { | |
| /// *priv1 = &.privates.priv1; | |
| /// ...; | |
| /// *privn = &.privates.privn; | |
| /// } | |
| /// \endcode | |
| static llvm::Value * | |
| emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, | |
| const OMPTaskDataTy &Data, QualType PrivatesQTy, | |
| ArrayRef<PrivateDataTy> Privates) { | |
| ASTContext &C = CGM.getContext(); | |
| FunctionArgList Args; | |
| auto *TaskPrivatesArg = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.getPointerType(PrivatesQTy).withConst().withRestrict(), | |
| ImplicitParamKind::Other); | |
| Args.push_back(TaskPrivatesArg); | |
| llvm::SmallDenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos; | |
| // Track BindingDecl positions separately since BindingDecl is not a VarDecl. | |
| llvm::SmallDenseMap<const BindingDecl *, unsigned> BindingDeclPos; | |
| unsigned Counter = 1; | |
| for (const Expr *E : Data.PrivateVars) { | |
| Args.push_back(ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.getPointerType(C.getPointerType(E->getType())) | |
| .withConst() | |
| .withRestrict(), | |
| ImplicitParamKind::Other)); | |
| const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl(); | |
| if (const auto *BD = dyn_cast<BindingDecl>(VD)) | |
| BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter; | |
| else | |
| PrivateVarsPos[cast<VarDecl>(VD)] = Counter; | |
| ++Counter; | |
| } | |
| for (const Expr *E : Data.FirstprivateVars) { | |
| Args.push_back(ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.getPointerType(C.getPointerType(E->getType())) | |
| .withConst() | |
| .withRestrict(), | |
| ImplicitParamKind::Other)); | |
| const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl(); | |
| if (const auto *BD = dyn_cast<BindingDecl>(VD)) | |
| BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter; | |
| else | |
| PrivateVarsPos[cast<VarDecl>(VD)] = Counter; | |
| ++Counter; | |
| } | |
| for (const Expr *E : Data.LastprivateVars) { | |
| Args.push_back(ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.getPointerType(C.getPointerType(E->getType())) | |
| .withConst() | |
| .withRestrict(), | |
| ImplicitParamKind::Other)); | |
| const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl(); | |
| if (const auto *BD = dyn_cast<BindingDecl>(VD)) | |
| BindingDeclPos[cast<BindingDecl>(BD->getCanonicalDecl())] = Counter; | |
| else | |
| PrivateVarsPos[cast<VarDecl>(VD)] = Counter; | |
| ++Counter; | |
| } | |
| for (const VarDecl *VD : Data.PrivateLocals) { | |
| QualType Ty = VD->getType().getNonReferenceType(); | |
| if (VD->getType()->isLValueReferenceType()) | |
| Ty = C.getPointerType(Ty); | |
| if (isAllocatableDecl(VD)) | |
| Ty = C.getPointerType(Ty); | |
| Args.push_back(ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.getPointerType(C.getPointerType(Ty)).withConst().withRestrict(), | |
| ImplicitParamKind::Other)); | |
| PrivateVarsPos[VD] = Counter; | |
| ++Counter; | |
| } | |
| const auto &TaskPrivatesMapFnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *TaskPrivatesMapTy = | |
| CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo); | |
| std::string Name = | |
| CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""}); | |
| auto *TaskPrivatesMap = llvm::Function::Create( | |
| TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name, | |
| &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap, | |
| TaskPrivatesMapFnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| TaskPrivatesMap->addFnAttr("sample-profile-suffix-elision-policy", | |
| "selected"); | |
| if (CGM.getCodeGenOpts().OptimizationLevel != 0) { | |
| TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline); | |
| TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone); | |
| TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); | |
| } | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, | |
| TaskPrivatesMapFnInfo, Args, Loc, Loc); | |
| // *privi = &.privates.privi; | |
| LValue Base = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(TaskPrivatesArg), | |
| TaskPrivatesArg->getType()->castAs<PointerType>()); | |
| const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl(); | |
| Counter = 0; | |
| for (const FieldDecl *Field : PrivatesQTyRD->fields()) { | |
| LValue FieldLVal = CGF.EmitLValueForField(Base, Field); | |
| // Lookup by the original declaration (BindingDecl or VarDecl). | |
| const ValueDecl *LookupVD; | |
| if (Privates[Counter].second.OriginalRef) { | |
| LookupVD = | |
| cast<DeclRefExpr>(Privates[Counter].second.OriginalRef)->getDecl(); | |
| } else { | |
| LookupVD = Privates[Counter].second.Original; | |
| } | |
| // For BindingDecls, the privates record now stores each binding's type | |
| // directly (not the full DecompositionDecl), so FieldLVal is already | |
| // correct. | |
| unsigned Position; | |
| if (const auto *BD = dyn_cast<BindingDecl>(LookupVD)) { | |
| Position = | |
| BindingDeclPos.lookup(cast<BindingDecl>(BD->getCanonicalDecl())); | |
| assert(Position && "binding not in privates mapping"); | |
| } else { | |
| Position = PrivateVarsPos[cast<VarDecl>(LookupVD)]; | |
| } | |
| const VarDecl *VD = Args[Position]; | |
| LValue RefLVal = | |
| CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); | |
| LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( | |
| RefLVal.getAddress(), RefLVal.getType()->castAs<PointerType>()); | |
| CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); | |
| ++Counter; | |
| } | |
| CGF.FinishFunction(); | |
| return TaskPrivatesMap; | |
| } | |
| /// Emit initialization for private variables in task-based directives. | |
| static void emitPrivatesInit(CodeGenFunction &CGF, | |
| const OMPExecutableDirective &D, | |
| Address KmpTaskSharedsPtr, LValue TDBase, | |
| const RecordDecl *KmpTaskTWithPrivatesQTyRD, | |
| QualType SharedsTy, QualType SharedsPtrTy, | |
| const OMPTaskDataTy &Data, | |
| ArrayRef<PrivateDataTy> Privates, bool ForDup) { | |
| ASTContext &C = CGF.getContext(); | |
| auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); | |
| OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind()) | |
| ? OMPD_taskloop | |
| : OMPD_task; | |
| const CapturedStmt &CS = *D.getCapturedStmt(Kind); | |
| CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS); | |
| LValue SrcBase; | |
| bool IsTargetTask = | |
| isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) || | |
| isOpenMPTargetExecutionDirective(D.getDirectiveKind()); | |
| // For target-based directives skip 4 firstprivate arrays BasePointersArray, | |
| // PointersArray, SizesArray, and MappersArray. The original variables for | |
| // these arrays are not captured and we get their addresses explicitly. | |
| if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) || | |
| (IsTargetTask && KmpTaskSharedsPtr.isValid())) { | |
| SrcBase = CGF.MakeAddrLValue( | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy), | |
| CGF.ConvertTypeForMem(SharedsTy)), | |
| SharedsTy); | |
| } | |
| FI = FI->getType()->castAsRecordDecl()->field_begin(); | |
| for (const PrivateDataTy &Pair : Privates) { | |
| // Do not initialize private locals. | |
| if (Pair.second.isLocalPrivate()) { | |
| ++FI; | |
| continue; | |
| } | |
| const VarDecl *VD = Pair.second.PrivateCopy; | |
| const Expr *Init = VD->getAnyInitializer(); | |
| if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) && | |
| !CGF.isTrivialInitializer(Init)))) { | |
| LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); | |
| if (const VarDecl *Elem = Pair.second.PrivateElemInit) { | |
| const VarDecl *OriginalVD = Pair.second.Original; | |
| // Check if the variable is the target-based BasePointersArray, | |
| // PointersArray, SizesArray, or MappersArray. | |
| LValue SharedRefLValue; | |
| QualType Type = PrivateLValue.getType(); | |
| const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD); | |
| if (IsTargetTask && !SharedField) { | |
| assert(isa<ImplicitParamDecl>(OriginalVD) && | |
| isa<CapturedDecl>(OriginalVD->getDeclContext()) && | |
| cast<CapturedDecl>(OriginalVD->getDeclContext()) | |
| ->getNumParams() == 0 && | |
| isa<TranslationUnitDecl>( | |
| cast<CapturedDecl>(OriginalVD->getDeclContext()) | |
| ->getDeclContext()) && | |
| "Expected artificial target data variable."); | |
| SharedRefLValue = | |
| CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); | |
| } else if (ForDup) { | |
| SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); | |
| // For BindingDecls, access the specific binding field within the | |
| // captured DecompositionDecl. | |
| if (Pair.second.OriginalRef) { | |
| if (const auto *DRE = | |
| dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) { | |
| if (const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) { | |
| // Emit the binding subobject (member or array element) with | |
| // the decomposed decl temporarily mapped to the capture. | |
| const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl()); | |
| auto It = CGF.findLocalDecl(DD); | |
| bool WasMapped = It != CGF.localDeclMapEnd(); | |
| Address Saved = WasMapped ? It->second : Address::invalid(); | |
| if (WasMapped) | |
| It->second = SharedRefLValue.getAddress(); | |
| else | |
| CGF.insertLocalDecl(DD, SharedRefLValue.getAddress()); | |
| SharedRefLValue = CGF.EmitLValue(BD->getBinding()); | |
| if (WasMapped) { | |
| auto RestoreIt = CGF.findLocalDecl(DD); | |
| RestoreIt->second = Saved; | |
| } else { | |
| CGF.eraseLocalDecl(DD); | |
| } | |
| } | |
| } | |
| } | |
| bool IsBinding = | |
| Pair.second.OriginalRef && | |
| isa<BindingDecl>( | |
| cast<DeclRefExpr>(Pair.second.OriginalRef)->getDecl()); | |
| SharedRefLValue = CGF.MakeAddrLValue( | |
| SharedRefLValue.getAddress().withAlignment( | |
| IsBinding ? C.toCharUnitsFromBits( | |
| C.getTypeAlign(SharedRefLValue.getType())) | |
| : C.getDeclAlign(OriginalVD)), | |
| SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl), | |
| SharedRefLValue.getTBAAInfo()); | |
| } else if (CGF.LambdaCaptureFields.count( | |
| Pair.second.Original->getCanonicalDecl()) > 0 || | |
| isa_and_nonnull<BlockDecl>(CGF.CurCodeDecl)) { | |
| SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); | |
| } else { | |
| // Processing for implicitly captured variables. | |
| InlinedOpenMPRegionRAII Region( | |
| CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, | |
| /*HasCancel=*/false, /*NoInheritance=*/true); | |
| SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef); | |
| } | |
| if (Type->isArrayType()) { | |
| // Initialize firstprivate array. | |
| if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { | |
| // Perform simple memcpy. | |
| CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); | |
| } else { | |
| // Initialize firstprivate array using element-by-element | |
| // initialization. | |
| CGF.EmitOMPAggregateAssign( | |
| PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type, | |
| [&CGF, Elem, Init, &CapturesInfo](Address DestElement, | |
| Address SrcElement) { | |
| // Clean up any temporaries needed by the initialization. | |
| CodeGenFunction::OMPPrivateScope InitScope(CGF); | |
| InitScope.addPrivate(Elem, SrcElement); | |
| (void)InitScope.Privatize(); | |
| // Emit initialization for single element. | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII( | |
| CGF, &CapturesInfo); | |
| CGF.EmitAnyExprToMem(Init, DestElement, | |
| Init->getType().getQualifiers(), | |
| /*IsInitializer=*/false); | |
| }); | |
| } | |
| } else { | |
| CodeGenFunction::OMPPrivateScope InitScope(CGF); | |
| InitScope.addPrivate(Elem, SharedRefLValue.getAddress()); | |
| (void)InitScope.Privatize(); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); | |
| CGF.EmitExprAsInit(Init, VD, PrivateLValue, | |
| /*capturedByInit=*/false); | |
| } | |
| } else { | |
| CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); | |
| } | |
| } else if (const VarDecl *OriginalVD = Pair.second.Original) { | |
| // Handle array bindings without initializers (firstprivate only). | |
| // For private clause (PrivateElemInit is nullptr), skip initialization. | |
| // Check if OriginalRef is a BindingDecl with array type. | |
| const BindingDecl *BD = nullptr; | |
| if (Pair.second.OriginalRef) { | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(Pair.second.OriginalRef)) { | |
| BD = dyn_cast<BindingDecl>(DRE->getDecl()); | |
| } | |
| } | |
| if (BD && BD->getType()->isArrayType() && Pair.second.PrivateElemInit) { | |
| LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); | |
| QualType Type = PrivateLValue.getType(); | |
| const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD); | |
| // Lambda to temporarily map DecompositionDecl and emit the binding | |
| // expression. | |
| auto EmitBindingWithTempMap = [&CGF](const BindingDecl *BD, | |
| Address DDAddr) -> LValue { | |
| const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl()); | |
| auto It = CGF.findLocalDecl(DD); | |
| bool WasMapped = It != CGF.localDeclMapEnd(); | |
| Address Saved = WasMapped ? It->second : Address::invalid(); | |
| if (WasMapped) | |
| It->second = DDAddr; | |
| else | |
| CGF.insertLocalDecl(DD, DDAddr); | |
| LValue Result = CGF.EmitLValue(BD->getBinding()); | |
| // Restore the mapping | |
| if (WasMapped) { | |
| auto RestoreIt = CGF.findLocalDecl(DD); | |
| RestoreIt->second = Saved; | |
| } else { | |
| CGF.eraseLocalDecl(DD); | |
| } | |
| return Result; | |
| }; | |
| LValue SharedRefLValue; | |
| if (ForDup) { | |
| SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); | |
| SharedRefLValue = | |
| EmitBindingWithTempMap(BD, SharedRefLValue.getAddress()); | |
| SharedRefLValue = CGF.MakeAddrLValue( | |
| SharedRefLValue.getAddress().withAlignment( | |
| C.getDeclAlign(OriginalVD)), | |
| SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl), | |
| SharedRefLValue.getTBAAInfo()); | |
| } else { | |
| // For !ForDup (first task), emit binding from parent scope using | |
| // InlinedOpenMPRegionRAII to access the correct scope | |
| InlinedOpenMPRegionRAII Region( | |
| CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown, | |
| /*HasCancel=*/false, /*NoInheritance=*/true); | |
| // Get the decomposed decl address from parent scope | |
| const VarDecl *DD = cast<VarDecl>(BD->getDecomposedDecl()); | |
| Address DDAddr = CGF.GetAddrOfLocalVar(DD); | |
| SharedRefLValue = EmitBindingWithTempMap(BD, DDAddr); | |
| } | |
| // Perform simple memcpy for array binding | |
| CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); | |
| } | |
| } | |
| ++FI; | |
| } | |
| } | |
| /// Check if duplication function is required for taskloops. | |
| static bool checkInitIsRequired(CodeGenFunction &CGF, | |
| ArrayRef<PrivateDataTy> Privates) { | |
| bool InitRequired = false; | |
| for (const PrivateDataTy &Pair : Privates) { | |
| if (Pair.second.isLocalPrivate()) | |
| continue; | |
| const VarDecl *VD = Pair.second.PrivateCopy; | |
| const Expr *Init = VD->getAnyInitializer(); | |
| InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Init) && | |
| !CGF.isTrivialInitializer(Init)); | |
| if (InitRequired) | |
| break; | |
| } | |
| return InitRequired; | |
| } | |
| /// Emit task_dup function (for initialization of | |
| /// private/firstprivate/lastprivate vars and last_iter flag) | |
| /// \code | |
| /// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int | |
| /// lastpriv) { | |
| /// // setup lastprivate flag | |
| /// task_dst->last = lastpriv; | |
| /// // could be constructor calls here... | |
| /// } | |
| /// \endcode | |
| static llvm::Value * | |
| emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, | |
| const OMPExecutableDirective &D, | |
| QualType KmpTaskTWithPrivatesPtrQTy, | |
| const RecordDecl *KmpTaskTWithPrivatesQTyRD, | |
| const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, | |
| QualType SharedsPtrTy, const OMPTaskDataTy &Data, | |
| ArrayRef<PrivateDataTy> Privates, bool WithLastIter) { | |
| ASTContext &C = CGM.getContext(); | |
| auto *DstArg = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy, | |
| ImplicitParamKind::Other); | |
| auto *SrcArg = ImplicitParamDecl::Create( | |
| C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy, | |
| ImplicitParamKind::Other); | |
| auto *LastprivArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, | |
| ImplicitParamKind::Other); | |
| FunctionArgList Args{DstArg, SrcArg, LastprivArg}; | |
| const auto &TaskDupFnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""}); | |
| auto *TaskDup = llvm::Function::Create( | |
| TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| TaskDup->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| TaskDup->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, | |
| Loc); | |
| LValue TDBase = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(DstArg), | |
| KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); | |
| // task_dst->liter = lastpriv; | |
| if (WithLastIter) { | |
| auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter); | |
| LValue Base = CGF.EmitLValueForField( | |
| TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); | |
| llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( | |
| CGF.GetAddrOfLocalVar(LastprivArg), /*Volatile=*/false, C.IntTy, Loc); | |
| CGF.EmitStoreOfScalar(Lastpriv, LILVal); | |
| } | |
| // Emit initial values for private copies (if any). | |
| assert(!Privates.empty()); | |
| Address KmpTaskSharedsPtr = Address::invalid(); | |
| if (!Data.FirstprivateVars.empty()) { | |
| LValue TDBase = CGF.EmitLoadOfPointerLValue( | |
| CGF.GetAddrOfLocalVar(SrcArg), | |
| KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); | |
| LValue Base = CGF.EmitLValueForField( | |
| TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| KmpTaskSharedsPtr = Address( | |
| CGF.EmitLoadOfScalar(CGF.EmitLValueForField( | |
| Base, *std::next(KmpTaskTQTyRD->field_begin(), | |
| KmpTaskTShareds)), | |
| Loc), | |
| CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); | |
| } | |
| emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, | |
| SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true); | |
| CGF.FinishFunction(); | |
| return TaskDup; | |
| } | |
| /// Checks if destructor function is required to be generated. | |
| /// \return true if cleanups are required, false otherwise. | |
| static bool | |
| checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD, | |
| ArrayRef<PrivateDataTy> Privates) { | |
| for (const PrivateDataTy &P : Privates) { | |
| if (P.second.isLocalPrivate()) | |
| continue; | |
| QualType Ty = P.second.Original->getType().getNonReferenceType(); | |
| if (Ty.isDestructedType()) | |
| return true; | |
| } | |
| return false; | |
| } | |
| namespace { | |
| /// Loop generator for OpenMP iterator expression. | |
| class OMPIteratorGeneratorScope final | |
| : public CodeGenFunction::OMPPrivateScope { | |
| CodeGenFunction &CGF; | |
| const OMPIteratorExpr *E = nullptr; | |
| SmallVector<CodeGenFunction::JumpDest, 4> ContDests; | |
| SmallVector<CodeGenFunction::JumpDest, 4> ExitDests; | |
| OMPIteratorGeneratorScope() = delete; | |
| OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete; | |
| public: | |
| OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E) | |
| : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) { | |
| if (!E) | |
| return; | |
| SmallVector<llvm::Value *, 4> Uppers; | |
| for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) { | |
| Uppers.push_back(CGF.EmitScalarExpr(E->getHelper(I).Upper)); | |
| const auto *VD = cast<VarDecl>(E->getIteratorDecl(I)); | |
| addPrivate(VD, CGF.CreateMemTemp(VD->getType(), VD->getName())); | |
| const OMPIteratorHelperData &HelperData = E->getHelper(I); | |
| addPrivate( | |
| HelperData.CounterVD, | |
| CGF.CreateMemTemp(HelperData.CounterVD->getType(), "counter.addr")); | |
| } | |
| Privatize(); | |
| for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) { | |
| const OMPIteratorHelperData &HelperData = E->getHelper(I); | |
| LValue CLVal = | |
| CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(HelperData.CounterVD), | |
| HelperData.CounterVD->getType()); | |
| // Counter = 0; | |
| CGF.EmitStoreOfScalar( | |
| llvm::ConstantInt::get(CLVal.getAddress().getElementType(), 0), | |
| CLVal); | |
| CodeGenFunction::JumpDest &ContDest = | |
| ContDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.cont")); | |
| CodeGenFunction::JumpDest &ExitDest = | |
| ExitDests.emplace_back(CGF.getJumpDestInCurrentScope("iter.exit")); | |
| // N = <number-of_iterations>; | |
| llvm::Value *N = Uppers[I]; | |
| // cont: | |
| // if (Counter < N) goto body; else goto exit; | |
| CGF.EmitBlock(ContDest.getBlock()); | |
| auto *CVal = | |
| CGF.EmitLoadOfScalar(CLVal, HelperData.CounterVD->getLocation()); | |
| llvm::Value *Cmp = | |
| HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType() | |
| ? CGF.Builder.CreateICmpSLT(CVal, N) | |
| : CGF.Builder.CreateICmpULT(CVal, N); | |
| llvm::BasicBlock *BodyBB = CGF.createBasicBlock("iter.body"); | |
| CGF.Builder.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock()); | |
| // body: | |
| CGF.EmitBlock(BodyBB); | |
| // Iteri = Begini + Counter * Stepi; | |
| CGF.EmitIgnoredExpr(HelperData.Update); | |
| } | |
| } | |
| ~OMPIteratorGeneratorScope() { | |
| if (!E) | |
| return; | |
| for (unsigned I = E->numOfIterators(); I > 0; --I) { | |
| // Counter = Counter + 1; | |
| const OMPIteratorHelperData &HelperData = E->getHelper(I - 1); | |
| CGF.EmitIgnoredExpr(HelperData.CounterUpdate); | |
| // goto cont; | |
| CGF.EmitBranchThroughCleanup(ContDests[I - 1]); | |
| // exit: | |
| CGF.EmitBlock(ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1); | |
| } | |
| } | |
| }; | |
| } // namespace | |
| static std::pair<llvm::Value *, llvm::Value *> | |
| getPointerAndSize(CodeGenFunction &CGF, const Expr *E) { | |
| const auto *OASE = dyn_cast<OMPArrayShapingExpr>(E); | |
| llvm::Value *Addr; | |
| if (OASE) { | |
| const Expr *Base = OASE->getBase(); | |
| Addr = CGF.EmitScalarExpr(Base); | |
| } else { | |
| Addr = CGF.EmitLValue(E).getPointer(CGF); | |
| } | |
| llvm::Value *SizeVal; | |
| QualType Ty = E->getType(); | |
| if (OASE) { | |
| SizeVal = CGF.getTypeSize(OASE->getBase()->getType()->getPointeeType()); | |
| for (const Expr *SE : OASE->getDimensions()) { | |
| llvm::Value *Sz = CGF.EmitScalarExpr(SE); | |
| Sz = CGF.EmitScalarConversion( | |
| Sz, SE->getType(), CGF.getContext().getSizeType(), SE->getExprLoc()); | |
| SizeVal = CGF.Builder.CreateNUWMul(SizeVal, Sz); | |
| } | |
| } else if (const auto *ASE = | |
| dyn_cast<ArraySectionExpr>(E->IgnoreParenImpCasts())) { | |
| LValue UpAddrLVal = CGF.EmitArraySectionExpr(ASE, /*IsLowerBound=*/false); | |
| Address UpAddrAddress = UpAddrLVal.getAddress(); | |
| llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( | |
| UpAddrAddress.getElementType(), UpAddrAddress.emitRawPointer(CGF), | |
| /*Idx0=*/1); | |
| SizeVal = CGF.Builder.CreatePtrDiff(UpAddr, Addr, "", /*IsNUW=*/true); | |
| } else { | |
| SizeVal = CGF.getTypeSize(Ty); | |
| } | |
| return std::make_pair(Addr, SizeVal); | |
| } | |
| /// Builds kmp_depend_info, if it is not built yet, and builds flags type. | |
| static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) { | |
| QualType FlagsTy = C.getIntTypeForBitwidth(32, /*Signed=*/false); | |
| if (KmpTaskAffinityInfoTy.isNull()) { | |
| RecordDecl *KmpAffinityInfoRD = | |
| C.buildImplicitRecord("kmp_task_affinity_info_t"); | |
| KmpAffinityInfoRD->startDefinition(); | |
| addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getIntPtrType()); | |
| addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType()); | |
| addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy); | |
| KmpAffinityInfoRD->completeDefinition(); | |
| KmpTaskAffinityInfoTy = C.getCanonicalTagType(KmpAffinityInfoRD); | |
| } | |
| } | |
| CGOpenMPRuntime::TaskResultTy | |
| CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, | |
| const OMPExecutableDirective &D, | |
| llvm::Function *TaskFunction, QualType SharedsTy, | |
| Address Shareds, const OMPTaskDataTy &Data) { | |
| ASTContext &C = CGM.getContext(); | |
| llvm::SmallVector<PrivateDataTy, 4> Privates; | |
| // Aggregate privates and sort them by the alignment. | |
| const auto *I = Data.PrivateCopies.begin(); | |
| for (const Expr *E : Data.PrivateVars) { | |
| const auto *Decl = cast<DeclRefExpr>(E)->getDecl(); | |
| const auto *VD = getOriginalVarDecl(Decl); | |
| const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); | |
| Privates.emplace_back(C.getDeclAlign(VD), | |
| PrivateHelpersTy(E, VD, CopyVD, | |
| /*PrivateElemInit=*/nullptr)); | |
| ++I; | |
| } | |
| I = Data.FirstprivateCopies.begin(); | |
| const auto *IElemInitRef = Data.FirstprivateInits.begin(); | |
| for (const Expr *E : Data.FirstprivateVars) { | |
| const auto *Decl = cast<DeclRefExpr>(E)->getDecl(); | |
| const auto *VD = getOriginalVarDecl(Decl); | |
| const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); | |
| const auto *InitVD = | |
| cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()); | |
| Privates.emplace_back(C.getDeclAlign(VD), | |
| PrivateHelpersTy(E, VD, CopyVD, InitVD)); | |
| ++I; | |
| ++IElemInitRef; | |
| } | |
| I = Data.LastprivateCopies.begin(); | |
| for (const Expr *E : Data.LastprivateVars) { | |
| const auto *Decl = cast<DeclRefExpr>(E)->getDecl(); | |
| const auto *VD = getOriginalVarDecl(Decl); | |
| const auto *CopyVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()); | |
| Privates.emplace_back(C.getDeclAlign(VD), | |
| PrivateHelpersTy(E, VD, CopyVD, | |
| /*PrivateElemInit=*/nullptr)); | |
| ++I; | |
| } | |
| for (const VarDecl *VD : Data.PrivateLocals) { | |
| if (isAllocatableDecl(VD)) | |
| Privates.emplace_back(CGM.getPointerAlign(), PrivateHelpersTy(VD)); | |
| else | |
| Privates.emplace_back(C.getDeclAlign(VD), PrivateHelpersTy(VD)); | |
| } | |
| llvm::stable_sort(Privates, | |
| [](const PrivateDataTy &L, const PrivateDataTy &R) { | |
| return L.first > R.first; | |
| }); | |
| QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); | |
| // Build type kmp_routine_entry_t (if not built yet). | |
| emitKmpRoutineEntryT(KmpInt32Ty); | |
| // Build type kmp_task_t (if not built yet). | |
| if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) { | |
| if (SavedKmpTaskloopTQTy.isNull()) { | |
| SavedKmpTaskloopTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl( | |
| CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy)); | |
| } | |
| KmpTaskTQTy = SavedKmpTaskloopTQTy; | |
| } else { | |
| assert((D.getDirectiveKind() == OMPD_task || | |
| isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || | |
| isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && | |
| "Expected taskloop, task or target directive"); | |
| if (SavedKmpTaskTQTy.isNull()) { | |
| SavedKmpTaskTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl( | |
| CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy)); | |
| } | |
| KmpTaskTQTy = SavedKmpTaskTQTy; | |
| } | |
| const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl(); | |
| // Build particular struct kmp_task_t for the given task. | |
| const RecordDecl *KmpTaskTWithPrivatesQTyRD = | |
| createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates); | |
| CanQualType KmpTaskTWithPrivatesQTy = | |
| C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD); | |
| QualType KmpTaskTWithPrivatesPtrQTy = | |
| C.getPointerType(KmpTaskTWithPrivatesQTy); | |
| llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0); | |
| llvm::Value *KmpTaskTWithPrivatesTySize = | |
| CGF.getTypeSize(KmpTaskTWithPrivatesQTy); | |
| QualType SharedsPtrTy = C.getPointerType(SharedsTy); | |
| // Emit initial values for private copies (if any). | |
| llvm::Value *TaskPrivatesMap = nullptr; | |
| llvm::Type *TaskPrivatesMapTy = | |
| std::next(TaskFunction->arg_begin(), 3)->getType(); | |
| if (!Privates.empty()) { | |
| auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| TaskPrivatesMap = | |
| emitTaskPrivateMappingFunction(CGM, Loc, Data, FI->getType(), Privates); | |
| TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| TaskPrivatesMap, TaskPrivatesMapTy); | |
| } else { | |
| TaskPrivatesMap = llvm::ConstantPointerNull::get( | |
| cast<llvm::PointerType>(TaskPrivatesMapTy)); | |
| } | |
| // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid, | |
| // kmp_task_t *tt); | |
| llvm::Function *TaskEntry = emitProxyTaskFunction( | |
| CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, | |
| KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction, | |
| TaskPrivatesMap); | |
| // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, | |
| // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, | |
| // kmp_routine_entry_t *task_entry); | |
| // Task flags. Format is taken from | |
| // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h, | |
| // description of kmp_tasking_flags struct. | |
| enum { | |
| TiedFlag = 0x1, | |
| FinalFlag = 0x2, | |
| DestructorsFlag = 0x8, | |
| PriorityFlag = 0x20, | |
| DetachableFlag = 0x40, | |
| FreeAgentFlag = 0x80, | |
| TransparentFlag = 0x100, | |
| }; | |
| unsigned Flags = Data.Tied ? TiedFlag : 0; | |
| bool NeedsCleanup = false; | |
| if (!Privates.empty()) { | |
| NeedsCleanup = | |
| checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates); | |
| if (NeedsCleanup) | |
| Flags = Flags | DestructorsFlag; | |
| } | |
| if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) { | |
| OpenMPThreadsetKind Kind = Clause->getThreadsetKind(); | |
| if (Kind == OMPC_THREADSET_omp_pool) | |
| Flags = Flags | FreeAgentFlag; | |
| } | |
| if (D.getSingleClause<OMPTransparentClause>()) | |
| Flags |= TransparentFlag; | |
| if (Data.Priority.getInt()) | |
| Flags = Flags | PriorityFlag; | |
| if (D.hasClausesOfKind<OMPDetachClause>()) | |
| Flags = Flags | DetachableFlag; | |
| llvm::Value *TaskFlags = | |
| Data.Final.getPointer() | |
| ? CGF.Builder.CreateSelect(Data.Final.getPointer(), | |
| CGF.Builder.getInt32(FinalFlag), | |
| CGF.Builder.getInt32(/*C=*/0)) | |
| : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); | |
| TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); | |
| llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy)); | |
| SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), | |
| getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, | |
| SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| TaskEntry, KmpRoutineEntryPtrTy)}; | |
| llvm::Value *NewTask; | |
| if (D.hasClausesOfKind<OMPNowaitClause>()) { | |
| // Check if we have any device clause associated with the directive. | |
| const Expr *Device = nullptr; | |
| if (auto *C = D.getSingleClause<OMPDeviceClause>()) | |
| Device = C->getDevice(); | |
| // Emit device ID if any otherwise use default value. | |
| llvm::Value *DeviceID; | |
| if (Device) | |
| DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), | |
| CGF.Int64Ty, /*isSigned=*/true); | |
| else | |
| DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); | |
| AllocArgs.push_back(DeviceID); | |
| NewTask = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_target_task_alloc), | |
| AllocArgs); | |
| } else { | |
| NewTask = | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_task_alloc), | |
| AllocArgs); | |
| } | |
| // Emit detach clause initialization. | |
| // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid, | |
| // task_descriptor); | |
| if (const auto *DC = D.getSingleClause<OMPDetachClause>()) { | |
| const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts(); | |
| LValue EvtLVal = CGF.EmitLValue(Evt); | |
| // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, | |
| // int gtid, kmp_task_t *task); | |
| llvm::Value *Loc = emitUpdateLocation(CGF, DC->getBeginLoc()); | |
| llvm::Value *Tid = getThreadID(CGF, DC->getBeginLoc()); | |
| Tid = CGF.Builder.CreateIntCast(Tid, CGF.IntTy, /*isSigned=*/false); | |
| llvm::Value *EvtVal = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_task_allow_completion_event), | |
| {Loc, Tid, NewTask}); | |
| EvtVal = CGF.EmitScalarConversion(EvtVal, C.VoidPtrTy, Evt->getType(), | |
| Evt->getExprLoc()); | |
| CGF.EmitStoreOfScalar(EvtVal, EvtLVal); | |
| } | |
| // Process affinity clauses. | |
| if (D.hasClausesOfKind<OMPAffinityClause>()) { | |
| // Process list of affinity data. | |
| ASTContext &C = CGM.getContext(); | |
| Address AffinitiesArray = Address::invalid(); | |
| // Calculate number of elements to form the array of affinity data. | |
| llvm::Value *NumOfElements = nullptr; | |
| unsigned NumAffinities = 0; | |
| for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) { | |
| if (const Expr *Modifier = C->getModifier()) { | |
| const auto *IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts()); | |
| for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) { | |
| llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); | |
| Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); | |
| NumOfElements = | |
| NumOfElements ? CGF.Builder.CreateNUWMul(NumOfElements, Sz) : Sz; | |
| } | |
| } else { | |
| NumAffinities += C->varlist_size(); | |
| } | |
| } | |
| getKmpAffinityType(CGM.getContext(), KmpTaskAffinityInfoTy); | |
| // Fields ids in kmp_task_affinity_info record. | |
| enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags }; | |
| QualType KmpTaskAffinityInfoArrayTy; | |
| if (NumOfElements) { | |
| NumOfElements = CGF.Builder.CreateNUWAdd( | |
| llvm::ConstantInt::get(CGF.SizeTy, NumAffinities), NumOfElements); | |
| auto *OVE = new (C) OpaqueValueExpr( | |
| Loc, | |
| C.getIntTypeForBitwidth(C.getTypeSize(C.getSizeType()), /*Signed=*/0), | |
| VK_PRValue); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, | |
| RValue::get(NumOfElements)); | |
| KmpTaskAffinityInfoArrayTy = C.getVariableArrayType( | |
| KmpTaskAffinityInfoTy, OVE, ArraySizeModifier::Normal, | |
| /*IndexTypeQuals=*/0); | |
| // Properly emit variable-sized array. | |
| auto *PD = ImplicitParamDecl::Create(C, KmpTaskAffinityInfoArrayTy, | |
| ImplicitParamKind::Other); | |
| CGF.EmitVarDecl(*PD); | |
| AffinitiesArray = CGF.GetAddrOfLocalVar(PD); | |
| NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| } else { | |
| KmpTaskAffinityInfoArrayTy = C.getConstantArrayType( | |
| KmpTaskAffinityInfoTy, | |
| llvm::APInt(C.getTypeSize(C.getSizeType()), NumAffinities), nullptr, | |
| ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); | |
| AffinitiesArray = CGF.CreateMemTempWithoutCast(KmpTaskAffinityInfoArrayTy, | |
| ".affs.arr.addr"); | |
| AffinitiesArray = CGF.Builder.CreateConstArrayGEP(AffinitiesArray, 0); | |
| NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumAffinities, | |
| /*isSigned=*/false); | |
| } | |
| const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl(); | |
| // Fill array by elements without iterators. | |
| unsigned Pos = 0; | |
| bool HasIterator = false; | |
| for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) { | |
| if (C->getModifier()) { | |
| HasIterator = true; | |
| continue; | |
| } | |
| for (const Expr *E : C->varlist()) { | |
| llvm::Value *Addr; | |
| llvm::Value *Size; | |
| std::tie(Addr, Size) = getPointerAndSize(CGF, E); | |
| LValue Base = | |
| CGF.MakeAddrLValue(CGF.Builder.CreateConstGEP(AffinitiesArray, Pos), | |
| KmpTaskAffinityInfoTy); | |
| // affs[i].base_addr = &<Affinities[i].second>; | |
| LValue BaseAddrLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); | |
| CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), | |
| BaseAddrLVal); | |
| // affs[i].len = sizeof(<Affinities[i].second>); | |
| LValue LenLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpAffinityInfoRD->field_begin(), Len)); | |
| CGF.EmitStoreOfScalar(Size, LenLVal); | |
| ++Pos; | |
| } | |
| } | |
| LValue PosLVal; | |
| if (HasIterator) { | |
| PosLVal = CGF.MakeAddrLValue( | |
| CGF.CreateMemTempWithoutCast(C.getSizeType(), "affs.counter.addr"), | |
| C.getSizeType()); | |
| CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); | |
| } | |
| // Process elements with iterators. | |
| for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) { | |
| const Expr *Modifier = C->getModifier(); | |
| if (!Modifier) | |
| continue; | |
| OMPIteratorGeneratorScope IteratorScope( | |
| CGF, cast_or_null<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts())); | |
| for (const Expr *E : C->varlist()) { | |
| llvm::Value *Addr; | |
| llvm::Value *Size; | |
| std::tie(Addr, Size) = getPointerAndSize(CGF, E); | |
| llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); | |
| LValue Base = | |
| CGF.MakeAddrLValue(CGF.Builder.CreateGEP(CGF, AffinitiesArray, Idx), | |
| KmpTaskAffinityInfoTy); | |
| // affs[i].base_addr = &<Affinities[i].second>; | |
| LValue BaseAddrLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr)); | |
| CGF.EmitStoreOfScalar(CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy), | |
| BaseAddrLVal); | |
| // affs[i].len = sizeof(<Affinities[i].second>); | |
| LValue LenLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpAffinityInfoRD->field_begin(), Len)); | |
| CGF.EmitStoreOfScalar(Size, LenLVal); | |
| Idx = CGF.Builder.CreateNUWAdd( | |
| Idx, llvm::ConstantInt::get(Idx->getType(), 1)); | |
| CGF.EmitStoreOfScalar(Idx, PosLVal); | |
| } | |
| } | |
| // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, | |
| // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 | |
| // naffins, kmp_task_affinity_info_t *affin_list); | |
| llvm::Value *LocRef = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *GTid = getThreadID(CGF, Loc); | |
| llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| AffinitiesArray.emitRawPointer(CGF), CGM.VoidPtrTy); | |
| // FIXME: Emit the function and ignore its result for now unless the | |
| // runtime function is properly implemented. | |
| (void)CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_reg_task_with_affinity), | |
| {LocRef, GTid, NewTask, NumOfElements, AffinListPtr}); | |
| } | |
| llvm::Value *NewTaskNewTaskTTy = | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| NewTask, KmpTaskTWithPrivatesPtrTy); | |
| LValue Base = CGF.MakeNaturalAlignRawAddrLValue(NewTaskNewTaskTTy, | |
| KmpTaskTWithPrivatesQTy); | |
| LValue TDBase = | |
| CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); | |
| // Fill the data in the resulting kmp_task_t record. | |
| // Copy shareds if there are any. | |
| Address KmpTaskSharedsPtr = Address::invalid(); | |
| if (!SharedsTy->castAsRecordDecl()->field_empty()) { | |
| KmpTaskSharedsPtr = Address( | |
| CGF.EmitLoadOfScalar( | |
| CGF.EmitLValueForField( | |
| TDBase, | |
| *std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)), | |
| Loc), | |
| CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy)); | |
| LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); | |
| LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); | |
| CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); | |
| } | |
| // Emit initial values for private copies (if any). | |
| TaskResultTy Result; | |
| if (!Privates.empty()) { | |
| emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, | |
| SharedsTy, SharedsPtrTy, Data, Privates, | |
| /*ForDup=*/false); | |
| if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && | |
| (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { | |
| Result.TaskDupFn = emitTaskDupFunction( | |
| CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD, | |
| KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates, | |
| /*WithLastIter=*/!Data.LastprivateVars.empty()); | |
| } | |
| } | |
| // Fields of union "kmp_cmplrdata_t" for destructors and priority. | |
| enum { Priority = 0, Destructors = 1 }; | |
| // Provide pointer to function with destructors for privates. | |
| auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1); | |
| const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl(); | |
| assert(KmpCmplrdataUD->isUnion()); | |
| if (NeedsCleanup) { | |
| llvm::Value *DestructorFn = emitDestructorsFunction( | |
| CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, | |
| KmpTaskTWithPrivatesQTy); | |
| LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); | |
| LValue DestructorsLV = CGF.EmitLValueForField( | |
| Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors)); | |
| CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| DestructorFn, KmpRoutineEntryPtrTy), | |
| DestructorsLV); | |
| } | |
| // Set priority. | |
| if (Data.Priority.getInt()) { | |
| LValue Data2LV = CGF.EmitLValueForField( | |
| TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2)); | |
| LValue PriorityLV = CGF.EmitLValueForField( | |
| Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority)); | |
| CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); | |
| } | |
| Result.NewTask = NewTask; | |
| Result.TaskEntry = TaskEntry; | |
| Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy; | |
| Result.TDBase = TDBase; | |
| Result.KmpTaskTQTyRD = KmpTaskTQTyRD; | |
| return Result; | |
| } | |
| /// Translates internal dependency kind into the runtime kind. | |
| static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) { | |
| RTLDependenceKindTy DepKind; | |
| switch (K) { | |
| case OMPC_DEPEND_in: | |
| DepKind = RTLDependenceKindTy::DepIn; | |
| break; | |
| // Out and InOut dependencies must use the same code. | |
| case OMPC_DEPEND_out: | |
| case OMPC_DEPEND_inout: | |
| DepKind = RTLDependenceKindTy::DepInOut; | |
| break; | |
| case OMPC_DEPEND_mutexinoutset: | |
| DepKind = RTLDependenceKindTy::DepMutexInOutSet; | |
| break; | |
| case OMPC_DEPEND_inoutset: | |
| DepKind = RTLDependenceKindTy::DepInOutSet; | |
| break; | |
| case OMPC_DEPEND_outallmemory: | |
| DepKind = RTLDependenceKindTy::DepOmpAllMem; | |
| break; | |
| case OMPC_DEPEND_source: | |
| case OMPC_DEPEND_sink: | |
| case OMPC_DEPEND_depobj: | |
| case OMPC_DEPEND_inoutallmemory: | |
| case OMPC_DEPEND_unknown: | |
| llvm_unreachable("Unknown task dependence type"); | |
| } | |
| return DepKind; | |
| } | |
| /// Builds kmp_depend_info, if it is not built yet, and builds flags type. | |
| static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy, | |
| QualType &FlagsTy) { | |
| FlagsTy = C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false); | |
| if (KmpDependInfoTy.isNull()) { | |
| RecordDecl *KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info"); | |
| KmpDependInfoRD->startDefinition(); | |
| addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType()); | |
| addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType()); | |
| addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy); | |
| KmpDependInfoRD->completeDefinition(); | |
| KmpDependInfoTy = C.getCanonicalTagType(KmpDependInfoRD); | |
| } | |
| } | |
| std::pair<llvm::Value *, LValue> | |
| CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal, | |
| SourceLocation Loc) { | |
| ASTContext &C = CGM.getContext(); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl(); | |
| QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy); | |
| LValue Base = CGF.EmitLoadOfPointerLValue( | |
| DepobjLVal.getAddress().withElementType( | |
| CGF.ConvertTypeForMem(KmpDependInfoPtrTy)), | |
| KmpDependInfoPtrTy->castAs<PointerType>()); | |
| Address DepObjAddr = CGF.Builder.CreateGEP( | |
| CGF, Base.getAddress(), | |
| llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); | |
| LValue NumDepsBase = CGF.MakeAddrLValue( | |
| DepObjAddr, KmpDependInfoTy, Base.getBaseInfo(), Base.getTBAAInfo()); | |
| // NumDeps = deps[i].base_addr; | |
| LValue BaseAddrLVal = CGF.EmitLValueForField( | |
| NumDepsBase, | |
| *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::BaseAddr))); | |
| llvm::Value *NumDeps = CGF.EmitLoadOfScalar(BaseAddrLVal, Loc); | |
| return std::make_pair(NumDeps, Base); | |
| } | |
| static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy, | |
| llvm::PointerUnion<unsigned *, LValue *> Pos, | |
| const OMPTaskDataTy::DependData &Data, | |
| Address DependenciesArray) { | |
| CodeGenModule &CGM = CGF.CGM; | |
| ASTContext &C = CGM.getContext(); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl(); | |
| llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); | |
| OMPIteratorGeneratorScope IteratorScope( | |
| CGF, cast_or_null<OMPIteratorExpr>( | |
| Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts() | |
| : nullptr)); | |
| for (const Expr *E : Data.DepExprs) { | |
| llvm::Value *Addr; | |
| llvm::Value *Size; | |
| // The expression will be a nullptr in the 'omp_all_memory' case. | |
| if (E) { | |
| std::tie(Addr, Size) = getPointerAndSize(CGF, E); | |
| Addr = CGF.Builder.CreatePtrToInt(Addr, CGF.IntPtrTy); | |
| } else { | |
| Addr = llvm::ConstantInt::get(CGF.IntPtrTy, 0); | |
| Size = llvm::ConstantInt::get(CGF.SizeTy, 0); | |
| } | |
| LValue Base; | |
| if (unsigned *P = dyn_cast<unsigned *>(Pos)) { | |
| Base = CGF.MakeAddrLValue( | |
| CGF.Builder.CreateConstGEP(DependenciesArray, *P), KmpDependInfoTy); | |
| } else { | |
| assert(E && "Expected a non-null expression"); | |
| LValue &PosLVal = *cast<LValue *>(Pos); | |
| llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); | |
| Base = CGF.MakeAddrLValue( | |
| CGF.Builder.CreateGEP(CGF, DependenciesArray, Idx), KmpDependInfoTy); | |
| } | |
| // deps[i].base_addr = &<Dependencies[i].second>; | |
| LValue BaseAddrLVal = CGF.EmitLValueForField( | |
| Base, | |
| *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::BaseAddr))); | |
| CGF.EmitStoreOfScalar(Addr, BaseAddrLVal); | |
| // deps[i].len = sizeof(<Dependencies[i].second>); | |
| LValue LenLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::Len))); | |
| CGF.EmitStoreOfScalar(Size, LenLVal); | |
| // deps[i].flags = <Dependencies[i].first>; | |
| RTLDependenceKindTy DepKind = translateDependencyKind(Data.DepKind); | |
| LValue FlagsLVal = CGF.EmitLValueForField( | |
| Base, | |
| *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::Flags))); | |
| CGF.EmitStoreOfScalar( | |
| llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)), | |
| FlagsLVal); | |
| if (unsigned *P = dyn_cast<unsigned *>(Pos)) { | |
| ++(*P); | |
| } else { | |
| LValue &PosLVal = *cast<LValue *>(Pos); | |
| llvm::Value *Idx = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); | |
| Idx = CGF.Builder.CreateNUWAdd(Idx, | |
| llvm::ConstantInt::get(Idx->getType(), 1)); | |
| CGF.EmitStoreOfScalar(Idx, PosLVal); | |
| } | |
| } | |
| } | |
| SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes( | |
| CodeGenFunction &CGF, QualType &KmpDependInfoTy, | |
| const OMPTaskDataTy::DependData &Data) { | |
| assert(Data.DepKind == OMPC_DEPEND_depobj && | |
| "Expected depobj dependency kind."); | |
| SmallVector<llvm::Value *, 4> Sizes; | |
| SmallVector<LValue, 4> SizeLVals; | |
| ASTContext &C = CGF.getContext(); | |
| { | |
| OMPIteratorGeneratorScope IteratorScope( | |
| CGF, cast_or_null<OMPIteratorExpr>( | |
| Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts() | |
| : nullptr)); | |
| for (const Expr *E : Data.DepExprs) { | |
| llvm::Value *NumDeps; | |
| LValue Base; | |
| LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); | |
| std::tie(NumDeps, Base) = | |
| getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); | |
| LValue NumLVal = CGF.MakeAddrLValue( | |
| CGF.CreateMemTempWithoutCast(C.getUIntPtrType(), "depobj.size.addr"), | |
| C.getUIntPtrType()); | |
| CGF.Builder.CreateStore(llvm::ConstantInt::get(CGF.IntPtrTy, 0), | |
| NumLVal.getAddress()); | |
| llvm::Value *PrevVal = CGF.EmitLoadOfScalar(NumLVal, E->getExprLoc()); | |
| llvm::Value *Add = CGF.Builder.CreateNUWAdd(PrevVal, NumDeps); | |
| CGF.EmitStoreOfScalar(Add, NumLVal); | |
| SizeLVals.push_back(NumLVal); | |
| } | |
| } | |
| for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) { | |
| llvm::Value *Size = | |
| CGF.EmitLoadOfScalar(SizeLVals[I], Data.DepExprs[I]->getExprLoc()); | |
| Sizes.push_back(Size); | |
| } | |
| return Sizes; | |
| } | |
| void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF, | |
| QualType &KmpDependInfoTy, | |
| LValue PosLVal, | |
| const OMPTaskDataTy::DependData &Data, | |
| Address DependenciesArray) { | |
| assert(Data.DepKind == OMPC_DEPEND_depobj && | |
| "Expected depobj dependency kind."); | |
| llvm::Value *ElSize = CGF.getTypeSize(KmpDependInfoTy); | |
| { | |
| OMPIteratorGeneratorScope IteratorScope( | |
| CGF, cast_or_null<OMPIteratorExpr>( | |
| Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts() | |
| : nullptr)); | |
| for (const Expr *E : Data.DepExprs) { | |
| llvm::Value *NumDeps; | |
| LValue Base; | |
| LValue DepobjLVal = CGF.EmitLValue(E->IgnoreParenImpCasts()); | |
| std::tie(NumDeps, Base) = | |
| getDepobjElements(CGF, DepobjLVal, E->getExprLoc()); | |
| // memcopy dependency data. | |
| llvm::Value *Size = CGF.Builder.CreateNUWMul( | |
| ElSize, | |
| CGF.Builder.CreateIntCast(NumDeps, CGF.SizeTy, /*isSigned=*/false)); | |
| llvm::Value *Pos = CGF.EmitLoadOfScalar(PosLVal, E->getExprLoc()); | |
| Address DepAddr = CGF.Builder.CreateGEP(CGF, DependenciesArray, Pos); | |
| CGF.Builder.CreateMemCpy(DepAddr, Base.getAddress(), Size); | |
| // Increase pos. | |
| // pos += size; | |
| llvm::Value *Add = CGF.Builder.CreateNUWAdd(Pos, NumDeps); | |
| CGF.EmitStoreOfScalar(Add, PosLVal); | |
| } | |
| } | |
| } | |
| std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause( | |
| CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies, | |
| SourceLocation Loc) { | |
| if (llvm::all_of(Dependencies, [](const OMPTaskDataTy::DependData &D) { | |
| return D.DepExprs.empty(); | |
| })) | |
| return std::make_pair(nullptr, Address::invalid()); | |
| // Process list of dependencies. | |
| ASTContext &C = CGM.getContext(); | |
| Address DependenciesArray = Address::invalid(); | |
| llvm::Value *NumOfElements = nullptr; | |
| unsigned NumDependencies = std::accumulate( | |
| Dependencies.begin(), Dependencies.end(), 0, | |
| [](unsigned V, const OMPTaskDataTy::DependData &D) { | |
| return D.DepKind == OMPC_DEPEND_depobj | |
| ? V | |
| : (V + (D.IteratorExpr ? 0 : D.DepExprs.size())); | |
| }); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| bool HasDepobjDeps = false; | |
| bool HasRegularWithIterators = false; | |
| llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.IntPtrTy, 0); | |
| llvm::Value *NumOfRegularWithIterators = | |
| llvm::ConstantInt::get(CGF.IntPtrTy, 0); | |
| // Calculate number of depobj dependencies and regular deps with the | |
| // iterators. | |
| for (const OMPTaskDataTy::DependData &D : Dependencies) { | |
| if (D.DepKind == OMPC_DEPEND_depobj) { | |
| SmallVector<llvm::Value *, 4> Sizes = | |
| emitDepobjElementsSizes(CGF, KmpDependInfoTy, D); | |
| for (llvm::Value *Size : Sizes) { | |
| NumOfDepobjElements = | |
| CGF.Builder.CreateNUWAdd(NumOfDepobjElements, Size); | |
| } | |
| HasDepobjDeps = true; | |
| continue; | |
| } | |
| // Include number of iterations, if any. | |
| if (const auto *IE = cast_or_null<OMPIteratorExpr>(D.IteratorExpr)) { | |
| llvm::Value *ClauseIteratorSpace = | |
| llvm::ConstantInt::get(CGF.IntPtrTy, 1); | |
| for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) { | |
| llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); | |
| Sz = CGF.Builder.CreateIntCast(Sz, CGF.IntPtrTy, /*isSigned=*/false); | |
| ClauseIteratorSpace = CGF.Builder.CreateNUWMul(Sz, ClauseIteratorSpace); | |
| } | |
| llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul( | |
| ClauseIteratorSpace, | |
| llvm::ConstantInt::get(CGF.IntPtrTy, D.DepExprs.size())); | |
| NumOfRegularWithIterators = | |
| CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps); | |
| HasRegularWithIterators = true; | |
| continue; | |
| } | |
| } | |
| QualType KmpDependInfoArrayTy; | |
| if (HasDepobjDeps || HasRegularWithIterators) { | |
| NumOfElements = llvm::ConstantInt::get(CGM.IntPtrTy, NumDependencies, | |
| /*isSigned=*/false); | |
| if (HasDepobjDeps) { | |
| NumOfElements = | |
| CGF.Builder.CreateNUWAdd(NumOfDepobjElements, NumOfElements); | |
| } | |
| if (HasRegularWithIterators) { | |
| NumOfElements = | |
| CGF.Builder.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements); | |
| } | |
| auto *OVE = new (C) OpaqueValueExpr( | |
| Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0), | |
| VK_PRValue); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE, | |
| RValue::get(NumOfElements)); | |
| KmpDependInfoArrayTy = | |
| C.getVariableArrayType(KmpDependInfoTy, OVE, ArraySizeModifier::Normal, | |
| /*IndexTypeQuals=*/0); | |
| // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy); | |
| // Properly emit variable-sized array. | |
| auto *PD = ImplicitParamDecl::Create(C, KmpDependInfoArrayTy, | |
| ImplicitParamKind::Other); | |
| CGF.EmitVarDecl(*PD); | |
| DependenciesArray = CGF.GetAddrOfLocalVar(PD); | |
| NumOfElements = CGF.Builder.CreateIntCast(NumOfElements, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| } else { | |
| KmpDependInfoArrayTy = C.getConstantArrayType( | |
| KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), nullptr, | |
| ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); | |
| DependenciesArray = | |
| CGF.CreateMemTempWithoutCast(KmpDependInfoArrayTy, ".dep.arr.addr"); | |
| DependenciesArray = CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0); | |
| NumOfElements = llvm::ConstantInt::get(CGM.Int32Ty, NumDependencies, | |
| /*isSigned=*/false); | |
| } | |
| unsigned Pos = 0; | |
| for (const OMPTaskDataTy::DependData &Dep : Dependencies) { | |
| if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr) | |
| continue; | |
| emitDependData(CGF, KmpDependInfoTy, &Pos, Dep, DependenciesArray); | |
| } | |
| // Copy regular dependencies with iterators. | |
| LValue PosLVal = CGF.MakeAddrLValue( | |
| CGF.CreateMemTempWithoutCast(C.getSizeType(), "dep.counter.addr"), | |
| C.getSizeType()); | |
| CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Pos), PosLVal); | |
| for (const OMPTaskDataTy::DependData &Dep : Dependencies) { | |
| if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr) | |
| continue; | |
| emitDependData(CGF, KmpDependInfoTy, &PosLVal, Dep, DependenciesArray); | |
| } | |
| // Copy final depobj arrays without iterators. | |
| if (HasDepobjDeps) { | |
| for (const OMPTaskDataTy::DependData &Dep : Dependencies) { | |
| if (Dep.DepKind != OMPC_DEPEND_depobj) | |
| continue; | |
| emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Dep, DependenciesArray); | |
| } | |
| } | |
| DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| DependenciesArray, CGF.VoidPtrTy, CGF.Int8Ty); | |
| return std::make_pair(NumOfElements, DependenciesArray); | |
| } | |
| Address CGOpenMPRuntime::emitDepobjDependClause( | |
| CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies, | |
| SourceLocation Loc) { | |
| if (Dependencies.DepExprs.empty()) | |
| return Address::invalid(); | |
| // Process list of dependencies. | |
| ASTContext &C = CGM.getContext(); | |
| Address DependenciesArray = Address::invalid(); | |
| unsigned NumDependencies = Dependencies.DepExprs.size(); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl(); | |
| llvm::Value *Size; | |
| // Define type kmp_depend_info[<Dependencies.size()>]; | |
| // For depobj reserve one extra element to store the number of elements. | |
| // It is required to handle depobj(x) update(in) construct. | |
| // kmp_depend_info[<Dependencies.size()>] deps; | |
| llvm::Value *NumDepsVal; | |
| CharUnits Align = C.getTypeAlignInChars(KmpDependInfoTy); | |
| if (const auto *IE = | |
| cast_or_null<OMPIteratorExpr>(Dependencies.IteratorExpr)) { | |
| NumDepsVal = llvm::ConstantInt::get(CGF.SizeTy, 1); | |
| for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) { | |
| llvm::Value *Sz = CGF.EmitScalarExpr(IE->getHelper(I).Upper); | |
| Sz = CGF.Builder.CreateIntCast(Sz, CGF.SizeTy, /*isSigned=*/false); | |
| NumDepsVal = CGF.Builder.CreateNUWMul(NumDepsVal, Sz); | |
| } | |
| Size = CGF.Builder.CreateNUWAdd(llvm::ConstantInt::get(CGF.SizeTy, 1), | |
| NumDepsVal); | |
| CharUnits SizeInBytes = | |
| C.getTypeSizeInChars(KmpDependInfoTy).alignTo(Align); | |
| llvm::Value *RecSize = CGM.getSize(SizeInBytes); | |
| Size = CGF.Builder.CreateNUWMul(Size, RecSize); | |
| NumDepsVal = | |
| CGF.Builder.CreateIntCast(NumDepsVal, CGF.IntPtrTy, /*isSigned=*/false); | |
| } else { | |
| QualType KmpDependInfoArrayTy = C.getConstantArrayType( | |
| KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies + 1), | |
| nullptr, ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); | |
| CharUnits Sz = C.getTypeSizeInChars(KmpDependInfoArrayTy); | |
| Size = CGM.getSize(Sz.alignTo(Align)); | |
| NumDepsVal = llvm::ConstantInt::get(CGF.IntPtrTy, NumDependencies); | |
| } | |
| // Need to allocate on the dynamic memory. | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| // Use default allocator. | |
| llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| llvm::Value *Args[] = {ThreadID, Size, Allocator}; | |
| llvm::Value *Addr = | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_alloc), | |
| Args, ".dep.arr.addr"); | |
| llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(KmpDependInfoTy); | |
| Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Addr, CGF.Builder.getPtrTy(0)); | |
| DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align); | |
| // Write number of elements in the first element of array for depobj. | |
| LValue Base = CGF.MakeAddrLValue(DependenciesArray, KmpDependInfoTy); | |
| // deps[i].base_addr = NumDependencies; | |
| LValue BaseAddrLVal = CGF.EmitLValueForField( | |
| Base, | |
| *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::BaseAddr))); | |
| CGF.EmitStoreOfScalar(NumDepsVal, BaseAddrLVal); | |
| llvm::PointerUnion<unsigned *, LValue *> Pos; | |
| unsigned Idx = 1; | |
| LValue PosLVal; | |
| if (Dependencies.IteratorExpr) { | |
| PosLVal = CGF.MakeAddrLValue( | |
| CGF.CreateMemTempWithoutCast(C.getSizeType(), "iterator.counter.addr"), | |
| C.getSizeType()); | |
| CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGF.SizeTy, Idx), PosLVal, | |
| /*IsInit=*/true); | |
| Pos = &PosLVal; | |
| } else { | |
| Pos = &Idx; | |
| } | |
| emitDependData(CGF, KmpDependInfoTy, Pos, Dependencies, DependenciesArray); | |
| DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateConstGEP(DependenciesArray, 1), CGF.VoidPtrTy, | |
| CGF.Int8Ty); | |
| return DependenciesArray; | |
| } | |
| void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal, | |
| SourceLocation Loc) { | |
| ASTContext &C = CGM.getContext(); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| LValue Base = CGF.EmitLoadOfPointerLValue(DepobjLVal.getAddress(), | |
| C.VoidPtrTy.castAs<PointerType>()); | |
| QualType KmpDependInfoPtrTy = C.getPointerType(KmpDependInfoTy); | |
| Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Base.getAddress(), CGF.ConvertTypeForMem(KmpDependInfoPtrTy), | |
| CGF.ConvertTypeForMem(KmpDependInfoTy)); | |
| llvm::Value *DepObjAddr = CGF.Builder.CreateGEP( | |
| Addr.getElementType(), Addr.emitRawPointer(CGF), | |
| llvm::ConstantInt::get(CGF.IntPtrTy, -1, /*isSigned=*/true)); | |
| DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DepObjAddr, | |
| CGF.VoidPtrTy); | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| // Use default allocator. | |
| llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator}; | |
| // _kmpc_free(gtid, addr, nullptr); | |
| (void)CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_free), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitUpdateDependObjectsClause( | |
| CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind, | |
| SourceLocation Loc) { | |
| ASTContext &C = CGM.getContext(); | |
| QualType FlagsTy; | |
| getDependTypes(C, KmpDependInfoTy, FlagsTy); | |
| auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl(); | |
| llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); | |
| llvm::Value *NumDeps; | |
| LValue Base; | |
| std::tie(NumDeps, Base) = getDepobjElements(CGF, DepobjLVal, Loc); | |
| Address Begin = Base.getAddress(); | |
| // Cast from pointer to array type to pointer to single element. | |
| llvm::Value *End = CGF.Builder.CreateGEP(Begin.getElementType(), | |
| Begin.emitRawPointer(CGF), NumDeps); | |
| // The basic structure here is a while-do loop. | |
| llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.body"); | |
| llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.done"); | |
| llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); | |
| CGF.EmitBlock(BodyBB); | |
| llvm::PHINode *ElementPHI = | |
| CGF.Builder.CreatePHI(Begin.getType(), 2, "omp.elementPast"); | |
| ElementPHI->addIncoming(Begin.emitRawPointer(CGF), EntryBB); | |
| Begin = Begin.withPointer(ElementPHI, KnownNonNull); | |
| Base = CGF.MakeAddrLValue(Begin, KmpDependInfoTy, Base.getBaseInfo(), | |
| Base.getTBAAInfo()); | |
| // deps[i].flags = NewDepKind; | |
| RTLDependenceKindTy DepKind = translateDependencyKind(NewDepKind); | |
| LValue FlagsLVal = CGF.EmitLValueForField( | |
| Base, *std::next(KmpDependInfoRD->field_begin(), | |
| static_cast<unsigned int>(RTLDependInfoFields::Flags))); | |
| CGF.EmitStoreOfScalar( | |
| llvm::ConstantInt::get(LLVMFlagsTy, static_cast<unsigned int>(DepKind)), | |
| FlagsLVal); | |
| // Shift the address forward by one element. | |
| llvm::Value *ElementNext = | |
| CGF.Builder.CreateConstGEP(Begin, /*Index=*/1, "omp.elementNext") | |
| .emitRawPointer(CGF); | |
| ElementPHI->addIncoming(ElementNext, CGF.Builder.GetInsertBlock()); | |
| llvm::Value *IsEmpty = | |
| CGF.Builder.CreateICmpEQ(ElementNext, End, "omp.isempty"); | |
| CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); | |
| // Done. | |
| CGF.EmitBlock(DoneBB, /*IsFinished=*/true); | |
| } | |
| void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| const OMPExecutableDirective &D, | |
| llvm::Function *TaskFunction, | |
| QualType SharedsTy, Address Shareds, | |
| const Expr *IfCond, | |
| const OMPTaskDataTy &Data) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| TaskResultTy Result = | |
| emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); | |
| llvm::Value *NewTask = Result.NewTask; | |
| llvm::Function *TaskEntry = Result.TaskEntry; | |
| llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy; | |
| LValue TDBase = Result.TDBase; | |
| const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD; | |
| // Process list of dependences. | |
| Address DependenciesArray = Address::invalid(); | |
| llvm::Value *NumOfElements; | |
| std::tie(NumOfElements, DependenciesArray) = | |
| emitDependClause(CGF, Data.Dependences, Loc); | |
| // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc() | |
| // libcall. | |
| // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, | |
| // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, | |
| // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence | |
| // list is not empty | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask }; | |
| llvm::Value *DepTaskArgs[7]; | |
| if (!Data.Dependences.empty()) { | |
| DepTaskArgs[0] = UpLoc; | |
| DepTaskArgs[1] = ThreadID; | |
| DepTaskArgs[2] = NewTask; | |
| DepTaskArgs[3] = NumOfElements; | |
| DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF); | |
| DepTaskArgs[5] = CGF.Builder.getInt32(0); | |
| DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| } | |
| auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs, | |
| &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { | |
| if (!Data.Tied) { | |
| auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); | |
| LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); | |
| CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); | |
| } | |
| if (!Data.Dependences.empty()) { | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps), | |
| DepTaskArgs); | |
| } else { | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_omp_task), | |
| TaskArgs); | |
| } | |
| // Check if parent region is untied and build return for untied task; | |
| if (auto *Region = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) | |
| Region->emitUntiedSwitch(CGF); | |
| }; | |
| llvm::Value *DepWaitTaskArgs[7]; | |
| if (!Data.Dependences.empty()) { | |
| DepWaitTaskArgs[0] = UpLoc; | |
| DepWaitTaskArgs[1] = ThreadID; | |
| DepWaitTaskArgs[2] = NumOfElements; | |
| DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); | |
| DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); | |
| DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| DepWaitTaskArgs[6] = | |
| llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); | |
| } | |
| auto &M = CGM.getModule(); | |
| auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy, | |
| TaskEntry, &Data, &DepWaitTaskArgs, | |
| Loc](CodeGenFunction &CGF, PrePostActionTy &) { | |
| CodeGenFunction::RunCleanupsScope LocalScope(CGF); | |
| // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, | |
| // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 | |
| // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info | |
| // is specified. | |
| if (!Data.Dependences.empty()) | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_omp_taskwait_deps_51), | |
| DepWaitTaskArgs); | |
| // Call proxy_task_entry(gtid, new_task); | |
| auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy, | |
| Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; | |
| CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, | |
| OutlinedFnArgs); | |
| }; | |
| // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, | |
| // kmp_task_t *new_task); | |
| // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, | |
| // kmp_task_t *new_task); | |
| RegionCodeGenTy RCG(CodeGen); | |
| CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_omp_task_begin_if0), | |
| TaskArgs, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_omp_task_complete_if0), | |
| TaskArgs); | |
| RCG.setAction(Action); | |
| RCG(CGF); | |
| }; | |
| if (IfCond) { | |
| emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); | |
| } else { | |
| RegionCodeGenTy ThenRCG(ThenCodeGen); | |
| ThenRCG(CGF); | |
| } | |
| } | |
| void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| const OMPLoopDirective &D, | |
| llvm::Function *TaskFunction, | |
| QualType SharedsTy, Address Shareds, | |
| const Expr *IfCond, | |
| const OMPTaskDataTy &Data) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| TaskResultTy Result = | |
| emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); | |
| // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc() | |
| // libcall. | |
| // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int | |
| // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int | |
| // sched, kmp_uint64 grainsize, void *task_dup); | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *IfVal; | |
| if (IfCond) { | |
| IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, | |
| /*isSigned=*/true); | |
| } else { | |
| IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); | |
| } | |
| LValue LBLVal = CGF.EmitLValueForField( | |
| Result.TDBase, | |
| *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound)); | |
| const auto *LBVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl()); | |
| CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(), | |
| /*IsInitializer=*/true); | |
| LValue UBLVal = CGF.EmitLValueForField( | |
| Result.TDBase, | |
| *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound)); | |
| const auto *UBVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl()); | |
| CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(), | |
| /*IsInitializer=*/true); | |
| LValue StLVal = CGF.EmitLValueForField( | |
| Result.TDBase, | |
| *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride)); | |
| const auto *StVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl()); | |
| CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(), | |
| /*IsInitializer=*/true); | |
| // Store reductions address. | |
| LValue RedLVal = CGF.EmitLValueForField( | |
| Result.TDBase, | |
| *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions)); | |
| if (Data.Reductions) { | |
| CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); | |
| } else { | |
| CGF.EmitNullInitialization(RedLVal.getAddress(), | |
| CGF.getContext().VoidPtrTy); | |
| } | |
| enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 }; | |
| llvm::SmallVector<llvm::Value *, 12> TaskArgs{ | |
| UpLoc, | |
| ThreadID, | |
| Result.NewTask, | |
| IfVal, | |
| LBLVal.getPointer(CGF), | |
| UBLVal.getPointer(CGF), | |
| CGF.EmitLoadOfScalar(StLVal, Loc), | |
| llvm::ConstantInt::getSigned( | |
| CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler | |
| llvm::ConstantInt::getSigned( | |
| CGF.IntTy, Data.Schedule.getPointer() | |
| ? Data.Schedule.getInt() ? NumTasks : Grainsize | |
| : NoSchedule), | |
| Data.Schedule.getPointer() | |
| ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, | |
| /*isSigned=*/false) | |
| : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)}; | |
| if (Data.HasModifier) | |
| TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1)); | |
| TaskArgs.push_back(Result.TaskDupFn | |
| ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Result.TaskDupFn, CGF.VoidPtrTy) | |
| : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)); | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), Data.HasModifier | |
| ? OMPRTL___kmpc_taskloop_5 | |
| : OMPRTL___kmpc_taskloop), | |
| TaskArgs); | |
| } | |
| /// Emit reduction operation for each element of array (required for | |
| /// array sections) LHS op = RHS. | |
| /// \param Type Type of array. | |
| /// \param LHSVar Variable on the left side of the reduction operation | |
| /// (references element of array in original variable). | |
| /// \param RHSVar Variable on the right side of the reduction operation | |
| /// (references element of array in original variable). | |
| /// \param RedOpGen Generator of reduction operation with use of LHSVar and | |
| /// RHSVar. | |
| static void EmitOMPAggregateReduction( | |
| CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, | |
| const VarDecl *RHSVar, | |
| const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, | |
| const Expr *, const Expr *)> &RedOpGen, | |
| const Expr *XExpr = nullptr, const Expr *EExpr = nullptr, | |
| const Expr *UpExpr = nullptr) { | |
| // Perform element-by-element initialization. | |
| QualType ElementTy; | |
| Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); | |
| Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); | |
| // Drill down to the base element type on both arrays. | |
| const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); | |
| llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); | |
| llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF); | |
| llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF); | |
| // Cast from pointer to array type to pointer to single element. | |
| llvm::Value *LHSEnd = | |
| CGF.Builder.CreateGEP(LHSAddr.getElementType(), LHSBegin, NumElements); | |
| // The basic structure here is a while-do loop. | |
| llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); | |
| llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); | |
| llvm::Value *IsEmpty = | |
| CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); | |
| CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); | |
| // Enter the loop body, making that address the current address. | |
| llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); | |
| CGF.EmitBlock(BodyBB); | |
| CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); | |
| llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( | |
| RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast"); | |
| RHSElementPHI->addIncoming(RHSBegin, EntryBB); | |
| Address RHSElementCurrent( | |
| RHSElementPHI, RHSAddr.getElementType(), | |
| RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |
| llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( | |
| LHSBegin->getType(), 2, "omp.arraycpy.destElementPast"); | |
| LHSElementPHI->addIncoming(LHSBegin, EntryBB); | |
| Address LHSElementCurrent( | |
| LHSElementPHI, LHSAddr.getElementType(), | |
| LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); | |
| // Emit copy. | |
| CodeGenFunction::OMPPrivateScope Scope(CGF); | |
| Scope.addPrivate(LHSVar, LHSElementCurrent); | |
| Scope.addPrivate(RHSVar, RHSElementCurrent); | |
| Scope.Privatize(); | |
| RedOpGen(CGF, XExpr, EExpr, UpExpr); | |
| Scope.ForceCleanup(); | |
| // Shift the address forward by one element. | |
| llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( | |
| LHSAddr.getElementType(), LHSElementPHI, /*Idx0=*/1, | |
| "omp.arraycpy.dest.element"); | |
| llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( | |
| RHSAddr.getElementType(), RHSElementPHI, /*Idx0=*/1, | |
| "omp.arraycpy.src.element"); | |
| // Check whether we've reached the end. | |
| llvm::Value *Done = | |
| CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); | |
| CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); | |
| LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); | |
| RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); | |
| // Done. | |
| CGF.EmitBlock(DoneBB, /*IsFinished=*/true); | |
| } | |
| /// Emit reduction combiner. If the combiner is a simple expression emit it as | |
| /// is, otherwise consider it as combiner of UDR decl and emit it as a call of | |
| /// UDR combiner function. | |
| static void emitReductionCombiner(CodeGenFunction &CGF, | |
| const Expr *ReductionOp) { | |
| if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) | |
| if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee())) | |
| if (const auto *DRE = | |
| dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts())) | |
| if (const auto *DRD = | |
| dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) { | |
| std::pair<llvm::Function *, llvm::Function *> Reduction = | |
| CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); | |
| RValue Func = RValue::get(Reduction.first); | |
| CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); | |
| CGF.EmitIgnoredExpr(ReductionOp); | |
| return; | |
| } | |
| CGF.EmitIgnoredExpr(ReductionOp); | |
| } | |
| llvm::Function *CGOpenMPRuntime::emitReductionFunction( | |
| StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType, | |
| ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, | |
| ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) { | |
| ASTContext &C = CGM.getContext(); | |
| // void reduction_func(void *LHSArg, void *RHSArg); | |
| auto *LHSArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| auto *RHSArg = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| FunctionArgList Args{LHSArg, RHSArg}; | |
| const auto &CGFI = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| std::string Name = getReductionFuncName(ReducerName); | |
| auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI), | |
| llvm::GlobalValue::InternalLinkage, Name, | |
| &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| Fn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); | |
| // Dst = (void*[n])(LHSArg); | |
| // Src = (void*[n])(RHSArg); | |
| Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(LHSArg)), | |
| CGF.Builder.getPtrTy(0)), | |
| ArgsElemType, CGF.getPointerAlign()); | |
| Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(RHSArg)), | |
| CGF.Builder.getPtrTy(0)), | |
| ArgsElemType, CGF.getPointerAlign()); | |
| // ... | |
| // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); | |
| // ... | |
| CodeGenFunction::OMPPrivateScope Scope(CGF); | |
| const auto *IPriv = Privates.begin(); | |
| unsigned Idx = 0; | |
| for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) { | |
| const auto *RHSVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()); | |
| Scope.addPrivate(RHSVar, emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar)); | |
| const auto *LHSVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()); | |
| Scope.addPrivate(LHSVar, emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar)); | |
| QualType PrivTy = (*IPriv)->getType(); | |
| if (PrivTy->isVariablyModifiedType()) { | |
| // Get array size and emit VLA type. | |
| ++Idx; | |
| Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); | |
| llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); | |
| const VariableArrayType *VLA = | |
| CGF.getContext().getAsVariableArrayType(PrivTy); | |
| const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr()); | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap( | |
| CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); | |
| CGF.EmitVariablyModifiedType(PrivTy); | |
| } | |
| } | |
| Scope.Privatize(); | |
| IPriv = Privates.begin(); | |
| const auto *ILHS = LHSExprs.begin(); | |
| const auto *IRHS = RHSExprs.begin(); | |
| for (const Expr *E : ReductionOps) { | |
| if ((*IPriv)->getType()->isArrayType()) { | |
| // Emit reduction for array section. | |
| const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); | |
| const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); | |
| EmitOMPAggregateReduction( | |
| CGF, (*IPriv)->getType(), LHSVar, RHSVar, | |
| [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { | |
| emitReductionCombiner(CGF, E); | |
| }); | |
| } else { | |
| // Emit reduction for array subscript or single variable. | |
| emitReductionCombiner(CGF, E); | |
| } | |
| ++IPriv; | |
| ++ILHS; | |
| ++IRHS; | |
| } | |
| Scope.ForceCleanup(); | |
| CGF.FinishFunction(); | |
| return Fn; | |
| } | |
| void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, | |
| const Expr *ReductionOp, | |
| const Expr *PrivateRef, | |
| const DeclRefExpr *LHS, | |
| const DeclRefExpr *RHS) { | |
| if (PrivateRef->getType()->isArrayType()) { | |
| // Emit reduction for array section. | |
| const auto *LHSVar = cast<VarDecl>(LHS->getDecl()); | |
| const auto *RHSVar = cast<VarDecl>(RHS->getDecl()); | |
| EmitOMPAggregateReduction( | |
| CGF, PrivateRef->getType(), LHSVar, RHSVar, | |
| [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { | |
| emitReductionCombiner(CGF, ReductionOp); | |
| }); | |
| } else { | |
| // Emit reduction for array subscript or single variable. | |
| emitReductionCombiner(CGF, ReductionOp); | |
| } | |
| } | |
| static std::string generateUniqueName(CodeGenModule &CGM, | |
| llvm::StringRef Prefix, const Expr *Ref); | |
| void CGOpenMPRuntime::emitPrivateReduction( | |
| CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates, | |
| const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) { | |
| // Create a shared global variable (__shared_reduction_var) to accumulate the | |
| // final result. | |
| // | |
| // Call __kmpc_barrier to synchronize threads before initialization. | |
| // | |
| // The master thread (thread_id == 0) initializes __shared_reduction_var | |
| // with the identity value or initializer. | |
| // | |
| // Call __kmpc_barrier to synchronize before combining. | |
| // For each i: | |
| // - Thread enters critical section. | |
| // - Reads its private value from LHSExprs[i]. | |
| // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i], | |
| // Privates[i]). | |
| // - Exits critical section. | |
| // | |
| // Call __kmpc_barrier after combining. | |
| // | |
| // Each thread copies __shared_reduction_var[i] back to RHSExprs[i]. | |
| // | |
| // Final __kmpc_barrier to synchronize after broadcasting | |
| QualType PrivateType = Privates->getType(); | |
| llvm::Type *LLVMType = CGF.ConvertTypeForMem(PrivateType); | |
| const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOps); | |
| std::string ReductionVarNameStr; | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates->IgnoreParenCasts())) | |
| ReductionVarNameStr = | |
| generateUniqueName(CGM, DRE->getDecl()->getNameAsString(), Privates); | |
| else | |
| ReductionVarNameStr = "unnamed_priv_var"; | |
| // Create an internal shared variable | |
| std::string SharedName = | |
| CGM.getOpenMPRuntime().getName({"internal_pivate_", ReductionVarNameStr}); | |
| llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable( | |
| LLVMType, ".omp.reduction." + SharedName); | |
| SharedVar->setAlignment( | |
| llvm::MaybeAlign(CGF.getContext().getTypeAlign(PrivateType) / 8)); | |
| Address SharedResult = | |
| CGF.MakeNaturalAlignRawAddrLValue(SharedVar, PrivateType).getAddress(); | |
| llvm::Value *ThreadId = getThreadID(CGF, Loc); | |
| llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); | |
| llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId}; | |
| llvm::BasicBlock *InitBB = CGF.createBasicBlock("init"); | |
| llvm::BasicBlock *InitEndBB = CGF.createBasicBlock("init.end"); | |
| llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ( | |
| ThreadId, llvm::ConstantInt::get(ThreadId->getType(), 0)); | |
| CGF.Builder.CreateCondBr(IsWorker, InitBB, InitEndBB); | |
| CGF.EmitBlock(InitBB); | |
| auto EmitSharedInit = [&]() { | |
| if (UDR) { // Check if it's a User-Defined Reduction | |
| if (const Expr *UDRInitExpr = UDR->getInitializer()) { | |
| std::pair<llvm::Function *, llvm::Function *> FnPair = | |
| getUserDefinedReduction(UDR); | |
| llvm::Function *InitializerFn = FnPair.second; | |
| if (InitializerFn) { | |
| if (const auto *CE = | |
| dyn_cast<CallExpr>(UDRInitExpr->IgnoreParenImpCasts())) { | |
| const auto *OutDRE = cast<DeclRefExpr>( | |
| cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts()) | |
| ->getSubExpr()); | |
| const VarDecl *OutVD = cast<VarDecl>(OutDRE->getDecl()); | |
| CodeGenFunction::OMPPrivateScope LocalScope(CGF); | |
| LocalScope.addPrivate(OutVD, SharedResult); | |
| (void)LocalScope.Privatize(); | |
| if (const auto *OVE = dyn_cast<OpaqueValueExpr>( | |
| CE->getCallee()->IgnoreParenImpCasts())) { | |
| CodeGenFunction::OpaqueValueMapping OpaqueMap( | |
| CGF, OVE, RValue::get(InitializerFn)); | |
| CGF.EmitIgnoredExpr(CE); | |
| } else { | |
| CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult, | |
| PrivateType.getQualifiers(), | |
| /*IsInitializer=*/true); | |
| } | |
| } else { | |
| CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult, | |
| PrivateType.getQualifiers(), | |
| /*IsInitializer=*/true); | |
| } | |
| } else { | |
| CGF.EmitAnyExprToMem(UDRInitExpr, SharedResult, | |
| PrivateType.getQualifiers(), | |
| /*IsInitializer=*/true); | |
| } | |
| } else { | |
| // EmitNullInitialization handles default construction for C++ classes | |
| // and zeroing for scalars, which is a reasonable default. | |
| CGF.EmitNullInitialization(SharedResult, PrivateType); | |
| } | |
| return; // UDR initialization handled | |
| } | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(Privates)) { | |
| if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) { | |
| if (const Expr *InitExpr = VD->getInit()) { | |
| CGF.EmitAnyExprToMem(InitExpr, SharedResult, | |
| PrivateType.getQualifiers(), true); | |
| return; | |
| } | |
| } | |
| } | |
| CGF.EmitNullInitialization(SharedResult, PrivateType); | |
| }; | |
| EmitSharedInit(); | |
| CGF.Builder.CreateBr(InitEndBB); | |
| CGF.EmitBlock(InitEndBB); | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_barrier), | |
| BarrierArgs); | |
| const Expr *ReductionOp = ReductionOps; | |
| const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp); | |
| LValue SharedLV = CGF.MakeAddrLValue(SharedResult, PrivateType); | |
| LValue LHSLV = CGF.EmitLValue(Privates); | |
| auto EmitCriticalReduction = [&](auto ReductionGen) { | |
| std::string CriticalName = getName({"reduction_critical"}); | |
| emitCriticalRegion(CGF, CriticalName, ReductionGen, Loc); | |
| }; | |
| if (CurrentUDR) { | |
| // Handle user-defined reduction. | |
| auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| std::pair<llvm::Function *, llvm::Function *> FnPair = | |
| getUserDefinedReduction(CurrentUDR); | |
| if (FnPair.first) { | |
| if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) { | |
| const auto *OutDRE = cast<DeclRefExpr>( | |
| cast<UnaryOperator>(CE->getArg(0)->IgnoreParenImpCasts()) | |
| ->getSubExpr()); | |
| const auto *InDRE = cast<DeclRefExpr>( | |
| cast<UnaryOperator>(CE->getArg(1)->IgnoreParenImpCasts()) | |
| ->getSubExpr()); | |
| CodeGenFunction::OMPPrivateScope LocalScope(CGF); | |
| LocalScope.addPrivate(cast<VarDecl>(OutDRE->getDecl()), | |
| SharedLV.getAddress()); | |
| LocalScope.addPrivate(cast<VarDecl>(InDRE->getDecl()), | |
| LHSLV.getAddress()); | |
| (void)LocalScope.Privatize(); | |
| emitReductionCombiner(CGF, ReductionOp); | |
| } | |
| } | |
| }; | |
| EmitCriticalReduction(ReductionGen); | |
| } else { | |
| // Handle built-in reduction operations. | |
| #ifndef NDEBUG | |
| const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts(); | |
| if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr)) | |
| ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts(); | |
| const Expr *AssignRHS = nullptr; | |
| if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) { | |
| if (BinOp->getOpcode() == BO_Assign) | |
| AssignRHS = BinOp->getRHS(); | |
| } else if (const auto *OpCall = | |
| dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) { | |
| if (OpCall->getOperator() == OO_Equal) | |
| AssignRHS = OpCall->getArg(1); | |
| } | |
| assert(AssignRHS && | |
| "Private Variable Reduction : Invalid ReductionOp expression"); | |
| #endif | |
| auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| const auto *OmpOutDRE = | |
| dyn_cast<DeclRefExpr>(LHSExprs->IgnoreParenImpCasts()); | |
| const auto *OmpInDRE = | |
| dyn_cast<DeclRefExpr>(RHSExprs->IgnoreParenImpCasts()); | |
| assert( | |
| OmpOutDRE && OmpInDRE && | |
| "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs"); | |
| const VarDecl *OmpOutVD = cast<VarDecl>(OmpOutDRE->getDecl()); | |
| const VarDecl *OmpInVD = cast<VarDecl>(OmpInDRE->getDecl()); | |
| CodeGenFunction::OMPPrivateScope LocalScope(CGF); | |
| LocalScope.addPrivate(OmpOutVD, SharedLV.getAddress()); | |
| LocalScope.addPrivate(OmpInVD, LHSLV.getAddress()); | |
| (void)LocalScope.Privatize(); | |
| // Emit the actual reduction operation | |
| CGF.EmitIgnoredExpr(ReductionOp); | |
| }; | |
| EmitCriticalReduction(ReductionGen); | |
| } | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_barrier), | |
| BarrierArgs); | |
| // Broadcast final result | |
| bool IsAggregate = PrivateType->isAggregateType(); | |
| LValue SharedLV1 = CGF.MakeAddrLValue(SharedResult, PrivateType); | |
| llvm::Value *FinalResultVal = nullptr; | |
| Address FinalResultAddr = Address::invalid(); | |
| if (IsAggregate) | |
| FinalResultAddr = SharedResult; | |
| else | |
| FinalResultVal = CGF.EmitLoadOfScalar(SharedLV1, Loc); | |
| LValue TargetLHSLV = CGF.EmitLValue(RHSExprs); | |
| if (IsAggregate) { | |
| CGF.EmitAggregateCopy(TargetLHSLV, | |
| CGF.MakeAddrLValue(FinalResultAddr, PrivateType), | |
| PrivateType, AggValueSlot::DoesNotOverlap, false); | |
| } else { | |
| CGF.EmitStoreOfScalar(FinalResultVal, TargetLHSLV); | |
| } | |
| // Final synchronization barrier | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_barrier), | |
| BarrierArgs); | |
| // Combiner with original list item | |
| auto OriginalListCombiner = [&](CodeGenFunction &CGF, | |
| PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| emitSingleReductionCombiner(CGF, ReductionOps, Privates, | |
| cast<DeclRefExpr>(LHSExprs), | |
| cast<DeclRefExpr>(RHSExprs)); | |
| }; | |
| EmitCriticalReduction(OriginalListCombiner); | |
| } | |
| void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, | |
| ArrayRef<const Expr *> OrgPrivates, | |
| ArrayRef<const Expr *> OrgLHSExprs, | |
| ArrayRef<const Expr *> OrgRHSExprs, | |
| ArrayRef<const Expr *> OrgReductionOps, | |
| ReductionOptionsTy Options) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| bool WithNowait = Options.WithNowait; | |
| bool SimpleReduction = Options.SimpleReduction; | |
| // Next code should be emitted for reduction: | |
| // | |
| // static kmp_critical_name lock = { 0 }; | |
| // | |
| // void reduce_func(void *lhs[<n>], void *rhs[<n>]) { | |
| // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]); | |
| // ... | |
| // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1], | |
| // *(Type<n>-1*)rhs[<n>-1]); | |
| // } | |
| // | |
| // ... | |
| // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; | |
| // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), | |
| // RedList, reduce_func, &<lock>)) { | |
| // case 1: | |
| // ... | |
| // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); | |
| // ... | |
| // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); | |
| // break; | |
| // case 2: | |
| // ... | |
| // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); | |
| // ... | |
| // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);] | |
| // break; | |
| // default:; | |
| // } | |
| // | |
| // if SimpleReduction is true, only the next code is generated: | |
| // ... | |
| // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); | |
| // ... | |
| ASTContext &C = CGM.getContext(); | |
| if (SimpleReduction) { | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| const auto *IPriv = OrgPrivates.begin(); | |
| const auto *ILHS = OrgLHSExprs.begin(); | |
| const auto *IRHS = OrgRHSExprs.begin(); | |
| for (const Expr *E : OrgReductionOps) { | |
| emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), | |
| cast<DeclRefExpr>(*IRHS)); | |
| ++IPriv; | |
| ++ILHS; | |
| ++IRHS; | |
| } | |
| return; | |
| } | |
| // Filter out shared reduction variables based on IsPrivateVarReduction flag. | |
| // Only keep entries where the corresponding variable is not private. | |
| SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs, | |
| FilteredRHSExprs, FilteredReductionOps; | |
| for (unsigned I : llvm::seq<unsigned>( | |
| std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) { | |
| if (!Options.IsPrivateVarReduction[I]) { | |
| FilteredPrivates.emplace_back(OrgPrivates[I]); | |
| FilteredLHSExprs.emplace_back(OrgLHSExprs[I]); | |
| FilteredRHSExprs.emplace_back(OrgRHSExprs[I]); | |
| FilteredReductionOps.emplace_back(OrgReductionOps[I]); | |
| } | |
| } | |
| // Wrap filtered vectors in ArrayRef for downstream shared reduction | |
| // processing. | |
| ArrayRef<const Expr *> Privates = FilteredPrivates; | |
| ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs; | |
| ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs; | |
| ArrayRef<const Expr *> ReductionOps = FilteredReductionOps; | |
| // 1. Build a list of reduction variables. | |
| // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; | |
| auto Size = RHSExprs.size(); | |
| for (const Expr *E : Privates) { | |
| if (E->getType()->isVariablyModifiedType()) | |
| // Reserve place for array size. | |
| ++Size; | |
| } | |
| llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size); | |
| QualType ReductionArrayTy = C.getConstantArrayType( | |
| C.VoidPtrTy, ArraySize, nullptr, ArraySizeModifier::Normal, | |
| /*IndexTypeQuals=*/0); | |
| RawAddress ReductionList = | |
| CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); | |
| const auto *IPriv = Privates.begin(); | |
| unsigned Idx = 0; | |
| for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) { | |
| Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); | |
| CGF.Builder.CreateStore( | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), | |
| Elem); | |
| if ((*IPriv)->getType()->isVariablyModifiedType()) { | |
| // Store array size. | |
| ++Idx; | |
| Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); | |
| llvm::Value *Size = CGF.Builder.CreateIntCast( | |
| CGF.getVLASize( | |
| CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) | |
| .NumElts, | |
| CGF.SizeTy, /*isSigned=*/false); | |
| CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), | |
| Elem); | |
| } | |
| } | |
| // 2. Emit reduce_func(). | |
| llvm::Function *ReductionFn = emitReductionFunction( | |
| CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy), | |
| Privates, LHSExprs, RHSExprs, ReductionOps); | |
| // 3. Create static kmp_critical_name lock = { 0 }; | |
| std::string Name = getName({"reduction"}); | |
| llvm::Value *Lock = getCriticalRegionLock(Name); | |
| // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), | |
| // RedList, reduce_func, &<lock>); | |
| llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); | |
| llvm::Value *ThreadId = getThreadID(CGF, Loc); | |
| llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); | |
| llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| ReductionList.getPointer(), CGF.VoidPtrTy); | |
| llvm::Value *Args[] = { | |
| IdentTLoc, // ident_t *<loc> | |
| ThreadId, // i32 <gtid> | |
| CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> | |
| ReductionArrayTySize, // size_type sizeof(RedList) | |
| RL, // void *RedList | |
| ReductionFn, // void (*) (void *, void *) <reduce_func> | |
| Lock // kmp_critical_name *&<lock> | |
| }; | |
| llvm::Value *Res = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), | |
| WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce), | |
| Args); | |
| // 5. Build switch(res) | |
| llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); | |
| llvm::SwitchInst *SwInst = | |
| CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); | |
| // 6. Build case 1: | |
| // ... | |
| // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); | |
| // ... | |
| // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); | |
| // break; | |
| llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); | |
| SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); | |
| CGF.EmitBlock(Case1BB); | |
| // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); | |
| llvm::Value *EndArgs[] = { | |
| IdentTLoc, // ident_t *<loc> | |
| ThreadId, // i32 <gtid> | |
| Lock // kmp_critical_name *&<lock> | |
| }; | |
| auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps]( | |
| CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); | |
| const auto *IPriv = Privates.begin(); | |
| const auto *ILHS = LHSExprs.begin(); | |
| const auto *IRHS = RHSExprs.begin(); | |
| for (const Expr *E : ReductionOps) { | |
| RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), | |
| cast<DeclRefExpr>(*IRHS)); | |
| ++IPriv; | |
| ++ILHS; | |
| ++IRHS; | |
| } | |
| }; | |
| RegionCodeGenTy RCG(CodeGen); | |
| CommonActionTy Action( | |
| nullptr, {}, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait | |
| : OMPRTL___kmpc_end_reduce), | |
| EndArgs); | |
| RCG.setAction(Action); | |
| RCG(CGF); | |
| CGF.EmitBranch(DefaultBB); | |
| // 7. Build case 2: | |
| // ... | |
| // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); | |
| // ... | |
| // break; | |
| llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); | |
| SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); | |
| CGF.EmitBlock(Case2BB); | |
| auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps]( | |
| CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| const auto *ILHS = LHSExprs.begin(); | |
| const auto *IRHS = RHSExprs.begin(); | |
| const auto *IPriv = Privates.begin(); | |
| for (const Expr *E : ReductionOps) { | |
| const Expr *XExpr = nullptr; | |
| const Expr *EExpr = nullptr; | |
| const Expr *UpExpr = nullptr; | |
| BinaryOperatorKind BO = BO_Comma; | |
| if (const auto *BO = dyn_cast<BinaryOperator>(E)) { | |
| if (BO->getOpcode() == BO_Assign) { | |
| XExpr = BO->getLHS(); | |
| UpExpr = BO->getRHS(); | |
| } | |
| } | |
| // Try to emit update expression as a simple atomic. | |
| const Expr *RHSExpr = UpExpr; | |
| if (RHSExpr) { | |
| // Analyze RHS part of the whole expression. | |
| if (const auto *ACO = dyn_cast<AbstractConditionalOperator>( | |
| RHSExpr->IgnoreParenImpCasts())) { | |
| // If this is a conditional operator, analyze its condition for | |
| // min/max reduction operator. | |
| RHSExpr = ACO->getCond(); | |
| } | |
| if (const auto *BORHS = | |
| dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { | |
| EExpr = BORHS->getRHS(); | |
| BO = BORHS->getOpcode(); | |
| } | |
| } | |
| if (XExpr) { | |
| const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); | |
| auto &&AtomicRedGen = [BO, VD, | |
| Loc](CodeGenFunction &CGF, const Expr *XExpr, | |
| const Expr *EExpr, const Expr *UpExpr) { | |
| LValue X = CGF.EmitLValue(XExpr); | |
| RValue E; | |
| if (EExpr) | |
| E = CGF.EmitAnyExpr(EExpr); | |
| CGF.EmitOMPAtomicSimpleUpdateExpr( | |
| X, E, BO, /*IsXLHSInRHSPart=*/true, | |
| llvm::AtomicOrdering::Monotonic, Loc, | |
| [&CGF, UpExpr, VD, Loc](RValue XRValue) { | |
| CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |
| Address LHSTemp = CGF.CreateMemTemp(VD->getType()); | |
| CGF.emitOMPSimpleStore( | |
| CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, | |
| VD->getType().getNonReferenceType(), Loc); | |
| PrivateScope.addPrivate(VD, LHSTemp); | |
| (void)PrivateScope.Privatize(); | |
| return CGF.EmitAnyExpr(UpExpr); | |
| }); | |
| }; | |
| if ((*IPriv)->getType()->isArrayType()) { | |
| // Emit atomic reduction for array section. | |
| const auto *RHSVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); | |
| EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, | |
| AtomicRedGen, XExpr, EExpr, UpExpr); | |
| } else { | |
| // Emit atomic reduction for array subscript or single variable. | |
| AtomicRedGen(CGF, XExpr, EExpr, UpExpr); | |
| } | |
| } else { | |
| // Emit as a critical region. | |
| auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, | |
| const Expr *, const Expr *) { | |
| CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); | |
| std::string Name = RT.getName({"atomic_reduction"}); | |
| RT.emitCriticalRegion( | |
| CGF, Name, | |
| [=](CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| emitReductionCombiner(CGF, E); | |
| }, | |
| Loc); | |
| }; | |
| if ((*IPriv)->getType()->isArrayType()) { | |
| const auto *LHSVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); | |
| const auto *RHSVar = | |
| cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); | |
| EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, | |
| CritRedGen); | |
| } else { | |
| CritRedGen(CGF, nullptr, nullptr, nullptr); | |
| } | |
| } | |
| ++ILHS; | |
| ++IRHS; | |
| ++IPriv; | |
| } | |
| }; | |
| RegionCodeGenTy AtomicRCG(AtomicCodeGen); | |
| if (!WithNowait) { | |
| // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); | |
| llvm::Value *EndArgs[] = { | |
| IdentTLoc, // ident_t *<loc> | |
| ThreadId, // i32 <gtid> | |
| Lock // kmp_critical_name *&<lock> | |
| }; | |
| CommonActionTy Action(nullptr, {}, | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_end_reduce), | |
| EndArgs); | |
| AtomicRCG.setAction(Action); | |
| AtomicRCG(CGF); | |
| } else { | |
| AtomicRCG(CGF); | |
| } | |
| CGF.EmitBranch(DefaultBB); | |
| CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); | |
| assert(OrgLHSExprs.size() == OrgPrivates.size() && | |
| "PrivateVarReduction: Privates size mismatch"); | |
| assert(OrgLHSExprs.size() == OrgReductionOps.size() && | |
| "PrivateVarReduction: ReductionOps size mismatch"); | |
| for (unsigned I : llvm::seq<unsigned>( | |
| std::min(OrgReductionOps.size(), OrgLHSExprs.size()))) { | |
| if (Options.IsPrivateVarReduction[I]) | |
| emitPrivateReduction(CGF, Loc, OrgPrivates[I], OrgLHSExprs[I], | |
| OrgRHSExprs[I], OrgReductionOps[I]); | |
| } | |
| } | |
| /// Generates unique name for artificial threadprivate variables. | |
| /// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>" | |
| static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, | |
| const Expr *Ref) { | |
| SmallString<256> Buffer; | |
| llvm::raw_svector_ostream Out(Buffer); | |
| const clang::DeclRefExpr *DE; | |
| const VarDecl *D = ::getBaseDecl(Ref, DE); | |
| if (!D) { | |
| auto *DRE = cast<DeclRefExpr>(Ref); | |
| if (const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl())) { | |
| // For BindingDecls, use the decomposed declaration as the base. | |
| D = cast<VarDecl>(BD->getDecomposedDecl()); | |
| } else { | |
| D = cast<VarDecl>(DRE->getDecl()); | |
| } | |
| } | |
| D = D->getCanonicalDecl(); | |
| std::string Name = CGM.getOpenMPRuntime().getName( | |
| {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)}); | |
| Out << Prefix << Name << "_" | |
| << D->getCanonicalDecl()->getBeginLoc().getRawEncoding(); | |
| return std::string(Out.str()); | |
| } | |
| /// Emits reduction initializer function: | |
| /// \code | |
| /// void @.red_init(void* %arg, void* %orig) { | |
| /// %0 = bitcast void* %arg to <type>* | |
| /// store <type> <init>, <type>* %0 | |
| /// ret void | |
| /// } | |
| /// \endcode | |
| static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM, | |
| SourceLocation Loc, | |
| ReductionCodeGen &RCG, unsigned N) { | |
| ASTContext &C = CGM.getContext(); | |
| QualType VoidPtrTy = C.VoidPtrTy; | |
| VoidPtrTy.addRestrict(); | |
| FunctionArgList Args; | |
| auto *Param = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| VoidPtrTy, ImplicitParamKind::Other); | |
| auto *ParamOrig = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| VoidPtrTy, ImplicitParamKind::Other); | |
| Args.emplace_back(Param); | |
| Args.emplace_back(ParamOrig); | |
| const auto &FnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""}); | |
| auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, | |
| Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| Fn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); | |
| QualType PrivateType = RCG.getPrivateType(N); | |
| Address PrivateAddr = CGF.EmitLoadOfPointer( | |
| CGF.GetAddrOfLocalVar(Param).withElementType(CGF.Builder.getPtrTy(0)), | |
| C.getPointerType(PrivateType)->castAs<PointerType>()); | |
| llvm::Value *Size = nullptr; | |
| // If the size of the reduction item is non-constant, load it from global | |
| // threadprivate variable. | |
| if (RCG.getSizes(N).second) { | |
| Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( | |
| CGF, CGM.getContext().getSizeType(), | |
| generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); | |
| Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, | |
| CGM.getContext().getSizeType(), Loc); | |
| } | |
| RCG.emitAggregateType(CGF, N, Size); | |
| Address OrigAddr = Address::invalid(); | |
| // If initializer uses initializer from declare reduction construct, emit a | |
| // pointer to the address of the original reduction item (reuired by reduction | |
| // initializer) | |
| if (RCG.usesReductionInitializer(N)) { | |
| Address SharedAddr = CGF.GetAddrOfLocalVar(ParamOrig); | |
| OrigAddr = CGF.EmitLoadOfPointer( | |
| SharedAddr, | |
| CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr()); | |
| } | |
| // Emit the initializer: | |
| // %0 = bitcast void* %arg to <type>* | |
| // store <type> <init>, <type>* %0 | |
| RCG.emitInitialization(CGF, N, PrivateAddr, OrigAddr, | |
| [](CodeGenFunction &) { return false; }); | |
| CGF.FinishFunction(); | |
| return Fn; | |
| } | |
| /// Emits reduction combiner function: | |
| /// \code | |
| /// void @.red_comb(void* %arg0, void* %arg1) { | |
| /// %lhs = bitcast void* %arg0 to <type>* | |
| /// %rhs = bitcast void* %arg1 to <type>* | |
| /// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs) | |
| /// store <type> %2, <type>* %lhs | |
| /// ret void | |
| /// } | |
| /// \endcode | |
| static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM, | |
| SourceLocation Loc, | |
| ReductionCodeGen &RCG, unsigned N, | |
| const Expr *ReductionOp, | |
| const Expr *LHS, const Expr *RHS, | |
| const Expr *PrivateRef) { | |
| ASTContext &C = CGM.getContext(); | |
| const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl()); | |
| const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl()); | |
| FunctionArgList Args; | |
| auto *ParamInOut = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| auto *ParamIn = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| Args.emplace_back(ParamInOut); | |
| Args.emplace_back(ParamIn); | |
| const auto &FnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""}); | |
| auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, | |
| Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| Fn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); | |
| llvm::Value *Size = nullptr; | |
| // If the size of the reduction item is non-constant, load it from global | |
| // threadprivate variable. | |
| if (RCG.getSizes(N).second) { | |
| Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( | |
| CGF, CGM.getContext().getSizeType(), | |
| generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); | |
| Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, | |
| CGM.getContext().getSizeType(), Loc); | |
| } | |
| RCG.emitAggregateType(CGF, N, Size); | |
| // Remap lhs and rhs variables to the addresses of the function arguments. | |
| // %lhs = bitcast void* %arg0 to <type>* | |
| // %rhs = bitcast void* %arg1 to <type>* | |
| CodeGenFunction::OMPPrivateScope PrivateScope(CGF); | |
| PrivateScope.addPrivate( | |
| LHSVD, | |
| // Pull out the pointer to the variable. | |
| CGF.EmitLoadOfPointer( | |
| CGF.GetAddrOfLocalVar(ParamInOut) | |
| .withElementType(CGF.Builder.getPtrTy(0)), | |
| C.getPointerType(LHSVD->getType())->castAs<PointerType>())); | |
| PrivateScope.addPrivate( | |
| RHSVD, | |
| // Pull out the pointer to the variable. | |
| CGF.EmitLoadOfPointer( | |
| CGF.GetAddrOfLocalVar(ParamIn).withElementType( | |
| CGF.Builder.getPtrTy(0)), | |
| C.getPointerType(RHSVD->getType())->castAs<PointerType>())); | |
| PrivateScope.Privatize(); | |
| // Emit the combiner body: | |
| // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs) | |
| // store <type> %2, <type>* %lhs | |
| CGM.getOpenMPRuntime().emitSingleReductionCombiner( | |
| CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), | |
| cast<DeclRefExpr>(RHS)); | |
| CGF.FinishFunction(); | |
| return Fn; | |
| } | |
| /// Emits reduction finalizer function: | |
| /// \code | |
| /// void @.red_fini(void* %arg) { | |
| /// %0 = bitcast void* %arg to <type>* | |
| /// <destroy>(<type>* %0) | |
| /// ret void | |
| /// } | |
| /// \endcode | |
| static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM, | |
| SourceLocation Loc, | |
| ReductionCodeGen &RCG, unsigned N) { | |
| if (!RCG.needCleanups(N)) | |
| return nullptr; | |
| ASTContext &C = CGM.getContext(); | |
| FunctionArgList Args; | |
| auto *Param = | |
| ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, | |
| C.VoidPtrTy, ImplicitParamKind::Other); | |
| Args.emplace_back(Param); | |
| const auto &FnInfo = | |
| CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); | |
| llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); | |
| std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""}); | |
| auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, | |
| Name, &CGM.getModule()); | |
| CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); | |
| if (!CGM.getCodeGenOpts().SampleProfileFile.empty()) | |
| Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected"); | |
| Fn->setDoesNotRecurse(); | |
| CodeGenFunction CGF(CGM); | |
| CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); | |
| Address PrivateAddr = CGF.EmitLoadOfPointer( | |
| CGF.GetAddrOfLocalVar(Param), C.VoidPtrTy.castAs<PointerType>()); | |
| llvm::Value *Size = nullptr; | |
| // If the size of the reduction item is non-constant, load it from global | |
| // threadprivate variable. | |
| if (RCG.getSizes(N).second) { | |
| Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( | |
| CGF, CGM.getContext().getSizeType(), | |
| generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); | |
| Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, | |
| CGM.getContext().getSizeType(), Loc); | |
| } | |
| RCG.emitAggregateType(CGF, N, Size); | |
| // Emit the finalizer body: | |
| // <destroy>(<type>* %0) | |
| RCG.emitCleanups(CGF, N, PrivateAddr); | |
| CGF.FinishFunction(Loc); | |
| return Fn; | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, | |
| ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { | |
| if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) | |
| return nullptr; | |
| // Build typedef struct: | |
| // kmp_taskred_input { | |
| // void *reduce_shar; // shared reduction item | |
| // void *reduce_orig; // original reduction item used for initialization | |
| // size_t reduce_size; // size of data item | |
| // void *reduce_init; // data initialization routine | |
| // void *reduce_fini; // data finalization routine | |
| // void *reduce_comb; // data combiner routine | |
| // kmp_task_red_flags_t flags; // flags for additional info from compiler | |
| // } kmp_taskred_input_t; | |
| ASTContext &C = CGM.getContext(); | |
| RecordDecl *RD = C.buildImplicitRecord("kmp_taskred_input_t"); | |
| RD->startDefinition(); | |
| const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| const FieldDecl *OrigFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType()); | |
| const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); | |
| const FieldDecl *FlagsFD = addFieldToRecordDecl( | |
| C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false)); | |
| RD->completeDefinition(); | |
| CanQualType RDType = C.getCanonicalTagType(RD); | |
| unsigned Size = Data.ReductionVars.size(); | |
| llvm::APInt ArraySize(/*numBits=*/64, Size); | |
| QualType ArrayRDType = | |
| C.getConstantArrayType(RDType, ArraySize, nullptr, | |
| ArraySizeModifier::Normal, /*IndexTypeQuals=*/0); | |
| // kmp_task_red_input_t .rd_input.[Size]; | |
| RawAddress TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); | |
| ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs, | |
| Data.ReductionCopies, Data.ReductionOps); | |
| for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { | |
| // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt]; | |
| llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0), | |
| llvm::ConstantInt::get(CGM.SizeTy, Cnt)}; | |
| llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( | |
| TaskRedInput.getElementType(), TaskRedInput.getPointer(), Idxs, | |
| /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, | |
| ".rd_input.gep."); | |
| LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(GEP, RDType); | |
| // ElemLVal.reduce_shar = &Shareds[Cnt]; | |
| LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); | |
| RCG.emitSharedOrigLValue(CGF, Cnt); | |
| llvm::Value *Shared = RCG.getSharedLValue(Cnt).getPointer(CGF); | |
| CGF.EmitStoreOfScalar(Shared, SharedLVal); | |
| // ElemLVal.reduce_orig = &Origs[Cnt]; | |
| LValue OrigLVal = CGF.EmitLValueForField(ElemLVal, OrigFD); | |
| llvm::Value *Orig = RCG.getOrigLValue(Cnt).getPointer(CGF); | |
| CGF.EmitStoreOfScalar(Orig, OrigLVal); | |
| RCG.emitAggregateType(CGF, Cnt); | |
| llvm::Value *SizeValInChars; | |
| llvm::Value *SizeVal; | |
| std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt); | |
| // We use delayed creation/initialization for VLAs and array sections. It is | |
| // required because runtime does not provide the way to pass the sizes of | |
| // VLAs/array sections to initializer/combiner/finalizer functions. Instead | |
| // threadprivate global variables are used to store these values and use | |
| // them in the functions. | |
| bool DelayedCreation = !!SizeVal; | |
| SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, | |
| /*isSigned=*/false); | |
| LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); | |
| CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); | |
| // ElemLVal.reduce_init = init; | |
| LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); | |
| llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, Cnt); | |
| CGF.EmitStoreOfScalar(InitAddr, InitLVal); | |
| // ElemLVal.reduce_fini = fini; | |
| LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); | |
| llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt); | |
| llvm::Value *FiniAddr = | |
| Fini ? Fini : llvm::ConstantPointerNull::get(CGM.VoidPtrTy); | |
| CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); | |
| // ElemLVal.reduce_comb = comb; | |
| LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); | |
| llvm::Value *CombAddr = emitReduceCombFunction( | |
| CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt], | |
| RHSExprs[Cnt], Data.ReductionCopies[Cnt]); | |
| CGF.EmitStoreOfScalar(CombAddr, CombLVal); | |
| // ElemLVal.flags = 0; | |
| LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); | |
| if (DelayedCreation) { | |
| CGF.EmitStoreOfScalar( | |
| llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true), | |
| FlagsLVal); | |
| } else | |
| CGF.EmitNullInitialization(FlagsLVal.getAddress(), FlagsLVal.getType()); | |
| } | |
| if (Data.IsReductionWithTaskMod) { | |
| // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int | |
| // is_ws, int num, void *data); | |
| llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), | |
| CGM.IntTy, /*isSigned=*/true); | |
| llvm::Value *Args[] = { | |
| IdentTLoc, GTid, | |
| llvm::ConstantInt::get(CGM.IntTy, Data.IsWorksharingReduction ? 1 : 0, | |
| /*isSigned=*/true), | |
| llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true), | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| TaskRedInput.getPointer(), CGM.VoidPtrTy)}; | |
| return CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_taskred_modifier_init), | |
| Args); | |
| } | |
| // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data); | |
| llvm::Value *Args[] = { | |
| CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, | |
| /*isSigned=*/true), | |
| llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true), | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), | |
| CGM.VoidPtrTy)}; | |
| return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_taskred_init), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| bool IsWorksharingReduction) { | |
| // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int | |
| // is_ws, int num, void *data); | |
| llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *GTid = CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), | |
| CGM.IntTy, /*isSigned=*/true); | |
| llvm::Value *Args[] = {IdentTLoc, GTid, | |
| llvm::ConstantInt::get(CGM.IntTy, | |
| IsWorksharingReduction ? 1 : 0, | |
| /*isSigned=*/true)}; | |
| (void)CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_task_reduction_modifier_fini), | |
| Args); | |
| } | |
| void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| ReductionCodeGen &RCG, | |
| unsigned N) { | |
| auto Sizes = RCG.getSizes(N); | |
| // Emit threadprivate global variable if the type is non-constant | |
| // (Sizes.second = nullptr). | |
| if (Sizes.second) { | |
| llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, | |
| /*isSigned=*/false); | |
| Address SizeAddr = getAddrOfArtificialThreadPrivate( | |
| CGF, CGM.getContext().getSizeType(), | |
| generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); | |
| CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); | |
| } | |
| } | |
| Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| llvm::Value *ReductionsPtr, | |
| LValue SharedLVal) { | |
| // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void | |
| // *d); | |
| llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), | |
| CGM.IntTy, | |
| /*isSigned=*/true), | |
| ReductionsPtr, | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; | |
| return Address( | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_task_reduction_get_th_data), | |
| Args), | |
| CGF.Int8Ty, SharedLVal.getAlignment()); | |
| } | |
| void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| const OMPTaskDataTy &Data) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) { | |
| // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder. | |
| OMPBuilder.createTaskwait(CGF.Builder); | |
| } else { | |
| llvm::Value *ThreadID = getThreadID(CGF, Loc); | |
| llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); | |
| auto &M = CGM.getModule(); | |
| Address DependenciesArray = Address::invalid(); | |
| llvm::Value *NumOfElements; | |
| std::tie(NumOfElements, DependenciesArray) = | |
| emitDependClause(CGF, Data.Dependences, Loc); | |
| if (!Data.Dependences.empty()) { | |
| llvm::Value *DepWaitTaskArgs[7]; | |
| DepWaitTaskArgs[0] = UpLoc; | |
| DepWaitTaskArgs[1] = ThreadID; | |
| DepWaitTaskArgs[2] = NumOfElements; | |
| DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF); | |
| DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); | |
| DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| DepWaitTaskArgs[6] = | |
| llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause); | |
| CodeGenFunction::RunCleanupsScope LocalScope(CGF); | |
| // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid, | |
| // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 | |
| // ndeps_noalias, kmp_depend_info_t *noalias_dep_list, | |
| // kmp_int32 has_no_wait); if dependence info is specified. | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| M, OMPRTL___kmpc_omp_taskwait_deps_51), | |
| DepWaitTaskArgs); | |
| } else { | |
| // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 | |
| // global_tid); | |
| llvm::Value *Args[] = {UpLoc, ThreadID}; | |
| // Ignore return result until untied tasks are supported. | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait), | |
| Args); | |
| } | |
| } | |
| if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) | |
| Region->emitUntiedSwitch(CGF); | |
| } | |
| void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, | |
| OpenMPDirectiveKind InnerKind, | |
| const RegionCodeGenTy &CodeGen, | |
| bool HasCancel) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel, | |
| InnerKind != OMPD_critical && | |
| InnerKind != OMPD_master && | |
| InnerKind != OMPD_masked); | |
| CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); | |
| } | |
| namespace { | |
| enum RTCancelKind { | |
| CancelNoreq = 0, | |
| CancelParallel = 1, | |
| CancelLoop = 2, | |
| CancelSections = 3, | |
| CancelTaskgroup = 4 | |
| }; | |
| } // anonymous namespace | |
| static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) { | |
| RTCancelKind CancelKind = CancelNoreq; | |
| if (CancelRegion == OMPD_parallel) | |
| CancelKind = CancelParallel; | |
| else if (CancelRegion == OMPD_for) | |
| CancelKind = CancelLoop; | |
| else if (CancelRegion == OMPD_sections) | |
| CancelKind = CancelSections; | |
| else { | |
| assert(CancelRegion == OMPD_taskgroup); | |
| CancelKind = CancelTaskgroup; | |
| } | |
| return CancelKind; | |
| } | |
| void CGOpenMPRuntime::emitCancellationPointCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| OpenMPDirectiveKind CancelRegion) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 | |
| // global_tid, kmp_int32 cncl_kind); | |
| if (auto *OMPRegionInfo = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { | |
| // For 'cancellation point taskgroup', the task region info may not have a | |
| // cancel. This may instead happen in another adjacent task. | |
| if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) { | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), | |
| CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; | |
| // Ignore return result until untied tasks are supported. | |
| llvm::Value *Result = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_cancellationpoint), | |
| Args); | |
| // if (__kmpc_cancellationpoint()) { | |
| // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only | |
| // exit from construct; | |
| // } | |
| llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); | |
| llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); | |
| llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); | |
| CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); | |
| CGF.EmitBlock(ExitBB); | |
| if (CancelRegion == OMPD_parallel) | |
| emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); | |
| // exit from construct; | |
| CodeGenFunction::JumpDest CancelDest = | |
| CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); | |
| CGF.EmitBranchThroughCleanup(CancelDest); | |
| CGF.EmitBlock(ContBB, /*IsFinished=*/true); | |
| } | |
| } | |
| } | |
| void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| const Expr *IfCond, | |
| OpenMPDirectiveKind CancelRegion) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, | |
| // kmp_int32 cncl_kind); | |
| auto &M = CGM.getModule(); | |
| if (auto *OMPRegionInfo = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { | |
| auto &&ThenGen = [this, &M, Loc, CancelRegion, | |
| OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) { | |
| CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); | |
| llvm::Value *Args[] = { | |
| RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), | |
| CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; | |
| // Ignore return result until untied tasks are supported. | |
| llvm::Value *Result = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args); | |
| // if (__kmpc_cancel()) { | |
| // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only | |
| // exit from construct; | |
| // } | |
| llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); | |
| llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); | |
| llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); | |
| CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); | |
| CGF.EmitBlock(ExitBB); | |
| if (CancelRegion == OMPD_parallel) | |
| RT.emitBarrierCall(CGF, Loc, OMPD_unknown, /*EmitChecks=*/false); | |
| // exit from construct; | |
| CodeGenFunction::JumpDest CancelDest = | |
| CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); | |
| CGF.EmitBranchThroughCleanup(CancelDest); | |
| CGF.EmitBlock(ContBB, /*IsFinished=*/true); | |
| }; | |
| if (IfCond) { | |
| emitIfClause(CGF, IfCond, ThenGen, | |
| [](CodeGenFunction &, PrePostActionTy &) {}); | |
| } else { | |
| RegionCodeGenTy ThenRCG(ThenGen); | |
| ThenRCG(CGF); | |
| } | |
| } | |
| } | |
| namespace { | |
| /// Cleanup action for uses_allocators support. | |
| class OMPUsesAllocatorsActionTy final : public PrePostActionTy { | |
| ArrayRef<std::pair<const Expr *, const Expr *>> Allocators; | |
| const OMPExecutableDirective &D; | |
| bool IsOffloadEntry; | |
| public: | |
| OMPUsesAllocatorsActionTy( | |
| ArrayRef<std::pair<const Expr *, const Expr *>> Allocators, | |
| const OMPExecutableDirective &D, bool IsOffloadEntry) | |
| : Allocators(Allocators), D(D), IsOffloadEntry(IsOffloadEntry) {} | |
| void Enter(CodeGenFunction &CGF) override { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| for (const auto &AllocatorData : Allocators) { | |
| CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit( | |
| CGF, AllocatorData.first, AllocatorData.second); | |
| } | |
| // This kernel does not go through the device-side runtime | |
| // init/deinit sequence (that is GPU-only), but the runtime still | |
| // needs a '<kernel>_kernel_environment' global to know how the | |
| // kernel was configured, so emit it directly here. | |
| if (IsOffloadEntry) | |
| CGF.CGM.getOpenMPRuntime().emitHostKernelEnvironment(D, CGF); | |
| } | |
| void Exit(CodeGenFunction &CGF) override { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| for (const auto &AllocatorData : Allocators) { | |
| CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF, | |
| AllocatorData.first); | |
| } | |
| } | |
| }; | |
| } // namespace | |
| void CGOpenMPRuntime::emitHostKernelEnvironment(const OMPExecutableDirective &D, | |
| CodeGenFunction &CGF) { | |
| llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs Attrs; | |
| Attrs.ExecFlags = llvm::omp::OMPTgtExecModeFlags::OMP_TGT_EXEC_MODE_GENERIC; | |
| computeMinAndMaxThreadsAndTeams(D, CGF, Attrs); | |
| OMPBuilder.emitKernelEnvironment(CGF.Builder, Attrs); | |
| } | |
| StringRef | |
| CGOpenMPRuntime::getTargetRegionParentName(const FunctionDecl *FD) const { | |
| // Regions in a closure are named after its enclosing non-closure function | |
| // (CodeGenFunction::CurFuncDecl), if there is one. | |
| if (const auto *Parent = | |
| dyn_cast_or_null<FunctionDecl>(FD->getNonClosureContext())) | |
| FD = Parent; | |
| if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD)) | |
| return CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete)); | |
| if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(FD)) | |
| return CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete)); | |
| return CGM.getMangledName(GlobalDecl(FD)); | |
| } | |
| void CGOpenMPRuntime::emitTargetOutlinedFunction( | |
| const OMPExecutableDirective &D, StringRef ParentName, | |
| llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, | |
| bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { | |
| assert(!ParentName.empty() && "Invalid target entry parent name!"); | |
| HasEmittedTargetRegion = true; | |
| SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators; | |
| for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) { | |
| for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) { | |
| const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I); | |
| if (!D.AllocatorTraits) | |
| continue; | |
| Allocators.emplace_back(D.Allocator, D.AllocatorTraits); | |
| } | |
| } | |
| OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators, D, IsOffloadEntry); | |
| CodeGen.setAction(UsesAllocatorAction); | |
| emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID, | |
| IsOffloadEntry, CodeGen); | |
| } | |
| void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF, | |
| const Expr *Allocator, | |
| const Expr *AllocatorTraits) { | |
| llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); | |
| ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); | |
| // Use default memspace handle. | |
| llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); | |
| llvm::Value *NumTraits = llvm::ConstantInt::get( | |
| CGF.IntTy, cast<ConstantArrayType>( | |
| AllocatorTraits->getType()->getAsArrayTypeUnsafe()) | |
| ->getSize() | |
| .getLimitedValue()); | |
| LValue AllocatorTraitsLVal = CGF.EmitLValue(AllocatorTraits); | |
| Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| AllocatorTraitsLVal.getAddress(), CGF.VoidPtrPtrTy, CGF.VoidPtrTy); | |
| AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, CGF.getContext().VoidPtrTy, | |
| AllocatorTraitsLVal.getBaseInfo(), | |
| AllocatorTraitsLVal.getTBAAInfo()); | |
| llvm::Value *Traits = Addr.emitRawPointer(CGF); | |
| llvm::Value *AllocatorVal = | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_init_allocator), | |
| {ThreadId, MemSpaceHandle, NumTraits, Traits}); | |
| // Store to allocator. | |
| CGF.EmitAutoVarAlloca(*cast<VarDecl>( | |
| cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl())); | |
| LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); | |
| AllocatorVal = | |
| CGF.EmitScalarConversion(AllocatorVal, CGF.getContext().VoidPtrTy, | |
| Allocator->getType(), Allocator->getExprLoc()); | |
| CGF.EmitStoreOfScalar(AllocatorVal, AllocatorLVal); | |
| } | |
| void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF, | |
| const Expr *Allocator) { | |
| llvm::Value *ThreadId = getThreadID(CGF, Allocator->getExprLoc()); | |
| ThreadId = CGF.Builder.CreateIntCast(ThreadId, CGF.IntTy, /*isSigned=*/true); | |
| LValue AllocatorLVal = CGF.EmitLValue(Allocator->IgnoreParenImpCasts()); | |
| llvm::Value *AllocatorVal = | |
| CGF.EmitLoadOfScalar(AllocatorLVal, Allocator->getExprLoc()); | |
| AllocatorVal = CGF.EmitScalarConversion(AllocatorVal, Allocator->getType(), | |
| CGF.getContext().VoidPtrTy, | |
| Allocator->getExprLoc()); | |
| (void)CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), | |
| OMPRTL___kmpc_destroy_allocator), | |
| {ThreadId, AllocatorVal}); | |
| } | |
| void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams( | |
| const OMPExecutableDirective &D, CodeGenFunction &CGF, | |
| llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) { | |
| assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 && | |
| "invalid default attrs structure"); | |
| int32_t &MaxTeamsVal = Attrs.MaxTeams.front(); | |
| int32_t &MaxThreadsVal = Attrs.MaxThreads.front(); | |
| getNumTeamsExprForTargetDirective(CGF, D, Attrs.MinTeams.front(), | |
| MaxTeamsVal); | |
| getNumThreadsExprForTargetDirective(CGF, D, MaxThreadsVal, | |
| /*UpperBoundOnly=*/true); | |
| for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) { | |
| for (auto *A : C->getAttrs()) { | |
| int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1; | |
| int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1; | |
| if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(A)) | |
| CGM.handleCUDALaunchBoundsAttr(nullptr, Attr, &AttrMaxThreadsVal, | |
| &AttrMinBlocksVal, &AttrMaxBlocksVal); | |
| else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A)) | |
| CGM.handleAMDGPUFlatWorkGroupSizeAttr( | |
| nullptr, Attr, /*ReqdWGS=*/nullptr, &AttrMinThreadsVal, | |
| &AttrMaxThreadsVal); | |
| else | |
| continue; | |
| Attrs.MinThreads.front() = | |
| std::max(Attrs.MinThreads.front(), AttrMinThreadsVal); | |
| if (AttrMaxThreadsVal > 0) | |
| MaxThreadsVal = MaxThreadsVal > 0 | |
| ? std::min(MaxThreadsVal, AttrMaxThreadsVal) | |
| : AttrMaxThreadsVal; | |
| Attrs.MinTeams.front() = | |
| std::max(Attrs.MinTeams.front(), AttrMinBlocksVal); | |
| if (AttrMaxBlocksVal > 0) | |
| MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal) | |
| : AttrMaxBlocksVal; | |
| } | |
| } | |
| } | |
| void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( | |
| const OMPExecutableDirective &D, StringRef ParentName, | |
| llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, | |
| bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { | |
| llvm::TargetRegionEntryInfo EntryInfo = | |
| getEntryInfoFromPresumedLoc(CGM, OMPBuilder, D.getBeginLoc(), ParentName); | |
| CodeGenFunction CGF(CGM, true); | |
| llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction = | |
| [&CGF, &D, &CodeGen, this](StringRef EntryFnName) { | |
| const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target); | |
| CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU()) | |
| return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(CS, D); | |
| return CGF.GenerateOpenMPCapturedStmtFunction(CS, D); | |
| }; | |
| cantFail(OMPBuilder.emitTargetRegionFunction( | |
| EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn, | |
| OutlinedFnID)); | |
| if (!OutlinedFn) | |
| return; | |
| // A target body is entered once, from the kernel, and never re-entered by | |
| // the runtime, so it cannot occur in a cycle. | |
| OutlinedFn->setDoesNotRecurse(); | |
| CGM.getTargetCodeGenInfo().setTargetAttributes(nullptr, OutlinedFn, CGM); | |
| for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) { | |
| for (auto *A : C->getAttrs()) { | |
| if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(A)) | |
| CGM.handleAMDGPUWavesPerEUAttr(OutlinedFn, Attr); | |
| } | |
| } | |
| registerVTable(D); | |
| } | |
| /// Checks if the expression is constant or does not have non-trivial function | |
| /// calls. | |
| static bool isTrivial(ASTContext &Ctx, const Expr * E) { | |
| // We can skip constant expressions. | |
| // We can skip expressions with trivial calls or simple expressions. | |
| return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) || | |
| !E->hasNonTrivialCall(Ctx)) && | |
| !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true); | |
| } | |
| const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx, | |
| const Stmt *Body) { | |
| const Stmt *Child = Body->IgnoreContainers(); | |
| while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) { | |
| Child = nullptr; | |
| for (const Stmt *S : C->body()) { | |
| if (const auto *E = dyn_cast<Expr>(S)) { | |
| if (isTrivial(Ctx, E)) | |
| continue; | |
| } | |
| // Some of the statements can be ignored. | |
| if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) || | |
| isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S)) | |
| continue; | |
| // Analyze declarations. | |
| if (const auto *DS = dyn_cast<DeclStmt>(S)) { | |
| if (llvm::all_of(DS->decls(), [](const Decl *D) { | |
| if (isa<EmptyDecl>(D) || isa<DeclContext>(D) || | |
| isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) || | |
| isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) || | |
| isa<UsingDirectiveDecl>(D) || | |
| isa<OMPDeclareReductionDecl>(D) || | |
| isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D)) | |
| return true; | |
| const auto *VD = dyn_cast<VarDecl>(D); | |
| if (!VD) | |
| return false; | |
| return VD->hasGlobalStorage() || !VD->isUsed(); | |
| })) | |
| continue; | |
| } | |
| // Found multiple children - cannot get the one child only. | |
| if (Child) | |
| return nullptr; | |
| Child = S; | |
| } | |
| if (Child) | |
| Child = Child->IgnoreContainers(); | |
| } | |
| return Child; | |
| } | |
| const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal, | |
| int32_t &MaxTeamsVal) { | |
| OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind(); | |
| assert(isOpenMPTargetExecutionDirective(DirectiveKind) && | |
| "Expected target-based executable directive."); | |
| switch (DirectiveKind) { | |
| case OMPD_target: { | |
| const auto *CS = D.getInnermostCapturedStmt(); | |
| const auto *Body = | |
| CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); | |
| const Stmt *ChildStmt = | |
| CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); | |
| if (const auto *NestedDir = | |
| dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { | |
| if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) { | |
| if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) { | |
| const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>() | |
| ->getNumTeams() | |
| .front(); | |
| if (NumTeams->isIntegerConstantExpr(CGF.getContext())) | |
| if (auto Constant = | |
| NumTeams->getIntegerConstantExpr(CGF.getContext())) | |
| MinTeamsVal = MaxTeamsVal = Constant->getExtValue(); | |
| return NumTeams; | |
| } | |
| MinTeamsVal = MaxTeamsVal = 0; | |
| return nullptr; | |
| } | |
| MinTeamsVal = MaxTeamsVal = 1; | |
| return nullptr; | |
| } | |
| // A value of -1 is used to check if we need to emit no teams region | |
| MinTeamsVal = MaxTeamsVal = -1; | |
| return nullptr; | |
| } | |
| case OMPD_target_teams_loop: | |
| case OMPD_target_teams: | |
| case OMPD_target_teams_distribute: | |
| case OMPD_target_teams_distribute_simd: | |
| case OMPD_target_teams_distribute_parallel_for: | |
| case OMPD_target_teams_distribute_parallel_for_simd: { | |
| if (D.hasClausesOfKind<OMPNumTeamsClause>()) { | |
| const Expr *NumTeams = | |
| D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front(); | |
| if (NumTeams->isIntegerConstantExpr(CGF.getContext())) | |
| if (auto Constant = NumTeams->getIntegerConstantExpr(CGF.getContext())) | |
| MinTeamsVal = MaxTeamsVal = Constant->getExtValue(); | |
| return NumTeams; | |
| } | |
| MinTeamsVal = MaxTeamsVal = 0; | |
| return nullptr; | |
| } | |
| case OMPD_target_parallel: | |
| case OMPD_target_parallel_for: | |
| case OMPD_target_parallel_for_simd: | |
| case OMPD_target_parallel_loop: | |
| case OMPD_target_simd: | |
| case OMPD_target_loop: | |
| MinTeamsVal = MaxTeamsVal = 1; | |
| return nullptr; | |
| case OMPD_parallel: | |
| case OMPD_for: | |
| case OMPD_parallel_for: | |
| case OMPD_parallel_loop: | |
| case OMPD_parallel_master: | |
| case OMPD_parallel_sections: | |
| case OMPD_for_simd: | |
| case OMPD_parallel_for_simd: | |
| case OMPD_cancel: | |
| case OMPD_cancellation_point: | |
| case OMPD_ordered_standalone: | |
| case OMPD_ordered_blockassoc: | |
| case OMPD_threadprivate: | |
| case OMPD_allocate: | |
| case OMPD_task: | |
| case OMPD_simd: | |
| case OMPD_tile: | |
| case OMPD_unroll: | |
| case OMPD_sections: | |
| case OMPD_section: | |
| case OMPD_single: | |
| case OMPD_master: | |
| case OMPD_critical: | |
| case OMPD_taskyield: | |
| case OMPD_barrier: | |
| case OMPD_taskwait: | |
| case OMPD_taskgroup: | |
| case OMPD_atomic: | |
| case OMPD_flush: | |
| case OMPD_depobj: | |
| case OMPD_scan: | |
| case OMPD_teams: | |
| case OMPD_target_data: | |
| case OMPD_target_exit_data: | |
| case OMPD_target_enter_data: | |
| case OMPD_distribute: | |
| case OMPD_distribute_simd: | |
| case OMPD_distribute_parallel_for: | |
| case OMPD_distribute_parallel_for_simd: | |
| case OMPD_teams_distribute: | |
| case OMPD_teams_distribute_simd: | |
| case OMPD_teams_distribute_parallel_for: | |
| case OMPD_teams_distribute_parallel_for_simd: | |
| case OMPD_target_update: | |
| case OMPD_declare_simd: | |
| case OMPD_declare_variant: | |
| case OMPD_begin_declare_variant: | |
| case OMPD_end_declare_variant: | |
| case OMPD_declare_target: | |
| case OMPD_end_declare_target: | |
| case OMPD_declare_reduction: | |
| case OMPD_declare_mapper: | |
| case OMPD_taskloop: | |
| case OMPD_taskloop_simd: | |
| case OMPD_master_taskloop: | |
| case OMPD_master_taskloop_simd: | |
| case OMPD_parallel_master_taskloop: | |
| case OMPD_parallel_master_taskloop_simd: | |
| case OMPD_requires: | |
| case OMPD_metadirective: | |
| case OMPD_unknown: | |
| break; | |
| default: | |
| break; | |
| } | |
| llvm_unreachable("Unexpected directive kind."); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D) { | |
| assert(!CGF.getLangOpts().OpenMPIsTargetDevice && | |
| "Clauses associated with the teams directive expected to be emitted " | |
| "only for the host!"); | |
| CGBuilderTy &Bld = CGF.Builder; | |
| int32_t MinNT = -1, MaxNT = -1; | |
| const Expr *NumTeams = | |
| getNumTeamsExprForTargetDirective(CGF, D, MinNT, MaxNT); | |
| if (NumTeams != nullptr) { | |
| OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind(); | |
| switch (DirectiveKind) { | |
| case OMPD_target: { | |
| const auto *CS = D.getInnermostCapturedStmt(); | |
| CGOpenMPInnerExprInfo CGInfo(CGF, *CS); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, | |
| /*IgnoreResultAssign*/ true); | |
| return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, | |
| /*isSigned=*/true); | |
| } | |
| case OMPD_target_teams: | |
| case OMPD_target_teams_distribute: | |
| case OMPD_target_teams_distribute_simd: | |
| case OMPD_target_teams_distribute_parallel_for: | |
| case OMPD_target_teams_distribute_parallel_for_simd: { | |
| CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); | |
| llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(NumTeams, | |
| /*IgnoreResultAssign*/ true); | |
| return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, | |
| /*isSigned=*/true); | |
| } | |
| default: | |
| break; | |
| } | |
| } | |
| assert(MinNT == MaxNT && "Num threads ranges require handling here."); | |
| return llvm::ConstantInt::getSigned(CGF.Int32Ty, MinNT); | |
| } | |
| /// Merge the thread count upper bound \p Val into \p UpperBound. | |
| /// | |
| /// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once | |
| /// one has been seen whose value is not known at compile time, and otherwise | |
| /// the smallest constant bound found so far. | |
| /// | |
| /// Thread limiting clauses compose by taking the minimum, so a constant bound | |
| /// stays valid whatever the clauses that are not compile time constants | |
| /// evaluate to. That makes it correct to replace the 0 marker with \p Val, and | |
| /// necessary to keep a clause from raising a smaller bound found earlier. | |
| static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) { | |
| UpperBound = UpperBound > 0 ? std::min(UpperBound, Val) : Val; | |
| } | |
| /// Check for a num threads constant value (stored in \p DefaultVal), or | |
| /// expression (stored in \p E). If the value is conditional (via an if-clause), | |
| /// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are | |
| /// nullptr, no expression evaluation is perfomed. | |
| static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, | |
| const Expr **E, int32_t &UpperBound, | |
| bool UpperBoundOnly, llvm::Value **CondVal) { | |
| const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( | |
| CGF.getContext(), CS->getCapturedStmt()); | |
| const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child); | |
| if (!Dir) | |
| return; | |
| if (isOpenMPParallelDirective(Dir->getDirectiveKind())) { | |
| // Handle if clause. If if clause present, the number of threads is | |
| // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1. | |
| if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) { | |
| CGOpenMPInnerExprInfo CGInfo(CGF, *CS); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| const OMPIfClause *IfClause = nullptr; | |
| for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) { | |
| if (C->getNameModifier() == OMPD_unknown || | |
| C->getNameModifier() == OMPD_parallel) { | |
| IfClause = C; | |
| break; | |
| } | |
| } | |
| if (IfClause) { | |
| const Expr *CondExpr = IfClause->getCondition(); | |
| bool Result; | |
| if (CondExpr->EvaluateAsBooleanCondition(Result, CGF.getContext())) { | |
| if (!Result) { | |
| UpperBound = 1; | |
| return; | |
| } | |
| } else { | |
| CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange()); | |
| if (const auto *PreInit = | |
| cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) { | |
| for (const auto *I : PreInit->decls()) { | |
| if (!I->hasAttr<OMPCaptureNoInitAttr>()) { | |
| CGF.EmitVarDecl(cast<VarDecl>(*I)); | |
| } else { | |
| CodeGenFunction::AutoVarEmission Emission = | |
| CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); | |
| CGF.EmitAutoVarCleanups(Emission); | |
| } | |
| } | |
| *CondVal = CGF.EvaluateExprAsBool(CondExpr); | |
| } | |
| } | |
| } | |
| } | |
| // Check the value of num_threads clause iff if clause was not specified | |
| // or is not evaluated to false. | |
| if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) { | |
| CGOpenMPInnerExprInfo CGInfo(CGF, *CS); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| const auto *NumThreadsClause = | |
| Dir->getSingleClause<OMPNumThreadsClause>(); | |
| const Expr *NTExpr = NumThreadsClause->getNumThreads().front(); | |
| if (NTExpr->isIntegerConstantExpr(CGF.getContext())) | |
| if (auto Constant = NTExpr->getIntegerConstantExpr(CGF.getContext())) | |
| mergeThreadCountUpperBound( | |
| UpperBound, static_cast<int32_t>(Constant->getZExtValue())); | |
| // If we haven't found a upper bound, remember we saw a thread limiting | |
| // clause. | |
| if (UpperBound == -1) | |
| UpperBound = 0; | |
| if (!E) | |
| return; | |
| CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange()); | |
| if (const auto *PreInit = | |
| cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) { | |
| for (const auto *I : PreInit->decls()) { | |
| if (!I->hasAttr<OMPCaptureNoInitAttr>()) { | |
| CGF.EmitVarDecl(cast<VarDecl>(*I)); | |
| } else { | |
| CodeGenFunction::AutoVarEmission Emission = | |
| CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); | |
| CGF.EmitAutoVarCleanups(Emission); | |
| } | |
| } | |
| } | |
| *E = NTExpr; | |
| } | |
| return; | |
| } | |
| if (isOpenMPSimdDirective(Dir->getDirectiveKind())) | |
| UpperBound = 1; | |
| } | |
| const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound, | |
| bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) { | |
| assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) && | |
| "Clauses associated with the teams directive expected to be emitted " | |
| "only for the host!"); | |
| OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind(); | |
| assert(isOpenMPTargetExecutionDirective(DirectiveKind) && | |
| "Expected target-based executable directive."); | |
| const Expr *NT = nullptr; | |
| const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT; | |
| auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) { | |
| if (E->isIntegerConstantExpr(CGF.getContext())) { | |
| if (auto Constant = E->getIntegerConstantExpr(CGF.getContext())) | |
| mergeThreadCountUpperBound( | |
| UpperBound, static_cast<int32_t>(Constant->getZExtValue())); | |
| } | |
| // If we haven't found a upper bound, remember we saw a thread limiting | |
| // clause. | |
| if (UpperBound == -1) | |
| UpperBound = 0; | |
| if (EPtr) | |
| *EPtr = E; | |
| }; | |
| auto ReturnSequential = [&]() { | |
| UpperBound = 1; | |
| return NT; | |
| }; | |
| switch (DirectiveKind) { | |
| case OMPD_target: { | |
| const CapturedStmt *CS = D.getInnermostCapturedStmt(); | |
| getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); | |
| const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( | |
| CGF.getContext(), CS->getCapturedStmt()); | |
| // TODO: The standard is not clear how to resolve two thread limit clauses, | |
| // let's pick the teams one if it's present, otherwise the target one. | |
| const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); | |
| if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { | |
| if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) { | |
| ThreadLimitClause = TLC; | |
| if (ThreadLimitExpr) { | |
| CGOpenMPInnerExprInfo CGInfo(CGF, *CS); | |
| CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); | |
| CodeGenFunction::LexicalScope Scope( | |
| CGF, | |
| ThreadLimitClause->getThreadLimit().front()->getSourceRange()); | |
| if (const auto *PreInit = | |
| cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) { | |
| for (const auto *I : PreInit->decls()) { | |
| if (!I->hasAttr<OMPCaptureNoInitAttr>()) { | |
| CGF.EmitVarDecl(cast<VarDecl>(*I)); | |
| } else { | |
| CodeGenFunction::AutoVarEmission Emission = | |
| CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); | |
| CGF.EmitAutoVarCleanups(Emission); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| if (ThreadLimitClause) | |
| CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(), | |
| ThreadLimitExpr); | |
| if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { | |
| if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) && | |
| !isOpenMPDistributeDirective(Dir->getDirectiveKind())) { | |
| CS = Dir->getInnermostCapturedStmt(); | |
| // Now that the 'teams' level has been peeled off, the remainder is | |
| // shaped like a 'target teams' region, so pick up the num_threads of | |
| // the directive nested in it the same way the OMPD_target_teams case | |
| // below does. Without this the upper bound of a construct written as | |
| // 'target' / 'teams' / 'distribute parallel for' would stay at the | |
| // default, while every combined spelling of the same construct honors | |
| // the clause. Only the bound is taken here: passing null for the | |
| // expression and the condition keeps this from emitting anything, so | |
| // the value the host passes to the kernel launch is left as it was. | |
| getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly, | |
| /*CondVal=*/nullptr); | |
| const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( | |
| CGF.getContext(), CS->getCapturedStmt()); | |
| Dir = dyn_cast_or_null<OMPExecutableDirective>(Child); | |
| } | |
| if (Dir && isOpenMPParallelDirective(Dir->getDirectiveKind())) { | |
| CS = Dir->getInnermostCapturedStmt(); | |
| getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); | |
| } else if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind())) | |
| return ReturnSequential(); | |
| } | |
| return NT; | |
| } | |
| case OMPD_target_teams: { | |
| if (D.hasClausesOfKind<OMPThreadLimitClause>()) { | |
| CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); | |
| const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); | |
| CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(), | |
| ThreadLimitExpr); | |
| } | |
| const CapturedStmt *CS = D.getInnermostCapturedStmt(); | |
| getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); | |
| const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( | |
| CGF.getContext(), CS->getCapturedStmt()); | |
| if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { | |
| if (Dir->getDirectiveKind() == OMPD_distribute) { | |
| CS = Dir->getInnermostCapturedStmt(); | |
| getNumThreads(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal); | |
| } | |
| } | |
| return NT; | |
| } | |
| case OMPD_target_teams_distribute: | |
| if (D.hasClausesOfKind<OMPThreadLimitClause>()) { | |
| CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); | |
| const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); | |
| CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(), | |
| ThreadLimitExpr); | |
| } | |
| getNumThreads(CGF, D.getInnermostCapturedStmt(), NTPtr, UpperBound, | |
| UpperBoundOnly, CondVal); | |
| return NT; | |
| case OMPD_target_teams_loop: | |
| case OMPD_target_parallel_loop: | |
| case OMPD_target_parallel: | |
| case OMPD_target_parallel_for: | |
| case OMPD_target_parallel_for_simd: | |
| case OMPD_target_teams_distribute_parallel_for: | |
| case OMPD_target_teams_distribute_parallel_for_simd: { | |
| if (CondVal && D.hasClausesOfKind<OMPIfClause>()) { | |
| const OMPIfClause *IfClause = nullptr; | |
| for (const auto *C : D.getClausesOfKind<OMPIfClause>()) { | |
| if (C->getNameModifier() == OMPD_unknown || | |
| C->getNameModifier() == OMPD_parallel) { | |
| IfClause = C; | |
| break; | |
| } | |
| } | |
| if (IfClause) { | |
| const Expr *Cond = IfClause->getCondition(); | |
| bool Result; | |
| if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { | |
| if (!Result) | |
| return ReturnSequential(); | |
| } else { | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| *CondVal = CGF.EvaluateExprAsBool(Cond); | |
| } | |
| } | |
| } | |
| if (D.hasClausesOfKind<OMPThreadLimitClause>()) { | |
| CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); | |
| const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); | |
| CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(), | |
| ThreadLimitExpr); | |
| } | |
| if (D.hasClausesOfKind<OMPNumThreadsClause>()) { | |
| CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); | |
| const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>(); | |
| CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr); | |
| return NumThreadsClause->getNumThreads().front(); | |
| } | |
| return NT; | |
| } | |
| case OMPD_target_teams_distribute_simd: | |
| case OMPD_target_simd: | |
| case OMPD_target_loop: | |
| return ReturnSequential(); | |
| default: | |
| break; | |
| } | |
| llvm_unreachable("Unsupported directive kind."); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D) { | |
| llvm::Value *NumThreadsVal = nullptr; | |
| llvm::Value *CondVal = nullptr; | |
| llvm::Value *ThreadLimitVal = nullptr; | |
| const Expr *ThreadLimitExpr = nullptr; | |
| int32_t UpperBound = -1; | |
| const Expr *NT = getNumThreadsExprForTargetDirective( | |
| CGF, D, UpperBound, /* UpperBoundOnly */ false, &CondVal, | |
| &ThreadLimitExpr); | |
| // Thread limit expressions are used below, emit them. | |
| if (ThreadLimitExpr) { | |
| ThreadLimitVal = | |
| CGF.EmitScalarExpr(ThreadLimitExpr, /*IgnoreResultAssign=*/true); | |
| ThreadLimitVal = CGF.Builder.CreateIntCast(ThreadLimitVal, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| } | |
| // Generate the num teams expression. | |
| if (UpperBound == 1) { | |
| NumThreadsVal = CGF.Builder.getInt32(UpperBound); | |
| } else if (NT) { | |
| NumThreadsVal = CGF.EmitScalarExpr(NT, /*IgnoreResultAssign=*/true); | |
| NumThreadsVal = CGF.Builder.CreateIntCast(NumThreadsVal, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| } else if (ThreadLimitVal) { | |
| // If we do not have a num threads value but a thread limit, replace the | |
| // former with the latter. We know handled the thread limit expression. | |
| NumThreadsVal = ThreadLimitVal; | |
| ThreadLimitVal = nullptr; | |
| } else { | |
| // Default to "0" which means runtime choice. | |
| assert(!ThreadLimitVal && "Default not applicable with thread limit value"); | |
| NumThreadsVal = CGF.Builder.getInt32(0); | |
| } | |
| // Handle if clause. If if clause present, the number of threads is | |
| // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1. | |
| if (CondVal) { | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| NumThreadsVal = CGF.Builder.CreateSelect(CondVal, NumThreadsVal, | |
| CGF.Builder.getInt32(1)); | |
| } | |
| // If the thread limit and num teams expression were present, take the | |
| // minimum. | |
| if (ThreadLimitVal) { | |
| NumThreadsVal = CGF.Builder.CreateSelect( | |
| CGF.Builder.CreateICmpULT(ThreadLimitVal, NumThreadsVal), | |
| ThreadLimitVal, NumThreadsVal); | |
| } | |
| return NumThreadsVal; | |
| } | |
| namespace { | |
| LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); | |
| // Utility to handle information from clauses associated with a given | |
| // construct that use mappable expressions (e.g. 'map' clause, 'to' clause). | |
| // It provides a convenient interface to obtain the information and generate | |
| // code for that information. | |
| class MappableExprsHandler { | |
| public: | |
| /// Custom comparator for attach-pointer expressions that compares them by | |
| /// complexity (i.e. their component-depth) first, then by the order in which | |
| /// they were computed by collectAttachPtrExprInfo(), if they are semantically | |
| /// different. | |
| struct AttachPtrExprComparator { | |
| const MappableExprsHandler &Handler; | |
| // Cache of previous equality comparison results. | |
| mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool> | |
| CachedEqualityComparisons; | |
| AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {} | |
| AttachPtrExprComparator() = delete; | |
| // Return true iff LHS is "less than" RHS. | |
| bool operator()(const Expr *LHS, const Expr *RHS) const { | |
| if (LHS == RHS) | |
| return false; | |
| // First, compare by complexity (depth) | |
| const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(LHS); | |
| const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(RHS); | |
| std::optional<size_t> DepthLHS = | |
| (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second | |
| : std::nullopt; | |
| std::optional<size_t> DepthRHS = | |
| (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second | |
| : std::nullopt; | |
| // std::nullopt (no attach pointer) has lowest complexity | |
| if (!DepthLHS.has_value() && !DepthRHS.has_value()) { | |
| // Both have same complexity, now check semantic equality | |
| if (areEqual(LHS, RHS)) | |
| return false; | |
| // Different semantically, compare by computation order | |
| return wasComputedBefore(LHS, RHS); | |
| } | |
| if (!DepthLHS.has_value()) | |
| return true; // LHS has lower complexity | |
| if (!DepthRHS.has_value()) | |
| return false; // RHS has lower complexity | |
| // Both have values, compare by depth (lower depth = lower complexity) | |
| if (DepthLHS.value() != DepthRHS.value()) | |
| return DepthLHS.value() < DepthRHS.value(); | |
| // Same complexity, now check semantic equality | |
| if (areEqual(LHS, RHS)) | |
| return false; | |
| // Different semantically, compare by computation order | |
| return wasComputedBefore(LHS, RHS); | |
| } | |
| public: | |
| /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached | |
| /// results, if available, otherwise does a recursive semantic comparison. | |
| bool areEqual(const Expr *LHS, const Expr *RHS) const { | |
| // Check cache first for faster lookup | |
| const auto CachedResultIt = CachedEqualityComparisons.find({LHS, RHS}); | |
| if (CachedResultIt != CachedEqualityComparisons.end()) | |
| return CachedResultIt->second; | |
| bool ComparisonResult = areSemanticallyEqual(LHS, RHS); | |
| // Cache the result for future lookups (both orders since semantic | |
| // equality is commutative) | |
| CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult; | |
| CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult; | |
| return ComparisonResult; | |
| } | |
| /// Compare the two attach-ptr expressions by their computation order. | |
| /// Returns true iff LHS was computed before RHS by | |
| /// collectAttachPtrExprInfo(). | |
| bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const { | |
| const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(LHS); | |
| const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(RHS); | |
| return OrderLHS < OrderRHS; | |
| } | |
| private: | |
| /// Helper function to compare attach-pointer expressions semantically. | |
| /// This function handles various expression types that can be part of an | |
| /// attach-pointer. | |
| /// TODO: Not urgent, but we should ideally return true when comparing | |
| /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc. | |
| bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const { | |
| if (LHS == RHS) | |
| return true; | |
| // If only one is null, they aren't equal | |
| if (!LHS || !RHS) | |
| return false; | |
| ASTContext &Ctx = Handler.CGF.getContext(); | |
| // Strip away parentheses and no-op casts to get to the core expression | |
| LHS = LHS->IgnoreParenNoopCasts(Ctx); | |
| RHS = RHS->IgnoreParenNoopCasts(Ctx); | |
| // Direct pointer comparison of the underlying expressions | |
| if (LHS == RHS) | |
| return true; | |
| // Check if the expression classes match | |
| if (LHS->getStmtClass() != RHS->getStmtClass()) | |
| return false; | |
| // Handle DeclRefExpr (variable references) | |
| if (const auto *LD = dyn_cast<DeclRefExpr>(LHS)) { | |
| const auto *RD = dyn_cast<DeclRefExpr>(RHS); | |
| if (!RD) | |
| return false; | |
| return LD->getDecl()->getCanonicalDecl() == | |
| RD->getDecl()->getCanonicalDecl(); | |
| } | |
| // Handle ArraySubscriptExpr (array indexing like a[i]) | |
| if (const auto *LA = dyn_cast<ArraySubscriptExpr>(LHS)) { | |
| const auto *RA = dyn_cast<ArraySubscriptExpr>(RHS); | |
| if (!RA) | |
| return false; | |
| return areSemanticallyEqual(LA->getBase(), RA->getBase()) && | |
| areSemanticallyEqual(LA->getIdx(), RA->getIdx()); | |
| } | |
| // Handle MemberExpr (member access like s.m or p->m) | |
| if (const auto *LM = dyn_cast<MemberExpr>(LHS)) { | |
| const auto *RM = dyn_cast<MemberExpr>(RHS); | |
| if (!RM) | |
| return false; | |
| if (LM->getMemberDecl()->getCanonicalDecl() != | |
| RM->getMemberDecl()->getCanonicalDecl()) | |
| return false; | |
| return areSemanticallyEqual(LM->getBase(), RM->getBase()); | |
| } | |
| // Handle UnaryOperator (unary operations like *p, &x, etc.) | |
| if (const auto *LU = dyn_cast<UnaryOperator>(LHS)) { | |
| const auto *RU = dyn_cast<UnaryOperator>(RHS); | |
| if (!RU) | |
| return false; | |
| if (LU->getOpcode() != RU->getOpcode()) | |
| return false; | |
| return areSemanticallyEqual(LU->getSubExpr(), RU->getSubExpr()); | |
| } | |
| // Handle BinaryOperator (binary operations like p + offset) | |
| if (const auto *LB = dyn_cast<BinaryOperator>(LHS)) { | |
| const auto *RB = dyn_cast<BinaryOperator>(RHS); | |
| if (!RB) | |
| return false; | |
| if (LB->getOpcode() != RB->getOpcode()) | |
| return false; | |
| return areSemanticallyEqual(LB->getLHS(), RB->getLHS()) && | |
| areSemanticallyEqual(LB->getRHS(), RB->getRHS()); | |
| } | |
| // Handle ArraySectionExpr (array sections like a[0:1]) | |
| // Attach pointers should not contain array-sections, but currently we | |
| // don't emit an error. | |
| if (const auto *LAS = dyn_cast<ArraySectionExpr>(LHS)) { | |
| const auto *RAS = dyn_cast<ArraySectionExpr>(RHS); | |
| if (!RAS) | |
| return false; | |
| return areSemanticallyEqual(LAS->getBase(), RAS->getBase()) && | |
| areSemanticallyEqual(LAS->getLowerBound(), | |
| RAS->getLowerBound()) && | |
| areSemanticallyEqual(LAS->getLength(), RAS->getLength()); | |
| } | |
| // Handle CastExpr (explicit casts) | |
| if (const auto *LC = dyn_cast<CastExpr>(LHS)) { | |
| const auto *RC = dyn_cast<CastExpr>(RHS); | |
| if (!RC) | |
| return false; | |
| if (LC->getCastKind() != RC->getCastKind()) | |
| return false; | |
| return areSemanticallyEqual(LC->getSubExpr(), RC->getSubExpr()); | |
| } | |
| // Handle CXXThisExpr (this pointer) | |
| if (isa<CXXThisExpr>(LHS) && isa<CXXThisExpr>(RHS)) | |
| return true; | |
| // Handle IntegerLiteral (integer constants) | |
| if (const auto *LI = dyn_cast<IntegerLiteral>(LHS)) { | |
| const auto *RI = dyn_cast<IntegerLiteral>(RHS); | |
| if (!RI) | |
| return false; | |
| return LI->getValue() == RI->getValue(); | |
| } | |
| // Handle CharacterLiteral (character constants) | |
| if (const auto *LC = dyn_cast<CharacterLiteral>(LHS)) { | |
| const auto *RC = dyn_cast<CharacterLiteral>(RHS); | |
| if (!RC) | |
| return false; | |
| return LC->getValue() == RC->getValue(); | |
| } | |
| // Handle FloatingLiteral (floating point constants) | |
| if (const auto *LF = dyn_cast<FloatingLiteral>(LHS)) { | |
| const auto *RF = dyn_cast<FloatingLiteral>(RHS); | |
| if (!RF) | |
| return false; | |
| // Use bitwise comparison for floating point literals | |
| return LF->getValue().bitwiseIsEqual(RF->getValue()); | |
| } | |
| // Handle StringLiteral (string constants) | |
| if (const auto *LS = dyn_cast<StringLiteral>(LHS)) { | |
| const auto *RS = dyn_cast<StringLiteral>(RHS); | |
| if (!RS) | |
| return false; | |
| return LS->getString() == RS->getString(); | |
| } | |
| // Handle CXXNullPtrLiteralExpr (nullptr) | |
| if (isa<CXXNullPtrLiteralExpr>(LHS) && isa<CXXNullPtrLiteralExpr>(RHS)) | |
| return true; | |
| // Handle CXXBoolLiteralExpr (true/false) | |
| if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(LHS)) { | |
| const auto *RB = dyn_cast<CXXBoolLiteralExpr>(RHS); | |
| if (!RB) | |
| return false; | |
| return LB->getValue() == RB->getValue(); | |
| } | |
| // Fallback for other forms - use the existing comparison method | |
| return Expr::isSameComparisonOperand(LHS, RHS); | |
| } | |
| }; | |
| /// Get the offset of the OMP_MAP_MEMBER_OF field. | |
| static unsigned getFlagMemberOffset() { | |
| unsigned Offset = 0; | |
| for (uint64_t Remain = | |
| static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>( | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF); | |
| !(Remain & 1); Remain = Remain >> 1) | |
| Offset++; | |
| return Offset; | |
| } | |
| /// Class that holds debugging information for a data mapping to be passed to | |
| /// the runtime library. | |
| class MappingExprInfo { | |
| /// The variable declaration used for the data mapping. | |
| const ValueDecl *MapDecl = nullptr; | |
| /// The original expression used in the map clause, or null if there is | |
| /// none. | |
| const Expr *MapExpr = nullptr; | |
| public: | |
| MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr) | |
| : MapDecl(MapDecl), MapExpr(MapExpr) {} | |
| const ValueDecl *getMapDecl() const { return MapDecl; } | |
| const Expr *getMapExpr() const { return MapExpr; } | |
| }; | |
| using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy; | |
| using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy; | |
| using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy; | |
| using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy; | |
| using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy; | |
| using MapNonContiguousArrayTy = | |
| llvm::OpenMPIRBuilder::MapNonContiguousArrayTy; | |
| using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>; | |
| using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>; | |
| using MapData = | |
| std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef, | |
| OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, | |
| bool /*IsImplicit*/, const ValueDecl *, const Expr *>; | |
| using MapDataArrayTy = SmallVector<MapData, 4>; | |
| /// This structure contains combined information generated for mappable | |
| /// clauses, including base pointers, pointers, sizes, map types, user-defined | |
| /// mappers, and non-contiguous information. | |
| struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy { | |
| MapExprsArrayTy Exprs; | |
| MapValueDeclsArrayTy Mappers; | |
| MapValueDeclsArrayTy DevicePtrDecls; | |
| /// Append arrays in \a CurInfo. | |
| void append(MapCombinedInfoTy &CurInfo) { | |
| Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end()); | |
| DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(), | |
| CurInfo.DevicePtrDecls.end()); | |
| Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end()); | |
| llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo); | |
| } | |
| }; | |
| /// Map between a struct and the its lowest & highest elements which have been | |
| /// mapped. | |
| /// [ValueDecl *] --> {LE(FieldIndex, Pointer), | |
| /// HE(FieldIndex, Pointer)} | |
| struct StructRangeInfoTy { | |
| MapCombinedInfoTy PreliminaryMapData; | |
| std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = { | |
| 0, Address::invalid()}; | |
| std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = { | |
| 0, Address::invalid()}; | |
| Address Base = Address::invalid(); | |
| Address LB = Address::invalid(); | |
| bool IsArraySection = false; | |
| bool HasCompleteRecord = false; | |
| }; | |
| /// A struct to store the attach pointer and pointee information, to be used | |
| /// when emitting an attach entry. | |
| struct AttachInfoTy { | |
| Address AttachPtrAddr = Address::invalid(); | |
| Address AttachPteeAddr = Address::invalid(); | |
| const ValueDecl *AttachPtrDecl = nullptr; | |
| const Expr *AttachMapExpr = nullptr; | |
| bool isValid() const { | |
| return AttachPtrAddr.isValid() && AttachPteeAddr.isValid(); | |
| } | |
| }; | |
| /// Check if there's any component list where the attach pointer expression | |
| /// matches the given captured variable. | |
| bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const { | |
| for (const auto &AttachEntry : AttachPtrExprMap) { | |
| if (AttachEntry.second) { | |
| // Check if the attach pointer expression is a DeclRefExpr that | |
| // references the captured variable | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(AttachEntry.second)) | |
| if (DRE->getDecl() == VD) | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| /// Get the previously-cached attach pointer for a component list, if-any. | |
| const Expr *getAttachPtrExpr( | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components) | |
| const { | |
| const auto It = AttachPtrExprMap.find(Components); | |
| if (It != AttachPtrExprMap.end()) | |
| return It->second; | |
| return nullptr; | |
| } | |
| private: | |
| /// Kind that defines how a device pointer has to be returned. | |
| struct MapInfo { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components; | |
| OpenMPMapClauseKind MapType = OMPC_MAP_unknown; | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers; | |
| ArrayRef<OpenMPMotionModifierKind> MotionModifiers; | |
| bool ReturnDevicePointer = false; | |
| bool IsImplicit = false; | |
| const ValueDecl *Mapper = nullptr; | |
| const Expr *VarRef = nullptr; | |
| bool ForDeviceAddr = false; | |
| bool HasUdpFbNullify = false; | |
| MapInfo() = default; | |
| MapInfo( | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components, | |
| OpenMPMapClauseKind MapType, | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers, | |
| ArrayRef<OpenMPMotionModifierKind> MotionModifiers, | |
| bool ReturnDevicePointer, bool IsImplicit, | |
| const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr, | |
| bool ForDeviceAddr = false, bool HasUdpFbNullify = false) | |
| : Components(Components), MapType(MapType), MapModifiers(MapModifiers), | |
| MotionModifiers(MotionModifiers), | |
| ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit), | |
| Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr), | |
| HasUdpFbNullify(HasUdpFbNullify) {} | |
| }; | |
| /// The target directive from where the mappable clauses were extracted. It | |
| /// is either a executable directive or a user-defined mapper directive. | |
| llvm::PointerUnion<const OMPExecutableDirective *, | |
| const OMPDeclareMapperDecl *> | |
| CurDir; | |
| /// Function the directive is being generated for. | |
| CodeGenFunction &CGF; | |
| /// Set of all first private variables in the current directive. | |
| /// bool data is set to true if the variable is implicitly marked as | |
| /// firstprivate, false otherwise. | |
| llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls; | |
| /// Set of defaultmap clause kinds that use firstprivate behavior. | |
| llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds; | |
| /// Map between device pointer declarations and their expression components. | |
| /// The key value for declarations in 'this' is null. | |
| llvm::DenseMap< | |
| const ValueDecl *, | |
| SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>> | |
| DevPointersMap; | |
| /// Map between device addr declarations and their expression components. | |
| /// The key value for declarations in 'this' is null. | |
| llvm::DenseMap< | |
| const ValueDecl *, | |
| SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>> | |
| HasDevAddrsMap; | |
| /// Map between lambda declarations and their map type. | |
| llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap; | |
| /// Map from component lists to their attach pointer expressions. | |
| llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef, | |
| const Expr *> | |
| AttachPtrExprMap; | |
| /// Map from attach pointer expressions to their component depth. | |
| /// nullptr key has std::nullopt depth. This can be used to order attach-ptr | |
| /// expressions with increasing/decreasing depth. | |
| /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`. | |
| /// TODO: Not urgent, but we should ideally use the number of pointer | |
| /// dereferences in an expr as an indicator of its complexity, instead of the | |
| /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`, | |
| /// `*(p + 5 + 5)` together. | |
| llvm::DenseMap<const Expr *, std::optional<size_t>> | |
| AttachPtrComponentDepthMap = {{nullptr, std::nullopt}}; | |
| /// Map from attach pointer expressions to the order they were computed in, in | |
| /// collectAttachPtrExprInfo(). | |
| llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = { | |
| {nullptr, 0}}; | |
| /// An instance of attach-ptr-expr comparator that can be used throughout the | |
| /// lifetime of this handler. | |
| AttachPtrExprComparator AttachPtrComparator; | |
| llvm::Value *getExprTypeSize(const Expr *E) const { | |
| QualType ExprTy = E->getType().getCanonicalType(); | |
| // Calculate the size for array shaping expression. | |
| if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(E)) { | |
| llvm::Value *Size = | |
| CGF.getTypeSize(OAE->getBase()->getType()->getPointeeType()); | |
| for (const Expr *SE : OAE->getDimensions()) { | |
| llvm::Value *Sz = CGF.EmitScalarExpr(SE); | |
| Sz = CGF.EmitScalarConversion(Sz, SE->getType(), | |
| CGF.getContext().getSizeType(), | |
| SE->getExprLoc()); | |
| Size = CGF.Builder.CreateNUWMul(Size, Sz); | |
| } | |
| return Size; | |
| } | |
| // Reference types are ignored for mapping purposes. | |
| if (const auto *RefTy = ExprTy->getAs<ReferenceType>()) | |
| ExprTy = RefTy->getPointeeType().getCanonicalType(); | |
| // Given that an array section is considered a built-in type, we need to | |
| // do the calculation based on the length of the section instead of relying | |
| // on CGF.getTypeSize(E->getType()). | |
| if (const auto *OAE = dyn_cast<ArraySectionExpr>(E)) { | |
| QualType BaseTy = ArraySectionExpr::getBaseOriginalType( | |
| OAE->getBase()->IgnoreParenImpCasts()) | |
| .getCanonicalType(); | |
| // If there is no length associated with the expression and lower bound is | |
| // not specified too, that means we are using the whole length of the | |
| // base. | |
| if (!OAE->getLength() && OAE->getColonLocFirst().isValid() && | |
| !OAE->getLowerBound()) | |
| return CGF.getTypeSize(BaseTy); | |
| llvm::Value *ElemSize; | |
| if (const auto *PTy = BaseTy->getAs<PointerType>()) { | |
| ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); | |
| } else { | |
| const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr()); | |
| assert(ATy && "Expecting array type if not a pointer type."); | |
| ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); | |
| } | |
| // If we don't have a length at this point, that is because we have an | |
| // array section with a single element. | |
| if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid()) | |
| return ElemSize; | |
| if (const Expr *LenExpr = OAE->getLength()) { | |
| llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); | |
| LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), | |
| CGF.getContext().getSizeType(), | |
| LenExpr->getExprLoc()); | |
| return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); | |
| } | |
| assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() && | |
| OAE->getLowerBound() && "expected array_section[lb:]."); | |
| // Size = sizetype - lb * elemtype; | |
| llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); | |
| llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); | |
| LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), | |
| CGF.getContext().getSizeType(), | |
| OAE->getLowerBound()->getExprLoc()); | |
| LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); | |
| llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); | |
| llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); | |
| LengthVal = CGF.Builder.CreateSelect( | |
| Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); | |
| return LengthVal; | |
| } | |
| return CGF.getTypeSize(ExprTy); | |
| } | |
| /// Return the corresponding bits for a given map clause modifier. Add | |
| /// a flag marking the map as a pointer if requested. Add a flag marking the | |
| /// map as the first one of a series of maps that relate to the same map | |
| /// expression. | |
| OpenMPOffloadMappingFlags getMapTypeBits( | |
| OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers, | |
| ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit, | |
| bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const { | |
| OpenMPOffloadMappingFlags Bits = | |
| IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT | |
| : OpenMPOffloadMappingFlags::OMP_MAP_NONE; | |
| switch (MapType) { | |
| case OMPC_MAP_alloc: | |
| case OMPC_MAP_release: | |
| // alloc and release is the default behavior in the runtime library, i.e. | |
| // if we don't pass any bits alloc/release that is what the runtime is | |
| // going to do. Therefore, we don't need to signal anything for these two | |
| // type modifiers. | |
| break; | |
| case OMPC_MAP_to: | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO; | |
| break; | |
| case OMPC_MAP_from: | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM; | |
| break; | |
| case OMPC_MAP_tofrom: | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO | | |
| OpenMPOffloadMappingFlags::OMP_MAP_FROM; | |
| break; | |
| case OMPC_MAP_delete: | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE; | |
| break; | |
| case OMPC_MAP_unknown: | |
| llvm_unreachable("Unexpected map type!"); | |
| } | |
| if (AddPtrFlag) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; | |
| if (AddIsTargetParamFlag) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; | |
| if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always)) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS; | |
| if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close)) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE; | |
| if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) || | |
| llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present)) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT; | |
| if (llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold)) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD; | |
| if (IsNonContiguous) | |
| Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG; | |
| return Bits; | |
| } | |
| /// Return true if the provided expression is a final array section. A | |
| /// final array section, is one whose length can't be proved to be one. | |
| bool isFinalArraySectionExpression(const Expr *E) const { | |
| const auto *OASE = dyn_cast<ArraySectionExpr>(E); | |
| // It is not an array section and therefore not a unity-size one. | |
| if (!OASE) | |
| return false; | |
| // An array section with no colon always refer to a single element. | |
| if (OASE->getColonLocFirst().isInvalid()) | |
| return false; | |
| const Expr *Length = OASE->getLength(); | |
| // If we don't have a length we have to check if the array has size 1 | |
| // for this dimension. Also, we should always expect a length if the | |
| // base type is pointer. | |
| if (!Length) { | |
| QualType BaseQTy = ArraySectionExpr::getBaseOriginalType( | |
| OASE->getBase()->IgnoreParenImpCasts()) | |
| .getCanonicalType(); | |
| if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr())) | |
| return ATy->getSExtSize() != 1; | |
| // If we don't have a constant dimension length, we have to consider | |
| // the current section as having any size, so it is not necessarily | |
| // unitary. If it happen to be unity size, that's user fault. | |
| return true; | |
| } | |
| // Check if the length evaluates to 1. | |
| Expr::EvalResult Result; | |
| if (!Length->EvaluateAsInt(Result, CGF.getContext())) | |
| return true; // Can have more that size 1. | |
| llvm::APSInt ConstLength = Result.Val.getInt(); | |
| return ConstLength.getSExtValue() != 1; | |
| } | |
| /// Emit an attach entry into \p CombinedInfo, using the information from \p | |
| /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`, | |
| /// an attach entry has the following form: | |
| /// &p, &p[1], sizeof(void*), ATTACH | |
| void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo, | |
| const AttachInfoTy &AttachInfo) const { | |
| assert(AttachInfo.isValid() && | |
| "Expected valid attach pointer/pointee information!"); | |
| // Size is the size of the pointer itself - use pointer size, not BaseDecl | |
| // size | |
| llvm::Value *PointerSize = CGF.Builder.CreateIntCast( | |
| llvm::ConstantInt::get( | |
| CGF.CGM.SizeTy, CGF.getContext() | |
| .getTypeSizeInChars(CGF.getContext().VoidPtrTy) | |
| .getQuantity()), | |
| CGF.Int64Ty, /*isSigned=*/true); | |
| CombinedInfo.Exprs.emplace_back(AttachInfo.AttachPtrDecl, | |
| AttachInfo.AttachMapExpr); | |
| CombinedInfo.BasePointers.push_back( | |
| AttachInfo.AttachPtrAddr.emitRawPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back( | |
| AttachInfo.AttachPteeAddr.emitRawPointer(CGF)); | |
| CombinedInfo.Sizes.push_back(PointerSize); | |
| CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); | |
| // ATTACH entries themselves don't "have" a base attach-ptr. | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| CombinedInfo.NonContigInfo.Dims.push_back(1); | |
| } | |
| /// A helper class to copy structures with overlapped elements, i.e. those | |
| /// which have mappings of both "s" and "s.mem". Consecutive elements that | |
| /// are not explicitly copied have mapping nodes synthesized for them, | |
| /// taking care to avoid generating zero-sized copies. | |
| class CopyOverlappedEntryGaps { | |
| CodeGenFunction &CGF; | |
| MapCombinedInfoTy &CombinedInfo; | |
| OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE; | |
| const ValueDecl *MapDecl = nullptr; | |
| const Expr *MapExpr = nullptr; | |
| Address BP = Address::invalid(); | |
| bool IsNonContiguous = false; | |
| uint64_t DimSize = 0; | |
| // These elements track the position as the struct is iterated over | |
| // (in order of increasing element address). | |
| const RecordDecl *LastParent = nullptr; | |
| uint64_t Cursor = 0; | |
| unsigned LastIndex = -1u; | |
| Address LB = Address::invalid(); | |
| public: | |
| CopyOverlappedEntryGaps(CodeGenFunction &CGF, | |
| MapCombinedInfoTy &CombinedInfo, | |
| OpenMPOffloadMappingFlags Flags, | |
| const ValueDecl *MapDecl, const Expr *MapExpr, | |
| Address BP, Address LB, bool IsNonContiguous, | |
| uint64_t DimSize) | |
| : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl), | |
| MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous), | |
| DimSize(DimSize), LB(LB) {} | |
| void processField( | |
| const OMPClauseMappableExprCommon::MappableComponent &MC, | |
| const FieldDecl *FD, | |
| llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)> | |
| EmitMemberExprBase) { | |
| const RecordDecl *RD = FD->getParent(); | |
| const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD); | |
| uint64_t FieldOffset = RL.getFieldOffset(FD->getFieldIndex()); | |
| uint64_t FieldSize = | |
| CGF.getContext().getTypeSize(FD->getType().getCanonicalType()); | |
| Address ComponentLB = Address::invalid(); | |
| if (FD->getType()->isLValueReferenceType()) { | |
| const auto *ME = cast<MemberExpr>(MC.getAssociatedExpression()); | |
| LValue BaseLVal = EmitMemberExprBase(CGF, ME); | |
| ComponentLB = | |
| CGF.EmitLValueForFieldInitialization(BaseLVal, FD).getAddress(); | |
| } else { | |
| ComponentLB = | |
| CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()).getAddress(); | |
| } | |
| if (!LastParent) | |
| LastParent = RD; | |
| if (FD->getParent() == LastParent) { | |
| if (FD->getFieldIndex() != LastIndex + 1) | |
| copyUntilField(FD, ComponentLB); | |
| } else { | |
| LastParent = FD->getParent(); | |
| if (((int64_t)FieldOffset - (int64_t)Cursor) > 0) | |
| copyUntilField(FD, ComponentLB); | |
| } | |
| Cursor = FieldOffset + FieldSize; | |
| LastIndex = FD->getFieldIndex(); | |
| LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); | |
| } | |
| void copyUntilField(const FieldDecl *FD, Address ComponentLB) { | |
| llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF); | |
| llvm::Value *LBPtr = LB.emitRawPointer(CGF); | |
| llvm::Value *Size = CGF.Builder.CreatePtrDiff(ComponentLBPtr, LBPtr); | |
| copySizedChunk(LBPtr, Size); | |
| } | |
| void copyUntilEnd(Address HB) { | |
| if (LastParent) { | |
| const ASTRecordLayout &RL = | |
| CGF.getContext().getASTRecordLayout(LastParent); | |
| if ((uint64_t)CGF.getContext().toBits(RL.getSize()) <= Cursor) | |
| return; | |
| } | |
| llvm::Value *LBPtr = LB.emitRawPointer(CGF); | |
| llvm::Value *Size = CGF.Builder.CreatePtrDiff( | |
| CGF.Builder.CreateConstGEP(HB, 1).emitRawPointer(CGF), LBPtr); | |
| copySizedChunk(LBPtr, Size); | |
| } | |
| void copySizedChunk(llvm::Value *Base, llvm::Value *Size) { | |
| CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); | |
| CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(Base); | |
| CombinedInfo.Sizes.push_back( | |
| CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/false)); | |
| CombinedInfo.Types.push_back(Flags); | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize : 1); | |
| } | |
| }; | |
| /// Generate the base pointers, section pointers, sizes, map type bits, and | |
| /// user-defined mappers (all included in \a CombinedInfo) for the provided | |
| /// map type, map or motion modifiers, and expression components. | |
| /// \a IsFirstComponent should be set to true if the provided set of | |
| /// components is the first associated with a capture. | |
| void generateInfoForComponentList( | |
| OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers, | |
| ArrayRef<OpenMPMotionModifierKind> MotionModifiers, | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components, | |
| MapCombinedInfoTy &CombinedInfo, | |
| MapCombinedInfoTy &StructBaseCombinedInfo, | |
| StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo, | |
| bool IsFirstComponentList, bool IsImplicit, | |
| bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr, | |
| bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr, | |
| const Expr *MapExpr = nullptr, | |
| ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> | |
| OverlappedElements = {}) const { | |
| // The following summarizes what has to be generated for each map and the | |
| // types below. The generated information is expressed in this order: | |
| // base pointer, section pointer, size, flags | |
| // (to add to the ones that come from the map type and modifier). | |
| // Entries annotated with (+) are only generated for "target" constructs, | |
| // and only if the variable at the beginning of the expression is used in | |
| // the region. | |
| // | |
| // double d; | |
| // int i[100]; | |
| // float *p; | |
| // int **a = &i; | |
| // | |
| // struct S1 { | |
| // int i; | |
| // float f[50]; | |
| // } | |
| // struct S2 { | |
| // int i; | |
| // float f[50]; | |
| // S1 s; | |
| // double *p; | |
| // double *&pref; | |
| // struct S2 *ps; | |
| // int &ref; | |
| // } | |
| // S2 s; | |
| // S2 *ps; | |
| // | |
| // map(d) | |
| // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM | |
| // | |
| // map(i) | |
| // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM | |
| // | |
| // map(i[1:23]) | |
| // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM | |
| // | |
| // map(p) | |
| // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM | |
| // | |
| // map(p[1:24]) | |
| // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee | |
| // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both | |
| // // are present, and either is new | |
| // | |
| // map(([22])p) | |
| // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM | |
| // &p, p, sizeof(void*), ATTACH | |
| // | |
| // map((*a)[0:3]) | |
| // a, a, 0, TARGET_PARAM | IMPLICIT // (+) | |
| // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM | |
| // &(*a), &(*a)[0], sizeof(void*), ATTACH | |
| // (+) Only on target, if a is used in the region | |
| // Note: Since the attach base-pointer is `*a`, which is not a scalar | |
| // variable, it doesn't determine the clause on `a`. `a` is mapped using | |
| // a zero-length-array-section map by generateDefaultMapInfo, if it is | |
| // referenced in the target region, because it is a pointer. | |
| // | |
| // map(**a) | |
| // a, a, 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM | |
| // &(*a), &(*a)[0], sizeof(void*), ATTACH | |
| // (+) Only on target, if a is used in the region | |
| // | |
| // map(s) | |
| // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the | |
| // effect is supposed to be same as if the user had a map for every element | |
| // of the struct. We currently do a shallow-map of s. | |
| // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM | |
| // | |
| // map(s.i) | |
| // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM | |
| // | |
| // map(s.s.f) | |
| // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM | |
| // | |
| // map(s.p) | |
| // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM | |
| // | |
| // map(to: s.p[:22]) | |
| // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+) | |
| // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM | |
| // &(s.p), &(s.p[0]), sizeof(void*), ATTACH | |
| // | |
| // map(to: s.ref) | |
| // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*) | |
| // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***) | |
| // (*) alloc space for struct members, only this is a target parameter. | |
| // (**) map the pointer (nothing to be mapped in this example) (the compiler | |
| // optimizes this entry out, same in the examples below) | |
| // (***) map the pointee (map: to) | |
| // Note: ptr(s.ref) represents the referring pointer of s.ref | |
| // ptee(s.ref) represents the referenced pointee of s.ref | |
| // | |
| // map(to: s.pref) | |
| // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | |
| // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO | |
| // | |
| // map(to: s.pref[:22]) | |
| // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+) | |
| // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO | | |
| // FROM | IMPLICIT // (+) | |
| // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO | |
| // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH | |
| // | |
| // map(s.ps) | |
| // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | |
| // | |
| // map(from: s.ps->s.i) | |
| // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+) | |
| // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM | |
| // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH | |
| // | |
| // map(to: s.ps->ps) | |
| // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+) | |
| // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO | |
| // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH | |
| // | |
| // map(s.ps->ps->ps) | |
| // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+) | |
| // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO | |
| // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH | |
| // | |
| // map(to: s.ps->ps->s.f[:22]) | |
| // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+) | |
| // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO | |
| // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH | |
| // | |
| // map(ps) | |
| // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM | |
| // | |
| // map(ps->i) | |
| // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM | |
| // &ps, &(ps->i), sizeof(void*), ATTACH | |
| // | |
| // map(ps->s.f) | |
| // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM | |
| // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH | |
| // | |
| // map(from: ps->p) | |
| // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM | |
| // &ps, &(ps->p), sizeof(ps), ATTACH | |
| // | |
| // map(to: ps->p[:22]) | |
| // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO | |
| // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH | |
| // | |
| // map(ps->ps) | |
| // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM | |
| // &ps, &(ps->ps), sizeof(ps), ATTACH | |
| // | |
| // map(from: ps->ps->s.i) | |
| // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM | |
| // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH | |
| // | |
| // map(from: ps->ps->ps) | |
| // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM | |
| // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH | |
| // | |
| // map(ps->ps->ps->ps) | |
| // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM | |
| // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH | |
| // | |
| // map(to: ps->ps->ps->s.f[:22]) | |
| // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO | |
| // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH | |
| // | |
| // map(to: s.f[:22]) map(from: s.p[:33]) | |
| // On target, and if s is used in the region: | |
| // | |
| // &s, &(s.f[0]), 50*sizeof(float) + | |
| // sizeof(struct S1) + | |
| // sizeof(double*) (**), TARGET_PARAM | |
| // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO | |
| // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO | | |
| // FROM | IMPLICIT | |
| // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM | |
| // &(s.p), &(s.p[0]), sizeof(void*), ATTACH | |
| // (**) allocate contiguous space needed to fit all mapped members even if | |
| // we allocate space for members not mapped (in this example, | |
| // s.f[22..49] and s.s are not mapped, yet we must allocate space for | |
| // them as well because they fall between &s.f[0] and &s.p) | |
| // | |
| // On other constructs, and, if s is not used in the region, on target: | |
| // &s, &(s.f[0]), 22*sizeof(float), TO | |
| // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM | |
| // &(s.p), &(s.p[0]), sizeof(void*), ATTACH | |
| // | |
| // map(from: s.f[:22]) map(to: ps->p[:33]) | |
| // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM | |
| // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO | |
| // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH | |
| // | |
| // map(from: s.f[:22], s.s) map(to: ps->p[:33]) | |
| // &s, &(s.f[0]), 50*sizeof(float) + | |
| // sizeof(struct S1), TARGET_PARAM | |
| // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM | |
| // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM | |
| // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+) | |
| // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO | |
| // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH | |
| // | |
| // map(p[:100], p) | |
| // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM | |
| // p, &p[0], 100*sizeof(float), TO | FROM | |
| // &p, &p[0], sizeof(float*), ATTACH | |
| // Track if the map information being generated is the first for a capture. | |
| bool IsCaptureFirstInfo = IsFirstComponentList; | |
| // When the variable is on a declare target link or in a to clause with | |
| // unified memory, a reference is needed to hold the host/device address | |
| // of the variable. | |
| bool RequiresReference = false; | |
| // Scan the components from the base to the complete expression. | |
| auto CI = Components.rbegin(); | |
| auto CE = Components.rend(); | |
| auto I = CI; | |
| // Track if the map information being generated is the first for a list of | |
| // components. | |
| bool IsExpressionFirstInfo = true; | |
| bool FirstPointerInComplexData = false; | |
| Address BP = Address::invalid(); | |
| Address FinalLowestElem = Address::invalid(); | |
| const Expr *AssocExpr = I->getAssociatedExpression(); | |
| const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr); | |
| const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); | |
| const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr); | |
| // Get the pointer-attachment base-pointer for the given list, if any. | |
| const Expr *AttachPtrExpr = getAttachPtrExpr(Components); | |
| auto [AttachPtrAddr, AttachPteeBaseAddr] = | |
| getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF); | |
| bool HasAttachPtr = AttachPtrExpr != nullptr; | |
| bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr; | |
| bool SeenAttachPtr = FirstComponentIsForAttachPtr; | |
| if (FirstComponentIsForAttachPtr) { | |
| // No need to process AttachPtr here. It will be processed at the end | |
| // after we have computed the pointee's address. | |
| ++I; | |
| } else if (isa<MemberExpr>(AssocExpr)) { | |
| // The base is the 'this' pointer. The content of the pointer is going | |
| // to be the base of the field being mapped. | |
| BP = CGF.LoadCXXThisAddress(); | |
| } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) || | |
| (OASE && | |
| isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) { | |
| BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); | |
| } else if (OAShE && | |
| isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) { | |
| BP = Address( | |
| CGF.EmitScalarExpr(OAShE->getBase()), | |
| CGF.ConvertTypeForMem(OAShE->getBase()->getType()->getPointeeType()), | |
| CGF.getContext().getTypeAlignInChars(OAShE->getBase()->getType())); | |
| } else { | |
| // The base is the reference to the variable. | |
| // BP = &Var. | |
| BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(); | |
| if (const auto *VD = | |
| dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) { | |
| if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = | |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) { | |
| if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) || | |
| ((*Res == OMPDeclareTargetDeclAttr::MT_To || | |
| *Res == OMPDeclareTargetDeclAttr::MT_Enter) && | |
| CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { | |
| RequiresReference = true; | |
| BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); | |
| } | |
| } | |
| } | |
| // If the variable is a pointer and is being dereferenced (i.e. is not | |
| // the last component), the base has to be the pointer itself, not its | |
| // reference. References are ignored for mapping purposes. | |
| QualType Ty = | |
| I->getAssociatedDeclaration()->getType().getNonReferenceType(); | |
| if (Ty->isAnyPointerType() && std::next(I) != CE) { | |
| // No need to generate individual map information for the pointer, it | |
| // can be associated with the combined storage if shared memory mode is | |
| // active or the base declaration is not global variable. | |
| const auto *VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration()); | |
| if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || | |
| !VD || VD->hasLocalStorage() || HasAttachPtr) | |
| BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); | |
| else | |
| FirstPointerInComplexData = true; | |
| ++I; | |
| } | |
| } | |
| // Track whether a component of the list should be marked as MEMBER_OF some | |
| // combined entry (for partial structs). Only the first PTR_AND_OBJ entry | |
| // in a component list should be marked as MEMBER_OF, all subsequent entries | |
| // do not belong to the base struct. E.g. | |
| // struct S2 s; | |
| // s.ps->ps->ps->f[:] | |
| // (1) (2) (3) (4) | |
| // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a | |
| // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3) | |
| // is the pointee of ps(2) which is not member of struct s, so it should not | |
| // be marked as such (it is still PTR_AND_OBJ). | |
| // The variable is initialized to false so that PTR_AND_OBJ entries which | |
| // are not struct members are not considered (e.g. array of pointers to | |
| // data). | |
| bool ShouldBeMemberOf = false; | |
| // Variable keeping track of whether or not we have encountered a component | |
| // in the component list which is a member expression. Useful when we have a | |
| // pointer or a final array section, in which case it is the previous | |
| // component in the list which tells us whether we have a member expression. | |
| // E.g. X.f[:] | |
| // While processing the final array section "[:]" it is "f" which tells us | |
| // whether we are dealing with a member of a declared struct. | |
| const MemberExpr *EncounteredME = nullptr; | |
| // Track for the total number of dimension. Start from one for the dummy | |
| // dimension. | |
| uint64_t DimSize = 1; | |
| // Detects non-contiguous updates due to strided accesses. | |
| // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set | |
| // correctly when generating information to be passed to the runtime. The | |
| // flag is set to true if any array section has a stride not equal to 1, or | |
| // if the stride is not a constant expression (conservatively assumed | |
| // non-contiguous). | |
| bool IsNonContiguous = | |
| CombinedInfo.NonContigInfo.IsNonContiguous || | |
| any_of(Components, [&](const auto &Component) { | |
| const auto *OASE = | |
| dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression()); | |
| if (!OASE) | |
| return false; | |
| const Expr *StrideExpr = OASE->getStride(); | |
| if (!StrideExpr) | |
| return false; | |
| assert(StrideExpr->getType()->isIntegerType() && | |
| "Stride expression must be of integer type"); | |
| // If stride is not evaluatable as a constant, treat as | |
| // non-contiguous. | |
| const auto Constant = | |
| StrideExpr->getIntegerConstantExpr(CGF.getContext()); | |
| if (!Constant) | |
| return true; | |
| // Treat non-unitary strides as non-contiguous. | |
| return !Constant->isOne(); | |
| }); | |
| bool IsPrevMemberReference = false; | |
| bool IsPartialMapped = | |
| !PartialStruct.PreliminaryMapData.BasePointers.empty(); | |
| // We need to check if we will be encountering any MEs. If we do not | |
| // encounter any ME expression it means we will be mapping the whole struct. | |
| // In that case we need to skip adding an entry for the struct to the | |
| // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo | |
| // list only when generating all info for clauses. | |
| bool IsMappingWholeStruct = true; | |
| if (!GenerateAllInfoForClauses) { | |
| IsMappingWholeStruct = false; | |
| } else { | |
| for (auto TempI = I; TempI != CE; ++TempI) { | |
| const MemberExpr *PossibleME = | |
| dyn_cast<MemberExpr>(TempI->getAssociatedExpression()); | |
| if (PossibleME) { | |
| IsMappingWholeStruct = false; | |
| break; | |
| } | |
| } | |
| } | |
| bool SeenFirstNonBinOpExprAfterAttachPtr = false; | |
| for (; I != CE; ++I) { | |
| // If we have a valid attach-ptr, we skip processing all components until | |
| // after the attach-ptr. | |
| if (HasAttachPtr && !SeenAttachPtr) { | |
| SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr; | |
| continue; | |
| } | |
| // After finding the attach pointer, skip binary-ops, to skip past | |
| // expressions like (p + 10), for a map like map(*(p + 10)), where p is | |
| // the attach-ptr. | |
| if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) { | |
| const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression()); | |
| if (BO) | |
| continue; | |
| // Found the first non-binary-operator component after attach | |
| SeenFirstNonBinOpExprAfterAttachPtr = true; | |
| BP = AttachPteeBaseAddr; | |
| } | |
| // If the current component is member of a struct (parent struct) mark it. | |
| if (!EncounteredME) { | |
| EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression()); | |
| // If we encounter a PTR_AND_OBJ entry from now on it should be marked | |
| // as MEMBER_OF the parent struct. | |
| if (EncounteredME) { | |
| ShouldBeMemberOf = true; | |
| // Do not emit as complex pointer if this is actually not array-like | |
| // expression. | |
| if (FirstPointerInComplexData) { | |
| QualType Ty = std::prev(I) | |
| ->getAssociatedDeclaration() | |
| ->getType() | |
| .getNonReferenceType(); | |
| BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); | |
| FirstPointerInComplexData = false; | |
| } | |
| } | |
| } | |
| auto Next = std::next(I); | |
| // We need to generate the addresses and sizes if this is the last | |
| // component, if the component is a pointer or if it is an array section | |
| // whose length can't be proved to be one. If this is a pointer, it | |
| // becomes the base address for the following components. | |
| // A final array section, is one whose length can't be proved to be one. | |
| // If the map item is non-contiguous then we don't treat any array section | |
| // as final array section. | |
| bool IsFinalArraySection = | |
| !IsNonContiguous && | |
| isFinalArraySectionExpression(I->getAssociatedExpression()); | |
| // If we have a declaration for the mapping use that, otherwise use | |
| // the base declaration of the map clause. | |
| const ValueDecl *MapDecl = (I->getAssociatedDeclaration()) | |
| ? I->getAssociatedDeclaration() | |
| : BaseDecl; | |
| MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression() | |
| : MapExpr; | |
| // Get information on whether the element is a pointer. Have to do a | |
| // special treatment for array sections given that they are built-in | |
| // types. | |
| const auto *OASE = | |
| dyn_cast<ArraySectionExpr>(I->getAssociatedExpression()); | |
| const auto *OAShE = | |
| dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression()); | |
| const auto *UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression()); | |
| const auto *BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression()); | |
| bool IsPointer = | |
| OAShE || | |
| (OASE && ArraySectionExpr::getBaseOriginalType(OASE) | |
| .getCanonicalType() | |
| ->isAnyPointerType()) || | |
| I->getAssociatedExpression()->getType()->isAnyPointerType(); | |
| bool IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) && | |
| MapDecl && | |
| MapDecl->getType()->isLValueReferenceType(); | |
| bool IsNonDerefPointer = IsPointer && | |
| !(UO && UO->getOpcode() != UO_Deref) && !BO && | |
| !IsNonContiguous; | |
| if (OASE) | |
| ++DimSize; | |
| if (Next == CE || IsMemberReference || IsNonDerefPointer || | |
| IsFinalArraySection) { | |
| // If this is not the last component, we expect the pointer to be | |
| // associated with an array expression or member expression. | |
| assert((Next == CE || | |
| isa<MemberExpr>(Next->getAssociatedExpression()) || | |
| isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || | |
| isa<ArraySectionExpr>(Next->getAssociatedExpression()) || | |
| isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) || | |
| isa<UnaryOperator>(Next->getAssociatedExpression()) || | |
| isa<BinaryOperator>(Next->getAssociatedExpression())) && | |
| "Unexpected expression"); | |
| Address LB = Address::invalid(); | |
| Address LowestElem = Address::invalid(); | |
| auto &&EmitMemberExprBase = [](CodeGenFunction &CGF, | |
| const MemberExpr *E) { | |
| const Expr *BaseExpr = E->getBase(); | |
| // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a | |
| // scalar. | |
| LValue BaseLV; | |
| if (E->isArrow()) { | |
| LValueBaseInfo BaseInfo; | |
| TBAAAccessInfo TBAAInfo; | |
| Address Addr = | |
| CGF.EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); | |
| QualType PtrTy = BaseExpr->getType()->getPointeeType(); | |
| BaseLV = CGF.MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); | |
| } else { | |
| BaseLV = CGF.EmitOMPSharedLValue(BaseExpr); | |
| } | |
| return BaseLV; | |
| }; | |
| if (OAShE) { | |
| LowestElem = LB = | |
| Address(CGF.EmitScalarExpr(OAShE->getBase()), | |
| CGF.ConvertTypeForMem( | |
| OAShE->getBase()->getType()->getPointeeType()), | |
| CGF.getContext().getTypeAlignInChars( | |
| OAShE->getBase()->getType())); | |
| } else if (IsMemberReference) { | |
| const auto *ME = cast<MemberExpr>(I->getAssociatedExpression()); | |
| LValue BaseLVal = EmitMemberExprBase(CGF, ME); | |
| LowestElem = CGF.EmitLValueForFieldInitialization( | |
| BaseLVal, cast<FieldDecl>(MapDecl)) | |
| .getAddress(); | |
| LB = CGF.EmitLoadOfReferenceLValue(LowestElem, MapDecl->getType()) | |
| .getAddress(); | |
| } else { | |
| LowestElem = LB = | |
| CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) | |
| .getAddress(); | |
| } | |
| // Save the final LowestElem, to use it as the pointee in attach maps, | |
| // if emitted. | |
| if (Next == CE) | |
| FinalLowestElem = LowestElem; | |
| // If this component is a pointer inside the base struct then we don't | |
| // need to create any entry for it - it will be combined with the object | |
| // it is pointing to into a single PTR_AND_OBJ entry. | |
| bool IsMemberPointerOrAddr = | |
| EncounteredME && | |
| (((IsPointer || ForDeviceAddr) && | |
| I->getAssociatedExpression() == EncounteredME) || | |
| (IsPrevMemberReference && !IsPointer) || | |
| (IsMemberReference && Next != CE && | |
| !Next->getAssociatedExpression()->getType()->isPointerType())); | |
| if (!OverlappedElements.empty() && Next == CE) { | |
| // Handle base element with the info for overlapped elements. | |
| assert(!PartialStruct.Base.isValid() && "The base element is set."); | |
| assert(!IsPointer && | |
| "Unexpected base element with the pointer type."); | |
| // Mark the whole struct as the struct that requires allocation on the | |
| // device. | |
| PartialStruct.LowestElem = {0, LowestElem}; | |
| CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( | |
| I->getAssociatedExpression()->getType()); | |
| Address HB = CGF.Builder.CreateConstGEP( | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| LowestElem, CGF.VoidPtrTy, CGF.Int8Ty), | |
| TypeSize.getQuantity() - 1); | |
| PartialStruct.HighestElem = { | |
| std::numeric_limits<decltype( | |
| PartialStruct.HighestElem.first)>::max(), | |
| HB}; | |
| PartialStruct.Base = BP; | |
| PartialStruct.LB = LB; | |
| assert( | |
| PartialStruct.PreliminaryMapData.BasePointers.empty() && | |
| "Overlapped elements must be used only once for the variable."); | |
| std::swap(PartialStruct.PreliminaryMapData, CombinedInfo); | |
| // Emit data for non-overlapped data. | |
| OpenMPOffloadMappingFlags Flags = | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | | |
| getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit, | |
| /*AddPtrFlag=*/false, | |
| /*AddIsTargetParamFlag=*/false, IsNonContiguous); | |
| CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl, | |
| MapExpr, BP, LB, IsNonContiguous, | |
| DimSize); | |
| // Do bitcopy of all non-overlapped structure elements. | |
| for (OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| Component : OverlappedElements) { | |
| for (const OMPClauseMappableExprCommon::MappableComponent &MC : | |
| Component) { | |
| if (const ValueDecl *VD = MC.getAssociatedDeclaration()) { | |
| if (const auto *FD = dyn_cast<FieldDecl>(VD)) { | |
| CopyGaps.processField(MC, FD, EmitMemberExprBase); | |
| } | |
| } | |
| } | |
| } | |
| CopyGaps.copyUntilEnd(HB); | |
| break; | |
| } | |
| llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression()); | |
| // Skip adding an entry in the CurInfo of this combined entry if the | |
| // whole struct is currently being mapped. The struct needs to be added | |
| // in the first position before any data internal to the struct is being | |
| // mapped. | |
| // Skip adding an entry in the CurInfo of this combined entry if the | |
| // PartialStruct.PreliminaryMapData.BasePointers has been mapped. | |
| if ((!IsMemberPointerOrAddr && !IsPartialMapped) || | |
| (Next == CE && MapType != OMPC_MAP_unknown)) { | |
| if (!IsMappingWholeStruct) { | |
| CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); | |
| CombinedInfo.BasePointers.push_back(BP.emitRawPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); | |
| CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| Size, CGF.Int64Ty, /*isSigned=*/true)); | |
| CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize | |
| : 1); | |
| } else { | |
| StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr); | |
| StructBaseCombinedInfo.BasePointers.push_back( | |
| BP.emitRawPointer(CGF)); | |
| StructBaseCombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| StructBaseCombinedInfo.Pointers.push_back(LB.emitRawPointer(CGF)); | |
| StructBaseCombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| Size, CGF.Int64Ty, /*isSigned=*/true)); | |
| StructBaseCombinedInfo.NonContigInfo.Dims.push_back( | |
| IsNonContiguous ? DimSize : 1); | |
| } | |
| // If Mapper is valid, the last component inherits the mapper. | |
| bool HasMapper = Mapper && Next == CE; | |
| if (!IsMappingWholeStruct) | |
| CombinedInfo.Mappers.push_back(HasMapper ? Mapper : nullptr); | |
| else | |
| StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper | |
| : nullptr); | |
| // We need to add a pointer flag for each map that comes from the | |
| // same expression except for the first one. We also need to signal | |
| // this map is the first one that relates with the current capture | |
| // (there is a set of entries for each capture). | |
| OpenMPOffloadMappingFlags Flags = getMapTypeBits( | |
| MapType, MapModifiers, MotionModifiers, IsImplicit, | |
| !IsExpressionFirstInfo || RequiresReference || | |
| FirstPointerInComplexData || IsMemberReference, | |
| IsCaptureFirstInfo && !RequiresReference, IsNonContiguous); | |
| if (!IsExpressionFirstInfo || IsMemberReference) { | |
| // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well, | |
| // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags. | |
| if (IsPointer || (IsMemberReference && Next != CE)) | |
| Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO | | |
| OpenMPOffloadMappingFlags::OMP_MAP_FROM | | |
| OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS | | |
| OpenMPOffloadMappingFlags::OMP_MAP_DELETE | | |
| OpenMPOffloadMappingFlags::OMP_MAP_CLOSE); | |
| if (ShouldBeMemberOf) { | |
| // Set placeholder value MEMBER_OF=FFFF to indicate that the flag | |
| // should be later updated with the correct value of MEMBER_OF. | |
| Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF; | |
| // From now on, all subsequent PTR_AND_OBJ entries should not be | |
| // marked as MEMBER_OF. | |
| ShouldBeMemberOf = false; | |
| } | |
| } | |
| if (!IsMappingWholeStruct) { | |
| CombinedInfo.Types.push_back(Flags); | |
| // HasAttachPtr marks pointee entries, which have a base attach-ptr. | |
| CombinedInfo.HasAttachPtr.push_back(HasAttachPtr); | |
| } else { | |
| StructBaseCombinedInfo.Types.push_back(Flags); | |
| StructBaseCombinedInfo.HasAttachPtr.push_back(HasAttachPtr); | |
| } | |
| } | |
| // If we have encountered a member expression so far, keep track of the | |
| // mapped member. If the parent is "*this", then the value declaration | |
| // is nullptr. | |
| if (EncounteredME) { | |
| const auto *FD = cast<FieldDecl>(EncounteredME->getMemberDecl()); | |
| unsigned FieldIndex = FD->getFieldIndex(); | |
| // Update info about the lowest and highest elements for this struct | |
| if (!PartialStruct.Base.isValid()) { | |
| PartialStruct.LowestElem = {FieldIndex, LowestElem}; | |
| if (IsFinalArraySection && OASE) { | |
| Address HB = | |
| CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false) | |
| .getAddress(); | |
| PartialStruct.HighestElem = {FieldIndex, HB}; | |
| } else { | |
| PartialStruct.HighestElem = {FieldIndex, LowestElem}; | |
| } | |
| PartialStruct.Base = BP; | |
| PartialStruct.LB = BP; | |
| } else if (FieldIndex < PartialStruct.LowestElem.first) { | |
| PartialStruct.LowestElem = {FieldIndex, LowestElem}; | |
| } else if (FieldIndex > PartialStruct.HighestElem.first) { | |
| if (IsFinalArraySection && OASE) { | |
| Address HB = | |
| CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/false) | |
| .getAddress(); | |
| PartialStruct.HighestElem = {FieldIndex, HB}; | |
| } else { | |
| PartialStruct.HighestElem = {FieldIndex, LowestElem}; | |
| } | |
| } | |
| } | |
| // Need to emit combined struct for array sections. | |
| if (IsFinalArraySection || IsNonContiguous) | |
| PartialStruct.IsArraySection = true; | |
| // If we have a final array section, we are done with this expression. | |
| if (IsFinalArraySection) | |
| break; | |
| // The pointer becomes the base for the next element. | |
| if (Next != CE) | |
| BP = IsMemberReference ? LowestElem : LB; | |
| if (!IsPartialMapped) | |
| IsExpressionFirstInfo = false; | |
| IsCaptureFirstInfo = false; | |
| FirstPointerInComplexData = false; | |
| IsPrevMemberReference = IsMemberReference; | |
| } else if (FirstPointerInComplexData) { | |
| QualType Ty = Components.rbegin() | |
| ->getAssociatedDeclaration() | |
| ->getType() | |
| .getNonReferenceType(); | |
| BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); | |
| FirstPointerInComplexData = false; | |
| } | |
| } | |
| // If ran into the whole component - allocate the space for the whole | |
| // record. | |
| if (!EncounteredME) | |
| PartialStruct.HasCompleteRecord = true; | |
| // Populate ATTACH information for later processing by emitAttachEntry. | |
| if (shouldEmitAttachEntry(AttachPtrExpr, BaseDecl, CGF, CurDir)) { | |
| AttachInfo.AttachPtrAddr = AttachPtrAddr; | |
| AttachInfo.AttachPteeAddr = FinalLowestElem; | |
| AttachInfo.AttachPtrDecl = BaseDecl; | |
| AttachInfo.AttachMapExpr = MapExpr; | |
| } | |
| if (!IsNonContiguous) | |
| return; | |
| const ASTContext &Context = CGF.getContext(); | |
| // For supporting stride in array section, we need to initialize the first | |
| // dimension size as 1, first offset as 0, and first count as 1 | |
| MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 0)}; | |
| MapValuesArrayTy CurCounts; | |
| MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; | |
| MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.CGM.Int64Ty, 1)}; | |
| uint64_t ElementTypeSize; | |
| // Collect Size information for each dimension and get the element size as | |
| // the first Stride. For example, for `int arr[10][10]`, the DimSizes | |
| // should be [10, 10] and the first stride is 4 btyes. | |
| for (const OMPClauseMappableExprCommon::MappableComponent &Component : | |
| Components) { | |
| const Expr *AssocExpr = Component.getAssociatedExpression(); | |
| const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); | |
| if (!OASE) | |
| continue; | |
| QualType Ty = ArraySectionExpr::getBaseOriginalType(OASE->getBase()); | |
| auto *CAT = Context.getAsConstantArrayType(Ty); | |
| auto *VAT = Context.getAsVariableArrayType(Ty); | |
| // We need all the dimension size except for the last dimension. | |
| assert((VAT || CAT || &Component == &*Components.begin()) && | |
| "Should be either ConstantArray or VariableArray if not the " | |
| "first Component"); | |
| // Get element size if CurCounts is empty. | |
| if (CurCounts.empty()) { | |
| const Type *ElementType = nullptr; | |
| if (CAT) | |
| ElementType = CAT->getElementType().getTypePtr(); | |
| else if (VAT) | |
| ElementType = VAT->getElementType().getTypePtr(); | |
| else if (&Component == &*Components.begin()) { | |
| // If the base is a raw pointer (e.g. T *data with data[a:b:c]), | |
| // there was no earlier CAT/VAT/array handling to establish | |
| // ElementType. Capture the pointee type now so that subsequent | |
| // components (offset/length/stride) have a concrete element type to | |
| // work with. This makes pointer-backed sections behave consistently | |
| // with CAT/VAT/array bases. | |
| if (const auto *PtrType = Ty->getAs<PointerType>()) | |
| ElementType = PtrType->getPointeeType().getTypePtr(); | |
| } else { | |
| // Any component after the first should never have a raw pointer type; | |
| // by this point. ElementType must already be known (set above or in | |
| // prior array / CAT / VAT handling). | |
| assert(!Ty->isPointerType() && | |
| "Non-first components should not be raw pointers"); | |
| } | |
| // At this stage, if ElementType was a base pointer and we are in the | |
| // first iteration, it has been computed. | |
| if (ElementType) { | |
| // For the case that having pointer as base, we need to remove one | |
| // level of indirection. | |
| if (&Component != &*Components.begin()) | |
| ElementType = ElementType->getPointeeOrArrayElementType(); | |
| ElementTypeSize = | |
| Context.getTypeSizeInChars(ElementType).getQuantity(); | |
| CurCounts.push_back( | |
| llvm::ConstantInt::get(CGF.Int64Ty, ElementTypeSize)); | |
| } | |
| } | |
| // Get dimension value except for the last dimension since we don't need | |
| // it. | |
| if (DimSizes.size() < Components.size() - 1) { | |
| if (CAT) | |
| DimSizes.push_back( | |
| llvm::ConstantInt::get(CGF.Int64Ty, CAT->getZExtSize())); | |
| else if (VAT) | |
| DimSizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.EmitScalarExpr(VAT->getSizeExpr()), CGF.Int64Ty, | |
| /*IsSigned=*/false)); | |
| } | |
| } | |
| // Skip the dummy dimension since we have already have its information. | |
| auto *DI = DimSizes.begin() + 1; | |
| // Product of dimension. | |
| llvm::Value *DimProd = | |
| llvm::ConstantInt::get(CGF.CGM.Int64Ty, ElementTypeSize); | |
| // Collect info for non-contiguous. Notice that offset, count, and stride | |
| // are only meaningful for array-section, so we insert a null for anything | |
| // other than array-section. | |
| // Also, the size of offset, count, and stride are not the same as | |
| // pointers, base_pointers, sizes, or dims. Instead, the size of offset, | |
| // count, and stride are the same as the number of non-contiguous | |
| // declaration in target update to/from clause. | |
| for (const OMPClauseMappableExprCommon::MappableComponent &Component : | |
| Components) { | |
| const Expr *AssocExpr = Component.getAssociatedExpression(); | |
| if (const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) { | |
| llvm::Value *Offset = CGF.Builder.CreateIntCast( | |
| CGF.EmitScalarExpr(AE->getIdx()), CGF.Int64Ty, | |
| /*isSigned=*/false); | |
| CurOffsets.push_back(Offset); | |
| CurCounts.push_back(llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/1)); | |
| CurStrides.push_back(CurStrides.back()); | |
| continue; | |
| } | |
| const auto *OASE = dyn_cast<ArraySectionExpr>(AssocExpr); | |
| if (!OASE) | |
| continue; | |
| // Offset | |
| const Expr *OffsetExpr = OASE->getLowerBound(); | |
| llvm::Value *Offset = nullptr; | |
| if (!OffsetExpr) { | |
| // If offset is absent, then we just set it to zero. | |
| Offset = llvm::ConstantInt::get(CGF.Int64Ty, 0); | |
| } else { | |
| Offset = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(OffsetExpr), | |
| CGF.Int64Ty, | |
| /*isSigned=*/false); | |
| } | |
| // Count | |
| const Expr *CountExpr = OASE->getLength(); | |
| llvm::Value *Count = nullptr; | |
| if (!CountExpr) { | |
| // In Clang, once a high dimension is an array section, we construct all | |
| // the lower dimension as array section, however, for case like | |
| // arr[0:2][2], Clang construct the inner dimension as an array section | |
| // but it actually is not in an array section form according to spec. | |
| if (!OASE->getColonLocFirst().isValid() && | |
| !OASE->getColonLocSecond().isValid()) { | |
| Count = llvm::ConstantInt::get(CGF.Int64Ty, 1); | |
| } else { | |
| // OpenMP 5.0, 2.1.5 Array Sections, Description. | |
| // When the length is absent it defaults to ⌈(size − | |
| // lower-bound)/stride⌉, where size is the size of the array | |
| // dimension. | |
| const Expr *StrideExpr = OASE->getStride(); | |
| llvm::Value *Stride = | |
| StrideExpr | |
| ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), | |
| CGF.Int64Ty, /*isSigned=*/false) | |
| : nullptr; | |
| if (Stride) | |
| Count = CGF.Builder.CreateUDiv( | |
| CGF.Builder.CreateNUWSub(*DI, Offset), Stride); | |
| else | |
| Count = CGF.Builder.CreateNUWSub(*DI, Offset); | |
| } | |
| } else { | |
| Count = CGF.EmitScalarExpr(CountExpr); | |
| } | |
| Count = CGF.Builder.CreateIntCast(Count, CGF.Int64Ty, /*isSigned=*/false); | |
| CurCounts.push_back(Count); | |
| // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size | |
| // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size | |
| // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example: | |
| // Offset Count Stride | |
| // D0 0 4 1 (int) <- dummy dimension | |
| // D1 0 2 8 (2 * (1) * 4) | |
| // D2 100 2 20 (1 * (1 * 5) * 4) | |
| // D3 0 2 200 (2 * (1 * 5 * 4) * 4) | |
| const Expr *StrideExpr = OASE->getStride(); | |
| llvm::Value *Stride = | |
| StrideExpr | |
| ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(StrideExpr), | |
| CGF.Int64Ty, /*isSigned=*/false) | |
| : nullptr; | |
| DimProd = CGF.Builder.CreateNUWMul(DimProd, *(DI - 1)); | |
| if (Stride) | |
| CurStrides.push_back(CGF.Builder.CreateNUWMul(DimProd, Stride)); | |
| else | |
| CurStrides.push_back(DimProd); | |
| Offset = CGF.Builder.CreateNUWMul(DimProd, Offset); | |
| CurOffsets.push_back(Offset); | |
| if (DI != DimSizes.end()) | |
| ++DI; | |
| } | |
| CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets); | |
| CombinedInfo.NonContigInfo.Counts.push_back(CurCounts); | |
| CombinedInfo.NonContigInfo.Strides.push_back(CurStrides); | |
| } | |
| /// Return the adjusted map modifiers if the declaration a capture refers to | |
| /// appears in a first-private clause. This is expected to be used only with | |
| /// directives that start with 'target'. | |
| OpenMPOffloadMappingFlags | |
| getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const { | |
| assert(Cap.capturesVariable() && "Expected capture by reference only!"); | |
| // A first private variable captured by reference will use only the | |
| // 'private ptr' and 'map to' flag. Return the right flags if the captured | |
| // declaration is known as first-private in this handler. | |
| if (FirstPrivateDecls.count(Cap.getCapturedVar())) { | |
| if (Cap.getCapturedVar()->getType()->isAnyPointerType()) | |
| return OpenMPOffloadMappingFlags::OMP_MAP_TO | | |
| OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; | |
| return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE | | |
| OpenMPOffloadMappingFlags::OMP_MAP_TO; | |
| } | |
| auto I = LambdasMap.find(Cap.getCapturedVar()->getCanonicalDecl()); | |
| if (I != LambdasMap.end()) | |
| // for map(to: lambda): using user specified map type. | |
| return getMapTypeBits( | |
| I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(), | |
| /*MotionModifiers=*/{}, I->getSecond()->isImplicit(), | |
| /*AddPtrFlag=*/false, | |
| /*AddIsTargetParamFlag=*/false, | |
| /*isNonContiguous=*/false); | |
| return OpenMPOffloadMappingFlags::OMP_MAP_TO | | |
| OpenMPOffloadMappingFlags::OMP_MAP_FROM; | |
| } | |
| void getPlainLayout(const CXXRecordDecl *RD, | |
| llvm::SmallVectorImpl<const FieldDecl *> &Layout, | |
| bool AsBase) const { | |
| const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); | |
| llvm::StructType *St = | |
| AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType(); | |
| unsigned NumElements = St->getNumElements(); | |
| llvm::SmallVector< | |
| llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4> | |
| RecordLayout(NumElements); | |
| // Fill bases. | |
| for (const auto &I : RD->bases()) { | |
| if (I.isVirtual()) | |
| continue; | |
| QualType BaseTy = I.getType(); | |
| const auto *Base = BaseTy->getAsCXXRecordDecl(); | |
| // Ignore empty bases. | |
| if (CodeGenUtils::isEmptyRecordForLayout(CGF.getContext(), BaseTy) || | |
| CGF.getContext() | |
| .getASTRecordLayout(Base) | |
| .getNonVirtualSize() | |
| .isZero()) | |
| continue; | |
| unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base); | |
| RecordLayout[FieldIndex] = Base; | |
| } | |
| // Fill in virtual bases. | |
| for (const auto &I : RD->vbases()) { | |
| QualType BaseTy = I.getType(); | |
| // Ignore empty bases. | |
| if (CodeGenUtils::isEmptyRecordForLayout(CGF.getContext(), BaseTy)) | |
| continue; | |
| const auto *Base = BaseTy->getAsCXXRecordDecl(); | |
| unsigned FieldIndex = RL.getVirtualBaseIndex(Base); | |
| if (RecordLayout[FieldIndex]) | |
| continue; | |
| RecordLayout[FieldIndex] = Base; | |
| } | |
| // Fill in all the fields. | |
| assert(!RD->isUnion() && "Unexpected union."); | |
| for (const auto *Field : RD->fields()) { | |
| // Fill in non-bitfields. (Bitfields always use a zero pattern, which we | |
| // will fill in later.) | |
| if (!Field->isBitField() && | |
| !CodeGenUtils::isEmptyFieldForLayout(CGF.getContext(), Field)) { | |
| unsigned FieldIndex = RL.getLLVMFieldNo(Field); | |
| RecordLayout[FieldIndex] = Field; | |
| } | |
| } | |
| for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *> | |
| &Data : RecordLayout) { | |
| if (Data.isNull()) | |
| continue; | |
| if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Data)) | |
| getPlainLayout(Base, Layout, /*AsBase=*/true); | |
| else | |
| Layout.push_back(cast<const FieldDecl *>(Data)); | |
| } | |
| } | |
| /// Returns the address corresponding to \p PointerExpr. | |
| static Address getAttachPtrAddr(const Expr *PointerExpr, | |
| CodeGenFunction &CGF) { | |
| assert(PointerExpr && "Cannot get addr from null attach-ptr expr"); | |
| Address AttachPtrAddr = Address::invalid(); | |
| if (auto *DRE = dyn_cast<DeclRefExpr>(PointerExpr)) { | |
| // If the pointer is a variable, we can use its address directly. | |
| AttachPtrAddr = CGF.EmitLValue(DRE).getAddress(); | |
| } else if (auto *OASE = dyn_cast<ArraySectionExpr>(PointerExpr)) { | |
| AttachPtrAddr = | |
| CGF.EmitArraySectionExpr(OASE, /*IsLowerBound=*/true).getAddress(); | |
| } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(PointerExpr)) { | |
| AttachPtrAddr = CGF.EmitLValue(ASE).getAddress(); | |
| } else if (auto *ME = dyn_cast<MemberExpr>(PointerExpr)) { | |
| AttachPtrAddr = CGF.EmitMemberExpr(ME).getAddress(); | |
| } else if (auto *UO = dyn_cast<UnaryOperator>(PointerExpr)) { | |
| assert(UO->getOpcode() == UO_Deref && | |
| "Unexpected unary-operator on attach-ptr-expr"); | |
| AttachPtrAddr = CGF.EmitLValue(UO).getAddress(); | |
| } | |
| assert(AttachPtrAddr.isValid() && | |
| "Failed to get address for attach pointer expression"); | |
| return AttachPtrAddr; | |
| } | |
| /// Get the address of the attach pointer, and a load from it, to get the | |
| /// pointee base address. | |
| /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair | |
| /// contains invalid addresses if \p AttachPtrExpr is null. | |
| static std::pair<Address, Address> | |
| getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr, | |
| CodeGenFunction &CGF) { | |
| if (!AttachPtrExpr) | |
| return {Address::invalid(), Address::invalid()}; | |
| Address AttachPtrAddr = getAttachPtrAddr(AttachPtrExpr, CGF); | |
| assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr"); | |
| QualType AttachPtrType = | |
| OMPClauseMappableExprCommon::getComponentExprElementType(AttachPtrExpr) | |
| .getCanonicalType(); | |
| Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer( | |
| AttachPtrAddr, AttachPtrType->castAs<PointerType>()); | |
| assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr"); | |
| return {AttachPtrAddr, AttachPteeBaseAddr}; | |
| } | |
| /// Returns whether an attach entry should be emitted for a map on | |
| /// \p MapBaseDecl on the directive \p CurDir. | |
| static bool | |
| shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl, | |
| CodeGenFunction &CGF, | |
| llvm::PointerUnion<const OMPExecutableDirective *, | |
| const OMPDeclareMapperDecl *> | |
| CurDir) { | |
| if (!PointerExpr) | |
| return false; | |
| // Pointer attachment is needed at map-entering time or for declare | |
| // mappers. | |
| return isa<const OMPDeclareMapperDecl *>(CurDir) || | |
| isOpenMPTargetMapEnteringDirective( | |
| cast<const OMPExecutableDirective *>(CurDir) | |
| ->getDirectiveKind()); | |
| } | |
| /// Computes the attach-ptr expr for \p Components, and updates various maps | |
| /// with the information. | |
| /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr() | |
| /// with the OpenMPDirectiveKind extracted from \p CurDir. | |
| /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and | |
| /// AttachPtrExprMap. | |
| void collectAttachPtrExprInfo( | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components, | |
| llvm::PointerUnion<const OMPExecutableDirective *, | |
| const OMPDeclareMapperDecl *> | |
| CurDir) { | |
| OpenMPDirectiveKind CurDirectiveID = | |
| isa<const OMPDeclareMapperDecl *>(CurDir) | |
| ? OMPD_declare_mapper | |
| : cast<const OMPExecutableDirective *>(CurDir)->getDirectiveKind(); | |
| const auto &[AttachPtrExpr, Depth] = | |
| OMPClauseMappableExprCommon::findAttachPtrExpr(Components, | |
| CurDirectiveID); | |
| AttachPtrComputationOrderMap.try_emplace( | |
| AttachPtrExpr, AttachPtrComputationOrderMap.size()); | |
| AttachPtrComponentDepthMap.try_emplace(AttachPtrExpr, Depth); | |
| AttachPtrExprMap.try_emplace(Components, AttachPtrExpr); | |
| } | |
| /// Generate all the base pointers, section pointers, sizes, map types, and | |
| /// mappers for the extracted mappable expressions (all included in \a | |
| /// CombinedInfo). Also, for each item that relates with a device pointer, a | |
| /// pair of the relevant declaration and index where it occurs is appended to | |
| /// the device pointers info array. | |
| void generateAllInfoForClauses( | |
| ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo, | |
| llvm::OpenMPIRBuilder &OMPBuilder, | |
| const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet = | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const { | |
| // We have to process the component lists that relate with the same | |
| // declaration in a single chunk so that we can generate the map flags | |
| // correctly. Therefore, we organize all lists in a map. | |
| enum MapKind { Present, Allocs, Other, Total }; | |
| llvm::MapVector<CanonicalDeclPtr<const Decl>, | |
| SmallVector<SmallVector<MapInfo, 8>, 4>> | |
| Info; | |
| // Helper function to fill the information map for the different supported | |
| // clauses. | |
| auto &&InfoGen = | |
| [&Info, &SkipVarSet]( | |
| const ValueDecl *D, MapKind Kind, | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef L, | |
| OpenMPMapClauseKind MapType, | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers, | |
| ArrayRef<OpenMPMotionModifierKind> MotionModifiers, | |
| bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper, | |
| const Expr *VarRef = nullptr, bool ForDeviceAddr = false) { | |
| if (SkipVarSet.contains(D)) | |
| return; | |
| auto It = Info.try_emplace(D, Total).first; | |
| It->second[Kind].emplace_back( | |
| L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer, | |
| IsImplicit, Mapper, VarRef, ForDeviceAddr); | |
| }; | |
| for (const auto *Cl : Clauses) { | |
| const auto *C = dyn_cast<OMPMapClause>(Cl); | |
| if (!C) | |
| continue; | |
| MapKind Kind = Other; | |
| if (llvm::is_contained(C->getMapTypeModifiers(), | |
| OMPC_MAP_MODIFIER_present)) | |
| Kind = Present; | |
| else if (C->getMapType() == OMPC_MAP_alloc) | |
| Kind = Allocs; | |
| const auto *EI = C->getVarRefs().begin(); | |
| for (const auto L : C->component_lists()) { | |
| const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr; | |
| InfoGen(std::get<0>(L), Kind, std::get<1>(L), C->getMapType(), | |
| C->getMapTypeModifiers(), {}, | |
| /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L), | |
| E); | |
| ++EI; | |
| } | |
| } | |
| for (const auto *Cl : Clauses) { | |
| const auto *C = dyn_cast<OMPToClause>(Cl); | |
| if (!C) | |
| continue; | |
| MapKind Kind = Other; | |
| if (llvm::is_contained(C->getMotionModifiers(), | |
| OMPC_MOTION_MODIFIER_present)) | |
| Kind = Present; | |
| if (llvm::is_contained(C->getMotionModifiers(), | |
| OMPC_MOTION_MODIFIER_iterator)) { | |
| if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>( | |
| C->getIteratorModifier()->IgnoreParenImpCasts())) { | |
| const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0)); | |
| CGF.EmitVarDecl(*VD); | |
| } | |
| } | |
| const auto *EI = C->getVarRefs().begin(); | |
| for (const auto L : C->component_lists()) { | |
| InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_to, {}, | |
| C->getMotionModifiers(), /*ReturnDevicePointer=*/false, | |
| C->isImplicit(), std::get<2>(L), *EI); | |
| ++EI; | |
| } | |
| } | |
| for (const auto *Cl : Clauses) { | |
| const auto *C = dyn_cast<OMPFromClause>(Cl); | |
| if (!C) | |
| continue; | |
| MapKind Kind = Other; | |
| if (llvm::is_contained(C->getMotionModifiers(), | |
| OMPC_MOTION_MODIFIER_present)) | |
| Kind = Present; | |
| if (llvm::is_contained(C->getMotionModifiers(), | |
| OMPC_MOTION_MODIFIER_iterator)) { | |
| if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>( | |
| C->getIteratorModifier()->IgnoreParenImpCasts())) { | |
| const auto *VD = cast<VarDecl>(IteratorExpr->getIteratorDecl(0)); | |
| CGF.EmitVarDecl(*VD); | |
| } | |
| } | |
| const auto *EI = C->getVarRefs().begin(); | |
| for (const auto L : C->component_lists()) { | |
| InfoGen(std::get<0>(L), Kind, std::get<1>(L), OMPC_MAP_from, {}, | |
| C->getMotionModifiers(), | |
| /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(L), | |
| *EI); | |
| ++EI; | |
| } | |
| } | |
| // Look at the use_device_ptr and use_device_addr clauses information and | |
| // mark the existing map entries as such. If there is no map information for | |
| // an entry in the use_device_ptr and use_device_addr list, we create one | |
| // with map type 'return_param' and zero size section. It is the user's | |
| // fault if that was not mapped before. If there is no map information, then | |
| // we defer the emission of that entry until all the maps for the same VD | |
| // have been handled. | |
| MapCombinedInfoTy UseDeviceDataCombinedInfo; | |
| auto &&UseDeviceDataCombinedInfoGen = | |
| [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr, | |
| CodeGenFunction &CGF, bool IsDevAddr, | |
| bool HasUdpFbNullify = false) { | |
| UseDeviceDataCombinedInfo.Exprs.push_back(VD); | |
| UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr); | |
| UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD); | |
| UseDeviceDataCombinedInfo.DevicePointers.emplace_back( | |
| IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer); | |
| // FIXME: For use_device_addr on array-sections, this should | |
| // be the starting address of the section. | |
| // e.g. int *p; | |
| // ... use_device_addr(p[3]) | |
| // &p[0], &p[3], /*size=*/0, RETURN_PARAM | |
| UseDeviceDataCombinedInfo.Pointers.push_back(Ptr); | |
| UseDeviceDataCombinedInfo.Sizes.push_back( | |
| llvm::Constant::getNullValue(CGF.Int64Ty)); | |
| OpenMPOffloadMappingFlags Flags = | |
| OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; | |
| if (HasUdpFbNullify) | |
| Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY; | |
| UseDeviceDataCombinedInfo.Types.push_back(Flags); | |
| UseDeviceDataCombinedInfo.HasAttachPtr.push_back(false); | |
| UseDeviceDataCombinedInfo.Mappers.push_back(nullptr); | |
| }; | |
| auto &&MapInfoGen = | |
| [&UseDeviceDataCombinedInfoGen]( | |
| CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD, | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| Components, | |
| bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false, | |
| bool HasUdpFbNullify = false) { | |
| // We didn't find any match in our map information - generate a zero | |
| // size array section. | |
| llvm::Value *Ptr; | |
| if (IsDevAddr && !IEIsAttachPtrForDevAddr) { | |
| if (IE->isGLValue()) | |
| Ptr = CGF.EmitLValue(IE).getPointer(CGF); | |
| else | |
| Ptr = CGF.EmitScalarExpr(IE); | |
| } else { | |
| Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); | |
| } | |
| bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr; | |
| // For the purpose of address-translation, treat something like the | |
| // following: | |
| // int *p; | |
| // ... use_device_addr(p[1]) | |
| // equivalent to | |
| // ... use_device_ptr(p) | |
| UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr && | |
| !TreatDevAddrAsDevPtr, | |
| HasUdpFbNullify); | |
| }; | |
| auto &&IsMapInfoExist = | |
| [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE, | |
| const Expr *DesiredAttachPtrExpr, bool IsDevAddr, | |
| bool HasUdpFbNullify = false) -> bool { | |
| // We potentially have map information for this declaration already. | |
| // Look for the first set of components that refer to it. If found, | |
| // return true. | |
| // If the first component is a member expression, we have to look into | |
| // 'this', which maps to null in the map of map information. Otherwise | |
| // look directly for the information. | |
| auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD); | |
| if (It != Info.end()) { | |
| bool Found = false; | |
| for (auto &Data : It->second) { | |
| MapInfo *CI = nullptr; | |
| // We potentially have multiple maps for the same decl. We need to | |
| // only consider those for which the attach-ptr matches the desired | |
| // attach-ptr. | |
| auto *It = llvm::find_if(Data, [&](const MapInfo &MI) { | |
| if (MI.Components.back().getAssociatedDeclaration() != VD) | |
| return false; | |
| const Expr *MapAttachPtr = getAttachPtrExpr(MI.Components); | |
| bool Match = AttachPtrComparator.areEqual(MapAttachPtr, | |
| DesiredAttachPtrExpr); | |
| return Match; | |
| }); | |
| if (It != Data.end()) | |
| CI = &*It; | |
| if (CI) { | |
| if (IsDevAddr) { | |
| CI->ForDeviceAddr = true; | |
| CI->ReturnDevicePointer = true; | |
| CI->HasUdpFbNullify = HasUdpFbNullify; | |
| Found = true; | |
| break; | |
| } else { | |
| auto PrevCI = std::next(CI->Components.rbegin()); | |
| const auto *VarD = dyn_cast<VarDecl>(VD); | |
| const Expr *AttachPtrExpr = getAttachPtrExpr(CI->Components); | |
| if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || | |
| isa<MemberExpr>(IE) || | |
| !VD->getType().getNonReferenceType()->isPointerType() || | |
| PrevCI == CI->Components.rend() || | |
| isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD || | |
| VarD->hasLocalStorage() || | |
| (isa_and_nonnull<DeclRefExpr>(AttachPtrExpr) && | |
| VD == cast<DeclRefExpr>(AttachPtrExpr)->getDecl())) { | |
| CI->ForDeviceAddr = IsDevAddr; | |
| CI->ReturnDevicePointer = true; | |
| CI->HasUdpFbNullify = HasUdpFbNullify; | |
| Found = true; | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| return Found; | |
| } | |
| return false; | |
| }; | |
| // Look at the use_device_ptr clause information and mark the existing map | |
| // entries as such. If there is no map information for an entry in the | |
| // use_device_ptr list, we create one with map type 'alloc' and zero size | |
| // section. It is the user fault if that was not mapped before. If there is | |
| // no map information and the pointer is a struct member, then we defer the | |
| // emission of that entry until the whole struct has been processed. | |
| for (const auto *Cl : Clauses) { | |
| const auto *C = dyn_cast<OMPUseDevicePtrClause>(Cl); | |
| if (!C) | |
| continue; | |
| bool HasUdpFbNullify = | |
| C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify; | |
| for (const auto L : C->component_lists()) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components = | |
| std::get<1>(L); | |
| assert(!Components.empty() && | |
| "Not expecting empty list of components!"); | |
| const ValueDecl *VD = Components.back().getAssociatedDeclaration(); | |
| VD = cast<ValueDecl>(VD->getCanonicalDecl()); | |
| const Expr *IE = Components.back().getAssociatedExpression(); | |
| // For use_device_ptr, we match an existing map clause if its attach-ptr | |
| // is same as the use_device_ptr operand. e.g. | |
| // map expr | use_device_ptr expr | current behavior | |
| // ---------|---------------------|----------------- | |
| // p[1] | p | match | |
| // ps->a | ps | match | |
| // p | p | no match | |
| const Expr *UDPOperandExpr = | |
| Components.front().getAssociatedExpression(); | |
| if (IsMapInfoExist(CGF, VD, IE, | |
| /*DesiredAttachPtrExpr=*/UDPOperandExpr, | |
| /*IsDevAddr=*/false, HasUdpFbNullify)) | |
| continue; | |
| MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false, | |
| /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify); | |
| } | |
| } | |
| llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed; | |
| for (const auto *Cl : Clauses) { | |
| const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Cl); | |
| if (!C) | |
| continue; | |
| for (const auto L : C->component_lists()) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components = | |
| std::get<1>(L); | |
| assert(!std::get<1>(L).empty() && | |
| "Not expecting empty list of components!"); | |
| const ValueDecl *VD = std::get<1>(L).back().getAssociatedDeclaration(); | |
| if (!Processed.insert(VD).second) | |
| continue; | |
| VD = cast<ValueDecl>(VD->getCanonicalDecl()); | |
| // For use_device_addr, we match an existing map clause if the | |
| // use_device_addr operand's attach-ptr matches the map operand's | |
| // attach-ptr. | |
| // We chould also restrict to only match cases when there is a full | |
| // match between the map/use_device_addr clause exprs, but that may be | |
| // unnecessary. | |
| // | |
| // map expr | use_device_addr expr | current | possible restrictive/ | |
| // | | behavior | safer behavior | |
| // ---------|----------------------|-----------|----------------------- | |
| // p | p | match | match | |
| // p[0] | p[0] | match | match | |
| // p[0:1] | p[0] | match | no match | |
| // p[0:1] | p[2:1] | match | no match | |
| // p[1] | p[0] | match | no match | |
| // ps->a | ps->b | match | no match | |
| // p | p[0] | no match | no match | |
| // pp | pp[0][0] | no match | no match | |
| const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components); | |
| const Expr *IE = std::get<1>(L).back().getAssociatedExpression(); | |
| assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) && | |
| "use_device_addr operand has an attach-ptr, but does not match " | |
| "last component's expr."); | |
| if (IsMapInfoExist(CGF, VD, IE, | |
| /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr, | |
| /*IsDevAddr=*/true)) | |
| continue; | |
| MapInfoGen(CGF, IE, VD, Components, | |
| /*IsDevAddr=*/true, | |
| /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr); | |
| } | |
| } | |
| for (const auto &Data : Info) { | |
| MapCombinedInfoTy CurInfo; | |
| const Decl *D = Data.first; | |
| const ValueDecl *VD = cast_or_null<ValueDecl>(D); | |
| // Group component lists by their AttachPtrExpr and process them in order | |
| // of increasing complexity (nullptr first, then simple expressions like | |
| // p, then more complex ones like p[0], etc.) | |
| // | |
| // This is similar to how generateInfoForCaptureFromClauseInfo handles | |
| // grouping for target constructs. | |
| SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs; | |
| // First, collect all MapData entries with their attach-ptr exprs. | |
| for (const auto &M : Data.second) { | |
| for (const MapInfo &L : M) { | |
| assert(!L.Components.empty() && | |
| "Not expecting declaration with no component lists."); | |
| const Expr *AttachPtrExpr = getAttachPtrExpr(L.Components); | |
| AttachPtrMapInfoPairs.emplace_back(AttachPtrExpr, L); | |
| } | |
| } | |
| // Next, sort by increasing order of their complexity. | |
| llvm::stable_sort(AttachPtrMapInfoPairs, | |
| [this](const auto &LHS, const auto &RHS) { | |
| return AttachPtrComparator(LHS.first, RHS.first); | |
| }); | |
| // And finally, process them all in order, grouping those with | |
| // equivalent attach-ptr exprs together. | |
| auto *It = AttachPtrMapInfoPairs.begin(); | |
| while (It != AttachPtrMapInfoPairs.end()) { | |
| const Expr *AttachPtrExpr = It->first; | |
| SmallVector<MapInfo, 8> GroupLists; | |
| while (It != AttachPtrMapInfoPairs.end() && | |
| (It->first == AttachPtrExpr || | |
| AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) { | |
| GroupLists.push_back(It->second); | |
| ++It; | |
| } | |
| assert(!GroupLists.empty() && "GroupLists should not be empty"); | |
| StructRangeInfoTy PartialStruct; | |
| AttachInfoTy AttachInfo; | |
| MapCombinedInfoTy GroupCurInfo; | |
| // Current group's struct base information: | |
| MapCombinedInfoTy GroupStructBaseCurInfo; | |
| for (const MapInfo &L : GroupLists) { | |
| // Remember the current base pointer index. | |
| unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size(); | |
| unsigned StructBasePointersIdx = | |
| GroupStructBaseCurInfo.BasePointers.size(); | |
| GroupCurInfo.NonContigInfo.IsNonContiguous = | |
| L.Components.back().isNonContiguous(); | |
| generateInfoForComponentList( | |
| L.MapType, L.MapModifiers, L.MotionModifiers, L.Components, | |
| GroupCurInfo, GroupStructBaseCurInfo, PartialStruct, AttachInfo, | |
| /*IsFirstComponentList=*/false, L.IsImplicit, | |
| /*GenerateAllInfoForClauses*/ true, L.Mapper, L.ForDeviceAddr, VD, | |
| L.VarRef, /*OverlappedElements*/ {}); | |
| // If this entry relates to a device pointer, set the relevant | |
| // declaration and add the 'return pointer' flag. | |
| if (L.ReturnDevicePointer) { | |
| // Check whether a value was added to either GroupCurInfo or | |
| // GroupStructBaseCurInfo and error if no value was added to either | |
| // of them: | |
| assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() || | |
| StructBasePointersIdx < | |
| GroupStructBaseCurInfo.BasePointers.size()) && | |
| "Unexpected number of mapped base pointers."); | |
| // Choose a base pointer index which is always valid: | |
| const ValueDecl *RelevantVD = | |
| L.Components.back().getAssociatedDeclaration(); | |
| assert(RelevantVD && | |
| "No relevant declaration related with device pointer??"); | |
| // If GroupStructBaseCurInfo has been updated this iteration then | |
| // work on the first new entry added to it i.e. make sure that when | |
| // multiple values are added to any of the lists, the first value | |
| // added is being modified by the assignments below (not the last | |
| // value added). | |
| auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info, | |
| unsigned Idx) { | |
| Info.DevicePtrDecls[Idx] = RelevantVD; | |
| Info.DevicePointers[Idx] = L.ForDeviceAddr | |
| ? DeviceInfoTy::Address | |
| : DeviceInfoTy::Pointer; | |
| Info.Types[Idx] |= | |
| OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; | |
| if (L.HasUdpFbNullify) | |
| Info.Types[Idx] |= | |
| OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY; | |
| }; | |
| if (StructBasePointersIdx < | |
| GroupStructBaseCurInfo.BasePointers.size()) | |
| SetDevicePointerInfo(GroupStructBaseCurInfo, | |
| StructBasePointersIdx); | |
| else | |
| SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx); | |
| } | |
| } | |
| // Unify entries in one list making sure the struct mapping precedes the | |
| // individual fields: | |
| MapCombinedInfoTy GroupUnionCurInfo; | |
| GroupUnionCurInfo.append(GroupStructBaseCurInfo); | |
| GroupUnionCurInfo.append(GroupCurInfo); | |
| // If there is an entry in PartialStruct it means we have a struct with | |
| // individual members mapped. Emit an extra combined entry. | |
| if (PartialStruct.Base.isValid()) { | |
| // Prepend a synthetic dimension of length 1 to represent the | |
| // aggregated struct object. Using 1 (not 0, as 0 produced an | |
| // incorrect non-contiguous descriptor (DimSize==1), causing the | |
| // non-contiguous motion clause path to be skipped.) is important: | |
| // * It preserves the correct rank so targetDataUpdate() computes | |
| // DimSize == 2 for cases like strided array sections originating | |
| // from user-defined mappers (e.g. test with s.data[0:8:2]). | |
| GroupUnionCurInfo.NonContigInfo.Dims.insert( | |
| GroupUnionCurInfo.NonContigInfo.Dims.begin(), 1); | |
| emitCombinedEntry( | |
| CurInfo, GroupUnionCurInfo.Types, PartialStruct, AttachInfo, | |
| /*IsMapThis=*/!VD, OMPBuilder, VD, | |
| /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(), | |
| /*NotTargetParams=*/true); | |
| } | |
| // Append this group's results to the overall CurInfo in the correct | |
| // order: combined-entry -> original-field-entries -> attach-entry | |
| CurInfo.append(GroupUnionCurInfo); | |
| if (AttachInfo.isValid()) | |
| emitAttachEntry(CGF, CurInfo, AttachInfo); | |
| } | |
| // We need to append the results of this capture to what we already have. | |
| CombinedInfo.append(CurInfo); | |
| } | |
| // Append data for use_device_ptr/addr clauses. | |
| CombinedInfo.append(UseDeviceDataCombinedInfo); | |
| } | |
| public: | |
| MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) | |
| : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) { | |
| // Extract firstprivate clause information. | |
| for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>()) | |
| for (const auto *D : C->varlist()) { | |
| const ValueDecl *VD = cast<DeclRefExpr>(D)->getDecl(); | |
| if (const auto *BD = dyn_cast<BindingDecl>(VD)) | |
| VD = cast<VarDecl>(BD->getDecomposedDecl()); | |
| FirstPrivateDecls.try_emplace(cast<VarDecl>(VD), C->isImplicit()); | |
| } | |
| // Extract implicit firstprivates from uses_allocators clauses. | |
| for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) { | |
| for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) { | |
| OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I); | |
| if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(D.AllocatorTraits)) | |
| FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()), | |
| /*Implicit=*/true); | |
| else if (const auto *VD = dyn_cast<VarDecl>( | |
| cast<DeclRefExpr>(D.Allocator->IgnoreParenImpCasts()) | |
| ->getDecl())) | |
| FirstPrivateDecls.try_emplace(VD, /*Implicit=*/true); | |
| } | |
| } | |
| // Extract defaultmap clause information. | |
| for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>()) | |
| if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate) | |
| DefaultmapFirstprivateKinds.insert(C->getDefaultmapKind()); | |
| // Extract device pointer clause information. | |
| for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>()) | |
| for (auto L : C->component_lists()) | |
| DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L)); | |
| // Extract device addr clause information. | |
| for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>()) | |
| for (auto L : C->component_lists()) | |
| HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L)); | |
| // Extract map information. | |
| for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) { | |
| if (C->getMapType() != OMPC_MAP_to) | |
| continue; | |
| for (auto L : C->component_lists()) { | |
| const ValueDecl *VD = std::get<0>(L); | |
| const auto *RD = VD ? VD->getType() | |
| .getCanonicalType() | |
| .getNonReferenceType() | |
| ->getAsCXXRecordDecl() | |
| : nullptr; | |
| if (RD && RD->isLambda()) | |
| LambdasMap.try_emplace(std::get<0>(L), C); | |
| } | |
| } | |
| auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) { | |
| for (auto L : C->component_lists()) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components = | |
| std::get<1>(L); | |
| if (!Components.empty()) | |
| collectAttachPtrExprInfo(Components, CurDir); | |
| } | |
| }; | |
| // Populate the AttachPtrExprMap for all component lists from map-related | |
| // clauses. | |
| for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPToClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPFromClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>()) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| } | |
| /// Constructor for the declare mapper directive. | |
| MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) | |
| : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) { | |
| auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) { | |
| for (auto L : C->component_lists()) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components = | |
| std::get<1>(L); | |
| if (!Components.empty()) | |
| collectAttachPtrExprInfo(Components, CurDir); | |
| } | |
| }; | |
| // Populate the AttachPtrExprMap for all component lists from map-related | |
| // clauses in the declare mapper directive, to enable attach-style mapping | |
| // for mappers. | |
| for (const auto *Cl : Dir.clauses()) { | |
| if (const auto *C = dyn_cast<OMPMapClause>(Cl)) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| else if (const auto *C = dyn_cast<OMPToClause>(Cl)) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| else if (const auto *C = dyn_cast<OMPFromClause>(Cl)) | |
| CollectAttachPtrExprsForClauseComponents(C); | |
| } | |
| } | |
| /// Generate code for the combined entry if we have a partially mapped struct | |
| /// and take care of the mapping flags of the arguments corresponding to | |
| /// individual struct members. | |
| /// If a valid \p AttachInfo exists, its pointee addr will be updated to point | |
| /// to the combined-entry's begin address, if emitted. | |
| /// \p PartialStruct contains attach base-pointer information. | |
| /// \returns The index of the combined entry if one was added, std::nullopt | |
| /// otherwise. | |
| void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo, | |
| MapFlagsArrayTy &CurTypes, | |
| const StructRangeInfoTy &PartialStruct, | |
| AttachInfoTy &AttachInfo, bool IsMapThis, | |
| llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD, | |
| unsigned OffsetForMemberOfFlag, | |
| bool NotTargetParams) const { | |
| if (CurTypes.size() == 1 && | |
| ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) != | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) && | |
| !PartialStruct.IsArraySection) | |
| return; | |
| Address LBAddr = PartialStruct.LowestElem.second; | |
| Address HBAddr = PartialStruct.HighestElem.second; | |
| if (PartialStruct.HasCompleteRecord) { | |
| LBAddr = PartialStruct.LB; | |
| HBAddr = PartialStruct.LB; | |
| } | |
| CombinedInfo.Exprs.push_back(VD); | |
| // Base is the base of the struct | |
| CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| // Pointer is the address of the lowest element | |
| llvm::Value *LB = LBAddr.emitRawPointer(CGF); | |
| const CXXMethodDecl *MD = | |
| CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(CGF.CurFuncDecl) : nullptr; | |
| const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr; | |
| bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false; | |
| // There should not be a mapper for a combined entry. | |
| if (HasBaseClass) { | |
| // OpenMP 5.2 148:21: | |
| // If the target construct is within a class non-static member function, | |
| // and a variable is an accessible data member of the object for which the | |
| // non-static data member function is invoked, the variable is treated as | |
| // if the this[:1] expression had appeared in a map clause with a map-type | |
| // of tofrom. | |
| // Emit this[:1] | |
| CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF)); | |
| QualType Ty = MD->getFunctionObjectParameterType(); | |
| llvm::Value *Size = | |
| CGF.Builder.CreateIntCast(CGF.getTypeSize(Ty), CGF.Int64Ty, | |
| /*isSigned=*/true); | |
| CombinedInfo.Sizes.push_back(Size); | |
| } else { | |
| CombinedInfo.Pointers.push_back(LB); | |
| // Size is (addr of {highest+1} element) - (addr of lowest element) | |
| llvm::Value *HB = HBAddr.emitRawPointer(CGF); | |
| llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32( | |
| HBAddr.getElementType(), HB, /*Idx0=*/1); | |
| llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); | |
| llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); | |
| llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr); | |
| llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, | |
| /*isSigned=*/false); | |
| CombinedInfo.Sizes.push_back(Size); | |
| } | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| // Map type is always TARGET_PARAM, if generate info for captures. | |
| CombinedInfo.Types.push_back( | |
| NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE | |
| : !PartialStruct.PreliminaryMapData.BasePointers.empty() | |
| ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ | |
| : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM); | |
| // A combined entry has a base attach-ptr if its constituents do. e.g.: | |
| // map(s2.s1p->x, s2.s1p->y) | |
| // combined entry: | |
| // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC | |
| // here s2.s1p is the attach-ptr for the combined entry. | |
| // See the inline comments in emitUserDefinedMapper's definition for how | |
| // entries with an attach-ptr are treated. | |
| CombinedInfo.HasAttachPtr.push_back(AttachInfo.isValid()); | |
| // If any element has the present modifier, then make sure the runtime | |
| // doesn't attempt to allocate the struct. | |
| if (CurTypes.end() != | |
| llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) { | |
| return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>( | |
| Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT); | |
| })) | |
| CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT; | |
| // Remove TARGET_PARAM flag from the first element | |
| (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; | |
| // If any element has the ompx_hold modifier, then make sure the runtime | |
| // uses the hold reference count for the struct as a whole so that it won't | |
| // be unmapped by an extra dynamic reference count decrement. Add it to all | |
| // elements as well so the runtime knows which reference count to check | |
| // when determining whether it's time for device-to-host transfers of | |
| // individual elements. | |
| if (CurTypes.end() != | |
| llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags Type) { | |
| return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>( | |
| Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD); | |
| })) { | |
| CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD; | |
| for (auto &M : CurTypes) | |
| M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD; | |
| } | |
| // All other current entries will be MEMBER_OF the combined entry | |
| // (except for PTR_AND_OBJ entries which do not have a placeholder value | |
| // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected | |
| // to be handled by themselves, after all other maps). | |
| OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag( | |
| OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1); | |
| for (auto &M : CurTypes) | |
| OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag); | |
| // When we are emitting a combined entry. If there were any pending | |
| // attachments to be done, we do them to the begin address of the combined | |
| // entry. Note that this means only one attachment per combined-entry will | |
| // be done. So, for instance, if we have: | |
| // S *ps; | |
| // ... map(ps->a, ps->b) | |
| // When we are emitting a combined entry. If AttachInfo is valid, | |
| // update the pointee address to point to the begin address of the combined | |
| // entry. This ensures that if we have multiple maps like: | |
| // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like: | |
| // | |
| // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry | |
| // &ps[0], &ps->a, sizeof(ps->a), TO | FROM | |
| // &ps[0], &ps->b, sizeof(ps->b), TO | FROM | |
| // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB | |
| if (AttachInfo.isValid()) | |
| AttachInfo.AttachPteeAddr = LBAddr; | |
| } | |
| /// Generate all the base pointers, section pointers, sizes, map types, and | |
| /// mappers for the extracted mappable expressions (all included in \a | |
| /// CombinedInfo). Also, for each item that relates with a device pointer, a | |
| /// pair of the relevant declaration and index where it occurs is appended to | |
| /// the device pointers info array. | |
| void generateAllInfo( | |
| MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder, | |
| const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet = | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const { | |
| assert(isa<const OMPExecutableDirective *>(CurDir) && | |
| "Expect a executable directive"); | |
| const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir); | |
| generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder, | |
| SkipVarSet); | |
| } | |
| /// Generate all the base pointers, section pointers, sizes, map types, and | |
| /// mappers for the extracted map clauses of user-defined mapper (all included | |
| /// in \a CombinedInfo). | |
| void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo, | |
| llvm::OpenMPIRBuilder &OMPBuilder) const { | |
| assert(isa<const OMPDeclareMapperDecl *>(CurDir) && | |
| "Expect a declare mapper directive"); | |
| const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(CurDir); | |
| generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo, | |
| OMPBuilder); | |
| } | |
| /// Emit capture info for lambdas for variables captured by reference. | |
| void generateInfoForLambdaCaptures( | |
| const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo, | |
| llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const { | |
| QualType VDType = VD->getType().getCanonicalType().getNonReferenceType(); | |
| const auto *RD = VDType->getAsCXXRecordDecl(); | |
| if (!RD || !RD->isLambda()) | |
| return; | |
| Address VDAddr(Arg, CGF.ConvertTypeForMem(VDType), | |
| CGF.getContext().getDeclAlign(VD)); | |
| LValue VDLVal = CGF.MakeAddrLValue(VDAddr, VDType); | |
| llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures; | |
| FieldDecl *ThisCapture = nullptr; | |
| RD->getCaptureFields(Captures, ThisCapture); | |
| if (ThisCapture) { | |
| LValue ThisLVal = | |
| CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); | |
| LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); | |
| LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), | |
| VDLVal.getPointer(CGF)); | |
| CombinedInfo.Exprs.push_back(VD); | |
| CombinedInfo.BasePointers.push_back(ThisLVal.getPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(ThisLValVal.getPointer(CGF)); | |
| CombinedInfo.Sizes.push_back( | |
| CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), | |
| CGF.Int64Ty, /*isSigned=*/true)); | |
| CombinedInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ | | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | | |
| OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| } | |
| for (const LambdaCapture &LC : RD->captures()) { | |
| if (!LC.capturesVariable()) | |
| continue; | |
| const VarDecl *VD = cast<VarDecl>(LC.getCapturedVar()); | |
| if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType()) | |
| continue; | |
| auto It = Captures.find(VD); | |
| assert(It != Captures.end() && "Found lambda capture without field."); | |
| LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); | |
| if (LC.getCaptureKind() == LCK_ByRef) { | |
| LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); | |
| LambdaPointers.try_emplace(VarLVal.getPointer(CGF), | |
| VDLVal.getPointer(CGF)); | |
| CombinedInfo.Exprs.push_back(VD); | |
| CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(VarLValVal.getPointer(CGF)); | |
| CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.getTypeSize( | |
| VD->getType().getCanonicalType().getNonReferenceType()), | |
| CGF.Int64Ty, /*isSigned=*/true)); | |
| } else { | |
| RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); | |
| LambdaPointers.try_emplace(VarLVal.getPointer(CGF), | |
| VDLVal.getPointer(CGF)); | |
| CombinedInfo.Exprs.push_back(VD); | |
| CombinedInfo.BasePointers.push_back(VarLVal.getPointer(CGF)); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(VarRVal.getScalarVal()); | |
| CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); | |
| } | |
| CombinedInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ | | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | | |
| OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| } | |
| } | |
| /// Set correct indices for lambdas captures. | |
| void adjustMemberOfForLambdaCaptures( | |
| llvm::OpenMPIRBuilder &OMPBuilder, | |
| const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers, | |
| MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, | |
| MapFlagsArrayTy &Types) const { | |
| for (unsigned I = 0, E = Types.size(); I < E; ++I) { | |
| // Set correct member_of idx for all implicit lambda captures. | |
| if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ | | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | | |
| OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF | | |
| OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT)) | |
| continue; | |
| llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]); | |
| assert(BasePtr && "Unable to find base lambda address."); | |
| int TgtIdx = -1; | |
| for (unsigned J = I; J > 0; --J) { | |
| unsigned Idx = J - 1; | |
| if (Pointers[Idx] != BasePtr) | |
| continue; | |
| TgtIdx = Idx; | |
| break; | |
| } | |
| assert(TgtIdx != -1 && "Unable to find parent lambda."); | |
| // All other current entries will be MEMBER_OF the combined entry | |
| // (except for PTR_AND_OBJ entries which do not have a placeholder value | |
| // 0xFFFF in the MEMBER_OF field). | |
| OpenMPOffloadMappingFlags MemberOfFlag = | |
| OMPBuilder.getMemberOfFlag(TgtIdx); | |
| OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag); | |
| } | |
| } | |
| /// Populate component lists for non-lambda captured variables from map, | |
| /// is_device_ptr and has_device_addr clause info. | |
| void populateComponentListsForNonLambdaCaptureFromClauses( | |
| const ValueDecl *VD, MapDataArrayTy &DeclComponentLists, | |
| SmallVectorImpl< | |
| SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>> | |
| &StorageForImplicitlyAddedComponentLists) const { | |
| if (VD && LambdasMap.count(VD)) | |
| return; | |
| // For member fields list in is_device_ptr, store it in | |
| // DeclComponentLists for generating components info. | |
| static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown; | |
| auto It = DevPointersMap.find(VD); | |
| if (It != DevPointersMap.end()) | |
| for (const auto &MCL : It->second) | |
| DeclComponentLists.emplace_back(MCL, OMPC_MAP_to, Unknown, | |
| /*IsImpicit = */ true, nullptr, | |
| nullptr); | |
| auto I = HasDevAddrsMap.find(VD); | |
| if (I != HasDevAddrsMap.end()) | |
| for (const auto &MCL : I->second) | |
| DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom, Unknown, | |
| /*IsImpicit = */ true, nullptr, | |
| nullptr); | |
| assert(isa<const OMPExecutableDirective *>(CurDir) && | |
| "Expect a executable directive"); | |
| const auto *CurExecDir = cast<const OMPExecutableDirective *>(CurDir); | |
| for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) { | |
| const auto *EI = C->getVarRefs().begin(); | |
| for (const auto L : C->decl_component_lists(VD)) { | |
| const ValueDecl *VDecl, *Mapper; | |
| // The Expression is not correct if the mapping is implicit | |
| const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr; | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components; | |
| std::tie(VDecl, Components, Mapper) = L; | |
| assert(VDecl == VD && "We got information for the wrong declaration??"); | |
| assert(!Components.empty() && | |
| "Not expecting declaration with no component lists."); | |
| DeclComponentLists.emplace_back(Components, C->getMapType(), | |
| C->getMapTypeModifiers(), | |
| C->isImplicit(), Mapper, E); | |
| ++EI; | |
| } | |
| } | |
| // For the target construct, if there's a map with a base-pointer that's | |
| // a member of an implicitly captured struct, of the current class, | |
| // we need to emit an implicit map on the pointer. | |
| if (isOpenMPTargetExecutionDirective(CurExecDir->getDirectiveKind())) | |
| addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD( | |
| VD, DeclComponentLists, StorageForImplicitlyAddedComponentLists); | |
| llvm::stable_sort(DeclComponentLists, [](const MapData &LHS, | |
| const MapData &RHS) { | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(LHS); | |
| OpenMPMapClauseKind MapType = std::get<1>(RHS); | |
| bool HasPresent = | |
| llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present); | |
| bool HasAllocs = MapType == OMPC_MAP_alloc; | |
| MapModifiers = std::get<2>(RHS); | |
| MapType = std::get<1>(LHS); | |
| bool HasPresentR = | |
| llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present); | |
| bool HasAllocsR = MapType == OMPC_MAP_alloc; | |
| return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR); | |
| }); | |
| } | |
| /// On a target construct, if there's an implicit map on a struct, or that of | |
| /// this[:], and an explicit map with a member of that struct/class as the | |
| /// base-pointer, we need to make sure that base-pointer is implicitly mapped, | |
| /// to make sure we don't map the full struct/class. For example: | |
| /// | |
| /// \code | |
| /// struct S { | |
| /// int dummy[10000]; | |
| /// int *p; | |
| /// void f1() { | |
| /// #pragma omp target map(p[0:1]) | |
| /// (void)this; | |
| /// } | |
| /// }; S s; | |
| /// | |
| /// void f2() { | |
| /// #pragma omp target map(s.p[0:10]) | |
| /// (void)s; | |
| /// } | |
| /// \endcode | |
| /// | |
| /// Only `this-p` and `s.p` should be mapped in the two cases above. | |
| // | |
| // OpenMP 6.0: 7.9.6 map clause, pg 285 | |
| // If a list item with an implicitly determined data-mapping attribute does | |
| // not have any corresponding storage in the device data environment prior to | |
| // a task encountering the construct associated with the map clause, and one | |
| // or more contiguous parts of the original storage are either list items or | |
| // base pointers to list items that are explicitly mapped on the construct, | |
| // only those parts of the original storage will have corresponding storage in | |
| // the device data environment as a result of the map clauses on the | |
| // construct. | |
| void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD( | |
| const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists, | |
| SmallVectorImpl< | |
| SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>> | |
| &ComponentVectorStorage) const { | |
| bool IsThisCapture = CapturedVD == nullptr; | |
| for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| ComponentsWithAttachPtr = ComponentsAndAttachPtr.first; | |
| const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second; | |
| if (!AttachPtrExpr) | |
| continue; | |
| const auto *ME = dyn_cast<MemberExpr>(AttachPtrExpr); | |
| if (!ME) | |
| continue; | |
| const Expr *Base = ME->getBase()->IgnoreParenImpCasts(); | |
| // If we are handling a "this" capture, then we are looking for | |
| // attach-ptrs of form `this->p`, either explicitly or implicitly. | |
| if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Base)) | |
| continue; | |
| if (!IsThisCapture && (!isa<DeclRefExpr>(Base) || | |
| cast<DeclRefExpr>(Base)->getDecl() != CapturedVD)) | |
| continue; | |
| // For non-this captures, we are looking for attach-ptrs of form | |
| // `s.p`. | |
| // For non-this captures, we are looking for attach-ptrs like `s.p`. | |
| if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Base) || | |
| cast<DeclRefExpr>(Base)->getDecl() != CapturedVD)) | |
| continue; | |
| // Check if we have an existing map on either: | |
| // this[:], s, this->p, or s.p, in which case, we don't need to add | |
| // an implicit one for the attach-ptr s.p/this->p. | |
| bool FoundExistingMap = false; | |
| for (const MapData &ExistingL : DeclComponentLists) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| ExistingComponents = std::get<0>(ExistingL); | |
| if (ExistingComponents.empty()) | |
| continue; | |
| // First check if we have a map like map(this->p) or map(s.p). | |
| const auto &FirstComponent = ExistingComponents.front(); | |
| const Expr *FirstExpr = FirstComponent.getAssociatedExpression(); | |
| if (!FirstExpr) | |
| continue; | |
| // First check if we have a map like map(this->p) or map(s.p). | |
| if (AttachPtrComparator.areEqual(FirstExpr, AttachPtrExpr)) { | |
| FoundExistingMap = true; | |
| break; | |
| } | |
| // Check if we have a map like this[0:1] | |
| if (IsThisCapture) { | |
| if (const auto *OASE = dyn_cast<ArraySectionExpr>(FirstExpr)) { | |
| if (isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts())) { | |
| FoundExistingMap = true; | |
| break; | |
| } | |
| } | |
| continue; | |
| } | |
| // When the attach-ptr is something like `s.p`, check if | |
| // `s` itself is mapped explicitly. | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(FirstExpr)) { | |
| if (DRE->getDecl() == CapturedVD) { | |
| FoundExistingMap = true; | |
| break; | |
| } | |
| } | |
| } | |
| if (FoundExistingMap) | |
| continue; | |
| // If no base map is found, we need to create an implicit map for the | |
| // attach-pointer expr. | |
| ComponentVectorStorage.emplace_back(); | |
| auto &AttachPtrComponents = ComponentVectorStorage.back(); | |
| static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown; | |
| bool SeenAttachPtrComponent = false; | |
| // For creating a map on the attach-ptr `s.p/this->p`, we copy all | |
| // components from the component-list which has `s.p/this->p` | |
| // as the attach-ptr, starting from the component which matches | |
| // `s.p/this->p`. This way, we'll have component-lists of | |
| // `s.p` -> `s`, and `this->p` -> `this`. | |
| for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) { | |
| const auto &Component = ComponentsWithAttachPtr[i]; | |
| const Expr *ComponentExpr = Component.getAssociatedExpression(); | |
| if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr) | |
| continue; | |
| SeenAttachPtrComponent = true; | |
| AttachPtrComponents.emplace_back(Component.getAssociatedExpression(), | |
| Component.getAssociatedDeclaration(), | |
| Component.isNonContiguous()); | |
| } | |
| assert(!AttachPtrComponents.empty() && | |
| "Could not populate component-lists for mapping attach-ptr"); | |
| DeclComponentLists.emplace_back( | |
| AttachPtrComponents, OMPC_MAP_tofrom, Unknown, | |
| /*IsImplicit=*/true, /*mapper=*/nullptr, AttachPtrExpr); | |
| } | |
| } | |
| /// For a capture that has an associated clause, generate the base pointers, | |
| /// section pointers, sizes, map types, and mappers (all included in | |
| /// \a CurCaptureVarInfo). | |
| void generateInfoForCaptureFromClauseInfo( | |
| const MapDataArrayTy &DeclComponentListsFromClauses, | |
| const CapturedStmt::Capture *Cap, llvm::Value *Arg, | |
| MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder, | |
| unsigned OffsetForMemberOfFlag) const { | |
| assert(!Cap->capturesVariableArrayType() && | |
| "Not expecting to generate map info for a variable array type!"); | |
| // We need to know when we generating information for the first component | |
| const ValueDecl *VD = Cap->capturesThis() | |
| ? nullptr | |
| : Cap->getCapturedVar()->getCanonicalDecl(); | |
| // for map(to: lambda): skip here, processing it in | |
| // generateDefaultMapInfo | |
| if (LambdasMap.count(VD)) | |
| return; | |
| // If this declaration appears in a is_device_ptr clause we just have to | |
| // pass the pointer by value. If it is a reference to a declaration, we just | |
| // pass its value. | |
| if (VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) { | |
| CurCaptureVarInfo.Exprs.push_back(VD); | |
| CurCaptureVarInfo.BasePointers.emplace_back(Arg); | |
| CurCaptureVarInfo.DevicePtrDecls.emplace_back(VD); | |
| CurCaptureVarInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer); | |
| CurCaptureVarInfo.Pointers.push_back(Arg); | |
| CurCaptureVarInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.getTypeSize(CGF.getContext().VoidPtrTy), CGF.Int64Ty, | |
| /*isSigned=*/true)); | |
| CurCaptureVarInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | | |
| OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM); | |
| CurCaptureVarInfo.HasAttachPtr.push_back(false); | |
| CurCaptureVarInfo.Mappers.push_back(nullptr); | |
| return; | |
| } | |
| auto GenerateInfoForComponentLists = | |
| [&](ArrayRef<MapData> DeclComponentListsFromClauses, | |
| bool IsEligibleForTargetParamFlag) { | |
| MapCombinedInfoTy CurInfoForComponentLists; | |
| StructRangeInfoTy PartialStruct; | |
| AttachInfoTy AttachInfo; | |
| if (DeclComponentListsFromClauses.empty()) | |
| return; | |
| generateInfoForCaptureFromComponentLists( | |
| VD, DeclComponentListsFromClauses, CurInfoForComponentLists, | |
| PartialStruct, AttachInfo, IsEligibleForTargetParamFlag); | |
| // If there is an entry in PartialStruct it means we have a | |
| // struct with individual members mapped. Emit an extra combined | |
| // entry. | |
| if (PartialStruct.Base.isValid()) { | |
| CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData); | |
| emitCombinedEntry( | |
| CurCaptureVarInfo, CurInfoForComponentLists.Types, | |
| PartialStruct, AttachInfo, Cap->capturesThis(), OMPBuilder, | |
| /*VD=*/nullptr, OffsetForMemberOfFlag, | |
| /*NotTargetParams*/ !IsEligibleForTargetParamFlag); | |
| } | |
| // We do the appends to get the entries in the following order: | |
| // combined-entry -> individual-field-entries -> attach-entry, | |
| CurCaptureVarInfo.append(CurInfoForComponentLists); | |
| if (AttachInfo.isValid()) | |
| emitAttachEntry(CGF, CurCaptureVarInfo, AttachInfo); | |
| }; | |
| // Group component lists by their AttachPtrExpr and process them in order | |
| // of increasing complexity (nullptr first, then simple expressions like p, | |
| // then more complex ones like p[0], etc.) | |
| // | |
| // This ensure that we: | |
| // * handle maps that can contribute towards setting the kernel argument, | |
| // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d). | |
| // * allocate a single contiguous storage for all exprs with the same | |
| // captured var and having the same attach-ptr. | |
| // | |
| // Example: The map clauses below should be handled grouped together based | |
| // on their attachable-base-pointers: | |
| // map-clause | attachable-base-pointer | |
| // --------------------------+------------------------ | |
| // map(p, ps) | nullptr | |
| // map(p[0]) | p | |
| // map(p[0]->b, p[0]->c) | p[0] | |
| // map(ps->d, ps->e, ps->pt) | ps | |
| // map(ps->pt->d, ps->pt->e) | ps->pt | |
| // First, collect all MapData entries with their attach-ptr exprs. | |
| SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs; | |
| for (const MapData &L : DeclComponentListsFromClauses) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components = | |
| std::get<0>(L); | |
| const Expr *AttachPtrExpr = getAttachPtrExpr(Components); | |
| AttachPtrMapDataPairs.emplace_back(AttachPtrExpr, L); | |
| } | |
| // Next, sort by increasing order of their complexity. | |
| llvm::stable_sort(AttachPtrMapDataPairs, | |
| [this](const auto &LHS, const auto &RHS) { | |
| return AttachPtrComparator(LHS.first, RHS.first); | |
| }); | |
| bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty(); | |
| bool IsFirstGroup = true; | |
| // And finally, process them all in order, grouping those with | |
| // equivalent attach-ptr exprs together. | |
| auto *It = AttachPtrMapDataPairs.begin(); | |
| while (It != AttachPtrMapDataPairs.end()) { | |
| const Expr *AttachPtrExpr = It->first; | |
| MapDataArrayTy GroupLists; | |
| while (It != AttachPtrMapDataPairs.end() && | |
| (It->first == AttachPtrExpr || | |
| AttachPtrComparator.areEqual(It->first, AttachPtrExpr))) { | |
| GroupLists.push_back(It->second); | |
| ++It; | |
| } | |
| assert(!GroupLists.empty() && "GroupLists should not be empty"); | |
| // Determine if this group of component-lists is eligible for TARGET_PARAM | |
| // flag. Only the first group processed should be eligible, and only if no | |
| // default mapping was done. | |
| bool IsEligibleForTargetParamFlag = | |
| IsFirstGroup && NoDefaultMappingDoneForVD; | |
| GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag); | |
| IsFirstGroup = false; | |
| } | |
| } | |
| /// Generate the base pointers, section pointers, sizes, map types, and | |
| /// mappers associated to \a DeclComponentLists for a given capture | |
| /// \a VD (all included in \a CurComponentListInfo). | |
| void generateInfoForCaptureFromComponentLists( | |
| const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists, | |
| MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct, | |
| AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const { | |
| // Find overlapping elements (including the offset from the base element). | |
| llvm::SmallDenseMap< | |
| const MapData *, | |
| llvm::SmallVector< | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>, | |
| 4> | |
| OverlappedData; | |
| size_t Count = 0; | |
| for (const MapData &L : DeclComponentLists) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components; | |
| OpenMPMapClauseKind MapType; | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers; | |
| bool IsImplicit; | |
| const ValueDecl *Mapper; | |
| const Expr *VarRef; | |
| std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) = | |
| L; | |
| ++Count; | |
| for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(Count)) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components1; | |
| std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper, | |
| VarRef) = L1; | |
| auto CI = Components.rbegin(); | |
| auto CE = Components.rend(); | |
| auto SI = Components1.rbegin(); | |
| auto SE = Components1.rend(); | |
| for (; CI != CE && SI != SE; ++CI, ++SI) { | |
| if (CI->getAssociatedExpression()->getStmtClass() != | |
| SI->getAssociatedExpression()->getStmtClass()) | |
| break; | |
| // Are we dealing with different variables/fields? | |
| if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration()) | |
| break; | |
| } | |
| // Found overlapping if, at least for one component, reached the head | |
| // of the components list. | |
| if (CI == CE || SI == SE) { | |
| // Ignore it if it is the same component. | |
| if (CI == CE && SI == SE) | |
| continue; | |
| const auto It = (SI == SE) ? CI : SI; | |
| // If one component is a pointer and another one is a kind of | |
| // dereference of this pointer (array subscript, section, dereference, | |
| // etc.), it is not an overlapping. | |
| // Same, if one component is a base and another component is a | |
| // dereferenced pointer memberexpr with the same base. | |
| if (!isa<MemberExpr>(It->getAssociatedExpression()) || | |
| (std::prev(It)->getAssociatedDeclaration() && | |
| std::prev(It) | |
| ->getAssociatedDeclaration() | |
| ->getType() | |
| ->isPointerType()) || | |
| (It->getAssociatedDeclaration() && | |
| It->getAssociatedDeclaration()->getType()->isPointerType() && | |
| std::next(It) != CE && std::next(It) != SE)) | |
| continue; | |
| const MapData &BaseData = CI == CE ? L : L1; | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef SubData = | |
| SI == SE ? Components : Components1; | |
| OverlappedData[&BaseData].push_back(SubData); | |
| } | |
| } | |
| } | |
| // Sort the overlapped elements for each item. | |
| llvm::SmallVector<const FieldDecl *, 4> Layout; | |
| if (!OverlappedData.empty()) { | |
| const Type *BaseType = VD->getType().getCanonicalType().getTypePtr(); | |
| const Type *OrigType = BaseType->getPointeeOrArrayElementType(); | |
| while (BaseType != OrigType) { | |
| BaseType = OrigType->getCanonicalTypeInternal().getTypePtr(); | |
| OrigType = BaseType->getPointeeOrArrayElementType(); | |
| } | |
| if (const auto *CRD = BaseType->getAsCXXRecordDecl()) | |
| getPlainLayout(CRD, Layout, /*AsBase=*/false); | |
| else { | |
| const auto *RD = BaseType->getAsRecordDecl(); | |
| Layout.append(RD->field_begin(), RD->field_end()); | |
| } | |
| } | |
| for (auto &Pair : OverlappedData) { | |
| llvm::stable_sort( | |
| Pair.getSecond(), | |
| [&Layout]( | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef First, | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| Second) { | |
| auto CI = First.rbegin(); | |
| auto CE = First.rend(); | |
| auto SI = Second.rbegin(); | |
| auto SE = Second.rend(); | |
| for (; CI != CE && SI != SE; ++CI, ++SI) { | |
| if (CI->getAssociatedExpression()->getStmtClass() != | |
| SI->getAssociatedExpression()->getStmtClass()) | |
| break; | |
| // Are we dealing with different variables/fields? | |
| if (CI->getAssociatedDeclaration() != | |
| SI->getAssociatedDeclaration()) | |
| break; | |
| } | |
| // Lists contain the same elements. | |
| if (CI == CE && SI == SE) | |
| return false; | |
| // List with less elements is less than list with more elements. | |
| if (CI == CE || SI == SE) | |
| return CI == CE; | |
| const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration()); | |
| const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration()); | |
| if (FD1->getParent() == FD2->getParent()) | |
| return FD1->getFieldIndex() < FD2->getFieldIndex(); | |
| const auto *It = | |
| llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) { | |
| return FD == FD1 || FD == FD2; | |
| }); | |
| return *It == FD1; | |
| }); | |
| } | |
| // Associated with a capture, because the mapping flags depend on it. | |
| // Go through all of the elements with the overlapped elements. | |
| bool AddTargetParamFlag = IsListEligibleForTargetParamFlag; | |
| MapCombinedInfoTy StructBaseCombinedInfo; | |
| for (const auto &Pair : OverlappedData) { | |
| const MapData &L = *Pair.getFirst(); | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components; | |
| OpenMPMapClauseKind MapType; | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers; | |
| bool IsImplicit; | |
| const ValueDecl *Mapper; | |
| const Expr *VarRef; | |
| std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) = | |
| L; | |
| ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> | |
| OverlappedComponents = Pair.getSecond(); | |
| generateInfoForComponentList( | |
| MapType, MapModifiers, {}, Components, CurComponentListInfo, | |
| StructBaseCombinedInfo, PartialStruct, AttachInfo, AddTargetParamFlag, | |
| IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper, | |
| /*ForDeviceAddr=*/false, VD, VarRef, OverlappedComponents); | |
| AddTargetParamFlag = false; | |
| } | |
| // Go through other elements without overlapped elements. | |
| for (const MapData &L : DeclComponentLists) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef Components; | |
| OpenMPMapClauseKind MapType; | |
| ArrayRef<OpenMPMapModifierKind> MapModifiers; | |
| bool IsImplicit; | |
| const ValueDecl *Mapper; | |
| const Expr *VarRef; | |
| std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) = | |
| L; | |
| auto It = OverlappedData.find(&L); | |
| if (It == OverlappedData.end()) | |
| generateInfoForComponentList( | |
| MapType, MapModifiers, {}, Components, CurComponentListInfo, | |
| StructBaseCombinedInfo, PartialStruct, AttachInfo, | |
| AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false, | |
| Mapper, /*ForDeviceAddr=*/false, VD, VarRef, | |
| /*OverlappedElements*/ {}); | |
| AddTargetParamFlag = false; | |
| } | |
| } | |
| /// Check if a variable should be treated as firstprivate due to explicit | |
| /// firstprivate clause or defaultmap(firstprivate:...). | |
| bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const { | |
| // Check explicit firstprivate clauses (not implicit from defaultmap) | |
| auto I = FirstPrivateDecls.find(VD); | |
| if (I != FirstPrivateDecls.end() && !I->getSecond()) | |
| return true; // Explicit firstprivate only | |
| // Check defaultmap(firstprivate:scalar) for scalar types | |
| if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_scalar)) { | |
| if (Type->isScalarType()) | |
| return true; | |
| } | |
| // Check defaultmap(firstprivate:pointer) for pointer types | |
| if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_pointer)) { | |
| if (Type->isAnyPointerType()) | |
| return true; | |
| } | |
| // Check defaultmap(firstprivate:aggregate) for aggregate types | |
| if (DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_aggregate)) { | |
| if (Type->isAggregateType()) | |
| return true; | |
| } | |
| // Check defaultmap(firstprivate:all) for all types | |
| return DefaultmapFirstprivateKinds.count(OMPC_DEFAULTMAP_all); | |
| } | |
| /// Generate the default map information for a given capture \a CI, | |
| /// record field declaration \a RI and captured value \a CV. | |
| void generateDefaultMapInfo(const CapturedStmt::Capture &CI, | |
| const FieldDecl &RI, llvm::Value *CV, | |
| MapCombinedInfoTy &CombinedInfo) const { | |
| bool IsImplicit = true; | |
| // Do the default mapping. | |
| if (CI.capturesThis()) { | |
| CombinedInfo.Exprs.push_back(nullptr); | |
| CombinedInfo.BasePointers.push_back(CV); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(CV); | |
| const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr()); | |
| CombinedInfo.Sizes.push_back( | |
| CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), | |
| CGF.Int64Ty, /*isSigned=*/true)); | |
| // Default map type. | |
| CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO | | |
| OpenMPOffloadMappingFlags::OMP_MAP_FROM); | |
| } else if (CI.capturesVariableByCopy()) { | |
| const VarDecl *VD = CI.getCapturedVar(); | |
| CombinedInfo.Exprs.push_back(VD->getCanonicalDecl()); | |
| CombinedInfo.BasePointers.push_back(CV); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| CombinedInfo.Pointers.push_back(CV); | |
| bool IsFirstprivate = | |
| isEffectivelyFirstprivate(VD, RI.getType().getNonReferenceType()); | |
| if (!RI.getType()->isAnyPointerType()) { | |
| // We have to signal to the runtime captures passed by value that are | |
| // not pointers. | |
| CombinedInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); | |
| CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); | |
| } else if (IsFirstprivate) { | |
| // Firstprivate pointers should be passed by value (as literals) | |
| // without performing a present table lookup at runtime. | |
| CombinedInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); | |
| // Use zero size for pointer literals (just passing the pointer value) | |
| CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); | |
| } else { | |
| // Pointers are implicitly mapped with a zero size and no flags | |
| // (other than first map that is added for all implicit maps). | |
| CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE); | |
| CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); | |
| } | |
| auto I = FirstPrivateDecls.find(VD); | |
| if (I != FirstPrivateDecls.end()) | |
| IsImplicit = I->getSecond(); | |
| } else { | |
| assert(CI.capturesVariable() && "Expected captured reference."); | |
| const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr()); | |
| QualType ElementType = PtrTy->getPointeeType(); | |
| const VarDecl *VD = CI.getCapturedVar(); | |
| bool IsFirstprivate = isEffectivelyFirstprivate(VD, ElementType); | |
| CombinedInfo.Exprs.push_back(VD->getCanonicalDecl()); | |
| CombinedInfo.BasePointers.push_back(CV); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None); | |
| // For firstprivate pointers, pass by value instead of dereferencing | |
| if (IsFirstprivate && ElementType->isAnyPointerType()) { | |
| // Treat as a literal value (pass the pointer value itself) | |
| CombinedInfo.Pointers.push_back(CV); | |
| // Use zero size for pointer literals | |
| CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); | |
| CombinedInfo.Types.push_back( | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); | |
| } else { | |
| CombinedInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); | |
| // The default map type for a scalar/complex type is 'to' because by | |
| // default the value doesn't have to be retrieved. For an aggregate | |
| // type, the default is 'tofrom'. | |
| CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI)); | |
| CombinedInfo.Pointers.push_back(CV); | |
| } | |
| auto I = FirstPrivateDecls.find(VD); | |
| if (I != FirstPrivateDecls.end()) | |
| IsImplicit = I->getSecond(); | |
| } | |
| // Every default map produces a single argument which is a target parameter. | |
| CombinedInfo.Types.back() |= | |
| OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; | |
| // Add flag stating this is an implicit map. | |
| if (IsImplicit) | |
| CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT; | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| // No user-defined mapper for default mapping. | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| } | |
| }; | |
| } // anonymous namespace | |
| // Try to extract the base declaration from a `this->x` expression if possible. | |
| static ValueDecl *getDeclFromThisExpr(const Expr *E) { | |
| if (!E) | |
| return nullptr; | |
| if (const auto *OASE = dyn_cast<ArraySectionExpr>(E->IgnoreParenCasts())) | |
| if (const MemberExpr *ME = | |
| dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts())) | |
| return ME->getMemberDecl(); | |
| return nullptr; | |
| } | |
| /// Emit a string constant containing the names of the values mapped to the | |
| /// offloading runtime library. | |
| static llvm::Constant * | |
| emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder, | |
| MappableExprsHandler::MappingExprInfo &MapExprs) { | |
| uint32_t SrcLocStrSize; | |
| if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr()) | |
| return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize); | |
| SourceLocation Loc; | |
| if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) { | |
| if (const ValueDecl *VD = getDeclFromThisExpr(MapExprs.getMapExpr())) | |
| Loc = VD->getLocation(); | |
| else | |
| Loc = MapExprs.getMapExpr()->getExprLoc(); | |
| } else { | |
| Loc = MapExprs.getMapDecl()->getLocation(); | |
| } | |
| std::string ExprName; | |
| if (MapExprs.getMapExpr()) { | |
| PrintingPolicy P(CGF.getContext().getLangOpts()); | |
| llvm::raw_string_ostream OS(ExprName); | |
| MapExprs.getMapExpr()->printPretty(OS, nullptr, P); | |
| } else { | |
| ExprName = MapExprs.getMapDecl()->getNameAsString(); | |
| } | |
| std::string FileName; | |
| PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); | |
| if (auto *DbgInfo = CGF.getDebugInfo()) | |
| FileName = DbgInfo->remapDIPath(PLoc.getFilename()); | |
| else | |
| FileName = PLoc.getFilename(); | |
| return OMPBuilder.getOrCreateSrcLocStr(FileName, ExprName, PLoc.getLine(), | |
| PLoc.getColumn(), SrcLocStrSize); | |
| } | |
| /// Emit the arrays used to pass the captures and map information to the | |
| /// offloading runtime library. If there is no map or capture information, | |
| /// return nullptr by reference. | |
| static void emitOffloadingArraysAndArgs( | |
| CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, | |
| CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder, | |
| bool IsNonContiguous = false, bool ForEndCall = false) { | |
| CodeGenModule &CGM = CGF.CGM; | |
| using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |
| InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator()); | |
| InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint()); | |
| auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) { | |
| if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) { | |
| Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl); | |
| } | |
| }; | |
| auto CustomMapperCB = [&](unsigned int I) { | |
| llvm::Function *MFunc = nullptr; | |
| if (CombinedInfo.Mappers[I]) { | |
| Info.HasMapper = true; | |
| MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc( | |
| cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I])); | |
| } | |
| return MFunc; | |
| }; | |
| cantFail(OMPBuilder.emitOffloadingArraysAndArgs( | |
| AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, CustomMapperCB, | |
| IsNonContiguous, ForEndCall, DeviceAddrCB)); | |
| } | |
| /// Check for inner distribute directive. | |
| static const OMPExecutableDirective * | |
| getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) { | |
| const auto *CS = D.getInnermostCapturedStmt(); | |
| const auto *Body = | |
| CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); | |
| const Stmt *ChildStmt = | |
| CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body); | |
| if (const auto *NestedDir = | |
| dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { | |
| OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind(); | |
| switch (D.getDirectiveKind()) { | |
| case OMPD_target: | |
| // For now, treat 'target' with nested 'teams loop' as if it's | |
| // distributed (target teams distribute). | |
| if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop) | |
| return NestedDir; | |
| if (DKind == OMPD_teams) { | |
| Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers( | |
| /*IgnoreCaptured=*/true); | |
| if (!Body) | |
| return nullptr; | |
| ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body); | |
| if (const auto *NND = | |
| dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { | |
| DKind = NND->getDirectiveKind(); | |
| if (isOpenMPDistributeDirective(DKind)) | |
| return NND; | |
| } | |
| } | |
| return nullptr; | |
| case OMPD_target_teams: | |
| if (isOpenMPDistributeDirective(DKind)) | |
| return NestedDir; | |
| return nullptr; | |
| case OMPD_target_parallel: | |
| case OMPD_target_simd: | |
| case OMPD_target_loop: | |
| case OMPD_target_parallel_for: | |
| case OMPD_target_parallel_for_simd: | |
| return nullptr; | |
| case OMPD_target_teams_distribute: | |
| case OMPD_target_teams_distribute_simd: | |
| case OMPD_target_teams_distribute_parallel_for: | |
| case OMPD_target_teams_distribute_parallel_for_simd: | |
| case OMPD_parallel: | |
| case OMPD_for: | |
| case OMPD_parallel_for: | |
| case OMPD_parallel_master: | |
| case OMPD_parallel_sections: | |
| case OMPD_for_simd: | |
| case OMPD_parallel_for_simd: | |
| case OMPD_cancel: | |
| case OMPD_cancellation_point: | |
| case OMPD_ordered_standalone: | |
| case OMPD_ordered_blockassoc: | |
| case OMPD_threadprivate: | |
| case OMPD_allocate: | |
| case OMPD_task: | |
| case OMPD_simd: | |
| case OMPD_tile: | |
| case OMPD_unroll: | |
| case OMPD_sections: | |
| case OMPD_section: | |
| case OMPD_single: | |
| case OMPD_master: | |
| case OMPD_critical: | |
| case OMPD_taskyield: | |
| case OMPD_barrier: | |
| case OMPD_taskwait: | |
| case OMPD_taskgroup: | |
| case OMPD_atomic: | |
| case OMPD_flush: | |
| case OMPD_depobj: | |
| case OMPD_scan: | |
| case OMPD_teams: | |
| case OMPD_target_data: | |
| case OMPD_target_exit_data: | |
| case OMPD_target_enter_data: | |
| case OMPD_distribute: | |
| case OMPD_distribute_simd: | |
| case OMPD_distribute_parallel_for: | |
| case OMPD_distribute_parallel_for_simd: | |
| case OMPD_teams_distribute: | |
| case OMPD_teams_distribute_simd: | |
| case OMPD_teams_distribute_parallel_for: | |
| case OMPD_teams_distribute_parallel_for_simd: | |
| case OMPD_target_update: | |
| case OMPD_declare_simd: | |
| case OMPD_declare_variant: | |
| case OMPD_begin_declare_variant: | |
| case OMPD_end_declare_variant: | |
| case OMPD_declare_target: | |
| case OMPD_end_declare_target: | |
| case OMPD_declare_reduction: | |
| case OMPD_declare_mapper: | |
| case OMPD_taskloop: | |
| case OMPD_taskloop_simd: | |
| case OMPD_master_taskloop: | |
| case OMPD_master_taskloop_simd: | |
| case OMPD_parallel_master_taskloop: | |
| case OMPD_parallel_master_taskloop_simd: | |
| case OMPD_requires: | |
| case OMPD_metadirective: | |
| case OMPD_unknown: | |
| default: | |
| llvm_unreachable("Unexpected directive."); | |
| } | |
| } | |
| return nullptr; | |
| } | |
| /// Emit the user-defined mapper function. The code generation follows the | |
| /// pattern in the example below. | |
| /// \code | |
| /// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle, | |
| /// void *base, void *begin, | |
| /// int64_t size, int64_t type, | |
| /// void *name = nullptr) { | |
| /// // Allocate space for an array section first. | |
| /// if ((size > 1 || (base != begin)) && !maptype.IsDelete) | |
| /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, | |
| /// size*sizeof(Ty), clearToFromMember(type)); | |
| /// // Map members. | |
| /// for (unsigned i = 0; i < size; i++) { | |
| /// N = __tgt_mapper_num_components(rt_mapper_handle); | |
| /// // For each component specified by this mapper: | |
| /// for (auto c : begin[i]->all_components) { | |
| /// // MEMBER_OF grouping: tie this component to the current array element | |
| /// // (component N) by adding N<<48. Exceptions: | |
| /// // - ATTACH entries are not members of any struct storage range. | |
| /// // - Pointee entries (reached via a pointer member) occupy separate | |
| /// // storage; their inner MEMBER_OF bits are shifted by N instead. | |
| /// if (c.isAttach() || c.isPointee()) | |
| /// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0); | |
| /// else | |
| /// member_type = c.arg_type + N<<48; | |
| /// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map | |
| /// // clause are propagated to each component, except ATTACH entries | |
| /// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier | |
| /// // bits have no meaning for ATTACH). PRESENT is additionally | |
| /// // propagated to components with HasAttachPtr (the pointee data) at | |
| /// // OpenMP >= 6.0. | |
| /// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0; | |
| /// imported_modifier_bits = | |
| /// type & (ALWAYS | DELETE | CLOSE | present_bit); | |
| /// effective_type = c.isAttach() ? member_type | |
| /// : member_type | imported_modifier_bits; | |
| /// if (c.hasMapper()) | |
| /// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size, | |
| /// effective_type, c.arg_name); | |
| /// else | |
| /// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base, | |
| /// c.arg_begin, c.arg_size, effective_type, | |
| /// c.arg_name); | |
| /// } | |
| /// } | |
| /// // Delete the array section. | |
| /// if (size > 1 && maptype.IsDelete) | |
| /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, | |
| /// size*sizeof(Ty), clearToFromMember(type)); | |
| /// } | |
| /// \endcode | |
| void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D, | |
| CodeGenFunction *CGF) { | |
| if (UDMMap.count(D) > 0) | |
| return; | |
| ASTContext &C = CGM.getContext(); | |
| QualType Ty = D->getType(); | |
| auto *MapperVarDecl = | |
| cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl()); | |
| CharUnits ElementSize = C.getTypeSizeInChars(Ty); | |
| llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(Ty); | |
| CodeGenFunction MapperCGF(CGM); | |
| MappableExprsHandler::MapCombinedInfoTy CombinedInfo; | |
| auto PrivatizeAndGenMapInfoCB = | |
| [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI, | |
| llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & { | |
| MapperCGF.Builder.restoreIP(CodeGenIP); | |
| // Privatize the declared variable of mapper to be the current array | |
| // element. | |
| Address PtrCurrent( | |
| PtrPHI, ElemTy, | |
| Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign()) | |
| .getAlignment() | |
| .alignmentOfArrayElement(ElementSize)); | |
| CodeGenFunction::OMPPrivateScope Scope(MapperCGF); | |
| Scope.addPrivate(MapperVarDecl, PtrCurrent); | |
| (void)Scope.Privatize(); | |
| // Get map clause information. | |
| MappableExprsHandler MEHandler(*D, MapperCGF); | |
| MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder); | |
| auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) { | |
| return emitMappingInformation(MapperCGF, OMPBuilder, MapExpr); | |
| }; | |
| if (CGM.getCodeGenOpts().getDebugInfo() != | |
| llvm::codegenoptions::NoDebugInfo) { | |
| CombinedInfo.Names.resize(CombinedInfo.Exprs.size()); | |
| llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(), | |
| FillInfoMap); | |
| } | |
| return CombinedInfo; | |
| }; | |
| auto CustomMapperCB = [&](unsigned I) { | |
| llvm::Function *MapperFunc = nullptr; | |
| if (CombinedInfo.Mappers[I]) { | |
| // Call the corresponding mapper function. | |
| MapperFunc = getOrCreateUserDefinedMapperFunc( | |
| cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I])); | |
| assert(MapperFunc && "Expect a valid mapper function is available."); | |
| } | |
| return MapperFunc; | |
| }; | |
| SmallString<64> TyStr; | |
| llvm::raw_svector_ostream Out(TyStr); | |
| CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out); | |
| std::string Name = getName({"omp_mapper", TyStr, D->getName()}); | |
| // Propagate the PRESENT modifier to the pointee entries (those with | |
| // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does | |
| // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion | |
| // vs. map-type modifier divergence). | |
| bool PropagatePresentToPointeeStorage = CGM.getLangOpts().OpenMP >= 60; | |
| // Keep non-pointee storage present propagation disabled until Clang stops | |
| // emitting PTR_AND_OBJ maps and uses attach-style maps throughout. | |
| bool PropagatePresentToNonPointeeStorage = false; | |
| llvm::Function *NewFn = cantFail(OMPBuilder.emitUserDefinedMapper( | |
| PrivatizeAndGenMapInfoCB, ElemTy, Name, CustomMapperCB, | |
| /*PreserveMemberOfFlags=*/false, PropagatePresentToPointeeStorage, | |
| PropagatePresentToNonPointeeStorage)); | |
| UDMMap.try_emplace(D, NewFn); | |
| if (CGF) | |
| FunctionUDMMap[CGF->CurFn].push_back(D); | |
| } | |
| llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc( | |
| const OMPDeclareMapperDecl *D) { | |
| auto I = UDMMap.find(D); | |
| if (I != UDMMap.end()) | |
| return I->second; | |
| emitUserDefinedMapper(D); | |
| return UDMMap.lookup(D); | |
| } | |
| llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D)> | |
| SizeEmitter) { | |
| OpenMPDirectiveKind Kind = D.getDirectiveKind(); | |
| const OMPExecutableDirective *TD = &D; | |
| // Get nested teams distribute kind directive, if any. For now, treat | |
| // 'target_teams_loop' as if it's really a target_teams_distribute. | |
| if ((!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind)) && | |
| Kind != OMPD_target_teams_loop) | |
| TD = getNestedDistributeDirective(CGM.getContext(), D); | |
| if (!TD) | |
| return llvm::ConstantInt::get(CGF.Int64Ty, 0); | |
| const auto *LD = cast<OMPLoopDirective>(TD); | |
| if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) | |
| return NumIterations; | |
| return llvm::ConstantInt::get(CGF.Int64Ty, 0); | |
| } | |
| static void | |
| emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, | |
| const OMPExecutableDirective &D, | |
| llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, | |
| bool RequiresOuterTask, const CapturedStmt &CS, | |
| bool OffloadingMandatory, CodeGenFunction &CGF) { | |
| if (OffloadingMandatory) { | |
| CGF.Builder.CreateUnreachable(); | |
| } else { | |
| if (RequiresOuterTask) { | |
| CapturedVars.clear(); | |
| CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); | |
| } | |
| llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(), | |
| CapturedVars.end()); | |
| Args.push_back(llvm::Constant::getNullValue(CGF.Builder.getPtrTy())); | |
| OMPRuntime->emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, | |
| Args); | |
| } | |
| } | |
| static llvm::Value *emitDeviceID( | |
| llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, | |
| CodeGenFunction &CGF) { | |
| // Emit device ID if any. | |
| llvm::Value *DeviceID; | |
| if (Device.getPointer()) { | |
| assert((Device.getInt() == OMPC_DEVICE_unknown || | |
| Device.getInt() == OMPC_DEVICE_device_num) && | |
| "Expected device_num modifier."); | |
| llvm::Value *DevVal = CGF.EmitScalarExpr(Device.getPointer()); | |
| DeviceID = | |
| CGF.Builder.CreateIntCast(DevVal, CGF.Int64Ty, /*isSigned=*/true); | |
| } else { | |
| DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); | |
| } | |
| return DeviceID; | |
| } | |
| static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType> | |
| emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) { | |
| llvm::Value *DynGP = CGF.Builder.getInt32(0); | |
| auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort; | |
| if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) { | |
| CodeGenFunction::RunCleanupsScope DynGPScope(CGF); | |
| llvm::Value *DynGPVal = | |
| CGF.EmitScalarExpr(DynGPClause->getSize(), /*IgnoreResultAssign=*/true); | |
| DynGP = CGF.Builder.CreateIntCast(DynGPVal, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier(); | |
| switch (FallbackModifier) { | |
| case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort: | |
| DynGPFallback = OMPDynGroupprivateFallbackType::Abort; | |
| break; | |
| case OMPC_DYN_GROUPPRIVATE_FALLBACK_null: | |
| DynGPFallback = OMPDynGroupprivateFallbackType::Null; | |
| break; | |
| case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem: | |
| case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown: | |
| // This is the default for dyn_groupprivate. | |
| DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem; | |
| break; | |
| default: | |
| llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate"); | |
| } | |
| } else if (auto *OMPXDynCGClause = | |
| D.getSingleClause<OMPXDynCGroupMemClause>()) { | |
| CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF); | |
| llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(OMPXDynCGClause->getSize(), | |
| /*IgnoreResultAssign=*/true); | |
| DynGP = CGF.Builder.CreateIntCast(DynCGMemVal, CGF.Int32Ty, | |
| /*isSigned=*/false); | |
| } | |
| return {DynGP, DynGPFallback}; | |
| } | |
| static void genMapInfoForCaptures( | |
| MappableExprsHandler &MEHandler, CodeGenFunction &CGF, | |
| const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, | |
| llvm::OpenMPIRBuilder &OMPBuilder, | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet, | |
| MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) { | |
| llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers; | |
| auto RI = CS.getCapturedRecordDecl()->field_begin(); | |
| auto *CV = CapturedVars.begin(); | |
| for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(), | |
| CE = CS.capture_end(); | |
| CI != CE; ++CI, ++RI, ++CV) { | |
| MappableExprsHandler::MapCombinedInfoTy CurInfo; | |
| // VLA sizes are passed to the outlined region by copy and do not have map | |
| // information associated. | |
| if (CI->capturesVariableArrayType()) { | |
| CurInfo.Exprs.push_back(nullptr); | |
| CurInfo.BasePointers.push_back(*CV); | |
| CurInfo.DevicePtrDecls.push_back(nullptr); | |
| CurInfo.DevicePointers.push_back( | |
| MappableExprsHandler::DeviceInfoTy::None); | |
| CurInfo.Pointers.push_back(*CV); | |
| CurInfo.Sizes.push_back(CGF.Builder.CreateIntCast( | |
| CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); | |
| // Copy to the device as an argument. No need to retrieve it. | |
| CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL | | |
| OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | | |
| OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT); | |
| CurInfo.HasAttachPtr.push_back(false); | |
| CurInfo.Mappers.push_back(nullptr); | |
| } else { | |
| const ValueDecl *CapturedVD = | |
| CI->capturesThis() ? nullptr | |
| : CI->getCapturedVar()->getCanonicalDecl(); | |
| bool HasEntryWithCVAsAttachPtr = false; | |
| if (CapturedVD) | |
| HasEntryWithCVAsAttachPtr = | |
| MEHandler.hasAttachEntryForCapturedVar(CapturedVD); | |
| // Populate component lists for the captured variable from clauses. | |
| MappableExprsHandler::MapDataArrayTy DeclComponentLists; | |
| SmallVector< | |
| SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4> | |
| StorageForImplicitlyAddedComponentLists; | |
| MEHandler.populateComponentListsForNonLambdaCaptureFromClauses( | |
| CapturedVD, DeclComponentLists, | |
| StorageForImplicitlyAddedComponentLists); | |
| // OpenMP 6.0, 15.8, target construct, restrictions: | |
| // * A list item in a map clause that is specified on a target construct | |
| // must have a base variable or base pointer. | |
| // | |
| // Map clauses on a target construct must either have a base pointer, or a | |
| // base-variable. So, if we don't have a base-pointer, that means that it | |
| // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)` | |
| // etc. In such cases, we do not need to handle default map generation | |
| // for `s`. | |
| bool HasEntryWithoutAttachPtr = | |
| llvm::any_of(DeclComponentLists, [&](const auto &MapData) { | |
| OMPClauseMappableExprCommon::MappableExprComponentListRef | |
| Components = std::get<0>(MapData); | |
| return !MEHandler.getAttachPtrExpr(Components); | |
| }); | |
| // Generate default map info first if there's no direct map with CV as | |
| // the base-variable, or attach pointer. | |
| if (DeclComponentLists.empty() || | |
| (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr)) | |
| MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo); | |
| // If we have any information in the map clause, we use it, otherwise we | |
| // just do a default mapping. | |
| MEHandler.generateInfoForCaptureFromClauseInfo( | |
| DeclComponentLists, CI, *CV, CurInfo, OMPBuilder, | |
| /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size()); | |
| if (!CI->capturesThis()) | |
| MappedVarSet.insert(CI->getCapturedVar()); | |
| else | |
| MappedVarSet.insert(nullptr); | |
| // Generate correct mapping for variables captured by reference in | |
| // lambdas. | |
| if (CI->capturesVariable()) | |
| MEHandler.generateInfoForLambdaCaptures(CI->getCapturedVar(), *CV, | |
| CurInfo, LambdaPointers); | |
| } | |
| // We expect to have at least an element of information for this capture. | |
| assert(!CurInfo.BasePointers.empty() && | |
| "Non-existing map pointer for capture!"); | |
| assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() && | |
| CurInfo.BasePointers.size() == CurInfo.Sizes.size() && | |
| CurInfo.BasePointers.size() == CurInfo.Types.size() && | |
| CurInfo.BasePointers.size() == CurInfo.Mappers.size() && | |
| "Inconsistent map information sizes!"); | |
| // We need to append the results of this capture to what we already have. | |
| CombinedInfo.append(CurInfo); | |
| } | |
| // Adjust MEMBER_OF flags for the lambdas captures. | |
| MEHandler.adjustMemberOfForLambdaCaptures( | |
| OMPBuilder, LambdaPointers, CombinedInfo.BasePointers, | |
| CombinedInfo.Pointers, CombinedInfo.Types); | |
| } | |
| static void | |
| genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF, | |
| MappableExprsHandler::MapCombinedInfoTy &CombinedInfo, | |
| llvm::OpenMPIRBuilder &OMPBuilder, | |
| const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet = | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) { | |
| CodeGenModule &CGM = CGF.CGM; | |
| // Map any list items in a map clause that were not captures because they | |
| // weren't referenced within the construct. | |
| MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet); | |
| auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) { | |
| return emitMappingInformation(CGF, OMPBuilder, MapExpr); | |
| }; | |
| if (CGM.getCodeGenOpts().getDebugInfo() != | |
| llvm::codegenoptions::NoDebugInfo) { | |
| CombinedInfo.Names.resize(CombinedInfo.Exprs.size()); | |
| llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(), | |
| FillInfoMap); | |
| } | |
| } | |
| static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF, | |
| const CapturedStmt &CS, | |
| llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, | |
| llvm::OpenMPIRBuilder &OMPBuilder, | |
| MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) { | |
| // Get mappable expression information. | |
| MappableExprsHandler MEHandler(D, CGF); | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet; | |
| genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder, | |
| MappedVarSet, CombinedInfo); | |
| genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet); | |
| } | |
| template <typename ClauseTy> | |
| static void | |
| emitClauseForBareTargetDirective(CodeGenFunction &CGF, | |
| const OMPExecutableDirective &D, | |
| llvm::SmallVectorImpl<llvm::Value *> &Values) { | |
| const auto *C = D.getSingleClause<ClauseTy>(); | |
| assert(!C->varlist_empty() && | |
| "ompx_bare requires explicit num_teams and thread_limit"); | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| for (auto *E : C->varlist()) { | |
| llvm::Value *V = CGF.EmitScalarExpr(E); | |
| Values.push_back( | |
| CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true)); | |
| } | |
| } | |
| static void emitTargetCallKernelLaunch( | |
| CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, | |
| const OMPExecutableDirective &D, | |
| llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask, | |
| const CapturedStmt &CS, bool OffloadingMandatory, | |
| llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, | |
| llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo, | |
| llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray, | |
| llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D)> | |
| SizeEmitter, | |
| CodeGenFunction &CGF, CodeGenModule &CGM) { | |
| llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder(); | |
| // Fill up the arrays with all the captured variables. | |
| MappableExprsHandler::MapCombinedInfoTy CombinedInfo; | |
| CGOpenMPRuntime::TargetDataInfo Info; | |
| genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo); | |
| // Append a null entry for the implicit dyn_ptr argument. | |
| using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags; | |
| auto *NullPtr = llvm::Constant::getNullValue(CGF.Builder.getPtrTy()); | |
| CombinedInfo.BasePointers.push_back(NullPtr); | |
| CombinedInfo.Pointers.push_back(NullPtr); | |
| CombinedInfo.DevicePointers.push_back( | |
| llvm::OpenMPIRBuilder::DeviceInfoTy::None); | |
| CombinedInfo.Sizes.push_back(CGF.Builder.getInt64(0)); | |
| CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | | |
| OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); | |
| CombinedInfo.HasAttachPtr.push_back(false); | |
| if (!CombinedInfo.Names.empty()) | |
| CombinedInfo.Names.push_back(NullPtr); | |
| CombinedInfo.Exprs.push_back(nullptr); | |
| CombinedInfo.Mappers.push_back(nullptr); | |
| CombinedInfo.DevicePtrDecls.push_back(nullptr); | |
| emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder, | |
| /*IsNonContiguous=*/true, /*ForEndCall=*/false); | |
| InputInfo.NumberOfTargetItems = Info.NumberOfPtrs; | |
| InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray, | |
| CGF.VoidPtrTy, CGM.getPointerAlign()); | |
| InputInfo.PointersArray = | |
| Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); | |
| InputInfo.SizesArray = | |
| Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); | |
| InputInfo.MappersArray = | |
| Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); | |
| MapTypesArray = Info.RTArgs.MapTypesArray; | |
| MapNamesArray = Info.RTArgs.MapNamesArray; | |
| auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars, | |
| RequiresOuterTask, &CS, OffloadingMandatory, Device, | |
| OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray, | |
| SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) { | |
| bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor; | |
| if (IsReverseOffloading) { | |
| // Reverse offloading is not supported, so just execute on the host. | |
| // FIXME: This fallback solution is incorrect since it ignores the | |
| // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to | |
| // assert here and ensure SEMA emits an error. | |
| emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars, | |
| RequiresOuterTask, CS, OffloadingMandatory, CGF); | |
| return; | |
| } | |
| bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>(); | |
| unsigned NumTargetItems = InputInfo.NumberOfTargetItems; | |
| llvm::Value *BasePointersArray = | |
| InputInfo.BasePointersArray.emitRawPointer(CGF); | |
| llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF); | |
| llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF); | |
| llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF); | |
| auto &&EmitTargetCallFallbackCB = | |
| [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, | |
| OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) | |
| -> llvm::OpenMPIRBuilder::InsertPointTy { | |
| CGF.Builder.restoreIP(IP); | |
| emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars, | |
| RequiresOuterTask, CS, OffloadingMandatory, CGF); | |
| return CGF.Builder.saveIP(); | |
| }; | |
| bool IsBare = D.hasClausesOfKind<OMPXBareClause>(); | |
| SmallVector<llvm::Value *, 3> NumTeams; | |
| SmallVector<llvm::Value *, 3> NumThreads; | |
| if (IsBare) { | |
| emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, NumTeams); | |
| emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D, | |
| NumThreads); | |
| } else { | |
| NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D)); | |
| NumThreads.push_back( | |
| OMPRuntime->emitNumThreadsForTargetDirective(CGF, D)); | |
| } | |
| llvm::Value *DeviceID = emitDeviceID(Device, CGF); | |
| llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc()); | |
| llvm::Value *NumIterations = | |
| OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter); | |
| auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF); | |
| llvm::OpenMPIRBuilder::InsertPointTy AllocaIP( | |
| CGF.AllocaInsertPt->getIterator()); | |
| llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs( | |
| BasePointersArray, PointersArray, SizesArray, MapTypesArray, | |
| nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray); | |
| llvm::OpenMPIRBuilder::TargetKernelArgs Args( | |
| NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads, | |
| DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare, | |
| /*StrictThreads=*/IsBare, DynCGroupMemFallback); | |
| llvm::OpenMPIRBuilder::InsertPointTy AfterIP = | |
| cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch( | |
| CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID, | |
| RTLoc, AllocaIP)); | |
| CGF.Builder.restoreIP(AfterIP); | |
| }; | |
| if (RequiresOuterTask) | |
| CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); | |
| else | |
| OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); | |
| } | |
| static void | |
| emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn, | |
| const OMPExecutableDirective &D, | |
| llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, | |
| bool RequiresOuterTask, const CapturedStmt &CS, | |
| bool OffloadingMandatory, CodeGenFunction &CGF) { | |
| // Notify that the host version must be executed. | |
| auto &&ElseGen = | |
| [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, | |
| OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) { | |
| emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars, | |
| RequiresOuterTask, CS, OffloadingMandatory, CGF); | |
| }; | |
| if (RequiresOuterTask) { | |
| CodeGenFunction::OMPTargetDataInfo InputInfo; | |
| CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); | |
| } else { | |
| OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); | |
| } | |
| } | |
| void CGOpenMPRuntime::emitTargetCall( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, | |
| llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, | |
| llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D)> | |
| SizeEmitter) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice && | |
| CGM.getLangOpts().OpenMPOffloadMandatory; | |
| assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!"); | |
| const bool RequiresOuterTask = | |
| D.hasClausesOfKind<OMPDependClause>() || | |
| D.hasClausesOfKind<OMPNowaitClause>() || | |
| D.hasClausesOfKind<OMPInReductionClause>() || | |
| (CGM.getLangOpts().OpenMP >= 51 && | |
| needsTaskBasedThreadLimit(D.getDirectiveKind()) && | |
| D.hasClausesOfKind<OMPThreadLimitClause>()); | |
| llvm::SmallVector<llvm::Value *, 16> CapturedVars; | |
| const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target); | |
| auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, | |
| PrePostActionTy &) { | |
| CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); | |
| }; | |
| emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); | |
| CodeGenFunction::OMPTargetDataInfo InputInfo; | |
| llvm::Value *MapTypesArray = nullptr; | |
| llvm::Value *MapNamesArray = nullptr; | |
| auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars, | |
| RequiresOuterTask, &CS, OffloadingMandatory, Device, | |
| OutlinedFnID, &InputInfo, &MapTypesArray, | |
| &MapNamesArray, SizeEmitter](CodeGenFunction &CGF, | |
| PrePostActionTy &) { | |
| emitTargetCallKernelLaunch(this, OutlinedFn, D, CapturedVars, | |
| RequiresOuterTask, CS, OffloadingMandatory, | |
| Device, OutlinedFnID, InputInfo, MapTypesArray, | |
| MapNamesArray, SizeEmitter, CGF, CGM); | |
| }; | |
| auto &&TargetElseGen = | |
| [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS, | |
| OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) { | |
| emitTargetCallElse(this, OutlinedFn, D, CapturedVars, RequiresOuterTask, | |
| CS, OffloadingMandatory, CGF); | |
| }; | |
| // If we have a target function ID it means that we need to support | |
| // offloading, otherwise, just execute on the host. We need to execute on host | |
| // regardless of the conditional in the if clause if, e.g., the user do not | |
| // specify target triples. | |
| if (OutlinedFnID) { | |
| if (IfCond) { | |
| emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); | |
| } else { | |
| RegionCodeGenTy ThenRCG(TargetThenGen); | |
| ThenRCG(CGF); | |
| } | |
| } else { | |
| RegionCodeGenTy ElseRCG(TargetElseGen); | |
| ElseRCG(CGF); | |
| } | |
| } | |
| void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S, | |
| StringRef ParentName) { | |
| if (!S) | |
| return; | |
| // Register vtable from device for target data and target directives. | |
| // Add this block here since scanForTargetRegionsFunctions ignores | |
| // target data by checking if S is a executable directive (target). | |
| if (auto *E = dyn_cast<OMPExecutableDirective>(S); | |
| E && isOpenMPTargetDataManagementDirective(E->getDirectiveKind())) { | |
| // Don't need to check if it's device compile | |
| // since scanForTargetRegionsFunctions currently only called | |
| // in device compilation. | |
| registerVTable(*E); | |
| } | |
| // Codegen OMP target directives that offload compute to the device. | |
| bool RequiresDeviceCodegen = | |
| isa<OMPExecutableDirective>(S) && | |
| isOpenMPTargetExecutionDirective( | |
| cast<OMPExecutableDirective>(S)->getDirectiveKind()); | |
| if (RequiresDeviceCodegen) { | |
| const auto &E = *cast<OMPExecutableDirective>(S); | |
| llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc( | |
| CGM, OMPBuilder, E.getBeginLoc(), ParentName); | |
| // Is this a target region that should not be emitted as an entry point? If | |
| // so just signal we are done with this target region. | |
| if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo)) | |
| return; | |
| switch (E.getDirectiveKind()) { | |
| case OMPD_target: | |
| CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName, | |
| cast<OMPTargetDirective>(E)); | |
| break; | |
| case OMPD_target_parallel: | |
| CodeGenFunction::EmitOMPTargetParallelDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetParallelDirective>(E)); | |
| break; | |
| case OMPD_target_teams: | |
| CodeGenFunction::EmitOMPTargetTeamsDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetTeamsDirective>(E)); | |
| break; | |
| case OMPD_target_teams_distribute: | |
| CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E)); | |
| break; | |
| case OMPD_target_teams_distribute_simd: | |
| CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E)); | |
| break; | |
| case OMPD_target_parallel_for: | |
| CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetParallelForDirective>(E)); | |
| break; | |
| case OMPD_target_parallel_for_simd: | |
| CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E)); | |
| break; | |
| case OMPD_target_simd: | |
| case OMPD_target_loop: | |
| CodeGenFunction::EmitOMPTargetSimdDeviceFunction( | |
| CGM, ParentName, cast<OMPLoopDirective>(E)); | |
| break; | |
| case OMPD_target_teams_distribute_parallel_for: | |
| CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( | |
| CGM, ParentName, | |
| cast<OMPTargetTeamsDistributeParallelForDirective>(E)); | |
| break; | |
| case OMPD_target_teams_distribute_parallel_for_simd: | |
| CodeGenFunction:: | |
| EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( | |
| CGM, ParentName, | |
| cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E)); | |
| break; | |
| case OMPD_target_teams_loop: | |
| CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(E)); | |
| break; | |
| case OMPD_target_parallel_loop: | |
| CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction( | |
| CGM, ParentName, cast<OMPTargetParallelGenericLoopDirective>(E)); | |
| break; | |
| case OMPD_parallel: | |
| case OMPD_for: | |
| case OMPD_parallel_for: | |
| case OMPD_parallel_master: | |
| case OMPD_parallel_sections: | |
| case OMPD_for_simd: | |
| case OMPD_parallel_for_simd: | |
| case OMPD_cancel: | |
| case OMPD_cancellation_point: | |
| case OMPD_ordered_standalone: | |
| case OMPD_ordered_blockassoc: | |
| case OMPD_threadprivate: | |
| case OMPD_allocate: | |
| case OMPD_task: | |
| case OMPD_simd: | |
| case OMPD_tile: | |
| case OMPD_unroll: | |
| case OMPD_sections: | |
| case OMPD_section: | |
| case OMPD_single: | |
| case OMPD_master: | |
| case OMPD_critical: | |
| case OMPD_taskyield: | |
| case OMPD_barrier: | |
| case OMPD_taskwait: | |
| case OMPD_taskgroup: | |
| case OMPD_atomic: | |
| case OMPD_flush: | |
| case OMPD_depobj: | |
| case OMPD_scan: | |
| case OMPD_teams: | |
| case OMPD_target_data: | |
| case OMPD_target_exit_data: | |
| case OMPD_target_enter_data: | |
| case OMPD_distribute: | |
| case OMPD_distribute_simd: | |
| case OMPD_distribute_parallel_for: | |
| case OMPD_distribute_parallel_for_simd: | |
| case OMPD_teams_distribute: | |
| case OMPD_teams_distribute_simd: | |
| case OMPD_teams_distribute_parallel_for: | |
| case OMPD_teams_distribute_parallel_for_simd: | |
| case OMPD_target_update: | |
| case OMPD_declare_simd: | |
| case OMPD_declare_variant: | |
| case OMPD_begin_declare_variant: | |
| case OMPD_end_declare_variant: | |
| case OMPD_declare_target: | |
| case OMPD_end_declare_target: | |
| case OMPD_declare_reduction: | |
| case OMPD_declare_mapper: | |
| case OMPD_taskloop: | |
| case OMPD_taskloop_simd: | |
| case OMPD_master_taskloop: | |
| case OMPD_master_taskloop_simd: | |
| case OMPD_parallel_master_taskloop: | |
| case OMPD_parallel_master_taskloop_simd: | |
| case OMPD_requires: | |
| case OMPD_metadirective: | |
| case OMPD_unknown: | |
| default: | |
| llvm_unreachable("Unknown target directive for OpenMP device codegen."); | |
| } | |
| return; | |
| } | |
| // A metadirective has no associated statement; look into its variants. | |
| if (const auto *MD = dyn_cast<OMPMetaDirective>(S)) { | |
| for (const Stmt *V : MD->variants()) | |
| scanForTargetRegionsFunctions(V, ParentName); | |
| return; | |
| } | |
| if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) { | |
| if (!E->hasAssociatedStmt() || !E->getAssociatedStmt()) | |
| return; | |
| scanForTargetRegionsFunctions(E->getRawStmt(), ParentName); | |
| return; | |
| } | |
| // Default arguments and member initializers are not children of their uses. | |
| if (const auto *E = dyn_cast<CXXDefaultArgExpr>(S)) { | |
| scanForTargetRegionsFunctions(E->getExpr(), ParentName); | |
| return; | |
| } | |
| if (const auto *E = dyn_cast<CXXDefaultInitExpr>(S)) { | |
| scanForTargetRegionsFunctions(E->getExpr(), ParentName); | |
| return; | |
| } | |
| // Implicit and inheriting constructors are only reachable from their uses. | |
| const CXXConstructorDecl *Ctor = nullptr; | |
| if (const auto *E = dyn_cast<CXXConstructExpr>(S)) | |
| Ctor = E->getConstructor(); | |
| else if (const auto *E = dyn_cast<CXXInheritedCtorInitExpr>(S)) | |
| Ctor = E->getConstructor(); | |
| if (Ctor && (!Ctor->isUserProvided() || Ctor->isInheritingConstructor()) && | |
| Ctor->hasBody() && ScannedImplicitCtors.insert(Ctor).second) | |
| scanConstructorForTargetRegions(Ctor, ParentName); | |
| // If this is a lambda function, look into its body. | |
| if (const auto *L = dyn_cast<LambdaExpr>(S)) { | |
| const CXXMethodDecl *CallOp = L->getCallOperator(); | |
| // Specializations of a generic lambda are scanned as top-level decls. | |
| if (CallOp->isTemplated()) | |
| return; | |
| ParentName = getTargetRegionParentName(CallOp); | |
| S = L->getBody(); | |
| } | |
| // Keep looking for target regions recursively. | |
| for (const Stmt *II : S->children()) | |
| scanForTargetRegionsFunctions(II, ParentName); | |
| } | |
| void CGOpenMPRuntime::scanConstructorForTargetRegions( | |
| const CXXConstructorDecl *Ctor, StringRef ParentName) { | |
| const auto *Def = dyn_cast_or_null<CXXConstructorDecl>(Ctor->getDefinition()); | |
| if (!Def) | |
| return; | |
| for (const CXXCtorInitializer *Init : Def->inits()) | |
| scanForTargetRegionsFunctions(Init->getInit(), ParentName); | |
| scanForTargetRegionsFunctions(Def->getBody(), ParentName); | |
| } | |
| static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) { | |
| std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = | |
| OMPDeclareTargetDeclAttr::getDeviceType(VD); | |
| if (!DevTy) | |
| return false; | |
| // Do not emit device_type(nohost) functions for the host. | |
| if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) | |
| return true; | |
| // Do not emit device_type(host) functions for the device. | |
| if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host) | |
| return true; | |
| return false; | |
| } | |
| bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) { | |
| // If emitting code for the host, we do not process FD here. Instead we do | |
| // the normal code generation. | |
| if (!CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) | |
| if (isAssumedToBeNotEmitted(cast<ValueDecl>(FD), | |
| CGM.getLangOpts().OpenMPIsTargetDevice)) | |
| return true; | |
| return false; | |
| } | |
| const ValueDecl *VD = cast<ValueDecl>(GD.getDecl()); | |
| // Try to detect target regions in the function. | |
| if (const auto *FD = dyn_cast<FunctionDecl>(VD)) { | |
| StringRef Name = getTargetRegionParentName(FD); | |
| if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(FD)) | |
| scanConstructorForTargetRegions(Ctor, Name); | |
| else | |
| scanForTargetRegionsFunctions(FD->getBody(), Name); | |
| if (isAssumedToBeNotEmitted(cast<ValueDecl>(FD), | |
| CGM.getLangOpts().OpenMPIsTargetDevice)) | |
| return true; | |
| } | |
| // Do not emit function if it is not marked as declare target. | |
| return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) && | |
| AlreadyEmittedTargetDecls.count(VD) == 0; | |
| } | |
| bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) { | |
| if (!CGM.getLangOpts().OpenMPIsTargetDevice) | |
| return isAssumedToBeNotEmitted(cast<ValueDecl>(GD.getDecl()), | |
| /*IsDevice=*/false); | |
| // Check if there are Ctors/Dtors in this declaration and look for target | |
| // regions in it. We use the complete variant to produce the kernel name | |
| // mangling. | |
| const auto *VD = cast<VarDecl>(GD.getDecl()); | |
| QualType RDTy = VD->getType(); | |
| if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) { | |
| for (const CXXConstructorDecl *Ctor : RD->ctors()) | |
| scanConstructorForTargetRegions(Ctor, getTargetRegionParentName(Ctor)); | |
| if (const CXXDestructorDecl *Dtor = RD->getDestructor()) | |
| scanForTargetRegionsFunctions(Dtor->getBody(), | |
| getTargetRegionParentName(Dtor)); | |
| } | |
| // Lambdas in the initializer are reached from nowhere else. | |
| if (const Expr *Init = VD->getInit()) | |
| scanForTargetRegionsFunctions(Init, CGM.getMangledName(GD)); | |
| if (isAssumedToBeNotEmitted(VD, /*IsDevice=*/true)) | |
| return true; | |
| // Do not emit variable if it is not marked as declare target. | |
| std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = | |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); | |
| if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link || | |
| ((*Res == OMPDeclareTargetDeclAttr::MT_To || | |
| *Res == OMPDeclareTargetDeclAttr::MT_Enter) && | |
| HasRequiresUnifiedSharedMemory)) { | |
| DeferredGlobalVariables.insert(VD); | |
| return true; | |
| } | |
| return false; | |
| } | |
| void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD, | |
| llvm::Constant *Addr) { | |
| if (CGM.getLangOpts().OMPTargetTriples.empty() && | |
| !CGM.getLangOpts().OpenMPIsTargetDevice) | |
| return; | |
| std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = | |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); | |
| // If this is an 'extern' declaration we defer to the canonical definition and | |
| // do not emit an offloading entry. | |
| if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link && | |
| VD->hasExternalStorage()) | |
| return; | |
| // MT_Local variables use direct access with no host-device mapping. | |
| // No offload entry needed — the device global keeps its own initializer. | |
| if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local) | |
| return; | |
| if (!Res) { | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| // Register non-target variables being emitted in device code (debug info | |
| // may cause this). | |
| StringRef VarName = CGM.getMangledName(VD); | |
| EmittedNonTargetVariables.try_emplace(VarName, Addr); | |
| } | |
| return; | |
| } | |
| auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(VD); }; | |
| auto LinkageForVariable = [&VD, this]() { | |
| return CGM.getLLVMLinkageVarDefinition(VD); | |
| }; | |
| std::vector<llvm::GlobalVariable *> GeneratedRefs; | |
| OMPBuilder.registerTargetGlobalVariable( | |
| convertCaptureClause(VD), convertDeviceClause(VD), | |
| VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly, | |
| VD->isExternallyVisible(), | |
| getEntryInfoFromPresumedLoc(CGM, OMPBuilder, | |
| VD->getCanonicalDecl()->getBeginLoc()), | |
| CGM.getMangledName(VD), GeneratedRefs, CGM.getLangOpts().OpenMPSimd, | |
| CGM.getLangOpts().OMPTargetTriples, AddrOfGlobal, LinkageForVariable, | |
| CGM.getTypes().ConvertTypeForMem( | |
| CGM.getContext().getPointerType(VD->getType())), | |
| Addr); | |
| for (auto *ref : GeneratedRefs) | |
| CGM.addCompilerUsedGlobal(ref); | |
| } | |
| bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) { | |
| if (isa<FunctionDecl>(GD.getDecl()) || | |
| isa<OMPDeclareReductionDecl>(GD.getDecl())) | |
| return emitTargetFunctions(GD); | |
| return emitTargetGlobalVariable(GD); | |
| } | |
| void CGOpenMPRuntime::emitDeferredTargetDecls() const { | |
| for (const VarDecl *VD : DeferredGlobalVariables) { | |
| std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = | |
| OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); | |
| if (!Res) | |
| continue; | |
| // MT_Local and MT_To/MT_Enter without USM are always emitted. | |
| if (*Res == OMPDeclareTargetDeclAttr::MT_Local || | |
| ((*Res == OMPDeclareTargetDeclAttr::MT_To || | |
| *Res == OMPDeclareTargetDeclAttr::MT_Enter) && | |
| !HasRequiresUnifiedSharedMemory)) { | |
| CGM.EmitGlobal(VD); | |
| } else { | |
| assert((*Res == OMPDeclareTargetDeclAttr::MT_Link || | |
| ((*Res == OMPDeclareTargetDeclAttr::MT_To || | |
| *Res == OMPDeclareTargetDeclAttr::MT_Enter || | |
| *Res == OMPDeclareTargetDeclAttr::MT_Local) && | |
| HasRequiresUnifiedSharedMemory)) && | |
| "Expected link clause or to clause with unified memory."); | |
| (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); | |
| } | |
| } | |
| } | |
| void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D) const { | |
| assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && | |
| " Expected target-based directive."); | |
| } | |
| void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) { | |
| for (const OMPClause *Clause : D->clauselists()) { | |
| if (Clause->getClauseKind() == OMPC_unified_shared_memory) { | |
| HasRequiresUnifiedSharedMemory = true; | |
| OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true); | |
| } else if (const auto *AC = | |
| dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) { | |
| switch (AC->getAtomicDefaultMemOrderKind()) { | |
| case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel: | |
| RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease; | |
| break; | |
| case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst: | |
| RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent; | |
| break; | |
| case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed: | |
| RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic; | |
| break; | |
| case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown: | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const { | |
| return RequiresAtomicOrdering; | |
| } | |
| bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD, | |
| LangAS &AS) { | |
| if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>()) | |
| return false; | |
| const auto *A = VD->getAttr<OMPAllocateDeclAttr>(); | |
| switch(A->getAllocatorType()) { | |
| case OMPAllocateDeclAttr::OMPNullMemAlloc: | |
| case OMPAllocateDeclAttr::OMPDefaultMemAlloc: | |
| // Not supported, fallback to the default mem space. | |
| case OMPAllocateDeclAttr::OMPLargeCapMemAlloc: | |
| case OMPAllocateDeclAttr::OMPCGroupMemAlloc: | |
| case OMPAllocateDeclAttr::OMPHighBWMemAlloc: | |
| case OMPAllocateDeclAttr::OMPLowLatMemAlloc: | |
| case OMPAllocateDeclAttr::OMPThreadMemAlloc: | |
| case OMPAllocateDeclAttr::OMPConstMemAlloc: | |
| case OMPAllocateDeclAttr::OMPPTeamMemAlloc: | |
| AS = LangAS::Default; | |
| return true; | |
| case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc: | |
| llvm_unreachable("Expected predefined allocator for the variables with the " | |
| "static storage."); | |
| } | |
| return false; | |
| } | |
| bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const { | |
| return HasRequiresUnifiedSharedMemory; | |
| } | |
| CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII( | |
| CodeGenModule &CGM) | |
| : CGM(CGM) { | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice) { | |
| SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal; | |
| CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false; | |
| } | |
| } | |
| CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() { | |
| if (CGM.getLangOpts().OpenMPIsTargetDevice) | |
| CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal; | |
| } | |
| bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) { | |
| if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal) | |
| return true; | |
| const auto *D = cast<FunctionDecl>(GD.getDecl()); | |
| // Do not emit function if it is marked as declare target as it was already | |
| // emitted. | |
| if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) { | |
| if (D->hasBody() && AlreadyEmittedTargetDecls.count(D) == 0) { | |
| if (auto *F = dyn_cast_or_null<llvm::Function>( | |
| CGM.GetGlobalValue(CGM.getMangledName(GD)))) | |
| return !F->isDeclaration(); | |
| return false; | |
| } | |
| return true; | |
| } | |
| return !AlreadyEmittedTargetDecls.insert(D).second; | |
| } | |
| void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, | |
| const OMPExecutableDirective &D, | |
| SourceLocation Loc, | |
| llvm::Function *OutlinedFn, | |
| ArrayRef<llvm::Value *> CapturedVars) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); | |
| CodeGenFunction::RunCleanupsScope Scope(CGF); | |
| // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn); | |
| llvm::Value *Args[] = { | |
| RTLoc, | |
| CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars | |
| OutlinedFn}; | |
| llvm::SmallVector<llvm::Value *, 16> RealArgs; | |
| RealArgs.append(std::begin(Args), std::end(Args)); | |
| RealArgs.append(CapturedVars.begin(), CapturedVars.end()); | |
| llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_fork_teams); | |
| CGF.EmitRuntimeCall(RTLFn, RealArgs); | |
| } | |
| void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, | |
| const Expr *NumTeams, | |
| const Expr *ThreadLimit, | |
| SourceLocation Loc) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *NumTeamsVal = | |
| NumTeams | |
| ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), | |
| CGF.CGM.Int32Ty, /* isSigned = */ true) | |
| : CGF.Builder.getInt32(0); | |
| llvm::Value *ThreadLimitVal = | |
| ThreadLimit | |
| ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), | |
| CGF.CGM.Int32Ty, /* isSigned = */ true) | |
| : CGF.Builder.getInt32(0); | |
| // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit) | |
| llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, | |
| ThreadLimitVal}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_push_num_teams), | |
| PushNumTeamsArgs); | |
| } | |
| void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF, | |
| const Expr *ThreadLimit, | |
| SourceLocation Loc) { | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); | |
| llvm::Value *ThreadLimitVal = | |
| ThreadLimit | |
| ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), | |
| CGF.CGM.Int32Ty, /* isSigned = */ true) | |
| : CGF.Builder.getInt32(0); | |
| // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit) | |
| llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc), | |
| ThreadLimitVal}; | |
| CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_set_thread_limit), | |
| ThreadLimitArgs); | |
| } | |
| void CGOpenMPRuntime::emitTargetDataCalls( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, | |
| const Expr *Device, const RegionCodeGenTy &CodeGen, | |
| CGOpenMPRuntime::TargetDataInfo &Info) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| // Action used to replace the default codegen action and turn privatization | |
| // off. | |
| PrePostActionTy NoPrivAction; | |
| using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; | |
| llvm::Value *IfCondVal = nullptr; | |
| if (IfCond) | |
| IfCondVal = CGF.EvaluateExprAsBool(IfCond); | |
| // Emit device ID if any. | |
| llvm::Value *DeviceID = nullptr; | |
| if (Device) { | |
| DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), | |
| CGF.Int64Ty, /*isSigned=*/true); | |
| } else { | |
| DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); | |
| } | |
| // Fill up the arrays with all the mapped variables. | |
| MappableExprsHandler::MapCombinedInfoTy CombinedInfo; | |
| auto GenMapInfoCB = | |
| [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & { | |
| CGF.Builder.restoreIP(CodeGenIP); | |
| // Get map clause information. | |
| MappableExprsHandler MEHandler(D, CGF); | |
| MEHandler.generateAllInfo(CombinedInfo, OMPBuilder); | |
| auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) { | |
| return emitMappingInformation(CGF, OMPBuilder, MapExpr); | |
| }; | |
| if (CGM.getCodeGenOpts().getDebugInfo() != | |
| llvm::codegenoptions::NoDebugInfo) { | |
| CombinedInfo.Names.resize(CombinedInfo.Exprs.size()); | |
| llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(), | |
| FillInfoMap); | |
| } | |
| return CombinedInfo; | |
| }; | |
| using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy; | |
| auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) { | |
| CGF.Builder.restoreIP(CodeGenIP); | |
| switch (BodyGenType) { | |
| case BodyGenTy::Priv: | |
| if (!Info.CaptureDeviceAddrMap.empty()) | |
| CodeGen(CGF); | |
| break; | |
| case BodyGenTy::DupNoPriv: | |
| if (!Info.CaptureDeviceAddrMap.empty()) { | |
| CodeGen.setAction(NoPrivAction); | |
| CodeGen(CGF); | |
| } | |
| break; | |
| case BodyGenTy::NoPriv: | |
| if (Info.CaptureDeviceAddrMap.empty()) { | |
| CodeGen.setAction(NoPrivAction); | |
| CodeGen(CGF); | |
| } | |
| break; | |
| } | |
| return InsertPointTy(CGF.Builder.GetInsertPoint()); | |
| }; | |
| auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) { | |
| if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) { | |
| Info.CaptureDeviceAddrMap.try_emplace(DevVD, NewDecl); | |
| } | |
| }; | |
| auto CustomMapperCB = [&](unsigned int I) { | |
| llvm::Function *MFunc = nullptr; | |
| if (CombinedInfo.Mappers[I]) { | |
| Info.HasMapper = true; | |
| MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc( | |
| cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I])); | |
| } | |
| return MFunc; | |
| }; | |
| // Source location for the ident struct | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); | |
| InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator()); | |
| InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint()); | |
| llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder); | |
| llvm::OpenMPIRBuilder::InsertPointTy AfterIP = | |
| cantFail(OMPBuilder.createTargetData( | |
| OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID, | |
| IfCondVal, Info, GenMapInfoCB, CustomMapperCB, | |
| /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc)); | |
| CGF.Builder.restoreIP(AfterIP); | |
| } | |
| void CGOpenMPRuntime::emitTargetDataStandAloneCall( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, | |
| const Expr *Device) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| assert((isa<OMPTargetEnterDataDirective>(D) || | |
| isa<OMPTargetExitDataDirective>(D) || | |
| isa<OMPTargetUpdateDirective>(D)) && | |
| "Expecting either target enter, exit data, or update directives."); | |
| CodeGenFunction::OMPTargetDataInfo InputInfo; | |
| llvm::Value *MapTypesArray = nullptr; | |
| llvm::Value *MapNamesArray = nullptr; | |
| // Generate the code for the opening of the data environment. | |
| auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray, | |
| &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) { | |
| // Emit device ID if any. | |
| llvm::Value *DeviceID = nullptr; | |
| if (Device) { | |
| DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), | |
| CGF.Int64Ty, /*isSigned=*/true); | |
| } else { | |
| DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); | |
| } | |
| // Emit the number of elements in the offloading arrays. | |
| llvm::Constant *PointerNum = | |
| CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); | |
| // Source location for the ident struct | |
| llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc()); | |
| SmallVector<llvm::Value *, 13> OffloadingArgs( | |
| {RTLoc, DeviceID, PointerNum, | |
| InputInfo.BasePointersArray.emitRawPointer(CGF), | |
| InputInfo.PointersArray.emitRawPointer(CGF), | |
| InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray, | |
| InputInfo.MappersArray.emitRawPointer(CGF)}); | |
| // Select the right runtime function call for each standalone | |
| // directive. | |
| const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>(); | |
| RuntimeFunction RTLFn; | |
| switch (D.getDirectiveKind()) { | |
| case OMPD_target_enter_data: | |
| RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper | |
| : OMPRTL___tgt_target_data_begin_mapper; | |
| break; | |
| case OMPD_target_exit_data: | |
| RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper | |
| : OMPRTL___tgt_target_data_end_mapper; | |
| break; | |
| case OMPD_target_update: | |
| RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper | |
| : OMPRTL___tgt_target_data_update_mapper; | |
| break; | |
| case OMPD_parallel: | |
| case OMPD_for: | |
| case OMPD_parallel_for: | |
| case OMPD_parallel_master: | |
| case OMPD_parallel_sections: | |
| case OMPD_for_simd: | |
| case OMPD_parallel_for_simd: | |
| case OMPD_cancel: | |
| case OMPD_cancellation_point: | |
| case OMPD_ordered_standalone: | |
| case OMPD_ordered_blockassoc: | |
| case OMPD_threadprivate: | |
| case OMPD_allocate: | |
| case OMPD_task: | |
| case OMPD_simd: | |
| case OMPD_tile: | |
| case OMPD_unroll: | |
| case OMPD_sections: | |
| case OMPD_section: | |
| case OMPD_single: | |
| case OMPD_master: | |
| case OMPD_critical: | |
| case OMPD_taskyield: | |
| case OMPD_barrier: | |
| case OMPD_taskwait: | |
| case OMPD_taskgroup: | |
| case OMPD_atomic: | |
| case OMPD_flush: | |
| case OMPD_depobj: | |
| case OMPD_scan: | |
| case OMPD_teams: | |
| case OMPD_target_data: | |
| case OMPD_distribute: | |
| case OMPD_distribute_simd: | |
| case OMPD_distribute_parallel_for: | |
| case OMPD_distribute_parallel_for_simd: | |
| case OMPD_teams_distribute: | |
| case OMPD_teams_distribute_simd: | |
| case OMPD_teams_distribute_parallel_for: | |
| case OMPD_teams_distribute_parallel_for_simd: | |
| case OMPD_declare_simd: | |
| case OMPD_declare_variant: | |
| case OMPD_begin_declare_variant: | |
| case OMPD_end_declare_variant: | |
| case OMPD_declare_target: | |
| case OMPD_end_declare_target: | |
| case OMPD_declare_reduction: | |
| case OMPD_declare_mapper: | |
| case OMPD_taskloop: | |
| case OMPD_taskloop_simd: | |
| case OMPD_master_taskloop: | |
| case OMPD_master_taskloop_simd: | |
| case OMPD_parallel_master_taskloop: | |
| case OMPD_parallel_master_taskloop_simd: | |
| case OMPD_target: | |
| case OMPD_target_simd: | |
| case OMPD_target_teams_distribute: | |
| case OMPD_target_teams_distribute_simd: | |
| case OMPD_target_teams_distribute_parallel_for: | |
| case OMPD_target_teams_distribute_parallel_for_simd: | |
| case OMPD_target_teams: | |
| case OMPD_target_parallel: | |
| case OMPD_target_parallel_for: | |
| case OMPD_target_parallel_for_simd: | |
| case OMPD_requires: | |
| case OMPD_metadirective: | |
| case OMPD_unknown: | |
| default: | |
| llvm_unreachable("Unexpected standalone target data directive."); | |
| break; | |
| } | |
| if (HasNowait) { | |
| OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty)); | |
| OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy)); | |
| OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty)); | |
| OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy)); | |
| } | |
| CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn), | |
| OffloadingArgs); | |
| }; | |
| auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray, | |
| &MapNamesArray](CodeGenFunction &CGF, | |
| PrePostActionTy &) { | |
| // Fill up the arrays with all the mapped variables. | |
| MappableExprsHandler::MapCombinedInfoTy CombinedInfo; | |
| CGOpenMPRuntime::TargetDataInfo Info; | |
| MappableExprsHandler MEHandler(D, CGF); | |
| genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder); | |
| emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder, | |
| /*IsNonContiguous=*/true, /*ForEndCall=*/false); | |
| bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() || | |
| D.hasClausesOfKind<OMPNowaitClause>(); | |
| InputInfo.NumberOfTargetItems = Info.NumberOfPtrs; | |
| InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray, | |
| CGF.VoidPtrTy, CGM.getPointerAlign()); | |
| InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, | |
| CGM.getPointerAlign()); | |
| InputInfo.SizesArray = | |
| Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign()); | |
| InputInfo.MappersArray = | |
| Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign()); | |
| MapTypesArray = Info.RTArgs.MapTypesArray; | |
| MapNamesArray = Info.RTArgs.MapNamesArray; | |
| if (RequiresOuterTask) | |
| CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); | |
| else | |
| emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); | |
| }; | |
| if (IfCond) { | |
| emitIfClause(CGF, IfCond, TargetThenGen, | |
| [](CodeGenFunction &CGF, PrePostActionTy &) {}); | |
| } else { | |
| RegionCodeGenTy ThenRCG(TargetThenGen); | |
| ThenRCG(CGF); | |
| } | |
| } | |
| static unsigned | |
| evaluateCDTSize(const FunctionDecl *FD, | |
| ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) { | |
| // Every vector variant of a SIMD-enabled function has a vector length (VLEN). | |
| // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument | |
| // of that clause. The VLEN value must be power of 2. | |
| // In other case the notion of the function`s "characteristic data type" (CDT) | |
| // is used to compute the vector length. | |
| // CDT is defined in the following order: | |
| // a) For non-void function, the CDT is the return type. | |
| // b) If the function has any non-uniform, non-linear parameters, then the | |
| // CDT is the type of the first such parameter. | |
| // c) If the CDT determined by a) or b) above is struct, union, or class | |
| // type which is pass-by-value (except for the type that maps to the | |
| // built-in complex data type), the characteristic data type is int. | |
| // d) If none of the above three cases is applicable, the CDT is int. | |
| // The VLEN is then determined based on the CDT and the size of vector | |
| // register of that ISA for which current vector version is generated. The | |
| // VLEN is computed using the formula below: | |
| // VLEN = sizeof(vector_register) / sizeof(CDT), | |
| // where vector register size specified in section 3.2.1 Registers and the | |
| // Stack Frame of original AMD64 ABI document. | |
| QualType RetType = FD->getReturnType(); | |
| if (RetType.isNull()) | |
| return 0; | |
| ASTContext &C = FD->getASTContext(); | |
| QualType CDT; | |
| if (!RetType.isNull() && !RetType->isVoidType()) { | |
| CDT = RetType; | |
| } else { | |
| unsigned Offset = 0; | |
| if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) { | |
| if (ParamAttrs[Offset].Kind == | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) | |
| CDT = C.getPointerType(C.getCanonicalTagType(MD->getParent())); | |
| ++Offset; | |
| } | |
| if (CDT.isNull()) { | |
| for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) { | |
| if (ParamAttrs[I + Offset].Kind == | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) { | |
| CDT = FD->getParamDecl(I)->getType(); | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| if (CDT.isNull()) | |
| CDT = C.IntTy; | |
| CDT = CDT->getCanonicalTypeUnqualified(); | |
| if (CDT->isRecordType() || CDT->isUnionType()) | |
| CDT = C.IntTy; | |
| return C.getTypeSize(CDT); | |
| } | |
| // This are the Functions that are needed to mangle the name of the | |
| // vector functions generated by the compiler, according to the rules | |
| // defined in the "Vector Function ABI specifications for AArch64", | |
| // available at | |
| // https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi. | |
| /// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1). | |
| static bool getAArch64MTV(QualType QT, | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) { | |
| QT = QT.getCanonicalType(); | |
| if (QT->isVoidType()) | |
| return false; | |
| if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform) | |
| return false; | |
| if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal || | |
| Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef) | |
| return false; | |
| if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear || | |
| Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) && | |
| !QT->isReferenceType()) | |
| return false; | |
| return true; | |
| } | |
| /// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI. | |
| static bool getAArch64PBV(QualType QT, ASTContext &C) { | |
| QT = QT.getCanonicalType(); | |
| unsigned Size = C.getTypeSize(QT); | |
| // Only scalars and complex within 16 bytes wide set PVB to true. | |
| if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128) | |
| return false; | |
| if (QT->isFloatingType()) | |
| return true; | |
| if (QT->isIntegerType()) | |
| return true; | |
| if (QT->isPointerType()) | |
| return true; | |
| // TODO: Add support for complex types (section 3.1.2, item 2). | |
| return false; | |
| } | |
| /// Computes the lane size (LS) of a return type or of an input parameter, | |
| /// as defined by `LS(P)` in 3.2.1 of the AAVFABI. | |
| /// TODO: Add support for references, section 3.2.1, item 1. | |
| static unsigned getAArch64LS(QualType QT, | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind, | |
| ASTContext &C) { | |
| if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) { | |
| QualType PTy = QT.getCanonicalType()->getPointeeType(); | |
| if (getAArch64PBV(PTy, C)) | |
| return C.getTypeSize(PTy); | |
| } | |
| if (getAArch64PBV(QT, C)) | |
| return C.getTypeSize(QT); | |
| return C.getTypeSize(C.getUIntPtrType()); | |
| } | |
| // Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the | |
| // signature of the scalar function, as defined in 3.2.2 of the | |
| // AAVFABI. | |
| static std::tuple<unsigned, unsigned, bool> | |
| getNDSWDS(const FunctionDecl *FD, | |
| ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) { | |
| QualType RetType = FD->getReturnType().getCanonicalType(); | |
| ASTContext &C = FD->getASTContext(); | |
| bool OutputBecomesInput = false; | |
| llvm::SmallVector<unsigned, 8> Sizes; | |
| if (!RetType->isVoidType()) { | |
| Sizes.push_back(getAArch64LS( | |
| RetType, llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C)); | |
| if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {})) | |
| OutputBecomesInput = true; | |
| } | |
| for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) { | |
| QualType QT = FD->getParamDecl(I)->getType().getCanonicalType(); | |
| Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C)); | |
| } | |
| assert(!Sizes.empty() && "Unable to determine NDS and WDS."); | |
| // The LS of a function parameter / return value can only be a power | |
| // of 2, starting from 8 bits, up to 128. | |
| assert(llvm::all_of(Sizes, | |
| [](unsigned Size) { | |
| return Size == 8 || Size == 16 || Size == 32 || | |
| Size == 64 || Size == 128; | |
| }) && | |
| "Invalid size"); | |
| return std::make_tuple(*llvm::min_element(Sizes), *llvm::max_element(Sizes), | |
| OutputBecomesInput); | |
| } | |
| static llvm::OpenMPIRBuilder::DeclareSimdBranch | |
| convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) { | |
| switch (State) { | |
| case OMPDeclareSimdDeclAttr::BS_Undefined: | |
| return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined; | |
| case OMPDeclareSimdDeclAttr::BS_Inbranch: | |
| return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch; | |
| case OMPDeclareSimdDeclAttr::BS_Notinbranch: | |
| return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch; | |
| } | |
| llvm_unreachable("unexpected declare simd branch state"); | |
| } | |
| // Check the values provided via `simdlen` by the user. | |
| static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc, | |
| unsigned UserVLEN, unsigned WDS, char ISA) { | |
| // 1. A `simdlen(1)` doesn't produce vector signatures. | |
| if (UserVLEN == 1) { | |
| CGM.getDiags().Report(SLoc, diag::warn_simdlen_1_no_effect); | |
| return false; | |
| } | |
| // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced | |
| // SIMD. | |
| if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) { | |
| CGM.getDiags().Report(SLoc, diag::warn_simdlen_requires_power_of_2); | |
| return false; | |
| } | |
| // 3. Section 3.4.1: SVE fixed length must obey the architectural limits. | |
| if (ISA == 's' && UserVLEN != 0 && | |
| ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) { | |
| CGM.getDiags().Report(SLoc, diag::warn_simdlen_must_fit_lanes) << WDS; | |
| return false; | |
| } | |
| return true; | |
| } | |
| void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, | |
| llvm::Function *Fn) { | |
| ASTContext &C = CGM.getContext(); | |
| FD = FD->getMostRecentDecl(); | |
| while (FD) { | |
| // Map params to their positions in function decl. | |
| llvm::DenseMap<const Decl *, unsigned> ParamPositions; | |
| if (isa<CXXMethodDecl>(FD)) | |
| ParamPositions.try_emplace(FD, 0); | |
| unsigned ParamPos = ParamPositions.size(); | |
| for (const ParmVarDecl *P : FD->parameters()) { | |
| ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos); | |
| ++ParamPos; | |
| } | |
| for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) { | |
| llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs( | |
| ParamPositions.size()); | |
| // Mark uniform parameters. | |
| for (const Expr *E : Attr->uniforms()) { | |
| E = E->IgnoreParenImpCasts(); | |
| unsigned Pos; | |
| if (isa<CXXThisExpr>(E)) { | |
| Pos = ParamPositions[FD]; | |
| } else { | |
| const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) | |
| ->getCanonicalDecl(); | |
| auto It = ParamPositions.find(PVD); | |
| assert(It != ParamPositions.end() && "Function parameter not found"); | |
| Pos = It->second; | |
| } | |
| ParamAttrs[Pos].Kind = | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform; | |
| } | |
| // Get alignment info. | |
| auto *NI = Attr->alignments_begin(); | |
| for (const Expr *E : Attr->aligneds()) { | |
| E = E->IgnoreParenImpCasts(); | |
| unsigned Pos; | |
| QualType ParmTy; | |
| if (isa<CXXThisExpr>(E)) { | |
| Pos = ParamPositions[FD]; | |
| ParmTy = E->getType(); | |
| } else { | |
| const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) | |
| ->getCanonicalDecl(); | |
| auto It = ParamPositions.find(PVD); | |
| assert(It != ParamPositions.end() && "Function parameter not found"); | |
| Pos = It->second; | |
| ParmTy = PVD->getType(); | |
| } | |
| ParamAttrs[Pos].Alignment = | |
| (*NI) | |
| ? (*NI)->EvaluateKnownConstInt(C) | |
| : llvm::APSInt::getUnsigned( | |
| C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy)) | |
| .getQuantity()); | |
| ++NI; | |
| } | |
| // Mark linear parameters. | |
| auto *SI = Attr->steps_begin(); | |
| auto *MI = Attr->modifiers_begin(); | |
| for (const Expr *E : Attr->linears()) { | |
| E = E->IgnoreParenImpCasts(); | |
| unsigned Pos; | |
| bool IsReferenceType = false; | |
| // Rescaling factor needed to compute the linear parameter | |
| // value in the mangled name. | |
| unsigned PtrRescalingFactor = 1; | |
| if (isa<CXXThisExpr>(E)) { | |
| Pos = ParamPositions[FD]; | |
| auto *P = cast<PointerType>(E->getType()); | |
| PtrRescalingFactor = CGM.getContext() | |
| .getTypeSizeInChars(P->getPointeeType()) | |
| .getQuantity(); | |
| } else { | |
| const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) | |
| ->getCanonicalDecl(); | |
| auto It = ParamPositions.find(PVD); | |
| assert(It != ParamPositions.end() && "Function parameter not found"); | |
| Pos = It->second; | |
| if (auto *P = dyn_cast<PointerType>(PVD->getType())) | |
| PtrRescalingFactor = CGM.getContext() | |
| .getTypeSizeInChars(P->getPointeeType()) | |
| .getQuantity(); | |
| else if (PVD->getType()->isReferenceType()) { | |
| IsReferenceType = true; | |
| PtrRescalingFactor = | |
| CGM.getContext() | |
| .getTypeSizeInChars(PVD->getType().getNonReferenceType()) | |
| .getQuantity(); | |
| } | |
| } | |
| llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos]; | |
| if (*MI == OMPC_LINEAR_ref) | |
| ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef; | |
| else if (*MI == OMPC_LINEAR_uval) | |
| ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal; | |
| else if (IsReferenceType) | |
| ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal; | |
| else | |
| ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear; | |
| // Assuming a stride of 1, for `linear` without modifiers. | |
| ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1); | |
| if (*SI) { | |
| Expr::EvalResult Result; | |
| if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) { | |
| if (const auto *DRE = | |
| cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) { | |
| if (const auto *StridePVD = | |
| dyn_cast<ParmVarDecl>(DRE->getDecl())) { | |
| ParamAttr.HasVarStride = true; | |
| auto It = ParamPositions.find(StridePVD->getCanonicalDecl()); | |
| assert(It != ParamPositions.end() && | |
| "Function parameter not found"); | |
| ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second); | |
| } | |
| } | |
| } else { | |
| ParamAttr.StrideOrArg = Result.Val.getInt(); | |
| } | |
| } | |
| // If we are using a linear clause on a pointer, we need to | |
| // rescale the value of linear_step with the byte size of the | |
| // pointee type. | |
| if (!ParamAttr.HasVarStride && | |
| (ParamAttr.Kind == | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear || | |
| ParamAttr.Kind == | |
| llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)) | |
| ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor; | |
| ++SI; | |
| ++MI; | |
| } | |
| llvm::APSInt VLENVal; | |
| SourceLocation ExprLoc; | |
| const Expr *VLENExpr = Attr->getSimdlen(); | |
| if (VLENExpr) { | |
| VLENVal = VLENExpr->EvaluateKnownConstInt(C); | |
| ExprLoc = VLENExpr->getExprLoc(); | |
| } | |
| llvm::OpenMPIRBuilder::DeclareSimdBranch State = | |
| convertDeclareSimdBranch(Attr->getBranchState()); | |
| if (CGM.getTriple().isX86()) { | |
| unsigned NumElts = evaluateCDTSize(FD, ParamAttrs); | |
| assert(NumElts && "Non-zero simdlen/cdtsize expected"); | |
| OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElts, VLENVal, ParamAttrs, | |
| State); | |
| } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) { | |
| unsigned VLEN = VLENVal.getExtValue(); | |
| // Get basic data for building the vector signature. | |
| const auto Data = getNDSWDS(FD, ParamAttrs); | |
| const unsigned NDS = std::get<0>(Data); | |
| const unsigned WDS = std::get<1>(Data); | |
| const bool OutputBecomesInput = std::get<2>(Data); | |
| if (CGM.getTarget().hasFeature("sve")) { | |
| if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 's')) | |
| OMPBuilder.emitAArch64DeclareSimdFunction( | |
| Fn, VLEN, ParamAttrs, State, 's', NDS, OutputBecomesInput); | |
| } else if (CGM.getTarget().hasFeature("neon")) { | |
| if (validateAArch64Simdlen(CGM, ExprLoc, VLEN, WDS, 'n')) | |
| OMPBuilder.emitAArch64DeclareSimdFunction( | |
| Fn, VLEN, ParamAttrs, State, 'n', NDS, OutputBecomesInput); | |
| } | |
| } | |
| } | |
| FD = FD->getPreviousDecl(); | |
| } | |
| } | |
| namespace { | |
| /// Cleanup action for doacross support. | |
| class DoacrossCleanupTy final : public EHScopeStack::Cleanup { | |
| public: | |
| static const int DoacrossFinArgs = 2; | |
| private: | |
| llvm::FunctionCallee RTLFn; | |
| llvm::Value *Args[DoacrossFinArgs]; | |
| public: | |
| DoacrossCleanupTy(llvm::FunctionCallee RTLFn, | |
| ArrayRef<llvm::Value *> CallArgs) | |
| : RTLFn(RTLFn) { | |
| assert(CallArgs.size() == DoacrossFinArgs); | |
| std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args)); | |
| } | |
| void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| CGF.EmitRuntimeCall(RTLFn, Args); | |
| } | |
| }; | |
| } // namespace | |
| void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D, | |
| ArrayRef<Expr *> NumIterations) { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| ASTContext &C = CGM.getContext(); | |
| QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true); | |
| RecordDecl *RD; | |
| if (KmpDimTy.isNull()) { | |
| // Build struct kmp_dim { // loop bounds info casted to kmp_int64 | |
| // kmp_int64 lo; // lower | |
| // kmp_int64 up; // upper | |
| // kmp_int64 st; // stride | |
| // }; | |
| RD = C.buildImplicitRecord("kmp_dim"); | |
| RD->startDefinition(); | |
| addFieldToRecordDecl(C, RD, Int64Ty); | |
| addFieldToRecordDecl(C, RD, Int64Ty); | |
| addFieldToRecordDecl(C, RD, Int64Ty); | |
| RD->completeDefinition(); | |
| KmpDimTy = C.getCanonicalTagType(RD); | |
| } else { | |
| RD = KmpDimTy->castAsRecordDecl(); | |
| } | |
| llvm::APInt Size(/*numBits=*/32, NumIterations.size()); | |
| QualType ArrayTy = C.getConstantArrayType(KmpDimTy, Size, nullptr, | |
| ArraySizeModifier::Normal, 0); | |
| Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); | |
| CGF.EmitNullInitialization(DimsAddr, ArrayTy); | |
| enum { LowerFD = 0, UpperFD, StrideFD }; | |
| // Fill dims with data. | |
| for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) { | |
| LValue DimsLVal = CGF.MakeAddrLValue( | |
| CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); | |
| // dims.upper = num_iterations; | |
| LValue UpperLVal = CGF.EmitLValueForField( | |
| DimsLVal, *std::next(RD->field_begin(), UpperFD)); | |
| llvm::Value *NumIterVal = CGF.EmitScalarConversion( | |
| CGF.EmitScalarExpr(NumIterations[I]), NumIterations[I]->getType(), | |
| Int64Ty, NumIterations[I]->getExprLoc()); | |
| CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); | |
| // dims.stride = 1; | |
| LValue StrideLVal = CGF.EmitLValueForField( | |
| DimsLVal, *std::next(RD->field_begin(), StrideFD)); | |
| CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), | |
| StrideLVal); | |
| } | |
| // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, | |
| // kmp_int32 num_dims, struct kmp_dim * dims); | |
| llvm::Value *Args[] = { | |
| emitUpdateLocation(CGF, D.getBeginLoc()), | |
| getThreadID(CGF, D.getBeginLoc()), | |
| llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), | |
| CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).emitRawPointer(CGF), | |
| CGM.VoidPtrTy)}; | |
| llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_doacross_init); | |
| CGF.EmitRuntimeCall(RTLFn, Args); | |
| llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = { | |
| emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; | |
| llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_doacross_fini); | |
| CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, | |
| llvm::ArrayRef(FiniArgs)); | |
| } | |
| template <typename T> | |
| static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM, | |
| const T *C, llvm::Value *ULoc, | |
| llvm::Value *ThreadID) { | |
| QualType Int64Ty = | |
| CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); | |
| llvm::APInt Size(/*numBits=*/32, C->getNumLoops()); | |
| QualType ArrayTy = CGM.getContext().getConstantArrayType( | |
| Int64Ty, Size, nullptr, ArraySizeModifier::Normal, 0); | |
| Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); | |
| for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) { | |
| const Expr *CounterVal = C->getLoopData(I); | |
| assert(CounterVal); | |
| llvm::Value *CntVal = CGF.EmitScalarConversion( | |
| CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, | |
| CounterVal->getExprLoc()); | |
| CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), | |
| /*Volatile=*/false, Int64Ty); | |
| } | |
| llvm::Value *Args[] = { | |
| ULoc, ThreadID, | |
| CGF.Builder.CreateConstArrayGEP(CntAddr, 0).emitRawPointer(CGF)}; | |
| llvm::FunctionCallee RTLFn; | |
| llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder(); | |
| OMPDoacrossKind<T> ODK; | |
| if (ODK.isSource(C)) { | |
| RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), | |
| OMPRTL___kmpc_doacross_post); | |
| } else { | |
| assert(ODK.isSink(C) && "Expect sink modifier."); | |
| RTLFn = OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), | |
| OMPRTL___kmpc_doacross_wait); | |
| } | |
| CGF.EmitRuntimeCall(RTLFn, Args); | |
| } | |
| void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, | |
| const OMPDependClause *C) { | |
| return EmitDoacrossOrdered<OMPDependClause>( | |
| CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), | |
| getThreadID(CGF, C->getBeginLoc())); | |
| } | |
| void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, | |
| const OMPDoacrossClause *C) { | |
| return EmitDoacrossOrdered<OMPDoacrossClause>( | |
| CGF, CGM, C, emitUpdateLocation(CGF, C->getBeginLoc()), | |
| getThreadID(CGF, C->getBeginLoc())); | |
| } | |
| void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| llvm::FunctionCallee Callee, | |
| ArrayRef<llvm::Value *> Args) const { | |
| assert(Loc.isValid() && "Outlined function call location must be valid."); | |
| auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); | |
| if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) { | |
| if (Fn->doesNotThrow()) { | |
| CGF.EmitNounwindRuntimeCall(Fn, Args); | |
| return; | |
| } | |
| } | |
| CGF.EmitRuntimeCall(Callee, Args); | |
| } | |
| void CGOpenMPRuntime::emitOutlinedFunctionCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, | |
| ArrayRef<llvm::Value *> Args) const { | |
| emitCall(CGF, Loc, OutlinedFn, Args); | |
| } | |
| void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { | |
| if (const auto *FD = dyn_cast<FunctionDecl>(D)) | |
| if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD)) | |
| HasEmittedDeclareTargetRegion = true; | |
| } | |
| Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, | |
| const VarDecl *NativeParam, | |
| const VarDecl *TargetParam) const { | |
| return CGF.GetAddrOfLocalVar(NativeParam); | |
| } | |
| /// Return allocator value from expression, or return a null allocator (default | |
| /// when no allocator specified). | |
| static llvm::Value *getAllocatorVal(CodeGenFunction &CGF, | |
| const Expr *Allocator) { | |
| llvm::Value *AllocVal; | |
| if (Allocator) { | |
| AllocVal = CGF.EmitScalarExpr(Allocator); | |
| // According to the standard, the original allocator type is a enum | |
| // (integer). Convert to pointer type, if required. | |
| AllocVal = CGF.EmitScalarConversion(AllocVal, Allocator->getType(), | |
| CGF.getContext().VoidPtrTy, | |
| Allocator->getExprLoc()); | |
| } else { | |
| // If no allocator specified, it defaults to the null allocator. | |
| AllocVal = llvm::Constant::getNullValue( | |
| CGF.CGM.getTypes().ConvertType(CGF.getContext().VoidPtrTy)); | |
| } | |
| return AllocVal; | |
| } | |
| /// Return the alignment from an allocate directive if present. | |
| static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) { | |
| std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD); | |
| if (!AllocateAlignment) | |
| return nullptr; | |
| return llvm::ConstantInt::get(CGM.SizeTy, AllocateAlignment->getQuantity()); | |
| } | |
| Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, | |
| const VarDecl *VD) { | |
| if (!VD) | |
| return Address::invalid(); | |
| Address UntiedAddr = Address::invalid(); | |
| Address UntiedRealAddr = Address::invalid(); | |
| auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); | |
| if (It != FunctionToUntiedTaskStackMap.end()) { | |
| const UntiedLocalVarsAddressesMap &UntiedData = | |
| UntiedLocalVarsStack[It->second]; | |
| auto I = UntiedData.find(VD); | |
| if (I != UntiedData.end()) { | |
| UntiedAddr = I->second.first; | |
| UntiedRealAddr = I->second.second; | |
| } | |
| } | |
| const VarDecl *CVD = VD->getCanonicalDecl(); | |
| if (CVD->hasAttr<OMPAllocateDeclAttr>()) { | |
| // Use the default allocation. | |
| if (!isAllocatableDecl(VD)) | |
| return UntiedAddr; | |
| llvm::Value *Size; | |
| CharUnits Align = CGM.getContext().getDeclAlign(CVD); | |
| if (CVD->getType()->isVariablyModifiedType()) { | |
| Size = CGF.getTypeSize(CVD->getType()); | |
| // Align the size: ((size + align - 1) / align) * align | |
| Size = CGF.Builder.CreateNUWAdd( | |
| Size, CGM.getSize(Align - CharUnits::fromQuantity(1))); | |
| Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); | |
| Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); | |
| } else { | |
| CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType()); | |
| Size = CGM.getSize(Sz.alignTo(Align)); | |
| } | |
| llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); | |
| const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>(); | |
| const Expr *Allocator = AA->getAllocator(); | |
| llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator); | |
| llvm::Value *Alignment = getAlignmentValue(CGM, CVD); | |
| SmallVector<llvm::Value *, 4> Args; | |
| Args.push_back(ThreadID); | |
| if (Alignment) | |
| Args.push_back(Alignment); | |
| Args.push_back(Size); | |
| Args.push_back(AllocVal); | |
| llvm::omp::RuntimeFunction FnID = | |
| Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc; | |
| llvm::Value *Addr = CGF.EmitRuntimeCall( | |
| OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), FnID), Args, | |
| getName({CVD->getName(), ".void.addr"})); | |
| llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction( | |
| CGM.getModule(), OMPRTL___kmpc_free); | |
| QualType Ty = CGM.getContext().getPointerType(CVD->getType()); | |
| Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Addr, CGF.ConvertTypeForMem(Ty), getName({CVD->getName(), ".addr"})); | |
| if (UntiedAddr.isValid()) | |
| CGF.EmitStoreOfScalar(Addr, UntiedAddr, /*Volatile=*/false, Ty); | |
| // Cleanup action for allocate support. | |
| class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup { | |
| llvm::FunctionCallee RTLFn; | |
| SourceLocation::UIntTy LocEncoding; | |
| Address Addr; | |
| const Expr *AllocExpr; | |
| public: | |
| OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn, | |
| SourceLocation::UIntTy LocEncoding, Address Addr, | |
| const Expr *AllocExpr) | |
| : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr), | |
| AllocExpr(AllocExpr) {} | |
| void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { | |
| if (!CGF.HaveInsertPoint()) | |
| return; | |
| llvm::Value *Args[3]; | |
| Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID( | |
| CGF, SourceLocation::getFromRawEncoding(LocEncoding)); | |
| Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| Addr.emitRawPointer(CGF), CGF.VoidPtrTy); | |
| llvm::Value *AllocVal = getAllocatorVal(CGF, AllocExpr); | |
| Args[2] = AllocVal; | |
| CGF.EmitRuntimeCall(RTLFn, Args); | |
| } | |
| }; | |
| Address VDAddr = | |
| UntiedRealAddr.isValid() | |
| ? UntiedRealAddr | |
| : Address(Addr, CGF.ConvertTypeForMem(CVD->getType()), Align); | |
| CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>( | |
| NormalAndEHCleanup, FiniRTLFn, CVD->getLocation().getRawEncoding(), | |
| VDAddr, Allocator); | |
| if (UntiedRealAddr.isValid()) | |
| if (auto *Region = | |
| dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) | |
| Region->emitUntiedSwitch(CGF); | |
| return VDAddr; | |
| } | |
| return UntiedAddr; | |
| } | |
| bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF, | |
| const VarDecl *VD) const { | |
| auto It = FunctionToUntiedTaskStackMap.find(CGF.CurFn); | |
| if (It == FunctionToUntiedTaskStackMap.end()) | |
| return false; | |
| return UntiedLocalVarsStack[It->second].count(VD) > 0; | |
| } | |
| CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII( | |
| CodeGenModule &CGM, const OMPLoopDirective &S) | |
| : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) { | |
| assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode."); | |
| if (!NeedToPush) | |
| return; | |
| NontemporalDeclsSet &DS = | |
| CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back(); | |
| for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) { | |
| for (const Stmt *Ref : C->private_refs()) { | |
| const auto *SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts(); | |
| const ValueDecl *VD; | |
| if (const auto *DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) { | |
| VD = DRE->getDecl(); | |
| } else { | |
| const auto *ME = cast<MemberExpr>(SimpleRefExpr); | |
| assert((ME->isImplicitCXXThis() || | |
| isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) && | |
| "Expected member of current class."); | |
| VD = ME->getMemberDecl(); | |
| } | |
| DS.insert(VD); | |
| } | |
| } | |
| } | |
| CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() { | |
| if (!NeedToPush) | |
| return; | |
| CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back(); | |
| } | |
| CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII( | |
| CodeGenFunction &CGF, | |
| const llvm::MapVector<CanonicalDeclPtr<const VarDecl>, | |
| std::pair<Address, Address>> &LocalVars) | |
| : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) { | |
| if (!NeedToPush) | |
| return; | |
| CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace( | |
| CGF.CurFn, CGM.getOpenMPRuntime().UntiedLocalVarsStack.size()); | |
| CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(LocalVars); | |
| } | |
| CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() { | |
| if (!NeedToPush) | |
| return; | |
| CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back(); | |
| } | |
| bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const { | |
| assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode."); | |
| return llvm::any_of( | |
| CGM.getOpenMPRuntime().NontemporalDeclsStack, | |
| [VD](const NontemporalDeclsSet &Set) { return Set.contains(VD); }); | |
| } | |
| void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis( | |
| const OMPExecutableDirective &S, | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled) | |
| const { | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs; | |
| // Vars in target/task regions must be excluded completely. | |
| if (isOpenMPTargetExecutionDirective(S.getDirectiveKind()) || | |
| isOpenMPTaskingDirective(S.getDirectiveKind())) { | |
| SmallVector<OpenMPDirectiveKind, 4> CaptureRegions; | |
| getOpenMPCaptureRegions(CaptureRegions, S.getDirectiveKind()); | |
| const CapturedStmt *CS = S.getCapturedStmt(CaptureRegions.front()); | |
| for (const CapturedStmt::Capture &Cap : CS->captures()) { | |
| if (Cap.capturesVariable() || Cap.capturesVariableByCopy()) | |
| NeedToCheckForLPCs.insert(Cap.getCapturedVar()); | |
| } | |
| } | |
| // Exclude vars in private clauses. | |
| for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) { | |
| for (const Expr *Ref : C->varlist()) { | |
| if (!Ref->getType()->isScalarType()) | |
| continue; | |
| const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts()); | |
| if (!DRE) | |
| continue; | |
| NeedToCheckForLPCs.insert(DRE->getDecl()); | |
| } | |
| } | |
| for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) { | |
| for (const Expr *Ref : C->varlist()) { | |
| if (!Ref->getType()->isScalarType()) | |
| continue; | |
| const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts()); | |
| if (!DRE) | |
| continue; | |
| NeedToCheckForLPCs.insert(DRE->getDecl()); | |
| } | |
| } | |
| for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) { | |
| for (const Expr *Ref : C->varlist()) { | |
| if (!Ref->getType()->isScalarType()) | |
| continue; | |
| const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts()); | |
| if (!DRE) | |
| continue; | |
| NeedToCheckForLPCs.insert(DRE->getDecl()); | |
| } | |
| } | |
| for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) { | |
| for (const Expr *Ref : C->varlist()) { | |
| if (!Ref->getType()->isScalarType()) | |
| continue; | |
| const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts()); | |
| if (!DRE) | |
| continue; | |
| NeedToCheckForLPCs.insert(DRE->getDecl()); | |
| } | |
| } | |
| for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) { | |
| for (const Expr *Ref : C->varlist()) { | |
| if (!Ref->getType()->isScalarType()) | |
| continue; | |
| const auto *DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts()); | |
| if (!DRE) | |
| continue; | |
| NeedToCheckForLPCs.insert(DRE->getDecl()); | |
| } | |
| } | |
| for (const Decl *VD : NeedToCheckForLPCs) { | |
| for (const LastprivateConditionalData &Data : | |
| llvm::reverse(CGM.getOpenMPRuntime().LastprivateConditionalStack)) { | |
| if (Data.DeclToUniqueName.count(VD) > 0) { | |
| if (!Data.Disabled) | |
| NeedToAddForLPCsAsDisabled.insert(VD); | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal) | |
| : CGM(CGF.CGM), | |
| Action((CGM.getLangOpts().OpenMP >= 50 && | |
| llvm::any_of(S.getClausesOfKind<OMPLastprivateClause>(), | |
| [](const OMPLastprivateClause *C) { | |
| return C->getKind() == | |
| OMPC_LASTPRIVATE_conditional; | |
| })) | |
| ? ActionToDo::PushAsLastprivateConditional | |
| : ActionToDo::DoNotPush) { | |
| assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode."); | |
| if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush) | |
| return; | |
| assert(Action == ActionToDo::PushAsLastprivateConditional && | |
| "Expected a push action."); | |
| LastprivateConditionalData &Data = | |
| CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back(); | |
| for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) { | |
| if (C->getKind() != OMPC_LASTPRIVATE_conditional) | |
| continue; | |
| for (const Expr *Ref : C->varlist()) { | |
| Data.DeclToUniqueName.insert(std::make_pair( | |
| cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(), | |
| SmallString<16>(generateUniqueName(CGM, "pl_cond", Ref)))); | |
| } | |
| } | |
| Data.IVLVal = IVLVal; | |
| Data.Fn = CGF.CurFn; | |
| } | |
| CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &S) | |
| : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) { | |
| assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode."); | |
| if (CGM.getLangOpts().OpenMP < 50) | |
| return; | |
| llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled; | |
| tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled); | |
| if (!NeedToAddForLPCsAsDisabled.empty()) { | |
| Action = ActionToDo::DisableLastprivateConditional; | |
| LastprivateConditionalData &Data = | |
| CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back(); | |
| for (const Decl *VD : NeedToAddForLPCsAsDisabled) | |
| Data.DeclToUniqueName.try_emplace(VD); | |
| Data.Fn = CGF.CurFn; | |
| Data.Disabled = true; | |
| } | |
| } | |
| CGOpenMPRuntime::LastprivateConditionalRAII | |
| CGOpenMPRuntime::LastprivateConditionalRAII::disable( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &S) { | |
| return LastprivateConditionalRAII(CGF, S); | |
| } | |
| CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() { | |
| if (CGM.getLangOpts().OpenMP < 50) | |
| return; | |
| if (Action == ActionToDo::DisableLastprivateConditional) { | |
| assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled && | |
| "Expected list of disabled private vars."); | |
| CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back(); | |
| } | |
| if (Action == ActionToDo::PushAsLastprivateConditional) { | |
| assert( | |
| !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled && | |
| "Expected list of lastprivate conditional vars."); | |
| CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back(); | |
| } | |
| } | |
| Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF, | |
| const VarDecl *VD) { | |
| ASTContext &C = CGM.getContext(); | |
| auto I = LastprivateConditionalToTypes.try_emplace(CGF.CurFn).first; | |
| QualType NewType; | |
| const FieldDecl *VDField; | |
| const FieldDecl *FiredField; | |
| LValue BaseLVal; | |
| auto VI = I->getSecond().find(VD); | |
| if (VI == I->getSecond().end()) { | |
| RecordDecl *RD = C.buildImplicitRecord("lasprivate.conditional"); | |
| RD->startDefinition(); | |
| VDField = addFieldToRecordDecl(C, RD, VD->getType().getNonReferenceType()); | |
| FiredField = addFieldToRecordDecl(C, RD, C.CharTy); | |
| RD->completeDefinition(); | |
| NewType = C.getCanonicalTagType(RD); | |
| Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName()); | |
| BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl); | |
| I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal); | |
| } else { | |
| NewType = std::get<0>(VI->getSecond()); | |
| VDField = std::get<1>(VI->getSecond()); | |
| FiredField = std::get<2>(VI->getSecond()); | |
| BaseLVal = std::get<3>(VI->getSecond()); | |
| } | |
| LValue FiredLVal = | |
| CGF.EmitLValueForField(BaseLVal, FiredField); | |
| CGF.EmitStoreOfScalar( | |
| llvm::ConstantInt::getNullValue(CGF.ConvertTypeForMem(C.CharTy)), | |
| FiredLVal); | |
| return CGF.EmitLValueForField(BaseLVal, VDField).getAddress(); | |
| } | |
| namespace { | |
| /// Checks if the lastprivate conditional variable is referenced in LHS. | |
| class LastprivateConditionalRefChecker final | |
| : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> { | |
| ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM; | |
| const Expr *FoundE = nullptr; | |
| const Decl *FoundD = nullptr; | |
| StringRef UniqueDeclName; | |
| LValue IVLVal; | |
| llvm::Function *FoundFn = nullptr; | |
| SourceLocation Loc; | |
| public: | |
| bool VisitDeclRefExpr(const DeclRefExpr *E) { | |
| for (const CGOpenMPRuntime::LastprivateConditionalData &D : | |
| llvm::reverse(LPM)) { | |
| auto It = D.DeclToUniqueName.find(E->getDecl()); | |
| if (It == D.DeclToUniqueName.end()) | |
| continue; | |
| if (D.Disabled) | |
| return false; | |
| FoundE = E; | |
| FoundD = E->getDecl()->getCanonicalDecl(); | |
| UniqueDeclName = It->second; | |
| IVLVal = D.IVLVal; | |
| FoundFn = D.Fn; | |
| break; | |
| } | |
| return FoundE == E; | |
| } | |
| bool VisitMemberExpr(const MemberExpr *E) { | |
| if (!CodeGenFunction::IsWrappedCXXThis(E->getBase())) | |
| return false; | |
| for (const CGOpenMPRuntime::LastprivateConditionalData &D : | |
| llvm::reverse(LPM)) { | |
| auto It = D.DeclToUniqueName.find(E->getMemberDecl()); | |
| if (It == D.DeclToUniqueName.end()) | |
| continue; | |
| if (D.Disabled) | |
| return false; | |
| FoundE = E; | |
| FoundD = E->getMemberDecl()->getCanonicalDecl(); | |
| UniqueDeclName = It->second; | |
| IVLVal = D.IVLVal; | |
| FoundFn = D.Fn; | |
| break; | |
| } | |
| return FoundE == E; | |
| } | |
| bool VisitStmt(const Stmt *S) { | |
| for (const Stmt *Child : S->children()) { | |
| if (!Child) | |
| continue; | |
| if (const auto *E = dyn_cast<Expr>(Child)) | |
| if (!E->isGLValue()) | |
| continue; | |
| if (Visit(Child)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| explicit LastprivateConditionalRefChecker( | |
| ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM) | |
| : LPM(LPM) {} | |
| std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *> | |
| getFoundData() const { | |
| return std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn); | |
| } | |
| }; | |
| } // namespace | |
| void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF, | |
| LValue IVLVal, | |
| StringRef UniqueDeclName, | |
| LValue LVal, | |
| SourceLocation Loc) { | |
| // Last updated loop counter for the lastprivate conditional var. | |
| // int<xx> last_iv = 0; | |
| llvm::Type *LLIVTy = CGF.ConvertTypeForMem(IVLVal.getType()); | |
| llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable( | |
| LLIVTy, getName({UniqueDeclName, "iv"})); | |
| cast<llvm::GlobalVariable>(LastIV)->setAlignment( | |
| IVLVal.getAlignment().getAsAlign()); | |
| LValue LastIVLVal = | |
| CGF.MakeNaturalAlignRawAddrLValue(LastIV, IVLVal.getType()); | |
| // Last value of the lastprivate conditional. | |
| // decltype(priv_a) last_a; | |
| llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable( | |
| CGF.ConvertTypeForMem(LVal.getType()), UniqueDeclName); | |
| cast<llvm::GlobalVariable>(Last)->setAlignment( | |
| LVal.getAlignment().getAsAlign()); | |
| LValue LastLVal = | |
| CGF.MakeRawAddrLValue(Last, LVal.getType(), LVal.getAlignment()); | |
| // Global loop counter. Required to handle inner parallel-for regions. | |
| // iv | |
| llvm::Value *IVVal = CGF.EmitLoadOfScalar(IVLVal, Loc); | |
| // #pragma omp critical(a) | |
| // if (last_iv <= iv) { | |
| // last_iv = iv; | |
| // last_a = priv_a; | |
| // } | |
| auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal, | |
| Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { | |
| Action.Enter(CGF); | |
| llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(LastIVLVal, Loc); | |
| // (last_iv <= iv) ? Check if the variable is updated and store new | |
| // value in global var. | |
| llvm::Value *CmpRes; | |
| if (IVLVal.getType()->isSignedIntegerType()) { | |
| CmpRes = CGF.Builder.CreateICmpSLE(LastIVVal, IVVal); | |
| } else { | |
| assert(IVLVal.getType()->isUnsignedIntegerType() && | |
| "Loop iteration variable must be integer."); | |
| CmpRes = CGF.Builder.CreateICmpULE(LastIVVal, IVVal); | |
| } | |
| llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lp_cond_then"); | |
| llvm::BasicBlock *ExitBB = CGF.createBasicBlock("lp_cond_exit"); | |
| CGF.Builder.CreateCondBr(CmpRes, ThenBB, ExitBB); | |
| // { | |
| CGF.EmitBlock(ThenBB); | |
| // last_iv = iv; | |
| CGF.EmitStoreOfScalar(IVVal, LastIVLVal); | |
| // last_a = priv_a; | |
| switch (CGF.getEvaluationKind(LVal.getType())) { | |
| case TEK_Scalar: { | |
| llvm::Value *PrivVal = CGF.EmitLoadOfScalar(LVal, Loc); | |
| CGF.EmitStoreOfScalar(PrivVal, LastLVal); | |
| break; | |
| } | |
| case TEK_Complex: { | |
| CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(LVal, Loc); | |
| CGF.EmitStoreOfComplex(PrivVal, LastLVal, /*isInit=*/false); | |
| break; | |
| } | |
| case TEK_Aggregate: | |
| llvm_unreachable( | |
| "Aggregates are not supported in lastprivate conditional."); | |
| } | |
| // } | |
| CGF.EmitBranch(ExitBB); | |
| // There is no need to emit line number for unconditional branch. | |
| (void)ApplyDebugLocation::CreateEmpty(CGF); | |
| CGF.EmitBlock(ExitBB, /*IsFinished=*/true); | |
| }; | |
| if (CGM.getLangOpts().OpenMPSimd) { | |
| // Do not emit as a critical region as no parallel region could be emitted. | |
| RegionCodeGenTy ThenRCG(CodeGen); | |
| ThenRCG(CGF); | |
| } else { | |
| emitCriticalRegion(CGF, UniqueDeclName, CodeGen, Loc); | |
| } | |
| } | |
| void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF, | |
| const Expr *LHS) { | |
| if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) | |
| return; | |
| LastprivateConditionalRefChecker Checker(LastprivateConditionalStack); | |
| if (!Checker.Visit(LHS)) | |
| return; | |
| const Expr *FoundE; | |
| const Decl *FoundD; | |
| StringRef UniqueDeclName; | |
| LValue IVLVal; | |
| llvm::Function *FoundFn; | |
| std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) = | |
| Checker.getFoundData(); | |
| if (FoundFn != CGF.CurFn) { | |
| // Special codegen for inner parallel regions. | |
| // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1; | |
| auto It = LastprivateConditionalToTypes[FoundFn].find(FoundD); | |
| assert(It != LastprivateConditionalToTypes[FoundFn].end() && | |
| "Lastprivate conditional is not found in outer region."); | |
| QualType StructTy = std::get<0>(It->getSecond()); | |
| const FieldDecl* FiredDecl = std::get<2>(It->getSecond()); | |
| LValue PrivLVal = CGF.EmitLValue(FoundE); | |
| Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( | |
| PrivLVal.getAddress(), | |
| CGF.ConvertTypeForMem(CGF.getContext().getPointerType(StructTy)), | |
| CGF.ConvertTypeForMem(StructTy)); | |
| LValue BaseLVal = | |
| CGF.MakeAddrLValue(StructAddr, StructTy, AlignmentSource::Decl); | |
| LValue FiredLVal = CGF.EmitLValueForField(BaseLVal, FiredDecl); | |
| CGF.EmitAtomicStore(RValue::get(llvm::ConstantInt::get( | |
| CGF.ConvertTypeForMem(FiredDecl->getType()), 1)), | |
| FiredLVal, llvm::AtomicOrdering::Unordered, | |
| /*IsVolatile=*/true, /*isInit=*/false); | |
| return; | |
| } | |
| // Private address of the lastprivate conditional in the current context. | |
| // priv_a | |
| LValue LVal = CGF.EmitLValue(FoundE); | |
| emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal, | |
| FoundE->getExprLoc()); | |
| } | |
| void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) { | |
| if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty()) | |
| return; | |
| auto Range = llvm::reverse(LastprivateConditionalStack); | |
| auto It = llvm::find_if( | |
| Range, [](const LastprivateConditionalData &D) { return !D.Disabled; }); | |
| if (It == Range.end() || It->Fn != CGF.CurFn) | |
| return; | |
| auto LPCI = LastprivateConditionalToTypes.find(It->Fn); | |
| assert(LPCI != LastprivateConditionalToTypes.end() && | |
| "Lastprivates must be registered already."); | |
| SmallVector<OpenMPDirectiveKind, 4> CaptureRegions; | |
| getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind()); | |
| const CapturedStmt *CS = D.getCapturedStmt(CaptureRegions.back()); | |
| for (const auto &Pair : It->DeclToUniqueName) { | |
| const auto *VD = cast<VarDecl>(Pair.first->getCanonicalDecl()); | |
| if (!CS->capturesVariable(VD) || IgnoredDecls.contains(VD)) | |
| continue; | |
| auto I = LPCI->getSecond().find(Pair.first); | |
| assert(I != LPCI->getSecond().end() && | |
| "Lastprivate must be rehistered already."); | |
| // bool Cmp = priv_a.Fired != 0; | |
| LValue BaseLVal = std::get<3>(I->getSecond()); | |
| LValue FiredLVal = | |
| CGF.EmitLValueForField(BaseLVal, std::get<2>(I->getSecond())); | |
| llvm::Value *Res = CGF.EmitLoadOfScalar(FiredLVal, D.getBeginLoc()); | |
| llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Res); | |
| llvm::BasicBlock *ThenBB = CGF.createBasicBlock("lpc.then"); | |
| llvm::BasicBlock *DoneBB = CGF.createBasicBlock("lpc.done"); | |
| // if (Cmp) { | |
| CGF.Builder.CreateCondBr(Cmp, ThenBB, DoneBB); | |
| CGF.EmitBlock(ThenBB); | |
| Address Addr = CGF.GetAddrOfLocalVar(VD); | |
| LValue LVal; | |
| if (VD->getType()->isReferenceType()) | |
| LVal = CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), | |
| AlignmentSource::Decl); | |
| else | |
| LVal = CGF.MakeAddrLValue(Addr, VD->getType().getNonReferenceType(), | |
| AlignmentSource::Decl); | |
| emitLastprivateConditionalUpdate(CGF, It->IVLVal, Pair.second, LVal, | |
| D.getBeginLoc()); | |
| auto AL = ApplyDebugLocation::CreateArtificial(CGF); | |
| CGF.EmitBlock(DoneBB, /*IsFinal=*/true); | |
| // } | |
| } | |
| } | |
| void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate( | |
| CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD, | |
| SourceLocation Loc) { | |
| if (CGF.getLangOpts().OpenMP < 50) | |
| return; | |
| auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(VD); | |
| assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() && | |
| "Unknown lastprivate conditional variable."); | |
| StringRef UniqueName = It->second; | |
| llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(UniqueName); | |
| // The variable was not updated in the region - exit. | |
| if (!GV) | |
| return; | |
| LValue LPLVal = CGF.MakeRawAddrLValue( | |
| GV, PrivLVal.getType().getNonReferenceType(), PrivLVal.getAlignment()); | |
| llvm::Value *Res = CGF.EmitLoadOfScalar(LPLVal, Loc); | |
| CGF.EmitStoreOfScalar(Res, PrivLVal); | |
| } | |
| llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, | |
| const RegionCodeGenTy &CodeGen) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, | |
| const RegionCodeGenTy &CodeGen) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction( | |
| const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, | |
| const VarDecl *PartIDVar, const VarDecl *TaskTVar, | |
| OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, | |
| bool Tied, unsigned &NumberOfParts) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitParallelCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn, | |
| ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond, | |
| llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier, | |
| OpenMPSeverityClauseKind Severity, const Expr *Message) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitCriticalRegion( | |
| CodeGenFunction &CGF, StringRef CriticalName, | |
| const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, | |
| const Expr *Hint) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &MasterOpGen, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &MasterOpGen, | |
| SourceLocation Loc, | |
| const Expr *Filter) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskgroupRegion( | |
| CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitSingleRegion( | |
| CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, | |
| SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars, | |
| ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs, | |
| ArrayRef<const Expr *> AssignmentOps) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, | |
| const RegionCodeGenTy &OrderedOpGen, | |
| SourceLocation Loc, | |
| bool IsThreads) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| OpenMPDirectiveKind Kind, | |
| bool EmitChecks, | |
| bool ForceSimpleCall) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitForDispatchInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, | |
| bool Ordered, const DispatchRTInput &DispatchValues) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitForStaticInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, | |
| const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitDistributeStaticInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| unsigned IVSize, | |
| bool IVSigned) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| OpenMPDirectiveKind DKind) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| unsigned IVSize, bool IVSigned, | |
| Address IL, Address LB, | |
| Address UB, Address ST) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitNumThreadsClause( | |
| CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc, | |
| OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity, | |
| SourceLocation SeverityLoc, const Expr *Message, | |
| SourceLocation MessageLoc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, | |
| ProcBindKind ProcBind, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, | |
| const VarDecl *VD, | |
| Address VDAddr, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition( | |
| const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, | |
| CodeGenFunction *CGF) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate( | |
| CodeGenFunction &CGF, QualType VarType, StringRef Name) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, | |
| ArrayRef<const Expr *> Vars, | |
| SourceLocation Loc, | |
| llvm::AtomicOrdering AO) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, | |
| const OMPExecutableDirective &D, | |
| llvm::Function *TaskFunction, | |
| QualType SharedsTy, Address Shareds, | |
| const Expr *IfCond, | |
| const OMPTaskDataTy &Data) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskLoopCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, | |
| llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, | |
| const Expr *IfCond, const OMPTaskDataTy &Data) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitReduction( | |
| CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, | |
| ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, | |
| ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) { | |
| assert(Options.SimpleReduction && "Only simple reduction is expected."); | |
| CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, | |
| ReductionOps, Options); | |
| } | |
| llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit( | |
| CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, | |
| ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| bool IsWorksharingReduction) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| ReductionCodeGen &RCG, | |
| unsigned N) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| llvm::Value *ReductionsPtr, | |
| LValue SharedLVal) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, | |
| SourceLocation Loc, | |
| const OMPTaskDataTy &Data) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitCancellationPointCall( | |
| CodeGenFunction &CGF, SourceLocation Loc, | |
| OpenMPDirectiveKind CancelRegion) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, | |
| SourceLocation Loc, const Expr *IfCond, | |
| OpenMPDirectiveKind CancelRegion) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction( | |
| const OMPExecutableDirective &D, StringRef ParentName, | |
| llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, | |
| bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTargetCall( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, | |
| llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, | |
| llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device, | |
| llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D)> | |
| SizeEmitter) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) { | |
| return false; | |
| } | |
| void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, | |
| const OMPExecutableDirective &D, | |
| SourceLocation Loc, | |
| llvm::Function *OutlinedFn, | |
| ArrayRef<llvm::Value *> CapturedVars) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, | |
| const Expr *NumTeams, | |
| const Expr *ThreadLimit, | |
| SourceLocation Loc) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTargetDataCalls( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, | |
| const Expr *Device, const RegionCodeGenTy &CodeGen, | |
| CGOpenMPRuntime::TargetDataInfo &Info) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall( | |
| CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, | |
| const Expr *Device) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, | |
| const OMPLoopDirective &D, | |
| ArrayRef<Expr *> NumIterations) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, | |
| const OMPDependClause *C) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, | |
| const OMPDoacrossClause *C) { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| const VarDecl * | |
| CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD, | |
| const VarDecl *NativeParam) const { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } | |
| Address | |
| CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, | |
| const VarDecl *NativeParam, | |
| const VarDecl *TargetParam) const { | |
| llvm_unreachable("Not supported in SIMD-only mode"); | |
| } |