[AArch64] Don't tail call memset if it would convert to a bzero. (#98969)
Well, not quite that simple. We can tc memset since it returns the first
argument but bzero doesn't do that and therefore we can end up
miscompiling.
This patch also refactors the logic out of isInTailCallPosition() into the callers.
As a result memcpy and memmove are also modified to do the same thing
for consistency.
rdar://131419786
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 6f7ed22..362cc30 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -126,7 +126,8 @@
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM);
+bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg = false);
/// Test if given that the input instruction is in the tail call position, if
/// there is an attribute mismatch between the caller and the callee that will
@@ -144,7 +145,12 @@
/// optimization.
bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI);
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg = false);
+
+/// Returns true if the parent of \p CI returns CI's first argument after
+/// calling \p CI.
+bool funcReturnsFirstArgOfCall(const CallInst &CI);
DenseMap<const MachineBasicBlock *, int>
getEHScopeMembership(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 8e189e9..16ec65f 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1182,16 +1182,22 @@
/// stack arguments from being clobbered.
SDValue getStackArgumentTokenFactor(SDValue Chain);
- SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo = AAMDNodes(),
- AAResults *AA = nullptr);
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
+ SDValue
+ getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
+ SDValue Size, Align Alignment, bool isVol, bool AlwaysInline,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
+ MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
+ const AAMDNodes &AAInfo = AAMDNodes(), AAResults *AA = nullptr);
+ /* \p CI if not null is the memset call being lowered.
+ * \p OverrideTailCall is an optional parameter that can be used to override
+ * the tail call optimization decision. */
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
- SDValue Size, Align Alignment, bool isVol, bool isTailCall,
+ SDValue Size, Align Alignment, bool isVol,
+ const CallInst *CI, std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1199,7 +1205,7 @@
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
SDValue Size, Align Alignment, bool isVol,
- bool AlwaysInline, bool isTailCall,
+ bool AlwaysInline, const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo = AAMDNodes());
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index 7fc18639..128060e 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -532,7 +532,8 @@
/// between it and the return.
///
/// This function only tests target-independent requirements.
-bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM) {
+bool llvm::isInTailCallPosition(const CallBase &Call, const TargetMachine &TM,
+ bool ReturnsFirstArg) {
const BasicBlock *ExitBB = Call.getParent();
const Instruction *Term = ExitBB->getTerminator();
const ReturnInst *Ret = dyn_cast<ReturnInst>(Term);
@@ -575,7 +576,8 @@
const Function *F = ExitBB->getParent();
return returnTypeIsEligibleForTailCall(
- F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering());
+ F, &Call, Ret, *TM.getSubtargetImpl(*F)->getTargetLowering(),
+ ReturnsFirstArg);
}
bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
@@ -638,26 +640,11 @@
return CallerAttrs == CalleeAttrs;
}
-/// Check whether B is a bitcast of a pointer type to another pointer type,
-/// which is equal to A.
-static bool isPointerBitcastEqualTo(const Value *A, const Value *B) {
- assert(A && B && "Expected non-null inputs!");
-
- auto *BitCastIn = dyn_cast<BitCastInst>(B);
-
- if (!BitCastIn)
- return false;
-
- if (!A->getType()->isPointerTy() || !B->getType()->isPointerTy())
- return false;
-
- return A == BitCastIn->getOperand(0);
-}
-
bool llvm::returnTypeIsEligibleForTailCall(const Function *F,
const Instruction *I,
const ReturnInst *Ret,
- const TargetLoweringBase &TLI) {
+ const TargetLoweringBase &TLI,
+ bool ReturnsFirstArg) {
// If the block ends with a void return or unreachable, it doesn't matter
// what the call's return type is.
if (!Ret || Ret->getNumOperands() == 0) return true;
@@ -671,26 +658,11 @@
if (!attributesPermitTailCall(F, I, Ret, TLI, &AllowDifferingSizes))
return false;
- const Value *RetVal = Ret->getOperand(0), *CallVal = I;
- // Intrinsic like llvm.memcpy has no return value, but the expanded
- // libcall may or may not have return value. On most platforms, it
- // will be expanded as memcpy in libc, which returns the first
- // argument. On other platforms like arm-none-eabi, memcpy may be
- // expanded as library call without return value, like __aeabi_memcpy.
- const CallInst *Call = cast<CallInst>(I);
- if (Function *F = Call->getCalledFunction()) {
- Intrinsic::ID IID = F->getIntrinsicID();
- if (((IID == Intrinsic::memcpy &&
- TLI.getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy")) ||
- (IID == Intrinsic::memmove &&
- TLI.getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove")) ||
- (IID == Intrinsic::memset &&
- TLI.getLibcallName(RTLIB::MEMSET) == StringRef("memset"))) &&
- (RetVal == Call->getArgOperand(0) ||
- isPointerBitcastEqualTo(RetVal, Call->getArgOperand(0))))
- return true;
- }
+ // If the return value is the first argument of the call.
+ if (ReturnsFirstArg)
+ return true;
+ const Value *RetVal = Ret->getOperand(0), *CallVal = I;
SmallVector<unsigned, 4> RetPath, CallPath;
SmallVector<Type *, 4> RetSubTypes, CallSubTypes;
@@ -739,6 +711,15 @@
return true;
}
+bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) {
+ const ReturnInst *Ret = dyn_cast<ReturnInst>(CI.getParent()->getTerminator());
+ Value *RetVal = Ret ? Ret->getReturnValue() : nullptr;
+ bool ReturnsFirstArg = false;
+ if (RetVal && ((RetVal == CI.getArgOperand(0))))
+ ReturnsFirstArg = true;
+ return ReturnsFirstArg;
+}
+
static void collectEHScopeMembers(
DenseMap<const MachineBasicBlock *, int> &EHScopeMembership, int EHScope,
const MachineBasicBlock *MBB) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9bd0d1c..94349ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -75,6 +75,7 @@
#include <cstdint>
#include <cstdlib>
#include <limits>
+#include <optional>
#include <set>
#include <string>
#include <utility>
@@ -8236,12 +8237,11 @@
}
}
-SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
- SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
- MachinePointerInfo DstPtrInfo,
- MachinePointerInfo SrcPtrInfo,
- const AAMDNodes &AAInfo, AAResults *AA) {
+SDValue SelectionDAG::getMemcpy(
+ SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
+ Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
+ std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
+ MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) {
// Check to see if we should lower the memcpy to loads and stores first.
// For cases within the target-specified limits, this is the best choice.
ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
@@ -8296,6 +8296,18 @@
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemcpy =
+ TLI->getLibcallName(RTLIB::MEMCPY) == StringRef("memcpy");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemcpy);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
@@ -8304,7 +8316,7 @@
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
@@ -8352,7 +8364,8 @@
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool isTailCall,
+ bool isVol, const CallInst *CI,
+ std::optional<bool> OverrideTailCall,
MachinePointerInfo DstPtrInfo,
MachinePointerInfo SrcPtrInfo,
const AAMDNodes &AAInfo, AAResults *AA) {
@@ -8398,6 +8411,19 @@
Entry.Node = Size; Args.push_back(Entry);
// FIXME: pass in SDLoc
TargetLowering::CallLoweringInfo CLI(*this);
+
+ bool IsTailCall = false;
+ if (OverrideTailCall.has_value()) {
+ IsTailCall = *OverrideTailCall;
+ } else {
+ bool LowersToMemmove =
+ TLI->getLibcallName(RTLIB::MEMMOVE) == StringRef("memmove");
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+ IsTailCall = CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(),
+ ReturnsFirstArg && LowersToMemmove);
+ }
+
CLI.setDebugLoc(dl)
.setChain(Chain)
.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
@@ -8406,7 +8432,7 @@
TLI->getPointerTy(getDataLayout())),
std::move(Args))
.setDiscardResult()
- .setTailCall(isTailCall);
+ .setTailCall(IsTailCall);
std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
@@ -8454,7 +8480,8 @@
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
SDValue Src, SDValue Size, Align Alignment,
- bool isVol, bool AlwaysInline, bool isTailCall,
+ bool isVol, bool AlwaysInline,
+ const CallInst *CI,
MachinePointerInfo DstPtrInfo,
const AAMDNodes &AAInfo) {
// Check to see if we should lower the memset to stores first.
@@ -8514,8 +8541,9 @@
return Entry;
};
+ bool UseBZero = isNullConstant(Src) && BzeroName;
// If zeroing out and bzero is present, use it.
- if (isNullConstant(Src) && BzeroName) {
+ if (UseBZero) {
TargetLowering::ArgListTy Args;
Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
@@ -8533,8 +8561,16 @@
TLI->getPointerTy(DL)),
std::move(Args));
}
-
- CLI.setDiscardResult().setTailCall(isTailCall);
+ bool LowersToMemset =
+ TLI->getLibcallName(RTLIB::MEMSET) == StringRef("memset");
+ // If we're going to use bzero, make sure not to tail call unless the
+ // subsequent return doesn't need a value, as bzero doesn't return the first
+ // arg unlike memset.
+ bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI) && !UseBZero;
+ bool IsTailCall =
+ CI && CI->isTailCall() &&
+ isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg && LowersToMemset);
+ CLI.setDiscardResult().setTailCall(IsTailCall);
std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
return CallResult.second;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 51cbdd9..d5cbb73 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6460,14 +6460,14 @@
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memcpy DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MC = DAG.getMemcpy(
- Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
+ /* AlwaysInline */ false, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6482,13 +6482,13 @@
Align SrcAlign = MCI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MCI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memcpy DAG
// node.
- SDValue MC = DAG.getMemcpy(
- getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
- /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
+ SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
+ /* AlwaysInline */ true, &I, std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)),
+ I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MC);
return;
}
@@ -6500,11 +6500,10 @@
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align Alignment = MSI.getDestAlign().valueOrOne();
bool isVol = MSI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MS = DAG.getMemset(
Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
- isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
+ &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
updateDAGForMaybeTailCall(MS);
return;
}
@@ -6517,10 +6516,9 @@
// @llvm.memset defines 0 and 1 to both mean no alignment.
Align DstAlign = MSII.getDestAlign().valueOrOne();
bool isVol = MSII.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
- /* AlwaysInline */ true, isTC,
+ /* AlwaysInline */ true, &I,
MachinePointerInfo(I.getArgOperand(0)),
I.getAAMetadata());
updateDAGForMaybeTailCall(MC);
@@ -6536,12 +6534,12 @@
Align SrcAlign = MMI.getSourceAlign().valueOrOne();
Align Alignment = std::min(DstAlign, SrcAlign);
bool isVol = MMI.isVolatile();
- bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
// FIXME: Support passing different dest/src alignments to the memmove DAG
// node.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
- SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
- isTC, MachinePointerInfo(I.getArgOperand(0)),
+ SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
+ /* OverrideTailCall */ std::nullopt,
+ MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)),
I.getAAMetadata(), AA);
updateDAGForMaybeTailCall(MM);
@@ -9039,11 +9037,10 @@
// because the return pointer needs to be adjusted by the size of
// the copied memory.
SDValue Root = getMemoryRoot();
- SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, false, false,
- /*isTailCall=*/false,
- MachinePointerInfo(I.getArgOperand(0)),
- MachinePointerInfo(I.getArgOperand(1)),
- I.getAAMetadata());
+ SDValue MC = DAG.getMemcpy(
+ Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
+ MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
assert(MC.getNode() != nullptr &&
"** memcpy should not be lowered as TailCall in mempcpy context **");
DAG.setRoot(MC);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index df9b0ae..eef83a8 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -8578,7 +8578,7 @@
Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ false,
- /*isTailCall = */ false, DstInfo, MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, DstInfo, MachinePointerInfo());
MemOpChains.push_back(Cpy);
} else {
@@ -10878,8 +10878,9 @@
return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(VaListSize, DL, MVT::i32),
- Align(PtrSize), false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ Align(PtrSize), false, false, /*CI=*/nullptr,
+ std::nullopt, MachinePointerInfo(DestSV),
+ MachinePointerInfo(SrcSV));
}
SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index bb8e217..a79d8f7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3822,7 +3822,7 @@
DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ true,
- /*isTailCall = */ false, DstInfo,
+ /*CI=*/nullptr, std::nullopt, DstInfo,
MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
MemOpChains.push_back(Cpy);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 79cffc0..7aeaebc 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -178,7 +178,7 @@
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
bool
@@ -1038,10 +1038,10 @@
SDLoc DL(Op);
// Size of the va_list is 12 bytes as it has 3 pointers. Therefore,
// we need to memcopy 12 bytes from va_list to another similar list.
- return DAG.getMemcpy(Chain, DL, DestPtr, SrcPtr,
- DAG.getIntPtrConstant(12, DL), Align(4),
- /*isVolatile*/ false, false, false,
- MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
+ return DAG.getMemcpy(
+ Chain, DL, DestPtr, SrcPtr, DAG.getIntPtrConstant(12, DL), Align(4),
+ /*isVolatile*/ false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(DestSV), MachinePointerInfo(SrcSV));
}
SDValue HexagonTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 06fd7ac..f6763a3 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -648,7 +648,7 @@
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
/*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 79da36c..ba6be85 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4225,7 +4225,7 @@
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, /*isTailCall=*/IsTailCall,
+ /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index fc066f0..ba7b6c8 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -864,11 +864,12 @@
if (Flags.isByVal()) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i16);
- MemOp = DAG.getMemcpy(
- Chain, dl, PtrOff, Arg, SizeNode, Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false,
- /*AlwaysInline=*/true,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ MemOp = DAG.getMemcpy(Chain, dl, PtrOff, Arg, SizeNode,
+ Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false,
+ /*AlwaysInline=*/true,
+ /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
} else {
MemOp = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
}
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index ef70ef2..0f2047f 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -4506,7 +4506,7 @@
Chain = DAG.getMemcpy(
Chain, DL, Dst, Src, DAG.getConstant(MemCpySize, DL, PtrTy),
Align(Alignment), /*isVolatile=*/false, /*AlwaysInline=*/false,
- /*isTailCall=*/false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
MemOpChains.push_back(Chain);
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 4111145..a11ab93 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3904,8 +3904,8 @@
// 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2),
DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8),
- false, true, false, MachinePointerInfo(),
- MachinePointerInfo());
+ false, true, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo());
}
SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
@@ -5275,9 +5275,9 @@
SDValue Chain, ISD::ArgFlagsTy Flags,
SelectionDAG &DAG, const SDLoc &dl) {
SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
- return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode,
- Flags.getNonZeroByValAlign(), false, false, false,
- MachinePointerInfo(), MachinePointerInfo());
+ return DAG.getMemcpy(
+ Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), false, false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 953196a..fef1441 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -19932,7 +19932,7 @@
Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
/*IsVolatile=*/false,
- /*AlwaysInline=*/false, IsTailCall,
+ /*AlwaysInline=*/false, /*CI*/ nullptr, IsTailCall,
MachinePointerInfo(), MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 0dba6c4..50aa194 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -868,8 +868,8 @@
Chain = DAG.getMemcpy(Chain, dl, FIPtr, Arg, SizeNode, Alignment,
false, // isVolatile,
(Size <= 32), // AlwaysInline if size <= 32,
- false, // isTailCall
- MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
ByValArgs.push_back(FIPtr);
}
else {
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 0a7229a..b2b8814 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3951,7 +3951,7 @@
Subtarget.isTargetXPLINK64() ? getTargetMachine().getPointerSize(0) : 32;
return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(Sz, DL),
Align(8), /*isVolatile*/ false, /*AlwaysInline*/ false,
- /*isTailCall*/ false, MachinePointerInfo(DstSV),
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
MachinePointerInfo(SrcSV));
}
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index a793e59..f77076d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1123,10 +1123,11 @@
SDValue SizeNode =
DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
- Chain = DAG.getMemcpy(
- Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
- /*isVolatile*/ false, /*AlwaysInline=*/false,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode,
+ Out.Flags.getNonZeroByValAlign(),
+ /*isVolatile*/ false, /*AlwaysInline=*/false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
OutVal = FINode;
}
// Count the number of fixed args *after* legalization.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 9d651d4..bcdd31c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -25147,7 +25147,8 @@
Chain, DL, DstPtr, SrcPtr,
DAG.getIntPtrConstant(Subtarget.isTarget64BitLP64() ? 24 : 16, DL),
Align(Subtarget.isTarget64BitLP64() ? 8 : 4), /*isVolatile*/ false, false,
- false, MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(DstSV),
+ MachinePointerInfo(SrcSV));
}
// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index ed3fb13..3c49b6e 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -1240,7 +1240,7 @@
return DAG.getMemcpy(
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
/*isVolatile*/ false, /*AlwaysInline=*/true,
- /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
/// Return true if the calling convention is one that we can guarantee TCO for.
diff --git a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
index e5f07f2..055466a 100644
--- a/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -147,7 +147,7 @@
DAG.getConstant(Offset, dl, AddrVT)),
Val, DAG.getConstant(BytesLeft, dl, SizeVT), Alignment,
isVolatile, AlwaysInline,
- /* isTailCall */ false, DstPtrInfo.getWithOffset(Offset)));
+ /* CI */ nullptr, DstPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
@@ -255,7 +255,7 @@
DAG.getNode(ISD::ADD, dl, DstVT, Dst, DAG.getConstant(Offset, dl, DstVT)),
DAG.getNode(ISD::ADD, dl, SrcVT, Src, DAG.getConstant(Offset, dl, SrcVT)),
DAG.getConstant(BytesLeft, dl, SizeVT), Alignment, isVolatile,
- /*AlwaysInline*/ true, /*isTailCall*/ false,
+ /*AlwaysInline*/ true, /*CI=*/nullptr, std::nullopt,
DstPtrInfo.getWithOffset(Offset), SrcPtrInfo.getWithOffset(Offset)));
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Results);
}
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index c81da03..f4d3283 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1301,8 +1301,8 @@
InVals.push_back(FIN);
MemOps.push_back(DAG.getMemcpy(
Chain, dl, FIN, ArgDI.SDV, DAG.getConstant(Size, dl, MVT::i32),
- Alignment, false, false, false, MachinePointerInfo(),
- MachinePointerInfo()));
+ Alignment, false, false, /*CI=*/nullptr, std::nullopt,
+ MachinePointerInfo(), MachinePointerInfo()));
} else {
InVals.push_back(ArgDI.SDV);
}
@@ -1704,7 +1704,7 @@
bool isTail = isInTailCallPosition(DAG, ST, Chain);
return DAG.getMemmove(Chain, dl, ST->getBasePtr(), LD->getBasePtr(),
DAG.getConstant(StoreBits / 8, dl, MVT::i32),
- Alignment, false, isTail,
+ Alignment, false, nullptr, isTail,
ST->getPointerInfo(), LD->getPointerInfo());
}
}
diff --git a/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
new file mode 100644
index 0000000..34c6c63
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/no-tail-call-bzero-from-memset.ll
@@ -0,0 +1,20 @@
+; RUN: llc -o - %s | FileCheck %s
+; RUN: llc -global-isel -o - %s | FileCheck %s
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx15.0.0"
+
+define ptr @test() {
+; CHECK-LABEL: test:
+; CHECK: bl _bzero
+ %1 = tail call ptr @fn(i32 noundef 1) #3
+ tail call void @llvm.memset.p0.i64(ptr noundef nonnull align 1 dereferenceable(1000) %1, i8 noundef 0, i64 noundef 1000, i1 noundef false) #3
+ ret ptr %1
+}
+
+declare ptr @fn(i32 noundef)
+
+; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: write)
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
+
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: write) }
+attributes #3 = { nounwind optsize }
