[AArch64][TTI][NFC] Overload method 'getVectorInstrCost' to provide vector instruction itself, as a context information for cost estimation.
1) Overloaded (instruction-based) method is a wrapper around the current (opcode-based) method.
2) This patch also changes a few callsites (VectorCombine.cpp,
SLPVectorizer.cpp, CodeGenPrepare.cpp) to call the overloaded method.
3) This is a split of D128302.
Differential Revision: https://reviews.llvm.org/D131114
GitOrigin-RevId: bc8f2f36496a3fda32943f261eebb9b053e44016
diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h
index f613cc3..13c9a2e 100644
--- a/include/llvm/Analysis/TargetTransformInfo.h
+++ b/include/llvm/Analysis/TargetTransformInfo.h
@@ -1166,9 +1166,21 @@
/// \return The expected cost of vector Insert and Extract.
/// Use -1 to indicate that there is no information on the index value.
+ /// This is used when the instruction is not available; a typical use
+ /// case is to provision the cost of vectorization/scalarization in
+ /// vectorizer passes.
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index = -1) const;
+ /// \return The expected cost of vector Insert and Extract.
+ /// This is used when instruction is available, and implementation
+ /// asserts 'I' is not nullptr.
+ ///
+ /// A typical suitable use case is cost estimation when vector instruction
+ /// exists (e.g., from basic blocks during transformation).
+ InstructionCost getVectorInstrCost(const Instruction *I, Type *Val,
+ unsigned Index = -1) const;
+
/// \return The cost of replication shuffle of \p VF elements typed \p EltTy
/// \p ReplicationFactor times.
///
@@ -1747,6 +1759,8 @@
const Instruction *I) = 0;
virtual InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index) = 0;
+ virtual InstructionCost getVectorInstrCost(const Instruction *I, Type *Val,
+ unsigned Index) = 0;
virtual InstructionCost
getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
@@ -2305,6 +2319,10 @@
unsigned Index) override {
return Impl.getVectorInstrCost(Opcode, Val, Index);
}
+ InstructionCost getVectorInstrCost(const Instruction *I, Type *Val,
+ unsigned Index) override {
+ return Impl.getVectorInstrCost(I, Val, Index);
+ }
InstructionCost
getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
const APInt &DemandedDstElts,
diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h
index f3376a3..514a970 100644
--- a/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -575,6 +575,11 @@
return 1;
}
+ InstructionCost getVectorInstrCost(const Instruction *I, Type *Val,
+ unsigned Index) const {
+ return 1;
+ }
+
unsigned getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) {
@@ -1148,7 +1153,7 @@
if (auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2)))
if (CI->getValue().getActiveBits() <= 32)
Idx = CI->getZExtValue();
- return TargetTTI->getVectorInstrCost(Opcode, Ty, Idx);
+ return TargetTTI->getVectorInstrCost(IE, Ty, Idx);
}
case Instruction::ShuffleVector: {
auto *Shuffle = dyn_cast<ShuffleVectorInst>(U);
@@ -1238,7 +1243,7 @@
if (CI->getValue().getActiveBits() <= 32)
Idx = CI->getZExtValue();
Type *DstTy = U->getOperand(0)->getType();
- return TargetTTI->getVectorInstrCost(Opcode, DstTy, Idx);
+ return TargetTTI->getVectorInstrCost(EEI, DstTy, Idx);
}
}
// By default, just classify everything as 'basic'.
diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h
index 33c93e4..2b092d4 100644
--- a/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1159,6 +1159,11 @@
return LT.first;
}
+ InstructionCost getVectorInstrCost(const Instruction *I, Type *Val,
+ unsigned Index) {
+ return thisT()->getVectorInstrCost(I->getOpcode(), Val, Index);
+ }
+
InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
int VF,
const APInt &DemandedDstElts,
diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp
index 4d3d549..c86279e 100644
--- a/lib/Analysis/TargetTransformInfo.cpp
+++ b/lib/Analysis/TargetTransformInfo.cpp
@@ -869,11 +869,26 @@
InstructionCost TargetTransformInfo::getVectorInstrCost(unsigned Opcode,
Type *Val,
unsigned Index) const {
+ // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
+ // This is mentioned in the interface description and respected by all
+ // callers, but never asserted upon.
InstructionCost Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
+InstructionCost TargetTransformInfo::getVectorInstrCost(const Instruction *I,
+ Type *Val,
+ unsigned Index) const {
+ assert((I != nullptr) && "Expect not-null instruction pointer");
+ // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
+ // This is mentioned in the interface description and respected by all
+ // callers, but never asserted upon.
+ InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, Index);
+ assert(Cost >= 0 && "TTI should not produce negative costs!");
+ return Cost;
+}
+
InstructionCost TargetTransformInfo::getReplicationShuffleCost(
Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) {
diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp
index b100fbe..57423ff 100644
--- a/lib/CodeGen/CodeGenPrepare.cpp
+++ b/lib/CodeGen/CodeGenPrepare.cpp
@@ -7261,7 +7261,7 @@
// scalar to vector.
// The vector chain has to account for the combining cost.
InstructionCost ScalarCost =
- TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index);
+ TTI.getVectorInstrCost(Transition, PromotedType, Index);
InstructionCost VectorCost = StoreExtractCombineCost;
enum TargetTransformInfo::TargetCostKind CostKind =
TargetTransformInfo::TCK_RecipThroughput;
diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.h b/lib/Target/AArch64/AArch64TargetTransformInfo.h
index fcd0df6..c4e7135 100644
--- a/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -173,6 +173,7 @@
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
diff --git a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f2260c3..eeb3043 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -159,6 +159,7 @@
bool isInlineAsmSourceOfDivergence(const CallInst *CI,
ArrayRef<unsigned> Indices = {}) const;
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
unsigned Index);
bool isSourceOfDivergence(const Value *V) const;
diff --git a/lib/Target/AMDGPU/R600TargetTransformInfo.h b/lib/Target/AMDGPU/R600TargetTransformInfo.h
index 544292b..f1a198f 100644
--- a/lib/Target/AMDGPU/R600TargetTransformInfo.h
+++ b/lib/Target/AMDGPU/R600TargetTransformInfo.h
@@ -60,6 +60,7 @@
unsigned getMaxInterleaveFactor(unsigned VF);
InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
unsigned Index);
};
diff --git a/lib/Target/ARM/ARMTargetTransformInfo.h b/lib/Target/ARM/ARMTargetTransformInfo.h
index 5e39e85..a3aed48 100644
--- a/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -237,6 +237,7 @@
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
diff --git a/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 7bbaf7a..9d26364 100644
--- a/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -151,6 +151,7 @@
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
diff --git a/lib/Target/PowerPC/PPCTargetTransformInfo.h b/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 790eb0b..9a5b26b 100644
--- a/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -123,6 +123,7 @@
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src,
diff --git a/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 33317e7..0804d03 100644
--- a/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -107,6 +107,7 @@
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue);
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index fde58a9..10179ef 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -67,6 +67,7 @@
TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None,
ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
const Instruction *CxtI = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
diff --git a/lib/Target/X86/X86TargetTransformInfo.h b/lib/Target/X86/X86TargetTransformInfo.h
index bd3c3fb..9f83c04 100644
--- a/lib/Target/X86/X86TargetTransformInfo.h
+++ b/lib/Target/X86/X86TargetTransformInfo.h
@@ -146,6 +146,7 @@
CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr);
+ using BaseT::getVectorInstrCost;
InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val,
unsigned Index);
InstructionCost getScalarizationOverhead(VectorType *Ty,
diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d69d1e3..737fcb3 100644
--- a/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5882,8 +5882,7 @@
continue;
}
}
- Cost -= TTIRef.getVectorInstrCost(Instruction::ExtractElement,
- EE->getVectorOperandType(), Idx);
+ Cost -= TTIRef.getVectorInstrCost(EE, EE->getVectorOperandType(), Idx);
}
// Add a cost for subvector extracts/inserts if required.
for (const auto &Data : ExtractVectorsTys) {
@@ -6116,9 +6115,8 @@
for (unsigned I : E->ReuseShuffleIndices) {
if (ShuffleOrOp == Instruction::ExtractElement) {
auto *EE = cast<ExtractElementInst>(VL[I]);
- CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement,
- EE->getVectorOperandType(),
- *getExtractIndex(EE));
+ CommonCost -= TTI->getVectorInstrCost(
+ EE, EE->getVectorOperandType(), *getExtractIndex(EE));
} else {
CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement,
VecTy, Idx);
@@ -6129,9 +6127,8 @@
for (Value *V : VL) {
if (ShuffleOrOp == Instruction::ExtractElement) {
auto *EE = cast<ExtractElementInst>(V);
- CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement,
- EE->getVectorOperandType(),
- *getExtractIndex(EE));
+ CommonCost += TTI->getVectorInstrCost(
+ EE, EE->getVectorOperandType(), *getExtractIndex(EE));
} else {
--Idx;
CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement,
diff --git a/lib/Transforms/Vectorize/VectorCombine.cpp b/lib/Transforms/Vectorize/VectorCombine.cpp
index a389366..37901d8 100644
--- a/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -270,10 +270,8 @@
Type *VecTy = Ext0->getVectorOperand()->getType();
assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
- InstructionCost Cost0 =
- TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0);
- InstructionCost Cost1 =
- TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1);
+ InstructionCost Cost0 = TTI.getVectorInstrCost(Ext0, VecTy, Index0);
+ InstructionCost Cost1 = TTI.getVectorInstrCost(Ext1, VecTy, Index1);
// If both costs are invalid no shuffle is needed
if (!Cost0.isValid() && !Cost1.isValid())
@@ -337,10 +335,8 @@
unsigned Ext0Index = Ext0IndexC->getZExtValue();
unsigned Ext1Index = Ext1IndexC->getZExtValue();
- InstructionCost Extract0Cost =
- TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy, Ext0Index);
- InstructionCost Extract1Cost =
- TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy, Ext1Index);
+ InstructionCost Extract0Cost = TTI.getVectorInstrCost(Ext0, VecTy, Ext0Index);
+ InstructionCost Extract1Cost = TTI.getVectorInstrCost(Ext1, VecTy, Ext1Index);
// A more expensive extract will always be replaced by a splat shuffle.
// For example, if Ext0 is more expensive:
@@ -754,9 +750,8 @@
if (!VecTy)
return false;
- InstructionCost OldCost =
- TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0);
- OldCost += TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1);
+ InstructionCost OldCost = TTI.getVectorInstrCost(Ext0, VecTy, Index0);
+ OldCost += TTI.getVectorInstrCost(Ext1, VecTy, Index1);
OldCost +=
TTI.getCmpSelInstrCost(CmpOpcode, I0->getType(),
CmpInst::makeCmpResultType(I0->getType()), Pred) *
@@ -776,7 +771,7 @@
NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy,
ShufMask);
NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy);
- NewCost += TTI.getVectorInstrCost(Ext0->getOpcode(), CmpTy, CheapIndex);
+ NewCost += TTI.getVectorInstrCost(Ext0, CmpTy, CheapIndex);
// Aggressively form vector ops if the cost is equal because the transform
// may enable further optimization.