Google Git Google Git
Sign in
llvm/llvm-project/refs/heads/users/void/array-bounds-accessed/./llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
blob: 0bb91fa68751b862da03aceac520f127f27c9c78 [file] [edit]
//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "RISCVTargetTransformInfo.h"
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCVVectorUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/CodeGen/BasicTTIImpl.h"
#include "llvm/CodeGen/CostTable.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include <cmath>
#include <limits>
#include <optional>
using namespace llvm;
using namespace llvm::PatternMatch;
#define DEBUG_TYPE "riscvtti"
InstructionCost
RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
TTI::TargetCostKind CostKind) const {
// Check if the type is valid for all CostKind
if (!VT.isVector())
return InstructionCost::getInvalid();
size_t NumInstr = OpCodes.size();
if (CostKind == TTI::TCK_CodeSize)
return NumInstr;
InstructionCost LMULCost = TLI->getLMULCost(VT);
if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
return LMULCost * NumInstr;
InstructionCost Cost = 0;
for (auto Op : OpCodes) {
switch (Op) {
case RISCV::VRGATHER_VI:
Cost += TLI->getVRGatherVICost(VT);
break;
case RISCV::VRGATHER_VV:
Cost += TLI->getVRGatherVVCost(VT);
break;
case RISCV::VSLIDEUP_VI:
case RISCV::VSLIDEDOWN_VI:
Cost += TLI->getVSlideVICost(VT);
break;
case RISCV::VSLIDEUP_VX:
case RISCV::VSLIDEDOWN_VX:
Cost += TLI->getVSlideVXCost(VT);
break;
case RISCV::VREDMAX_VS:
case RISCV::VREDMIN_VS:
case RISCV::VREDMAXU_VS:
case RISCV::VREDMINU_VS:
case RISCV::VREDSUM_VS:
case RISCV::VREDAND_VS:
case RISCV::VREDOR_VS:
case RISCV::VREDXOR_VS:
case RISCV::VFREDMAX_VS:
case RISCV::VFREDMIN_VS:
case RISCV::VFREDUSUM_VS: {
unsigned VL = VT.getVectorMinNumElements();
if (!VT.isFixedLengthVector())
VL *= *getVScaleForTuning();
Cost += Log2_32_Ceil(VL);
break;
}
case RISCV::VFREDOSUM_VS: {
unsigned VL = VT.getVectorMinNumElements();
if (!VT.isFixedLengthVector())
VL *= *getVScaleForTuning();
Cost += VL;
break;
}
case RISCV::VMV_X_S:
case RISCV::VFMV_F_S:
// Domain crossings from vector -> scalar are usually more expensive.
Cost += 2;
break;
case RISCV::VMV_S_X:
case RISCV::VFMV_S_F:
case RISCV::VMOR_MM:
case RISCV::VMXOR_MM:
case RISCV::VMAND_MM:
case RISCV::VMANDN_MM:
case RISCV::VMNAND_MM:
case RISCV::VCPOP_M:
case RISCV::VFIRST_M:
Cost += 1;
break;
case RISCV::VDIV_VV:
case RISCV::VREM_VV:
Cost += LMULCost * TTI::TCC_Expensive;
break;
default:
Cost += LMULCost;
}
}
return Cost;
}
static InstructionCost getIntImmCostImpl(const DataLayout &DL,
const RISCVSubtarget *ST,
const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind,
bool FreeZeroes) {
assert(Ty->isIntegerTy() &&
"getIntImmCost can only estimate cost of materialising integers");
// We have a Zero register, so 0 is always free.
if (Imm == 0)
return TTI::TCC_Free;
// Otherwise, we check how many instructions it will take to materialise.
return RISCVMatInt::getIntMatCost(Imm, DL.getTypeSizeInBits(Ty), *ST,
/*CompressionCost=*/false, FreeZeroes);
}
InstructionCost
RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind) const {
return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind, false);
}
// Look for patterns of shift followed by AND that can be turned into a pair of
// shifts. We won't need to materialize an immediate for the AND so these can
// be considered free.
static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
uint64_t Mask = Imm.getZExtValue();
auto *BO = dyn_cast<BinaryOperator>(Inst->getOperand(0));
if (!BO || !BO->hasOneUse())
return false;
if (BO->getOpcode() != Instruction::Shl)
return false;
if (!isa<ConstantInt>(BO->getOperand(1)))
return false;
unsigned ShAmt = cast<ConstantInt>(BO->getOperand(1))->getZExtValue();
// (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
// is a mask shifted by c2 bits with c3 leading zeros.
if (isShiftedMask_64(Mask)) {
unsigned Trailing = llvm::countr_zero(Mask);
if (ShAmt == Trailing)
return true;
}
return false;
}
// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
// the type will be split so only the lower 32 bits need to be compared using
// (srai/srli X, C) == C2.
static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
if (!Inst->hasOneUse())
return false;
// Look for equality comparison.
auto *Cmp = dyn_cast<ICmpInst>(*Inst->user_begin());
if (!Cmp || !Cmp->isEquality())
return false;
// Right hand side of comparison should be a constant.
auto *C = dyn_cast<ConstantInt>(Cmp->getOperand(1));
if (!C)
return false;
uint64_t Mask = Imm.getZExtValue();
// Mask should be of the form -(1 << C) in the lower 32 bits.
if (!isUInt<32>(Mask) || !isPowerOf2_32(-uint32_t(Mask)))
return false;
// Comparison constant should be a subset of Mask.
uint64_t CmpC = C->getZExtValue();
if ((CmpC & Mask) != CmpC)
return false;
// We'll need to sign extend the comparison constant and shift it right. Make
// sure the new constant can use addi/xori+seqz/snez.
unsigned ShiftBits = llvm::countr_zero(Mask);
int64_t NewCmpC = SignExtend64<32>(CmpC) >> ShiftBits;
return NewCmpC >= -2048 && NewCmpC <= 2048;
}
InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind,
Instruction *Inst) const {
assert(Ty->isIntegerTy() &&
"getIntImmCost can only estimate cost of materialising integers");
// We have a Zero register, so 0 is always free.
if (Imm == 0)
return TTI::TCC_Free;
// Some instructions in RISC-V can take a 12-bit immediate. Some of these are
// commutative, in others the immediate comes from a specific argument index.
bool Takes12BitImm = false;
unsigned ImmArgIdx = ~0U;
switch (Opcode) {
case Instruction::GetElementPtr:
// Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
// split up large offsets in GEP into better parts than ConstantHoisting
// can.
return TTI::TCC_Free;
case Instruction::Store: {
// Use the materialization cost regardless of if it's the address or the
// value that is constant, except for if the store is misaligned and
// misaligned accesses are not legal (experience shows constant hoisting
// can sometimes be harmful in such cases).
if (Idx == 1 || !Inst)
return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind,
/*FreeZeroes=*/true);
StoreInst *ST = cast<StoreInst>(Inst);
if (!getTLI()->allowsMemoryAccessForAlignment(
Ty->getContext(), DL, getTLI()->getValueType(DL, Ty),
ST->getPointerAddressSpace(), ST->getAlign()))
return TTI::TCC_Free;
return getIntImmCostImpl(getDataLayout(), getST(), Imm, Ty, CostKind,
/*FreeZeroes=*/true);
}
case Instruction::Load:
// If the address is a constant, use the materialization cost.
return getIntImmCost(Imm, Ty, CostKind);
case Instruction::And:
// zext.h
if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
return TTI::TCC_Free;
// zext.w
if (Imm == UINT64_C(0xffffffff) && (!ST->is64Bit() || ST->hasStdExtZba()))
return TTI::TCC_Free;
// bclri
if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
return TTI::TCC_Free;
if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
canUseShiftPair(Inst, Imm))
return TTI::TCC_Free;
if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
canUseShiftCmp(Inst, Imm))
return TTI::TCC_Free;
Takes12BitImm = true;
break;
case Instruction::Add:
Takes12BitImm = true;
break;
case Instruction::Or:
case Instruction::Xor:
// bseti/binvi
if (ST->hasStdExtZbs() && Imm.isPowerOf2())
return TTI::TCC_Free;
Takes12BitImm = true;
break;
case Instruction::Mul:
// Power of 2 is a shift. Negated power of 2 is a shift and a negate.
if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
return TTI::TCC_Free;
// One more or less than a power of 2 can use SLLI+ADD/SUB.
if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
return TTI::TCC_Free;
// FIXME: There is no MULI instruction.
Takes12BitImm = true;
break;
case Instruction::Sub:
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
Takes12BitImm = true;
ImmArgIdx = 1;
break;
default:
break;
}
if (Takes12BitImm) {
// Check immediate is the correct argument...
if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
// ... and fits into the 12-bit immediate.
if (Imm.getSignificantBits() <= 64 &&
getTLI()->isLegalAddImmediate(Imm.getSExtValue())) {
return TTI::TCC_Free;
}
}
// Otherwise, use the full materialisation cost.
return getIntImmCost(Imm, Ty, CostKind);
}
// By default, prevent hoisting.
return TTI::TCC_Free;
}
InstructionCost
RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
const APInt &Imm, Type *Ty,
TTI::TargetCostKind CostKind) const {
// Prevent hoisting in unknown cases.
return TTI::TCC_Free;
}
bool RISCVTTIImpl::hasActiveVectorLength() const {
return ST->hasVInstructions();
}
TargetTransformInfo::PopcntSupportKind
RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
}
InstructionCost RISCVTTIImpl::getPartialReductionCost(
unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
if (Opcode == Instruction::FAdd)
return InstructionCost::getInvalid();
// zve32x is broken for partial_reduce_umla, but let's make sure we
// don't generate them.
// vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
// additionally supported by widening the i32 partial sums to i64 (see
// lowerPARTIAL_REDUCE_MLA). VF is the number of i8 input elements, so the
// reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(8) ||
(!AccumType->isIntegerTy(32) && !AccumType->isIntegerTy(64)))
return InstructionCost::getInvalid();
unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
if (!VF.isKnownMultipleOf(Ratio))
return InstructionCost::getInvalid();
// Cost of the vdot4a* itself, which operates on the i32 intermediate type
// holding VF/4 elements.
Type *DotTp = VectorType::get(Type::getInt32Ty(AccumType->getContext()),
VF.divideCoefficientBy(4));
std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(DotTp);
// Note: Asuming all vdot4a* variants are equal cost
InstructionCost Cost =
DotLT.first *
getRISCVInstructionCost(RISCV::VDOT4A_VV, DotLT.second, CostKind);
// Account for reducing the i32 partial sums down to the i64 accumulator's
// element count and accumulating into it (see lowerPARTIAL_REDUCE_MLA), which
// has two shapes depending on the accumulator's LMUL.
if (AccumType->isIntegerTy(64)) {
LLVMContext &Ctx = AccumType->getContext();
Type *I32Ty = Type::getInt32Ty(Ctx);
ElementCount AccVF = VF.divideCoefficientBy(Ratio);
std::pair<InstructionCost, MVT> AccLT =
getTypeLegalizationCost(VectorType::get(AccumType, AccVF));
// When the i32 subvectors of a single-vector scalable accumulator are a
// fractional LMUL, extracting the high subvector would need a vslidedown,
// so instead the i32 dot result is widened to i64 first (vsext.vf2 /
// vzext.vf2) and then reduced and accumulated with register-aligned i64
// vadd.vv.
bool WidenFirst = false;
if (VF.isScalable() && AccLT.second.isScalableVector()) {
MVT NarrowMVT = AccLT.second.changeVectorElementType(MVT::i32);
WidenFirst =
RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(NarrowMVT))
.second;
}
if (WidenFirst) {
// The widened i64 dot result has VF/4 elements, i.e. twice the
// accumulator's element count, so the reduction plus the accumulate are
// two i64 vadd.vv.
std::pair<InstructionCost, MVT> WideLT = getTypeLegalizationCost(
VectorType::get(AccumType, VF.divideCoefficientBy(4)));
Cost +=
WideLT.first * getRISCVInstructionCost(RISCV::VSEXT_VF2,
WideLT.second, CostKind) +
2 * AccLT.first *
getRISCVInstructionCost(RISCV::VADD_VV, AccLT.second, CostKind);
} else {
// Otherwise the scale-4 i32 sums are halved with a single i32 vadd.vv,
// then widened and added into the i64 result with a vwadd.wv.
std::pair<InstructionCost, MVT> RedLT =
getTypeLegalizationCost(VectorType::get(I32Ty, AccVF));
Cost += RedLT.first * getRISCVInstructionCost(RISCV::VADD_VV,
RedLT.second, CostKind) +
AccLT.first * getRISCVInstructionCost(RISCV::VWADD_WV,
AccLT.second, CostKind);
// Fixed-length vectors extract the high i32 subvector with a vslidedown.
if (VF.isFixed())
Cost += DotLT.first * getRISCVInstructionCost(RISCV::VSLIDEDOWN_VI,
DotLT.second, CostKind);
}
}
return Cost;
}
bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
// Currently, the ExpandReductions pass can't expand scalable-vector
// reductions, but we still request expansion as RVV doesn't support certain
// reductions and the SelectionDAG can't legalize them either.
switch (II->getIntrinsicID()) {
default:
return false;
// These reductions have no equivalent in RVV
case Intrinsic::vector_reduce_mul:
case Intrinsic::vector_reduce_fmul:
return true;
}
}
std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
if (ST->hasVInstructions())
if (unsigned MinVLen = ST->getRealMinVLen();
MinVLen >= RISCV::RVVBitsPerBlock)
return MinVLen / RISCV::RVVBitsPerBlock;
return BaseT::getVScaleForTuning();
}
TypeSize
RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
unsigned LMUL = llvm::bit_floor(
std::clamp<unsigned>(ST->getCLOpts().v_register_bit_width_lmul, 1, 8));
switch (K) {
case TargetTransformInfo::RGK_Scalar:
return TypeSize::getFixed(ST->getXLen());
case TargetTransformInfo::RGK_FixedWidthVector:
return TypeSize::getFixed(
ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
case TargetTransformInfo::RGK_ScalableVector:
return TypeSize::getScalable(
(ST->hasVInstructions() &&
ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
? LMUL * RISCV::RVVBitsPerBlock
: 0);
}
llvm_unreachable("Unsupported register kind");
}
InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
const TTI::TargetCostKind CostKind) const {
switch (CostKind) {
case TTI::TCK_CodeSize:
case TTI::TCK_SizeAndLatency:
// Always 2 instructions
return 2;
case TTI::TCK_Latency:
case TTI::TCK_RecipThroughput:
// Depending on the memory model the address generation will
// require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
// have a way of getting this information here, so conservatively
// require both.
// In practice, these are generally implemented together.
return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
}
llvm_unreachable("Unsupported cost kind");
}
InstructionCost
RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
TTI::TargetCostKind CostKind) const {
// Add a cost of address generation + the cost of the load. The address
// is expected to be a PC relative offset to a constant pool entry
// using auipc/addi.
InstructionCost Cost = 0;
Cost = getStaticDataAddrGenerationCost(CostKind) +
getMemoryOpCost(Instruction::Load, Ty, DL.getABITypeAlign(Ty),
/*AddressSpace=*/0, CostKind);
// Estimate the amount of 4 byte instructions that could fit
// instead of the constant pool, ignoring any extra padding.
if (CostKind == TTI::TCK_CodeSize)
Cost += ((InstructionCost)DL.getTypeAllocSize(Ty)) / 4;
return Cost;
}
static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
unsigned Size = Mask.size();
if (!isPowerOf2_32(Size))
return false;
for (unsigned I = 0; I != Size; ++I) {
if (static_cast<unsigned>(Mask[I]) == I)
continue;
if (Mask[I] != 0)
return false;
if (Size % I != 0)
return false;
for (unsigned J = I + 1; J != Size; ++J)
// Check the pattern is repeated.
if (static_cast<unsigned>(Mask[J]) != J % I)
return false;
SubVectorSize = I;
return true;
}
// That means Mask is <0, 1, 2, 3>. This is not a concatenation.
return false;
}
static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
LLVMContext &C) {
assert((DataVT.getScalarSizeInBits() != 8 ||
DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
MVT IndexVT = DataVT.changeTypeToInteger();
if (IndexVT.getScalarType().bitsGT(ST.getXLenVT()))
IndexVT = IndexVT.changeVectorElementType(MVT::i16);
return cast<VectorType>(EVT(IndexVT).getTypeForEVT(C));
}
/// Attempt to approximate the cost of a shuffle which will require splitting
/// during legalization. Note that processShuffleMasks is not an exact proxy
/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
/// reasonably close upperbound.
static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
MVT LegalVT, VectorType *Tp,
ArrayRef<int> Mask,
TTI::TargetCostKind CostKind) {
assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
"Expected fixed vector type and non-empty mask");
unsigned LegalNumElts = LegalVT.getVectorNumElements();
// Number of destination vectors after legalization:
unsigned NumOfDests = divideCeil(Mask.size(), LegalNumElts);
// We are going to permute multiple sources and the result will be in
// multiple destinations. Providing an accurate cost only for splits where
// the element type remains the same.
if (NumOfDests <= 1 ||
LegalVT.getVectorElementType().getSizeInBits() !=
Tp->getElementType()->getPrimitiveSizeInBits() ||
LegalNumElts >= Tp->getElementCount().getFixedValue())
return InstructionCost::getInvalid();
unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Tp);
unsigned LegalVTSize = LegalVT.getStoreSize();
// Number of source vectors after legalization:
unsigned NumOfSrcs = divideCeil(VecTySize, LegalVTSize);
auto *SingleOpTy = FixedVectorType::get(Tp->getElementType(), LegalNumElts);
unsigned NormalizedVF = LegalNumElts * std::max(NumOfSrcs, NumOfDests);
unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
assert(NormalizedVF >= Mask.size() &&
"Normalized mask expected to be not shorter than original mask.");
copy(Mask, NormalizedMask.begin());
InstructionCost Cost = 0;
SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
processShuffleMasks(
NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
[&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
if (ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size()))
return;
if (!ReusedSingleSrcShuffles.insert(std::make_pair(RegMask, SrcReg))
.second)
return;
Cost += TTI.getShuffleCost(
TTI::SK_PermuteSingleSrc,
FixedVectorType::get(SingleOpTy->getElementType(), RegMask.size()),
SingleOpTy, CostKind, RegMask, 0, nullptr);
},
[&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
Cost += TTI.getShuffleCost(
TTI::SK_PermuteTwoSrc,
FixedVectorType::get(SingleOpTy->getElementType(), RegMask.size()),
SingleOpTy, CostKind, RegMask, 0, nullptr);
});
return Cost;
}
/// Try to perform better estimation of the permutation.
/// 1. Split the source/destination vectors into real registers.
/// 2. Do the mask analysis to identify which real registers are
/// permuted. If more than 1 source registers are used for the
/// destination register building, the cost for this destination register
/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
/// source register is used, build mask and calculate the cost as a cost
/// of PermuteSingleSrc.
/// Also, for the single register permute we try to identify if the
/// destination register is just a copy of the source register or the
/// copy of the previous destination register (the cost is
/// TTI::TCC_Basic). If the source register is just reused, the cost for
/// this operation is 0.
static InstructionCost
costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
std::optional<unsigned> VLen, VectorType *Tp,
ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
assert(LegalVT.isFixedLengthVector());
if (!VLen || Mask.empty())
return InstructionCost::getInvalid();
MVT ElemVT = LegalVT.getVectorElementType();
unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
LegalVT = TTI.getTypeLegalizationCost(
FixedVectorType::get(Tp->getElementType(), ElemsPerVReg))
.second;
// Number of destination vectors after legalization:
InstructionCost NumOfDests =
divideCeil(Mask.size(), LegalVT.getVectorNumElements());
if (NumOfDests <= 1 ||
LegalVT.getVectorElementType().getSizeInBits() !=
Tp->getElementType()->getPrimitiveSizeInBits() ||
LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
return InstructionCost::getInvalid();
unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Tp);
unsigned LegalVTSize = LegalVT.getStoreSize();
// Number of source vectors after legalization:
unsigned NumOfSrcs = divideCeil(VecTySize, LegalVTSize);
auto *SingleOpTy = FixedVectorType::get(Tp->getElementType(),
LegalVT.getVectorNumElements());
unsigned E = NumOfDests.getValue();
unsigned NormalizedVF =
LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E);
unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
assert(NormalizedVF >= Mask.size() &&
"Normalized mask expected to be not shorter than original mask.");
copy(Mask, NormalizedMask.begin());
InstructionCost Cost = 0;
int NumShuffles = 0;
SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
processShuffleMasks(
NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {},
[&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
if (ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size()))
return;
if (!ReusedSingleSrcShuffles.insert(std::make_pair(RegMask, SrcReg))
.second)
return;
++NumShuffles;
Cost += TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, SingleOpTy,
SingleOpTy, CostKind, RegMask, 0, nullptr);
},
[&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
Cost += TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, SingleOpTy,
SingleOpTy, CostKind, RegMask, 0, nullptr);
NumShuffles += 2;
});
// Note: check that we do not emit too many shuffles here to prevent code
// size explosion.
// TODO: investigate, if it can be improved by extra analysis of the masks
// to check if the code is more profitable.
if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
(NumOfDestRegs <= 2 && NumShuffles < 4))
return Cost;
return InstructionCost::getInvalid();
}
InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
ArrayRef<int> Mask,
TTI::TargetCostKind CostKind) const {
// Avoid missing masks and length changing shuffles
if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
return InstructionCost::getInvalid();
int NumElts = Tp->getNumElements();
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
// Avoid scalarization cases
if (!LT.second.isFixedLengthVector())
return InstructionCost::getInvalid();
// Requires moving elements between parts, which requires additional
// unmodeled instructions.
if (LT.first != 1)
return InstructionCost::getInvalid();
auto GetSlideOpcode = [&](int SlideAmt) {
assert(SlideAmt != 0);
bool IsVI = isUInt<5>(std::abs(SlideAmt));
if (SlideAmt < 0)
return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
};
std::array<std::pair<int, int>, 2> SrcInfo;
if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
return InstructionCost::getInvalid();
if (SrcInfo[1].second == 0)
std::swap(SrcInfo[0], SrcInfo[1]);
if (ST->hasStdExtZvzip() && LT.second.getScalarSizeInBits() != 1) {
unsigned Factor;
if (isPairEven(SrcInfo, Mask, Factor) && Factor == 1)
return getRISCVInstructionCost(RISCV::VPAIRE_VV, LT.second, CostKind);
if (isPairOdd(SrcInfo, Mask, Factor) && Factor == 1)
return getRISCVInstructionCost(RISCV::VPAIRO_VV, LT.second, CostKind);
}
InstructionCost FirstSlideCost = 0;
if (SrcInfo[0].second != 0) {
unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
FirstSlideCost = getRISCVInstructionCost(Opcode, LT.second, CostKind);
}
if (SrcInfo[1].first == -1)
return FirstSlideCost;
InstructionCost SecondSlideCost = 0;
if (SrcInfo[1].second != 0) {
unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
SecondSlideCost = getRISCVInstructionCost(Opcode, LT.second, CostKind);
} else {
SecondSlideCost =
getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second, CostKind);
}
auto EC = Tp->getElementCount();
VectorType *MaskTy =
VectorType::get(IntegerType::getInt1Ty(Tp->getContext()), EC);
InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);
return FirstSlideCost + SecondSlideCost + MaskCost;
}
MVT RISCVTTIImpl::getZvzipVZIPCostVT(MVT InterleavedVT) const {
if (!InterleavedVT.getVectorElementCount().isKnownEven())
return MVT::INVALID_SIMPLE_VALUE_TYPE;
MVT CostVT = InterleavedVT;
if (InterleavedVT.isFixedLengthVector()) {
MVT SourceVT = InterleavedVT.getHalfNumVectorElementsVT();
CostVT = TLI->getContainerForFixedLengthVector(SourceVT)
.getDoubleNumVectorElementsVT();
}
unsigned EltBits = CostVT.getScalarSizeInBits();
unsigned MinSize = CostVT.getSizeInBits().getKnownMinValue();
unsigned LMULOctuple = MinSize / (RISCV::RVVBitsPerBlock / 8);
// Perform the 2 * SEW <= LMUL * min(ELEN, VLEN) check.
if (EltBits * 16 >
LMULOctuple * std::min(ST->getELen(), ST->getRealMinVLen()))
return MVT::INVALID_SIMPLE_VALUE_TYPE;
return CostVT;
}
MVT RISCVTTIImpl::getZvzipVUNZIPCostVT(MVT InterleavedVT) const {
if (!InterleavedVT.getVectorElementCount().isKnownEven())
return MVT::INVALID_SIMPLE_VALUE_TYPE;
MVT CostVT = InterleavedVT;
// lowerZvzipVUNZIP widens the source container if halving it would produce
// an illegal result type. Apply the same rule here so the cost uses the
// source LMUL selected by ISel.
if (InterleavedVT.isFixedLengthVector()) {
CostVT = TLI->getContainerForFixedLengthVector(InterleavedVT);
if (CostVT.getVectorMinNumElements() == 1 ||
!TLI->isTypeLegal(CostVT.getHalfNumVectorElementsVT()))
CostVT = CostVT.getDoubleNumVectorElementsVT();
}
MVT DeinterleavedVT = CostVT.getHalfNumVectorElementsVT();
if (RISCVTargetLowering::getLMUL(DeinterleavedVT) == RISCVVType::LMUL_8)
return MVT::INVALID_SIMPLE_VALUE_TYPE;
return CostVT;
}
InstructionCost RISCVTTIImpl::getVZIPCost(TTI::ShuffleKind Kind,
VectorType *DstTy, VectorType *SrcTy,
ArrayRef<int> Mask,
TTI::TargetCostKind CostKind) const {
assert(isa<FixedVectorType>(SrcTy) && "Expected fixed-length vector types");
if (!ST->hasStdExtZvzip() || Mask.size() < 2 ||
DstTy->getScalarSizeInBits() == 1)
return InstructionCost::getInvalid();
unsigned NumSrcElts = SrcTy->getElementCount().getKnownMinValue();
unsigned NumDstElts = DstTy->getElementCount().getKnownMinValue();
unsigned NumInputElts;
if (Kind == TTI::SK_PermuteSingleSrc) {
// A single-source vzip shuffle interleaves two halves of the same vector,
// so its source and destination must have the same number of elements.
if (NumSrcElts != NumDstElts)
return InstructionCost::getInvalid();
NumInputElts = NumSrcElts;
} else if (Kind == TTI::SK_PermuteTwoSrc) {
// Accept both forms that can be lowered with vzip.vv:
//
// 1. The destination has twice as many elements as each source, e.g.
// %r = shufflevector <4 x i32> %a, <4 x i32> %b,
// <8 x i32> <0, 4, 1, 5, 2, 6, 3, 7>
// The two operands are already the half-sized inputs of vzip.vv.
//
// 2. Each source has the same number of elements as the destination, e.g.
// %r = shufflevector <4 x i32> %a, <4 x i32> %b,
// <4 x i32> <0, 4, 1, 5>
// ISel first extracts a <2 x i32> half from each source and then uses
// those halves as the inputs of vzip.vv.
if (NumSrcElts != NumDstElts && NumSrcElts * 2 != NumDstElts)
return InstructionCost::getInvalid();
NumInputElts = NumSrcElts * 2;
} else {
return InstructionCost::getInvalid();
}
// ISel deliberately excludes the two-element single-source identity case.
if (Kind == TTI::SK_PermuteSingleSrc && Mask.size() == 2 &&
ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))
return InstructionCost::getInvalid();
// A deinterleave shuffle for an interleaved load may have a mask such as
// %even = shufflevector <4 x i32> %wide, <4 x i32> poison,
// <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
// Because isInterleaveMask ignores poison lanes, this mask may also satisfy
// the interleave matcher. Do not cost it as vzip.vv.
unsigned DeinterleaveIndex;
if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2,
DeinterleaveIndex) &&
count_if(Mask, [](int Idx) { return Idx >= 0; }) > 1)
return InstructionCost::getInvalid();
// Require the two start indices found in the mask to be aligned to
// half-vector boundaries, with at least one equal to zero.
SmallVector<unsigned, 2> StartIndexes;
if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, NumInputElts, StartIndexes))
return InstructionCost::getInvalid();
unsigned HalfNumElts = Mask.size() / 2;
if ((StartIndexes[0] != 0 && StartIndexes[1] != 0) ||
StartIndexes[0] % HalfNumElts != 0 || StartIndexes[1] % HalfNumElts != 0)
return InstructionCost::getInvalid();
std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(DstTy);
MVT CostVT = getZvzipVZIPCostVT(DstLT.second);
if (!CostVT.isValid())
return InstructionCost::getInvalid();
InstructionCost Cost =
DstLT.first * getRISCVInstructionCost(RISCV::VZIP_VV, CostVT, CostKind);
// For a shuffle such as
// %r = shufflevector <4 x i32> %v, <4 x i32> poison,
// <4 x i32> <i32 0, i32 2, i32 1, i32 3>
// vzip.vv needs two <2 x i32> inputs. Include the cost of extracting each
// upper half; extracting a low half is free.
if (NumSrcElts == NumDstElts) {
auto *HalfSrcTy = FixedVectorType::getHalfElementsVectorType(
cast<FixedVectorType>(SrcTy));
for (unsigned Start : StartIndexes) {
unsigned ExtractIndex = Start % NumSrcElts;
if (ExtractIndex != 0)
Cost += getShuffleCost(TTI::SK_ExtractSubvector, HalfSrcTy, SrcTy,
CostKind, {}, ExtractIndex, HalfSrcTy);
}
}
return Cost;
}
InstructionCost RISCVTTIImpl::getShuffleCost(
TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
TTI::VectorInstrContext VIC) const {
assert((Mask.empty() || DstTy->isScalableTy() ||
Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
"Expected the Mask to match the return size if given");
assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
"Expected the same scalar types");
Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);
if (VIC == TTI::VectorInstrContext::SplatOpFolded &&
ST->sinkSplatOperands() && Kind == TTI::SK_Broadcast)
return TTI::TCC_Free;
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
// For now, skip all fixed vector cost analysis when P extension is available
// to avoid crashes in getMinRVVVectorSizeInBits()
if (ST->hasStdExtP() && isa<FixedVectorType>(SrcTy))
return 1;
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
// First, handle cases where having a fixed length vector enables us to
// give a more accurate cost than falling back to generic scalable codegen.
// TODO: Each of these cases hints at a modeling gap around scalable vectors.
if (auto *FVTp = dyn_cast<FixedVectorType>(SrcTy);
FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
*this, LT.second, ST->getRealVLen(),
Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
if (VRegSplittingCost.isValid())
return VRegSplittingCost;
switch (Kind) {
default:
break;
case TTI::SK_PermuteSingleSrc: {
if (Mask.size() >= 2) {
MVT EltTp = LT.second.getVectorElementType();
// Try the vzip.vv cost first, otherwise model the shuffle as a
// vwaddu.vv/vwmaccu.vx sequence below if the element size < ELEN.
if (InstructionCost VZipCost =
getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
VZipCost.isValid())
return VZipCost;
unsigned Index;
if (ST->hasStdExtZvzip() && EltTp.getScalarSizeInBits() != 1 &&
ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index)) {
unsigned Opcode = Index == 0 ? RISCV::VUNZIPE_V : RISCV::VUNZIPO_V;
if (MVT CostVT = getZvzipVUNZIPCostVT(LT.second); CostVT.isValid())
return LT.first * getRISCVInstructionCost(Opcode, CostVT, CostKind);
}
// If the size of the element is < ELEN then shuffles of interleaves and
// deinterleaves of 2 vectors can be lowered into the following
// sequences
if (EltTp.getScalarSizeInBits() < ST->getELen()) {
// Example sequence:
// vsetivli zero, 4, e8, mf4, ta, ma (ignored)
// vwaddu.vv v10, v8, v9
// li a0, -1 (ignored)
// vwmaccu.vx v10, a0, v9
if (ShuffleVectorInst::isInterleaveMask(Mask, 2, Mask.size()))
return 2 * LT.first * TLI->getLMULCost(LT.second);
if (Mask[0] == 0 || Mask[0] == 1) {
auto DeinterleaveMask = createStrideMask(Mask[0], 2, Mask.size());
// Example sequence:
// vnsrl.wi v10, v8, 0
if (equal(DeinterleaveMask, Mask))
return LT.first * getRISCVInstructionCost(RISCV::VNSRL_WI,
LT.second, CostKind);
}
}
int SubVectorSize;
if (LT.second.getScalarSizeInBits() != 1 &&
isRepeatedConcatMask(Mask, SubVectorSize)) {
InstructionCost Cost = 0;
unsigned NumSlides = Log2_32(Mask.size() / SubVectorSize);
// The cost of extraction from a subvector is 0 if the index is 0.
for (unsigned I = 0; I != NumSlides; ++I) {
unsigned InsertIndex = SubVectorSize * (1 << I);
FixedVectorType *SubTp =
FixedVectorType::get(SrcTy->getElementType(), InsertIndex);
FixedVectorType *DestTp =
FixedVectorType::getDoubleElementsVectorType(SubTp);
std::pair<InstructionCost, MVT> DestLT =
getTypeLegalizationCost(DestTp);
// Add the cost of whole vector register move because the
// destination vector register group for vslideup cannot overlap the
// source.
Cost += DestLT.first * TLI->getLMULCost(DestLT.second);
Cost += getShuffleCost(TTI::SK_InsertSubvector, DestTp, DestTp,
CostKind, {}, InsertIndex, SubTp);
}
return Cost;
}
}
if (InstructionCost SlideCost = getSlideCost(FVTp, Mask, CostKind);
SlideCost.isValid())
return SlideCost;
// vrgather + cost of generating the mask constant.
// We model this for an unknown mask with a single vrgather.
if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
LT.second.getVectorNumElements() <= 256)) {
VectorType *IdxTy =
getVRGatherIndexType(LT.second, *ST, SrcTy->getContext());
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
return IndexCost +
getRISCVInstructionCost(RISCV::VRGATHER_VV, LT.second, CostKind);
}
break;
}
case TTI::SK_Transpose:
case TTI::SK_PermuteTwoSrc: {
if (InstructionCost SlideCost = getSlideCost(FVTp, Mask, CostKind);
SlideCost.isValid())
return SlideCost;
if (InstructionCost VZipCost =
getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
VZipCost.isValid())
return VZipCost;
// 2 x (vrgather + cost of generating the mask constant) + cost of mask
// register for the second vrgather. We model this for an unknown
// (shuffle) mask.
if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
LT.second.getVectorNumElements() <= 256)) {
auto &C = SrcTy->getContext();
auto EC = SrcTy->getElementCount();
VectorType *IdxTy = getVRGatherIndexType(LT.second, *ST, C);
VectorType *MaskTy = VectorType::get(IntegerType::getInt1Ty(C), EC);
InstructionCost IndexCost = getConstantPoolLoadCost(IdxTy, CostKind);
InstructionCost MaskCost = getConstantPoolLoadCost(MaskTy, CostKind);
return 2 * IndexCost +
getRISCVInstructionCost({RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
LT.second, CostKind) +
MaskCost;
}
break;
}
}
auto shouldSplit = [](TTI::ShuffleKind Kind) {
switch (Kind) {
default:
return false;
case TTI::SK_PermuteSingleSrc:
case TTI::SK_Transpose:
case TTI::SK_PermuteTwoSrc:
return true;
}
};
if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
shouldSplit(Kind)) {
InstructionCost SplitCost =
costShuffleViaSplitting(*this, LT.second, FVTp, Mask, CostKind);
if (SplitCost.isValid())
return SplitCost;
}
}
// Handle scalable vectors (and fixed vectors legalized to scalable vectors).
switch (Kind) {
default:
// Fallthrough to generic handling.
// TODO: Most of these cases will return getInvalid in generic code, and
// must be implemented here.
break;
case TTI::SK_ExtractSubvector:
// Extract at zero is always a subregister extract
if (Index == 0)
return TTI::TCC_Free;
// If we're extracting a subvector of at most m1 size at a sub-register
// boundary - which unfortunately we need exact vlen to identify - this is
// a subregister extract at worst and thus won't require a vslidedown.
// TODO: Extend for aligned m2, m4 subvector extracts
// TODO: Extend for misalgined (but contained) extracts
// TODO: Extend for scalable subvector types
if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(SubTp);
SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
if (std::optional<unsigned> VLen = ST->getRealVLen();
VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
SubLT.second.getSizeInBits() <= *VLen)
return TTI::TCC_Free;
}
// Example sequence:
// vsetivli zero, 4, e8, mf2, tu, ma (ignored)
// vslidedown.vi v8, v9, 2
return LT.first *
getRISCVInstructionCost(RISCV::VSLIDEDOWN_VI, LT.second, CostKind);
case TTI::SK_InsertSubvector:
// Example sequence:
// vsetivli zero, 4, e8, mf2, tu, ma (ignored)
// vslideup.vi v8, v9, 2
LT = getTypeLegalizationCost(DstTy);
return LT.first *
getRISCVInstructionCost(RISCV::VSLIDEUP_VI, LT.second, CostKind);
case TTI::SK_Select: {
// Example sequence:
// li a0, 90
// vsetivli zero, 8, e8, mf2, ta, ma (ignored)
// vmv.s.x v0, a0
// vmerge.vvm v8, v9, v8, v0
// We use 2 for the cost of the mask materialization as this is the true
// cost for small masks and most shuffles are small. At worst, this cost
// should be a very small constant for the constant pool load. As such,
// we may bias towards large selects slightly more than truly warranted.
return LT.first *
(1 + getRISCVInstructionCost({RISCV::VMV_S_X, RISCV::VMERGE_VVM},
LT.second, CostKind));
}
case TTI::SK_Broadcast: {
// Check for broadcast loads, which are synthesized by optimized zero-stride
// loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
bool IsLoad = !Args.empty() && isa<LoadInst>(Args[0]);
if (IsLoad && LT.second.isVector() &&
isLegalBroadcastLoad(SrcTy->getElementType(),
LT.second.getVectorElementCount()))
return 0;
bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(Args[0]) ==
Instruction::InsertElement);
if (LT.second.getScalarSizeInBits() == 1) {
if (HasScalar) {
// Example sequence:
// andi a0, a0, 1
// vsetivli zero, 2, e8, mf8, ta, ma (ignored)
// vmv.v.x v8, a0
// vmsne.vi v0, v8, 0
return LT.first *
(1 + getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI},
LT.second, CostKind));
}
// Example sequence:
// vsetivli zero, 2, e8, mf8, ta, mu (ignored)
// vmv.v.i v8, 0
// vmerge.vim v8, v8, 1, v0
// vmv.x.s a0, v8
// andi a0, a0, 1
// vmv.v.x v8, a0
// vmsne.vi v0, v8, 0
return LT.first *
(1 + getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM,
RISCV::VMV_X_S, RISCV::VMV_V_X,
RISCV::VMSNE_VI},
LT.second, CostKind));
}
if (HasScalar) {
// Example sequence:
// vmv.v.x v8, a0
return LT.first *
getRISCVInstructionCost(RISCV::VMV_V_X, LT.second, CostKind);
}
// Example sequence:
// vrgather.vi v9, v8, 0
return LT.first *
getRISCVInstructionCost(RISCV::VRGATHER_VI, LT.second, CostKind);
}
case TTI::SK_Splice: {
// vslidedown+vslideup.
// TODO: Multiplying by LT.first implies this legalizes into multiple copies
// of similar code, but I think we expand through memory.
unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
if (Index >= 0 && Index < 32)
Opcodes[0] = RISCV::VSLIDEDOWN_VI;
else if (Index < 0 && Index > -32)
Opcodes[1] = RISCV::VSLIDEUP_VI;
return LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
case TTI::SK_Reverse: {
if (!LT.second.isVector())
return InstructionCost::getInvalid();
// TODO: Cases to improve here:
// * Illegal vector types
// * i64 on RV32
if (SrcTy->getElementType()->isIntegerTy(1)) {
VectorType *WideTy =
VectorType::get(IntegerType::get(SrcTy->getContext(), 8),
cast<VectorType>(SrcTy)->getElementCount());
return getCastInstrCost(Instruction::ZExt, WideTy, SrcTy,
TTI::CastContextHint::None, CostKind) +
getShuffleCost(TTI::SK_Reverse, WideTy, WideTy, CostKind, {}, 0,
nullptr) +
getCastInstrCost(Instruction::Trunc, SrcTy, WideTy,
TTI::CastContextHint::None, CostKind);
}
MVT ContainerVT = LT.second;
if (LT.second.isFixedLengthVector())
ContainerVT = TLI->getContainerForFixedLengthVector(LT.second);
MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
if (ContainerVT.bitsLE(M1VT)) {
// Example sequence:
// csrr a0, vlenb
// srli a0, a0, 3
// addi a0, a0, -1
// vsetvli a1, zero, e8, mf8, ta, mu (ignored)
// vid.v v9
// vrsub.vx v10, v9, a0
// vrgather.vv v9, v8, v10
InstructionCost LenCost = 3;
if (LT.second.isFixedLengthVector())
// vrsub.vi has a 5 bit immediate field, otherwise an li suffices
LenCost = isInt<5>(LT.second.getVectorNumElements() - 1) ? 0 : 1;
unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
if (LT.second.isFixedLengthVector() &&
isInt<5>(LT.second.getVectorNumElements() - 1))
Opcodes[1] = RISCV::VRSUB_VI;
InstructionCost GatherCost =
getRISCVInstructionCost(Opcodes, LT.second, CostKind);
return LT.first * (LenCost + GatherCost);
}
// At high LMUL, we split into a series of M1 reverses (see
// lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
// the resulting gap at the bottom (for fixed vectors only). The important
// bit is that the cost scales linearly, not quadratically with LMUL.
unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
InstructionCost FixedCost =
getRISCVInstructionCost(M1Opcodes, M1VT, CostKind) + 3;
unsigned Ratio =
ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
InstructionCost GatherCost =
getRISCVInstructionCost({RISCV::VRGATHER_VV}, M1VT, CostKind) * Ratio;
InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
getRISCVInstructionCost({RISCV::VSLIDEDOWN_VX}, LT.second, CostKind);
return FixedCost + LT.first * (GatherCost + SlideCost);
}
}
return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
SubTp);
}
static unsigned isM1OrSmaller(MVT VT) {
RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
LMUL == RISCVVType::VLMUL::LMUL_F4 ||
LMUL == RISCVVType::VLMUL::LMUL_F2 ||
LMUL == RISCVVType::VLMUL::LMUL_1);
}
InstructionCost RISCVTTIImpl::getScalarizationOverhead(
VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
TTI::VectorInstrContext VIC) const {
if (isa<ScalableVectorType>(Ty))
return InstructionCost::getInvalid();
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
// For now, skip all fixed vector cost analysis when P extension is available
// to avoid crashes in getMinRVVVectorSizeInBits()
if (ST->hasStdExtP() && isa<FixedVectorType>(Ty)) {
return 1; // Treat as single instruction cost for now
}
// A build_vector (which is m1 sized or smaller) can be done in no
// worse than one vslide1down.vx per element in the type. We could
// in theory do an explode_vector in the inverse manner, but our
// lowering today does not have a first class node for this pattern.
InstructionCost Cost = BaseT::getScalarizationOverhead(
Ty, DemandedElts, Insert, Extract, CostKind);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
if (Ty->getScalarSizeInBits() == 1) {
auto *WideVecTy = cast<VectorType>(Ty->getWithNewBitWidth(8));
// Note: Implicit scalar anyextend is assumed to be free since the i1
// must be stored in a GPR.
return getScalarizationOverhead(WideVecTy, DemandedElts, Insert, Extract,
CostKind) +
getCastInstrCost(Instruction::Trunc, Ty, WideVecTy,
TTI::CastContextHint::None, CostKind, nullptr);
}
assert(LT.second.isFixedLengthVector());
MVT ContainerVT = TLI->getContainerForFixedLengthVector(LT.second);
if (isM1OrSmaller(ContainerVT)) {
InstructionCost BV =
cast<FixedVectorType>(Ty)->getNumElements() *
getRISCVInstructionCost(RISCV::VSLIDE1DOWN_VX, LT.second, CostKind);
if (BV < Cost)
Cost = BV;
}
}
return Cost;
}
InstructionCost
RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
Type *DataTy = MICA.getDataType();
Align Alignment = MICA.getAlignment();
switch (MICA.getID()) {
case Intrinsic::vp_load_ff: {
EVT DataTypeVT = TLI->getValueType(DL, DataTy);
if (!TLI->isLegalFirstFaultLoad(DataTypeVT, Alignment))
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
unsigned AS = MICA.getAddressSpace();
return getMemoryOpCost(Instruction::Load, DataTy, Alignment, AS, CostKind,
{TTI::OK_AnyValue, TTI::OP_None}, nullptr);
}
case Intrinsic::experimental_vp_strided_load:
case Intrinsic::experimental_vp_strided_store:
return getStridedMemoryOpCost(MICA, CostKind);
case Intrinsic::masked_compressstore:
case Intrinsic::masked_expandload:
return getExpandCompressMemoryOpCost(MICA, CostKind);
case Intrinsic::vp_scatter:
case Intrinsic::vp_gather:
case Intrinsic::masked_scatter:
case Intrinsic::masked_gather:
return getGatherScatterOpCost(MICA, CostKind);
case Intrinsic::vp_load:
case Intrinsic::vp_store:
case Intrinsic::masked_load:
case Intrinsic::masked_store:
return getMaskedMemoryOpCost(MICA, CostKind);
}
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
}
InstructionCost
RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
: Instruction::Store;
Type *Src = MICA.getDataType();
Align Alignment = MICA.getAlignment();
unsigned AddressSpace = MICA.getAddressSpace();
if (!isLegalMaskedLoadStore(Src, Alignment) ||
CostKind != TTI::TCK_RecipThroughput)
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Splitting involves additional evl arithmetic and vl toggles.
InstructionCost SplitCost = 0;
if (MICA.getID() == Intrinsic::vp_load ||
MICA.getID() == Intrinsic::vp_store) {
auto LT = getTypeLegalizationCost(Src);
if (LT.first > 1)
SplitCost += LT.first * TTI::TCC_Expensive;
}
return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
}
InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
bool UseMaskForCond, bool UseMaskForGaps) const {
// The interleaved memory access pass will lower (de)interleave ops combined
// with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
// only support masking per-iteration (i.e. condition), not per-segment (i.e.
// gap).
if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
auto *VTy = cast<VectorType>(VecTy);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VTy);
// Need to make sure type has't been scalarized
if (LT.second.isVector()) {
if (CostKind == TTI::TCK_CodeSize)
return LT.first * TTI::TCC_Basic;
auto *SubVecTy =
VectorType::get(VTy->getElementType(),
VTy->getElementCount().divideCoefficientBy(Factor));
if (VTy->getElementCount().isKnownMultipleOf(Factor) &&
TLI->isLegalInterleavedAccessType(SubVecTy, Factor, Alignment,
AddressSpace, DL)) {
// Some processors optimize segment loads/stores as N * DLEN sized
// load ops + Factor * LMUL shuffle ops.
if (ST->hasOptimizedSegmentLoadStore(Factor)) {
unsigned VecSizeInBits =
getEstimatedVLFor(VTy) * VTy->getScalarSizeInBits();
unsigned VLENForTuning =
*getVScaleForTuning() * RISCV::RVVBitsPerBlock;
unsigned DLENForTuning = VLENForTuning / ST->getDLenFactor();
InstructionCost Cost = divideCeil(VecSizeInBits, DLENForTuning);
MVT SubVecVT = getTLI()->getValueType(DL, SubVecTy).getSimpleVT();
Cost += Factor * TLI->getLMULCost(SubVecVT);
return Cost;
}
// Otherwise, the cost is proportional to the number of elements (VL *
// Factor ops).
unsigned NumLoads = getEstimatedVLFor(VTy);
return NumLoads * TTI::TCC_Basic;
}
}
}
// TODO: Return the cost of interleaved accesses for scalable vector when
// unable to convert to segment accesses instructions.
if (isa<ScalableVectorType>(VecTy))
return InstructionCost::getInvalid();
auto *FVTy = cast<FixedVectorType>(VecTy);
// When gaps are only at the tail, for interleaved load, we can emit a wide
// masked load and shufflevectors. For interleaved store, we can emit
// shufflevectors and a wide masked store. The interleaved memory access pass
// will lower them into vlsseg/vssseg intrinsics.
if (UseMaskForGaps) {
assert(llvm::is_sorted(Indices) && "Indices must be sorted");
assert(llvm::adjacent_find(Indices) == Indices.end() &&
"Indices should not contain duplicate elements");
unsigned NumOfFields = Indices.size();
bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
if (IsTailGapOnly &&
NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(FVTy);
if (LT.second.isVector() &&
FVTy->getElementCount().isKnownMultipleOf(Factor)) {
auto *SubVecTy = VectorType::get(
FVTy->getElementType(),
FVTy->getElementCount().divideCoefficientBy(Factor));
if (TLI->isLegalInterleavedAccessType(SubVecTy, NumOfFields, Alignment,
AddressSpace, DL)) {
// The cost is proportional to the total number of element accesses.
unsigned NumAccesses = getEstimatedVLFor(FVTy);
return NumAccesses * TTI::TCC_Basic;
}
}
}
}
InstructionCost MemCost =
getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
unsigned VF = FVTy->getNumElements() / Factor;
// An interleaved load will look like this for Factor=3:
// %wide.vec = load <12 x i32>, ptr %3, align 4
// %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
// %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
// %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
if (Opcode == Instruction::Load) {
InstructionCost Cost = MemCost;
for (unsigned Index : Indices) {
FixedVectorType *VecTy =
FixedVectorType::get(FVTy->getElementType(), VF * Factor);
auto Mask = createStrideMask(Index, Factor, VF);
Mask.resize(VF * Factor, -1);
InstructionCost ShuffleCost =
getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, VecTy, VecTy,
CostKind, Mask, 0, nullptr, {});
Cost += ShuffleCost;
}
return Cost;
}
// TODO: Model for NF > 2
// We'll need to enhance getShuffleCost to model shuffles that are just
// inserts and extracts into subvectors, since they won't have the full cost
// of a vrgather.
// An interleaved store for 3 vectors of 4 lanes will look like
// %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
// %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
// %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
// %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
// store <12 x i32> %interleaved.vec, ptr %10, align 4
if (Factor != 2)
return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
Alignment, AddressSpace, CostKind,
UseMaskForCond, UseMaskForGaps);
assert(Opcode == Instruction::Store && "Opcode must be a store");
// For an interleaving store of 2 vectors, we perform one large interleaving
// shuffle that goes into the wide store
auto Mask = createInterleaveMask(VF, Factor);
InstructionCost ShuffleCost =
getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, FVTy, FVTy,
CostKind, Mask, 0, nullptr, {});
return MemCost + ShuffleCost;
}
InstructionCost
RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
MICA.getID() == Intrinsic::vp_gather;
unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
Type *DataTy = MICA.getDataType();
Type *PtrTy = DataTy->getWithNewType(
DL.getAddressType(DataTy->getContext(), MICA.getAddressSpace()));
Align Alignment = MICA.getAlignment();
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if ((Opcode == Instruction::Load &&
!isLegalMaskedGather(DataTy, Align(Alignment))) ||
(Opcode == Instruction::Store &&
!isLegalMaskedScatter(DataTy, Align(Alignment))))
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
InstructionCost SplitCost = 0;
if (MICA.getID() == Intrinsic::vp_gather ||
MICA.getID() == Intrinsic::vp_scatter) {
auto DataLT = getTypeLegalizationCost(DataTy);
auto PtrLT = getTypeLegalizationCost(PtrTy);
if (DataLT.first > 1)
SplitCost += DataLT.first * TTI::TCC_Expensive;
if (PtrLT.first > 1)
SplitCost += PtrLT.first * TTI::TCC_Expensive;
}
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
auto &VTy = *cast<VectorType>(DataTy);
unsigned NumLoads = getEstimatedVLFor(&VTy);
return SplitCost + NumLoads * TTI::TCC_Basic;
}
InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
? Instruction::Load
: Instruction::Store;
Type *DataTy = MICA.getDataType();
bool VariableMask = MICA.getVariableMask();
Align Alignment = MICA.getAlignment();
bool IsLegal = (Opcode == Instruction::Store &&
isLegalMaskedCompressStore(DataTy, Alignment)) ||
(Opcode == Instruction::Load &&
isLegalMaskedExpandLoad(DataTy, Alignment));
if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
// Example compressstore sequence:
// vsetivli zero, 8, e32, m2, ta, ma (ignored)
// vcompress.vm v10, v8, v0
// vcpop.m a1, v0
// vsetvli zero, a1, e32, m2, ta, ma
// vse32.v v10, (a0)
// Example expandload sequence:
// vsetivli zero, 8, e8, mf2, ta, ma (ignored)
// vcpop.m a1, v0
// vsetvli zero, a1, e32, m2, ta, ma
// vle32.v v10, (a0)
// vsetivli zero, 8, e32, m2, ta, ma
// viota.m v12, v0
// vrgather.vv v8, v10, v12, v0.t
auto MemOpCost =
getMemoryOpCost(Opcode, DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
auto LT = getTypeLegalizationCost(DataTy);
SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
if (VariableMask)
Opcodes.push_back(RISCV::VCPOP_M);
if (Opcode == Instruction::Store)
Opcodes.append({RISCV::VCOMPRESS_VM});
else
Opcodes.append({RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
return MemOpCost +
LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
InstructionCost
RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
Type *DataTy = MICA.getDataType();
Align Alignment = MICA.getAlignment();
if (!isLegalStridedLoadStore(DataTy, Alignment))
return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
if (CostKind == TTI::TCK_CodeSize)
return TTI::TCC_Basic;
// Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
InstructionCost SplitCost = 0;
auto LT = getTypeLegalizationCost(DataTy);
if (LT.first > 1)
SplitCost += LT.first * TTI::TCC_Expensive;
// Cost is proportional to the number of memory operations implied. For
// scalable vectors, we use an estimate on that number since we don't
// know exactly what VL will be.
auto &VTy = *cast<VectorType>(DataTy);
unsigned NumLoads = getEstimatedVLFor(&VTy);
// Performant implementations of the vector extension will coalesce
// elements if they fall on the same cache line
uint64_t CacheLineBytes = ST->getCacheLineSize();
if (!CacheLineBytes) // If no value, use default value of 64
CacheLineBytes = 64;
if (const ConstantInt *StrideCI =
dyn_cast_or_null<ConstantInt>(MICA.getStrideVal())) {
int64_t Stride = StrideCI->getSExtValue();
// Bail early to avoid UB with std:abs() call
if (Stride != std::numeric_limits<int64_t>::min() && Stride != 0) {
uint64_t AbsStride = (uint64_t)std::abs(Stride);
if (AbsStride < CacheLineBytes) {
uint64_t MaxCombines = ST->getMaxVectorCoalesceElts();
if ((CacheLineBytes / AbsStride) >= MaxCombines)
NumLoads = divideCeil(NumLoads, MaxCombines);
else
// If we were to calculate CacheLineBytes / AbsStride first, would
// lose accuracy
NumLoads = divideCeil((NumLoads * AbsStride), CacheLineBytes);
}
}
}
return SplitCost + NumLoads * TTI::TCC_Basic;
}
InstructionCost
RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
// FIXME: This is a property of the default vector convention, not
// all possible calling conventions. Fixing that will require
// some TTI API and SLP rework.
InstructionCost Cost = 0;
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
for (auto *Ty : Tys) {
if (!Ty->isVectorTy())
continue;
Align A = DL.getPrefTypeAlign(Ty);
Cost += getMemoryOpCost(Instruction::Store, Ty, A, 0, CostKind) +
getMemoryOpCost(Instruction::Load, Ty, A, 0, CostKind);
}
return Cost;
}
// Currently, these represent both throughput and codesize costs
// for the respective intrinsics. The costs in this table are simply
// instruction counts with the following adjustments made:
// * One vsetvli is considered free.
static const CostTblEntry VectorIntrinsicCostTable[]{
{Intrinsic::floor, MVT::f32, 9},
{Intrinsic::floor, MVT::f64, 9},
{Intrinsic::ceil, MVT::f32, 9},
{Intrinsic::ceil, MVT::f64, 9},
{Intrinsic::trunc, MVT::f32, 7},
{Intrinsic::trunc, MVT::f64, 7},
{Intrinsic::round, MVT::f32, 9},
{Intrinsic::round, MVT::f64, 9},
{Intrinsic::roundeven, MVT::f32, 9},
{Intrinsic::roundeven, MVT::f64, 9},
{Intrinsic::rint, MVT::f32, 7},
{Intrinsic::rint, MVT::f64, 7},
{Intrinsic::nearbyint, MVT::f32, 9},
{Intrinsic::nearbyint, MVT::f64, 9},
{Intrinsic::bswap, MVT::i16, 3},
{Intrinsic::bswap, MVT::i32, 12},
{Intrinsic::bswap, MVT::i64, 31},
{Intrinsic::bitreverse, MVT::i8, 17},
{Intrinsic::bitreverse, MVT::i16, 24},
{Intrinsic::bitreverse, MVT::i32, 33},
{Intrinsic::bitreverse, MVT::i64, 52},
{Intrinsic::ctpop, MVT::i8, 12},
{Intrinsic::ctpop, MVT::i16, 19},
{Intrinsic::ctpop, MVT::i32, 20},
{Intrinsic::ctpop, MVT::i64, 21},
{Intrinsic::ctlz, MVT::i8, 19},
{Intrinsic::ctlz, MVT::i16, 28},
{Intrinsic::ctlz, MVT::i32, 31},
{Intrinsic::ctlz, MVT::i64, 35},
{Intrinsic::cttz, MVT::i8, 16},
{Intrinsic::cttz, MVT::i16, 23},
{Intrinsic::cttz, MVT::i32, 24},
{Intrinsic::cttz, MVT::i64, 25},
};
InstructionCost
RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
TTI::TargetCostKind CostKind) const {
auto *RetTy = ICA.getReturnType();
switch (ICA.getID()) {
case Intrinsic::lrint:
case Intrinsic::llrint:
case Intrinsic::lround:
case Intrinsic::llround: {
auto LT = getTypeLegalizationCost(RetTy);
Type *SrcTy = ICA.getArgTypes().front();
auto SrcLT = getTypeLegalizationCost(SrcTy);
if (ST->hasVInstructions() && LT.second.isVector()) {
SmallVector<unsigned, 2> Ops;
unsigned SrcEltSz = DL.getTypeSizeInBits(SrcTy->getScalarType());
unsigned DstEltSz = DL.getTypeSizeInBits(RetTy->getScalarType());
if (LT.second.getVectorElementType() == MVT::bf16) {
if (!ST->hasVInstructionsBF16Minimal())
return InstructionCost::getInvalid();
if (DstEltSz == 32)
Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
else
Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
} else if (LT.second.getVectorElementType() == MVT::f16 &&
!ST->hasVInstructionsF16()) {
if (!ST->hasVInstructionsF16Minimal())
return InstructionCost::getInvalid();
if (DstEltSz == 32)
Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
else
Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
} else if (SrcEltSz > DstEltSz) {
Ops = {RISCV::VFNCVT_X_F_W};
} else if (SrcEltSz < DstEltSz) {
Ops = {RISCV::VFWCVT_X_F_V};
} else {
Ops = {RISCV::VFCVT_X_F_V};
}
// We need to use the source LMUL in the case of a narrowing op, and the
// destination LMUL otherwise.
if (SrcEltSz > DstEltSz)
return SrcLT.first *
getRISCVInstructionCost(Ops, SrcLT.second, CostKind);
return LT.first * getRISCVInstructionCost(Ops, LT.second, CostKind);
}
break;
}
case Intrinsic::ceil:
case Intrinsic::floor:
case Intrinsic::trunc:
case Intrinsic::rint:
case Intrinsic::round:
case Intrinsic::roundeven: {
// These all use the same code.
auto LT = getTypeLegalizationCost(RetTy);
if (!LT.second.isVector() && TLI->isOperationCustom(ISD::FCEIL, LT.second))
return LT.first * 8;
break;
}
case Intrinsic::umin:
case Intrinsic::umax:
case Intrinsic::smin:
case Intrinsic::smax: {
auto LT = getTypeLegalizationCost(RetTy);
if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
return LT.first;
if (ST->hasVInstructions() && LT.second.isVector()) {
unsigned Op;
switch (ICA.getID()) {
case Intrinsic::umin:
Op = RISCV::VMINU_VV;
break;
case Intrinsic::umax:
Op = RISCV::VMAXU_VV;
break;
case Intrinsic::smin:
Op = RISCV::VMIN_VV;
break;
case Intrinsic::smax:
Op = RISCV::VMAX_VV;
break;
}
return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
}
break;
}
case Intrinsic::sadd_sat:
case Intrinsic::ssub_sat:
case Intrinsic::uadd_sat:
case Intrinsic::usub_sat: {
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasVInstructions() && LT.second.isVector()) {
unsigned Op;
switch (ICA.getID()) {
case Intrinsic::sadd_sat:
Op = RISCV::VSADD_VV;
break;
case Intrinsic::ssub_sat:
Op = RISCV::VSSUB_VV;
break;
case Intrinsic::uadd_sat:
Op = RISCV::VSADDU_VV;
break;
case Intrinsic::usub_sat:
Op = RISCV::VSSUBU_VV;
break;
}
return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
}
break;
}
case Intrinsic::fma:
case Intrinsic::fmuladd: {
// TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasVInstructions() && LT.second.isVector())
return LT.first *
getRISCVInstructionCost(RISCV::VFMADD_VV, LT.second, CostKind);
break;
}
case Intrinsic::fabs: {
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasVInstructions() && LT.second.isVector()) {
// lui a0, 8
// addi a0, a0, -1
// vsetvli a1, zero, e16, m1, ta, ma
// vand.vx v8, v8, a0
// f16 with zvfhmin and bf16 with zvfhbmin
if (LT.second.getVectorElementType() == MVT::bf16 ||
(LT.second.getVectorElementType() == MVT::f16 &&
!ST->hasVInstructionsF16()))
return LT.first * getRISCVInstructionCost(RISCV::VAND_VX, LT.second,
CostKind) +
2;
else
return LT.first *
getRISCVInstructionCost(RISCV::VFSGNJX_VV, LT.second, CostKind);
}
break;
}
case Intrinsic::sqrt: {
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasVInstructions() && LT.second.isVector()) {
SmallVector<unsigned, 4> ConvOp;
SmallVector<unsigned, 2> FsqrtOp;
MVT ConvType = LT.second;
MVT FsqrtType = LT.second;
// f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
// will be spilt.
if (LT.second.getVectorElementType() == MVT::bf16) {
if (LT.second == MVT::nxv32bf16) {
ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
ConvType = MVT::nxv16f16;
FsqrtType = MVT::nxv16f32;
} else {
ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
FsqrtOp = {RISCV::VFSQRT_V};
FsqrtType = TLI->getTypeToPromoteTo(ISD::FSQRT, FsqrtType);
}
} else if (LT.second.getVectorElementType() == MVT::f16 &&
!ST->hasVInstructionsF16()) {
if (LT.second == MVT::nxv32f16) {
ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
ConvType = MVT::nxv16f16;
FsqrtType = MVT::nxv16f32;
} else {
ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
FsqrtOp = {RISCV::VFSQRT_V};
FsqrtType = TLI->getTypeToPromoteTo(ISD::FSQRT, FsqrtType);
}
} else {
FsqrtOp = {RISCV::VFSQRT_V};
}
return LT.first * (getRISCVInstructionCost(FsqrtOp, FsqrtType, CostKind) +
getRISCVInstructionCost(ConvOp, ConvType, CostKind));
}
break;
}
case Intrinsic::cttz:
case Intrinsic::ctlz:
case Intrinsic::ctpop: {
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasStdExtZvbb() && LT.second.isVector()) {
unsigned Op;
switch (ICA.getID()) {
case Intrinsic::cttz:
Op = RISCV::VCTZ_V;
break;
case Intrinsic::ctlz:
Op = RISCV::VCLZ_V;
break;
case Intrinsic::ctpop:
Op = RISCV::VCPOP_V;
break;
}
return LT.first * getRISCVInstructionCost(Op, LT.second, CostKind);
}
break;
}
case Intrinsic::abs: {
auto LT = getTypeLegalizationCost(RetTy);
if (ST->hasVInstructions() && LT.second.isVector()) {
// vabs.v v10, v8 (alias for vabd.vx v10, v8, zero)
if (ST->hasStdExtZvabd())
return LT.first *
getRISCVInstructionCost({RISCV::VABD_VX}, LT.second, CostKind);
// vrsub.vi v10, v8, 0
// vmax.vv v8, v8, v10
return LT.first *
getRISCVInstructionCost({RISCV::VRSUB_VI, RISCV::VMAX_VV},
LT.second, CostKind);
}
break;
}
case Intrinsic::fshl:
case Intrinsic::fshr: {
if (ICA.getArgs().empty())
break;
// Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
// When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
// instruction.
if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
ICA.getArgs()[0] == ICA.getArgs()[1] &&
(RetTy->getIntegerBitWidth() == 32 ||
RetTy->getIntegerBitWidth() == 64) &&
RetTy->getIntegerBitWidth() <= ST->getXLen()) {
return 1;
}
break;
}
case Intrinsic::clmul: {
auto LT = getTypeLegalizationCost(RetTy);
if (!LT.second.isVector() && ST->hasStdExtZvbc() && !ST->hasStdExtZbkc()) {
// TODO: Once custom lowering in this case for RV32 is added, this guard
// should be removed and the cost model should be updated.
if (!ST->is64Bit() || LT.second != MVT::i64)
break;
// vmv.s.x v8, a0
// vclmul.vx v8, v8, a1
// vmv.x.s a0, v8
MVT VecVT = MVT::getScalableVectorVT(LT.second, 1);
return LT.first * getRISCVInstructionCost(
{RISCV::VMV_S_X, RISCV::VCLMUL_VX, RISCV::VMV_X_S},
VecVT, CostKind);
}
break;
}
case Intrinsic::masked_udiv:
return getArithmeticInstrCost(Instruction::UDiv, ICA.getReturnType(),
CostKind);
case Intrinsic::masked_sdiv:
return getArithmeticInstrCost(Instruction::SDiv, ICA.getReturnType(),
CostKind);
case Intrinsic::masked_urem:
return getArithmeticInstrCost(Instruction::URem, ICA.getReturnType(),
CostKind);
case Intrinsic::masked_srem:
return getArithmeticInstrCost(Instruction::SRem, ICA.getReturnType(),
CostKind);
case Intrinsic::get_active_lane_mask: {
if (ST->hasVInstructions()) {
Type *ExpRetTy = VectorType::get(
ICA.getArgTypes()[0], cast<VectorType>(RetTy)->getElementCount());
auto LT = getTypeLegalizationCost(ExpRetTy);
// vid.v v8 // considered hoisted
// vsaddu.vx v8, v8, a0
// vmsltu.vx v0, v8, a1
return LT.first *
getRISCVInstructionCost({RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
LT.second, CostKind);
}
break;
}
// TODO: add more intrinsic
case Intrinsic::stepvector: {
auto LT = getTypeLegalizationCost(RetTy);
// Legalisation of illegal types involves an `index' instruction plus
// (LT.first - 1) vector adds.
if (ST->hasVInstructions())
return getRISCVInstructionCost(RISCV::VID_V, LT.second, CostKind) +
(LT.first - 1) *
getRISCVInstructionCost(RISCV::VADD_VX, LT.second, CostKind);
return 1 + (LT.first - 1);
}
case Intrinsic::vector_splice_left:
case Intrinsic::vector_splice_right: {
auto LT = getTypeLegalizationCost(RetTy);
// Constant offsets fall through to getShuffleCost.
if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(ICA.getArgs()[2]))
break;
if (ST->hasVInstructions() && LT.second.isVector()) {
return LT.first *
getRISCVInstructionCost({RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
LT.second, CostKind);
}
break;
}
case Intrinsic::experimental_cttz_elts: {
if (!ST->hasVInstructions())
break;
InstructionCost Cost = 0;
Type *ArgTy = ICA.getArgTypes()[0];
auto LT = getTypeLegalizationCost(ArgTy);
if (!LT.second.isVector())
break;
// If the element type is not i1, do a comparison with all-zeros.
if (LT.second.getVectorElementType() != MVT::i1)
Cost += getRISCVInstructionCost(RISCV::VMSNE_VI, LT.second, CostKind);
Cost += getRISCVInstructionCost(RISCV::VFIRST_M, LT.second, CostKind);
// If zero_is_poison is false, then we will generate additional
// cmp + select instructions to convert -1 to EVL.
Type *BoolTy = Type::getInt1Ty(RetTy->getContext());
if (ICA.getArgs().size() > 1 &&
cast<ConstantInt>(ICA.getArgs()[1])->isZero())
Cost += getCmpSelInstrCost(Instruction::ICmp, BoolTy, RetTy,
CmpInst::ICMP_SLT, CostKind) +
getCmpSelInstrCost(Instruction::Select, RetTy, BoolTy,
CmpInst::BAD_ICMP_PREDICATE, CostKind);
return LT.first * Cost;
}
case Intrinsic::experimental_vp_splice: {
// To support type-based query from vectorizer, set the index to 0.
// Note that index only change the cost from vslide.vx to vslide.vi and in
// current implementations they have same costs.
return getShuffleCost(TTI::SK_Splice, cast<VectorType>(ICA.getReturnType()),
cast<VectorType>(ICA.getArgTypes()[0]), CostKind, {},
0, cast<VectorType>(ICA.getReturnType()));
}
case Intrinsic::vp_merge: {
// If an operand is a binary op and the type is legal, RISCVVectorPeephole
// will likely fold the resulting vmerge.vvm away.
if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
getTypeLegalizationCost(RetTy).first == 1)
return TTI::TCC_Free;
break;
}
case Intrinsic::fptoui_sat:
case Intrinsic::fptosi_sat: {
InstructionCost Cost = 0;
bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
Type *SrcTy = ICA.getArgTypes()[0];
auto SrcLT = getTypeLegalizationCost(SrcTy);
auto DstLT = getTypeLegalizationCost(RetTy);
if (!SrcTy->isVectorTy())
break;
if (!SrcLT.first.isValid() || !DstLT.first.isValid())
return InstructionCost::getInvalid();
Cost +=
getCastInstrCost(IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
RetTy, SrcTy, TTI::CastContextHint::None, CostKind);
// Handle NaN.
// vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
// vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
Type *CondTy = RetTy->getWithNewBitWidth(1);
Cost += getCmpSelInstrCost(BinaryOperator::FCmp, SrcTy, CondTy,
CmpInst::FCMP_UNO, CostKind);
Cost += getCmpSelInstrCost(BinaryOperator::Select, RetTy, CondTy,
CmpInst::FCMP_UNO, CostKind);
return Cost;
}
case Intrinsic::experimental_vector_extract_last_active: {
auto *ValTy = cast<VectorType>(ICA.getArgTypes()[0]);
auto *MaskTy = cast<VectorType>(ICA.getArgTypes()[1]);
auto ValLT = getTypeLegalizationCost(ValTy);
auto MaskLT = getTypeLegalizationCost(MaskTy);
// TODO: Return cheaper cost when the entire lane is inactive.
// The expected asm sequence is:
// vcpop.m a0, v0
// beqz a0, exit # Return passthru when the entire lane is inactive.
// vid v10, v0.t
// vredmaxu.vs v10, v10, v10
// vmv.x.s a0, v10
// zext.b a0, a0
// vslidedown.vx v8, v8, a0
// vmv.x.s a0, v8
// exit:
// ...
// Find a suitable type for a stepvector.
ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(64));
unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
TLI->getVectorIdxTy(getDataLayout()), MaskTy->getElementCount(),
/*ZeroIsPoison=*/true, &VScaleRange);
EltWidth = std::max(EltWidth, MaskTy->getScalarSizeInBits());
Type *StepTy = Type::getIntNTy(MaskTy->getContext(), EltWidth);
auto *StepVecTy = VectorType::get(StepTy, ValTy->getElementCount());
auto StepLT = getTypeLegalizationCost(StepVecTy);
// Currently expandVectorFindLastActive cannot handle step vector split.
// So return invalid when the type needs split.
// FIXME: Remove this if expandVectorFindLastActive supports split vector.
if (StepLT.first > 1)
return InstructionCost::getInvalid();
InstructionCost Cost = 0;
unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
Cost += MaskLT.first *
getRISCVInstructionCost(RISCV::VCPOP_M, MaskLT.second, CostKind);
Cost += getCFInstrCost(Instruction::CondBr, CostKind, nullptr);
Cost += StepLT.first *
getRISCVInstructionCost(Opcodes, StepLT.second, CostKind);
Cost += getCastInstrCost(Instruction::ZExt,
Type::getInt64Ty(ValTy->getContext()), StepTy,
TTI::CastContextHint::None, CostKind, nullptr);
Cost += ValLT.first *
getRISCVInstructionCost({RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
ValLT.second, CostKind);
return Cost;
}
case Intrinsic::vector_interleave2:
case Intrinsic::vector_deinterleave2: {
if (!ST->hasStdExtZvzip())
break;
bool IsInterleave = ICA.getID() == Intrinsic::vector_interleave2;
Type *InterleavedTy = IsInterleave ? RetTy : ICA.getArgTypes().front();
// ISel does not select vzip.vv if either interleave2 input is undef.
if (IsInterleave && !ICA.isTypeBasedOnly() &&
any_of(ICA.getArgs(),
[](const Value *Arg) { return isa<UndefValue>(Arg); }))
break;
if (InterleavedTy->getScalarSizeInBits() == 1)
break;
if (auto *FVT = dyn_cast<FixedVectorType>(InterleavedTy)) {
auto *HalfFVT = FixedVectorType::getHalfElementsVectorType(FVT);
unsigned HalfVF = HalfFVT->getNumElements();
if (IsInterleave)
return getShuffleCost(TTI::SK_PermuteTwoSrc, FVT, HalfFVT, CostKind,
createInterleaveMask(HalfVF, 2), 0, nullptr);
InstructionCost Cost = 0;
for (unsigned Start = 0; Start != 2; ++Start)
Cost += getShuffleCost(TTI::SK_PermuteSingleSrc, HalfFVT, FVT, CostKind,
createStrideMask(Start, 2, HalfVF), 0, nullptr);
return Cost;
}
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(InterleavedTy);
if (!LT.second.isScalableVector())
break;
if (IsInterleave) {
if (MVT CostVT = getZvzipVZIPCostVT(LT.second); CostVT.isValid())
return LT.first *
getRISCVInstructionCost(RISCV::VZIP_VV, CostVT, CostKind);
} else if (MVT CostVT = getZvzipVUNZIPCostVT(LT.second); CostVT.isValid()) {
return LT.first *
getRISCVInstructionCost({RISCV::VUNZIPE_V, RISCV::VUNZIPO_V},
CostVT, CostKind);
}
break;
}
}
if (ST->hasVInstructions() && RetTy->isVectorTy()) {
if (auto LT = getTypeLegalizationCost(RetTy);
LT.second.isVector()) {
MVT EltTy = LT.second.getVectorElementType();
if (const auto *Entry = CostTableLookup(VectorIntrinsicCostTable,
ICA.getID(), EltTy))
return LT.first * Entry->Cost;
}
}
return BaseT::getIntrinsicInstrCost(ICA, CostKind);
}
InstructionCost
RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
const SCEV *Ptr,
TTI::TargetCostKind CostKind) const {
// Address computations for vector indexed load/store likely require an offset
// and/or scaling.
if (ST->hasVInstructions() && PtrTy->isVectorTy())
return getArithmeticInstrCost(Instruction::Add, PtrTy, CostKind);
return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
}
InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src,
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
const Instruction *I) const {
bool IsVectorType = isa<VectorType>(Dst) && isa<VectorType>(Src);
if (!IsVectorType)
return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
// For now, skip all fixed vector cost analysis when P extension is available
// to avoid crashes in getMinRVVVectorSizeInBits()
if (ST->hasStdExtP() &&
(isa<FixedVectorType>(Dst) || isa<FixedVectorType>(Src))) {
return 1; // Treat as single instruction cost for now
}
// FIXME: Need to compute legalizing cost for illegal types. The current
// code handles only legal types and those which can be trivially
// promoted to legal.
if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
Dst->getScalarSizeInBits() > ST->getELen())
return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
// Handle i1 source and dest cases *before* calling logic in BasicTTI.
// The shared implementation doesn't model vector widening during legalization
// and instead assumes scalarization. In order to scalarize an <N x i1>
// vector, we need to extend/trunc to/from i8. If we don't special case
// this, we can get an infinite recursion cycle.
switch (ISD) {
default:
break;
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
if (Src->getScalarSizeInBits() == 1) {
// We do not use vsext/vzext to extend from mask vector.
// Instead we use the following instructions to extend from mask vector:
// vmv.v.i v8, 0
// vmerge.vim v8, v8, -1, v0 (repeated per split)
return getRISCVInstructionCost(RISCV::VMV_V_I, DstLT.second, CostKind) +
DstLT.first * getRISCVInstructionCost(RISCV::VMERGE_VIM,
DstLT.second, CostKind) +
DstLT.first - 1;
}
break;
case ISD::TRUNCATE:
if (Dst->getScalarSizeInBits() == 1) {
// We do not use several vncvt to truncate to mask vector. So we could
// not use PowDiff to calculate it.
// Instead we use the following instructions to truncate to mask vector:
// vand.vi v8, v8, 1
// vmsne.vi v0, v8, 0
return SrcLT.first *
getRISCVInstructionCost({RISCV::VAND_VI, RISCV::VMSNE_VI},
SrcLT.second, CostKind) +
SrcLT.first - 1;
}
break;
};
// Our actual lowering for the case where a wider legal type is available
// uses promotion to the wider type. This is reflected in the result of
// getTypeLegalizationCost, but BasicTTI assumes the widened cases are
// scalarized if the legalized Src and Dst are not equal sized.
const DataLayout &DL = this->getDataLayout();
if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
!SrcLT.first.isValid() || !DstLT.first.isValid() ||
!TypeSize::isKnownLE(DL.getTypeSizeInBits(Src),
SrcLT.second.getSizeInBits()) ||
!TypeSize::isKnownLE(DL.getTypeSizeInBits(Dst),
DstLT.second.getSizeInBits()) ||
SrcLT.first > 1 || DstLT.first > 1)
return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
// The split cost is handled by the base getCastInstrCost
assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
int PowDiff = (int)Log2_32(DstLT.second.getScalarSizeInBits()) -
(int)Log2_32(SrcLT.second.getScalarSizeInBits());
switch (ISD) {
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND: {
if ((PowDiff < 1) || (PowDiff > 3))
return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
unsigned Op =
(ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
return getRISCVInstructionCost(Op, DstLT.second, CostKind);
}
case ISD::TRUNCATE:
case ISD::FP_EXTEND:
case ISD::FP_ROUND: {
// Counts of narrow/widen instructions.
unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
: (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
: RISCV::VFNCVT_F_F_W;
InstructionCost Cost = 0;
for (; SrcEltSize != DstEltSize;) {
MVT ElementMVT = (ISD == ISD::TRUNCATE)
? MVT::getIntegerVT(DstEltSize)
: MVT::getFloatingPointVT(DstEltSize);
MVT DstMVT = DstLT.second.changeVectorElementType(ElementMVT);
DstEltSize =
(DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
Cost += getRISCVInstructionCost(Op, DstMVT, CostKind);
}
return Cost;
}
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT: {
unsigned IsSigned = ISD == ISD::FP_TO_SINT;
unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
unsigned FWCVT =
IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
unsigned FNCVT =
IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
unsigned SrcEltSize = Src->getScalarSizeInBits();
unsigned DstEltSize = Dst->getScalarSizeInBits();
InstructionCost Cost = 0;
if ((SrcEltSize == 16) &&
(!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
// If the target only supports zvfhmin or it is fp16-to-i64 conversion
// pre-widening to f32 and then convert f32 to integer
VectorType *VecF32Ty =
VectorType::get(Type::getFloatTy(Dst->getContext()),
cast<VectorType>(Dst)->getElementCount());
std::pair<InstructionCost, MVT> VecF32LT =
getTypeLegalizationCost(VecF32Ty);
Cost +=
VecF32LT.first * getRISCVInstructionCost(RISCV::VFWCVT_F_F_V,
VecF32LT.second, CostKind);
Cost += getCastInstrCost(Opcode, Dst, VecF32Ty, CCH, CostKind, I);
return Cost;
}
if (DstEltSize == SrcEltSize)
Cost += getRISCVInstructionCost(FCVT, DstLT.second, CostKind);
else if (DstEltSize > SrcEltSize)
Cost += getRISCVInstructionCost(FWCVT, DstLT.second, CostKind);
else { // (SrcEltSize > DstEltSize)
// First do a narrowing conversion to an integer half the size, then
// truncate if needed.
MVT ElementVT = MVT::getIntegerVT(SrcEltSize / 2);
MVT VecVT = DstLT.second.changeVectorElementType(ElementVT);
Cost += getRISCVInstructionCost(FNCVT, VecVT, CostKind);
if ((SrcEltSize / 2) > DstEltSize) {
Type *VecTy = EVT(VecVT).getTypeForEVT(Dst->getContext());
Cost +=
getCastInstrCost(Instruction::Trunc, Dst, VecTy, CCH, CostKind, I);
}
}
return Cost;
}
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP: {
unsigned IsSigned = ISD == ISD::SINT_TO_FP;
unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
unsigned SrcEltSize = Src->getScalarSizeInBits();
unsigned DstEltSize = Dst->getScalarSizeInBits();
InstructionCost Cost = 0;
if ((DstEltSize == 16) &&
(!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
// If the target only supports zvfhmin or it is i64-to-fp16 conversion
// it is converted to f32 and then converted to f16
VectorType *VecF32Ty =
VectorType::get(Type::getFloatTy(Dst->getContext()),
cast<VectorType>(Dst)->getElementCount());
std::pair<InstructionCost, MVT> VecF32LT =
getTypeLegalizationCost(VecF32Ty);
Cost += getCastInstrCost(Opcode, VecF32Ty, Src, CCH, CostKind, I);
Cost += VecF32LT.first * getRISCVInstructionCost(RISCV::VFNCVT_F_F_W,
DstLT.second, CostKind);
return Cost;
}
if (DstEltSize == SrcEltSize)
Cost += getRISCVInstructionCost(FCVT, DstLT.second, CostKind);
else if (DstEltSize > SrcEltSize) {
if ((DstEltSize / 2) > SrcEltSize) {
VectorType *VecTy =
VectorType::get(IntegerType::get(Dst->getContext(), DstEltSize / 2),
cast<VectorType>(Dst)->getElementCount());
unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
Cost += getCastInstrCost(Op, VecTy, Src, CCH, CostKind, I);
}
Cost += getRISCVInstructionCost(FWCVT, DstLT.second, CostKind);
} else
Cost += getRISCVInstructionCost(FNCVT, DstLT.second, CostKind);
return Cost;
}
}
return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
}
unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
if (isa<ScalableVectorType>(Ty)) {
const unsigned EltSize = DL.getTypeSizeInBits(Ty->getElementType());
const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
}
return cast<FixedVectorType>(Ty)->getNumElements();
}
InstructionCost
RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
FastMathFlags FMF,
TTI::TargetCostKind CostKind) const {
if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
// Skip if scalar size of Ty is bigger than ELEN.
if (Ty->getScalarSizeInBits() > ST->getELen())
return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
if (Ty->getElementType()->isIntegerTy(1)) {
// SelectionDAGBuilder does following transforms:
// vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
// vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
if (IID == Intrinsic::umax || IID == Intrinsic::smin)
return getArithmeticReductionCost(Instruction::Or, Ty, FMF, CostKind);
else
return getArithmeticReductionCost(Instruction::And, Ty, FMF, CostKind);
}
if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
SmallVector<unsigned, 3> Opcodes;
InstructionCost ExtraCost = 0;
switch (IID) {
case Intrinsic::maximum:
if (FMF.noNaNs()) {
Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
} else {
Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
RISCV::VFMV_F_S};
// Cost of Canonical Nan + branch
// lui a0, 523264
// fmv.w.x fa0, a0
Type *DstTy = Ty->getScalarType();
const unsigned EltTyBits = DstTy->getScalarSizeInBits();
Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);
ExtraCost = 1 +
getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,
TTI::CastContextHint::None, CostKind) +
getCFInstrCost(Instruction::CondBr, CostKind);
}
break;
case Intrinsic::minimum:
if (FMF.noNaNs()) {
Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
} else {
Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
RISCV::VFMV_F_S};
// Cost of Canonical Nan + branch
// lui a0, 523264
// fmv.w.x fa0, a0
Type *DstTy = Ty->getScalarType();
const unsigned EltTyBits = DL.getTypeSizeInBits(DstTy);
Type *SrcTy = IntegerType::getIntNTy(DstTy->getContext(), EltTyBits);
ExtraCost = 1 +
getCastInstrCost(Instruction::UIToFP, DstTy, SrcTy,
TTI::CastContextHint::None, CostKind) +
getCFInstrCost(Instruction::CondBr, CostKind);
}
break;
}
return ExtraCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
// IR Reduction is composed by one rvv reduction instruction and vmv
unsigned SplitOp;
SmallVector<unsigned, 3> Opcodes;
switch (IID) {
default:
llvm_unreachable("Unsupported intrinsic");
case Intrinsic::smax:
SplitOp = RISCV::VMAX_VV;
Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
break;
case Intrinsic::smin:
SplitOp = RISCV::VMIN_VV;
Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
break;
case Intrinsic::umax:
SplitOp = RISCV::VMAXU_VV;
Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
break;
case Intrinsic::umin:
SplitOp = RISCV::VMINU_VV;
Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
break;
case Intrinsic::maxnum:
SplitOp = RISCV::VFMAX_VV;
Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
break;
case Intrinsic::minnum:
SplitOp = RISCV::VFMIN_VV;
Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
break;
}
// Add a cost for data larger than LMUL8
InstructionCost SplitCost =
(LT.first > 1) ? (LT.first - 1) *
getRISCVInstructionCost(SplitOp, LT.second, CostKind)
: 0;
return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
InstructionCost
RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
std::optional<FastMathFlags> FMF,
TTI::TargetCostKind CostKind) const {
if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
// Skip if scalar size of Ty is bigger than ELEN.
if (Ty->getScalarSizeInBits() > ST->getELen())
return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
ISD != ISD::FADD)
return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
Type *ElementTy = Ty->getElementType();
if (ElementTy->isIntegerTy(1)) {
// Example sequences:
// vfirst.m a0, v0
// seqz a0, a0
if (LT.second == MVT::v1i1)
return getRISCVInstructionCost(RISCV::VFIRST_M, LT.second, CostKind) +
getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,
CmpInst::ICMP_EQ, CostKind);
if (ISD == ISD::AND) {
// Example sequences:
// vmand.mm v8, v9, v8 ; needed every time type is split
// vmnot.m v8, v0 ; alias for vmnand
// vcpop.m a0, v8
// seqz a0, a0
// See the discussion: https://github.com/llvm/llvm-project/pull/119160
// For LMUL <= 8, there is no splitting,
// the sequences are vmnot, vcpop and seqz.
// When LMUL > 8 and split = 1,
// the sequences are vmnand, vcpop and seqz.
// When LMUL > 8 and split > 1,
// the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
return ((LT.first > 2) ? (LT.first - 2) : 0) *
getRISCVInstructionCost(RISCV::VMAND_MM, LT.second, CostKind) +
getRISCVInstructionCost(RISCV::VMNAND_MM, LT.second, CostKind) +
getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +
getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,
CmpInst::ICMP_EQ, CostKind);
} else if (ISD == ISD::XOR || ISD == ISD::ADD) {
// Example sequences:
// vsetvli a0, zero, e8, mf8, ta, ma
// vmxor.mm v8, v0, v8 ; needed every time type is split
// vcpop.m a0, v8
// andi a0, a0, 1
return (LT.first - 1) *
getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind) +
getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) + 1;
} else {
assert(ISD == ISD::OR);
// Example sequences:
// vsetvli a0, zero, e8, mf8, ta, ma
// vmor.mm v8, v9, v8 ; needed every time type is split
// vcpop.m a0, v0
// snez a0, a0
return (LT.first - 1) *
getRISCVInstructionCost(RISCV::VMOR_MM, LT.second, CostKind) +
getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind) +
getCmpSelInstrCost(Instruction::ICmp, ElementTy, ElementTy,
CmpInst::ICMP_NE, CostKind);
}
}
// IR Reduction of or/and is composed by one vmv and one rvv reduction
// instruction, and others is composed by two vmv and one rvv reduction
// instruction
unsigned SplitOp;
SmallVector<unsigned, 3> Opcodes;
switch (ISD) {
case ISD::ADD:
SplitOp = RISCV::VADD_VV;
Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
break;
case ISD::OR:
SplitOp = RISCV::VOR_VV;
Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
break;
case ISD::XOR:
SplitOp = RISCV::VXOR_VV;
Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
break;
case ISD::AND:
SplitOp = RISCV::VAND_VV;
Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
break;
case ISD::FADD:
// We can't promote f16/bf16 fadd reductions.
if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
LT.second.getScalarType() == MVT::bf16)
return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
if (TTI::requiresOrderedReduction(FMF)) {
Opcodes.push_back(RISCV::VFMV_S_F);
for (unsigned i = 0; i < LT.first.getValue(); i++)
Opcodes.push_back(RISCV::VFREDOSUM_VS);
Opcodes.push_back(RISCV::VFMV_F_S);
return getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
SplitOp = RISCV::VFADD_VV;
Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
break;
}
// Add a cost for data larger than LMUL8
InstructionCost SplitCost =
(LT.first > 1) ? (LT.first - 1) *
getRISCVInstructionCost(SplitOp, LT.second, CostKind)
: 0;
return SplitCost + getRISCVInstructionCost(Opcodes, LT.second, CostKind);
}
InstructionCost RISCVTTIImpl::getExtendedReductionCost(
unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())
return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,
FMF, CostKind);
// Skip if scalar size of ResTy is bigger than ELEN.
if (ResTy->getScalarSizeInBits() > ST->getELen())
return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,
FMF, CostKind);
if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,
FMF, CostKind);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
if (IsUnsigned && Opcode == Instruction::Add &&
LT.second.isFixedLengthVectorOf(MVT::i1)) {
// Represent vector_reduce_add(ZExt(<n x i1>)) as
// ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
return LT.first *
getRISCVInstructionCost(RISCV::VCPOP_M, LT.second, CostKind);
}
if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, ValTy,
FMF, CostKind);
return (LT.first - 1) +
getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);
}
InstructionCost
RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
TTI::TargetCostKind CostKind) const {
assert(OpInfo.isConstant() && "non constant operand?");
if (!isa<VectorType>(Ty))
// FIXME: We need to account for immediate materialization here, but doing
// a decent job requires more knowledge about the immediate than we
// currently have here.
return 0;
if (OpInfo.isUniform())
// vmv.v.i, vmv.v.x, or vfmv.v.f
// We ignore the cost of the scalar constant materialization to be consistent
// with how we treat scalar constants themselves just above.
return 1;
return getConstantPoolLoadCost(Ty, CostKind);
}
InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
TTI::OperandValueInfo OpInfo,
const Instruction *I) const {
EVT VT = TLI->getValueType(DL, Src, true);
// Type legalization can't handle structs, and load latency isn't handled here
if (VT == MVT::Other ||
(Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
CostKind, OpInfo, I);
InstructionCost Cost = 0;
if (Opcode == Instruction::Store && OpInfo.isConstant())
Cost += getStoreImmCost(Src, OpInfo, CostKind);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Src);
InstructionCost BaseCost = [&]() {
InstructionCost Cost = LT.first;
if (CostKind != TTI::TCK_RecipThroughput)
return Cost;
// Our actual lowering for the case where a wider legal type is available
// uses the a VL predicated load on the wider type. This is reflected in
// the result of getTypeLegalizationCost, but BasicTTI assumes the
// widened cases are scalarized.
const DataLayout &DL = this->getDataLayout();
if (Src->isVectorTy() && LT.second.isVector() &&
TypeSize::isKnownLT(DL.getTypeStoreSizeInBits(Src),
LT.second.getSizeInBits()))
return Cost;
return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
CostKind, OpInfo, I);
}();
// Assume memory ops cost scale with the number of vector registers
// possible accessed by the instruction. Note that BasicTTI already
// handles the LT.first term for us.
if (ST->hasVInstructions() && LT.second.isVector() &&
CostKind != TTI::TCK_CodeSize)
BaseCost *= TLI->getLMULCost(LT.second);
return Cost + BaseCost;
}
InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
TTI::OperandValueInfo Op2Info, const Instruction *I) const {
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
if (isa<FixedVectorType>(ValTy) && !ST->useRVVForFixedLengthVectors())
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
// Skip if scalar size of ValTy is bigger than ELEN.
if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
auto GetConstantMatCost =
[&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
if (OpInfo.isUniform())
// We return 0 we currently ignore the cost of materializing scalar
// constants in GPRs.
return 0;
return getConstantPoolLoadCost(ValTy, CostKind);
};
InstructionCost ConstantMatCost;
if (Op1Info.isConstant())
ConstantMatCost += GetConstantMatCost(Op1Info);
if (Op2Info.isConstant())
ConstantMatCost += GetConstantMatCost(Op2Info);
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
if (Opcode == Instruction::Select && LT.second.isVector()) {
if (CondTy->isVectorTy()) {
if (ValTy->getScalarSizeInBits() == 1) {
// vmandn.mm v8, v8, v9
// vmand.mm v9, v0, v9
// vmor.mm v0, v9, v8
return ConstantMatCost +
LT.first *
getRISCVInstructionCost(
{RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
LT.second, CostKind);
}
// vselect and max/min are supported natively.
return ConstantMatCost +
LT.first * getRISCVInstructionCost(RISCV::VMERGE_VVM, LT.second,
CostKind);
}
if (ValTy->getScalarSizeInBits() == 1) {
// vmv.v.x v9, a0
// vmsne.vi v9, v9, 0
// vmandn.mm v8, v8, v9
// vmand.mm v9, v0, v9
// vmor.mm v0, v9, v8
MVT InterimVT = LT.second.changeVectorElementType(MVT::i8);
return ConstantMatCost +
LT.first *
getRISCVInstructionCost({RISCV::VMV_V_X, RISCV::VMSNE_VI},
InterimVT, CostKind) +
LT.first * getRISCVInstructionCost(
{RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
LT.second, CostKind);
}
// vmv.v.x v10, a0
// vmsne.vi v0, v10, 0
// vmerge.vvm v8, v9, v8, v0
return ConstantMatCost +
LT.first * getRISCVInstructionCost(
{RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
LT.second, CostKind);
}
if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
CmpInst::isIntPredicate(VecPred)) {
// Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
// provided they incur the same cost across all implementations
return ConstantMatCost + LT.first * getRISCVInstructionCost(RISCV::VMSLT_VV,
LT.second,
CostKind);
}
if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
CmpInst::isFPPredicate(VecPred)) {
// Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
return ConstantMatCost +
getRISCVInstructionCost(RISCV::VMXOR_MM, LT.second, CostKind);
// If we do not support the input floating point vector type, use the base
// one which will calculate as:
// ScalarizeCost + Num * Cost for fixed vector,
// InvalidCost for scalable vector.
if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
(ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
(ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
// Assuming vector fp compare and mask instructions are all the same cost
// until a need arises to differentiate them.
switch (VecPred) {
case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
return ConstantMatCost +
LT.first * getRISCVInstructionCost(
{RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
LT.second, CostKind);
case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
return ConstantMatCost +
LT.first *
getRISCVInstructionCost({RISCV::VMFLT_VV, RISCV::VMNAND_MM},
LT.second, CostKind);
case CmpInst::FCMP_OEQ: // vmfeq.vv
case CmpInst::FCMP_OGT: // vmflt.vv
case CmpInst::FCMP_OGE: // vmfle.vv
case CmpInst::FCMP_OLT: // vmflt.vv
case CmpInst::FCMP_OLE: // vmfle.vv
case CmpInst::FCMP_UNE: // vmfne.vv
return ConstantMatCost +
LT.first *
getRISCVInstructionCost(RISCV::VMFLT_VV, LT.second, CostKind);
default:
break;
}
}
// With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
// instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
// generate a conditional branch + mv. The cost of scalar (icmp + select) will
// be (0 + select instr cost).
if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(I) &&
ValTy->isIntegerTy() && !I->user_empty()) {
if (all_of(I->users(), [&](const User *U) {
return match(U, m_Select(m_Specific(I), m_Value(), m_Value())) &&
U->getType()->isIntegerTy() &&
!isa<ConstantData>(U->getOperand(1)) &&
!isa<ConstantData>(U->getOperand(2));
}))
return 0;
}
// TODO: Add cost for scalar type.
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
Op1Info, Op2Info, I);
}
InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
TTI::TargetCostKind CostKind,
const Instruction *I) const {
if (CostKind != TTI::TCK_RecipThroughput)
return Opcode == Instruction::PHI ? 0 : 1;
// Branches are assumed to be predicted.
return 0;
}
InstructionCost RISCVTTIImpl::getVectorInstrCost(
unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
assert(Val->isVectorTy() && "This must be a vector type");
// TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
// For now, skip all fixed vector cost analysis when P extension is available
// to avoid crashes in getMinRVVVectorSizeInBits()
if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
return 1; // Treat as single instruction cost for now
}
if (Opcode != Instruction::ExtractElement &&
Opcode != Instruction::InsertElement)
return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
VIC);
// Scalar splat operand can be folded for vector ops that support splatting
// the scalar operand, so the explicit insertelement is free in this context.
if (Opcode == Instruction::InsertElement &&
VIC == TTI::VectorInstrContext::SplatOpFolded &&
ST->sinkSplatOperands() && Index == 0)
return TTI::TCC_Free;
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Val);
// This type is legalized to a scalar type.
if (!LT.second.isVector()) {
auto *FixedVecTy = cast<FixedVectorType>(Val);
// If Index is a known constant, cost is zero.
if (Index != -1U)
return 0;
// Extract/InsertElement with non-constant index is very costly when
// scalarized; estimate cost of loads/stores sequence via the stack:
// ExtractElement cost: store vector to stack, load scalar;
// InsertElement cost: store vector to stack, store scalar, load vector.
Type *ElemTy = FixedVecTy->getElementType();
auto NumElems = FixedVecTy->getNumElements();
auto Align = DL.getPrefTypeAlign(ElemTy);
InstructionCost LoadCost =
getMemoryOpCost(Instruction::Load, ElemTy, Align, 0, CostKind);
InstructionCost StoreCost =
getMemoryOpCost(Instruction::Store, ElemTy, Align, 0, CostKind);
return Opcode == Instruction::ExtractElement
? StoreCost * NumElems + LoadCost
: (StoreCost + LoadCost) * NumElems + StoreCost;
}
// For unsupported scalable vector.
if (LT.second.isScalableVector() && !LT.first.isValid())
return LT.first;
// Mask vector extract/insert is expanded via e8.
if (Val->getScalarSizeInBits() == 1) {
VectorType *WideTy =
VectorType::get(IntegerType::get(Val->getContext(), 8),
cast<VectorType>(Val)->getElementCount());
if (Opcode == Instruction::ExtractElement) {
InstructionCost ExtendCost
= getCastInstrCost(Instruction::ZExt, WideTy, Val,
TTI::CastContextHint::None, CostKind);
InstructionCost ExtractCost
= getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr);
return ExtendCost + ExtractCost;
}
InstructionCost ExtendCost
= getCastInstrCost(Instruction::ZExt, WideTy, Val,
TTI::CastContextHint::None, CostKind);
InstructionCost InsertCost
= getVectorInstrCost(Opcode, WideTy, CostKind, Index, nullptr, nullptr);
InstructionCost TruncCost
= getCastInstrCost(Instruction::Trunc, Val, WideTy,
TTI::CastContextHint::None, CostKind);
return ExtendCost + InsertCost + TruncCost;
}
// In RVV, we could use vslidedown + vmv.x.s to extract element from vector
// and vslideup + vmv.s.x to insert element to vector.
unsigned MoveOpc;
if (LT.second.isFloatingPoint())
MoveOpc = Opcode == Instruction::InsertElement ? RISCV::VFMV_S_F
: RISCV::VFMV_F_S;
else
MoveOpc =
Opcode == Instruction::InsertElement ? RISCV::VMV_S_X : RISCV::VMV_X_S;
InstructionCost BaseCost =
getRISCVInstructionCost(MoveOpc, LT.second, CostKind);
// When insertelement we should add the index with 1 as the input of vslideup.
InstructionCost SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
if (Index != -1U) {
// The type may be split. For fixed-width vectors we can normalize the
// index to the new type.
if (LT.second.isFixedLengthVector()) {
unsigned Width = LT.second.getVectorNumElements();
Index = Index % Width;
}
// If exact VLEN is known, we will insert/extract into the appropriate
// subvector with no additional subvector insert/extract cost.
if (auto VLEN = ST->getRealVLen()) {
unsigned EltSize = LT.second.getScalarSizeInBits();
unsigned M1Max = *VLEN / EltSize;
Index = Index % M1Max;
}
if (Index == 0)
// We can extract/insert the first element without vslidedown/vslideup.
SlideCost = 0;
else if (Opcode == Instruction::InsertElement)
SlideCost = 1; // With a constant index, we do not need to use addi.
}
// When the vector needs to split into multiple register groups and the index
// exceeds single vector register group, we need to insert/extract the element
// via stack.
if (LT.first > 1 &&
((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
LT.second.isScalableVector()))) {
Type *ScalarType = Val->getScalarType();
Align VecAlign = DL.getPrefTypeAlign(Val);
Align SclAlign = DL.getPrefTypeAlign(ScalarType);
// Extra addi for unknown index.
InstructionCost IdxCost = Index == -1U ? 1 : 0;
// Store all split vectors into stack and load the target element.
if (Opcode == Instruction::ExtractElement)
return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +
getMemoryOpCost(Instruction::Load, ScalarType, SclAlign, 0,
CostKind) +
IdxCost;
// Store all split vectors into stack and store the target element and load
// vectors back.
return getMemoryOpCost(Instruction::Store, Val, VecAlign, 0, CostKind) +
getMemoryOpCost(Instruction::Load, Val, VecAlign, 0, CostKind) +
getMemoryOpCost(Instruction::Store, ScalarType, SclAlign, 0,
CostKind) +
IdxCost;
}
// Extract i64 in the target that has XLEN=32 need more instruction.
if (Val->getScalarType()->isIntegerTy() &&
ST->getXLen() < Val->getScalarSizeInBits()) {
// For extractelement, we need the following instructions:
// vsetivli zero, 1, e64, m1, ta, mu (not count)
// vslidedown.vx v8, v8, a0
// vmv.x.s a0, v8
// li a1, 32
// vsrl.vx v8, v8, a1
// vmv.x.s a1, v8
// For insertelement, we need the following instructions:
// vsetivli zero, 2, e32, m4, ta, ma (don't count)
// vslide1down.vx v12, v8, a0
// vslide1down.vx v12, v12, a1
// addi a0, a2, 1
// vsetvli zero, a0, e64, m4, tu, ma (don't count)
// vslideup.vx v8, v12, a2
// TODO: should we count these special vsetvlis?
BaseCost =
Opcode == Instruction::InsertElement
? getRISCVInstructionCost({RISCV::VSLIDE1DOWN_VX,
RISCV::VSLIDE1DOWN_VX,
RISCV::VSLIDEUP_VX},
LT.second, CostKind)
: getRISCVInstructionCost({RISCV::VSLIDEDOWN_VX, RISCV::VMV_X_S,
RISCV::VSRL_VX, RISCV::VMV_X_S},
LT.second, CostKind);
}
return BaseCost + SlideCost;
}
InstructionCost
RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
TTI::TargetCostKind CostKind,
unsigned Index) const {
if (isa<FixedVectorType>(Val))
return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
Index);
// TODO: This code replicates what LoopVectorize.cpp used to do when asking
// for the cost of extracting the last lane of a scalable vector. It probably
// needs a more accurate cost.
ElementCount EC = cast<VectorType>(Val)->getElementCount();
assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
return getVectorInstrCost(Opcode, Val, CostKind,
EC.getKnownMinValue() - 1 - Index, nullptr,
nullptr);
}
/// Check to see if this instruction is expected to be combined to a simpler
/// operation during/before lowering. If so return the cost of the combined
/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
/// than the cost of a `udiv`
std::optional<InstructionCost>
RISCVTTIImpl::getCombinedArithmeticInstructionCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
ArrayRef<const Value *> Args, const Instruction *CtxI) const {
// Vector unsigned division/remainder will be simplified to shifts/masks.
if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
if (Opcode == Instruction::UDiv)
return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, Opd1Info,
Opd2Info.getNoProps());
// UREM
return getArithmeticInstrCost(Instruction::And, Ty, CostKind, Opd1Info,
Opd2Info.getNoProps());
}
return std::nullopt;
}
InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
ArrayRef<const Value *> Args, const Instruction *CtxI) const {
// TODO: Handle more cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
Args, CtxI);
if (isa<FixedVectorType>(Ty) && !ST->useRVVForFixedLengthVectors())
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
Args, CtxI);
// Skip if scalar size of Ty is bigger than ELEN.
if (isa<VectorType>(Ty) && Ty->getScalarSizeInBits() > ST->getELen())
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
Args, CtxI);
if (std::optional<InstructionCost> CombinedCost =
getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Op1Info,
Op2Info, Args, CtxI))
return *CombinedCost;
// Legalize the type.
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
// TODO: Handle scalar type.
if (!LT.second.isVector()) {
static const CostTblEntry DivTbl[]{
{ISD::UDIV, MVT::i32, TTI::TCC_Expensive},
{ISD::UDIV, MVT::i64, TTI::TCC_Expensive},
{ISD::SDIV, MVT::i32, TTI::TCC_Expensive},
{ISD::SDIV, MVT::i64, TTI::TCC_Expensive},
{ISD::UREM, MVT::i32, TTI::TCC_Expensive},
{ISD::UREM, MVT::i64, TTI::TCC_Expensive},
{ISD::SREM, MVT::i32, TTI::TCC_Expensive},
{ISD::SREM, MVT::i64, TTI::TCC_Expensive}};
if (TLI->isOperationLegalOrPromote(ISDOpcode, LT.second))
if (const auto *Entry = CostTableLookup(DivTbl, ISDOpcode, LT.second))
return Entry->Cost * LT.first;
return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
Args, CtxI);
}
// f16 with zvfhmin and bf16 will be promoted to f32.
// FIXME: nxv32[b]f16 will be custom lowered and split.
InstructionCost CastCost = 0;
if ((LT.second.getVectorElementType() == MVT::f16 ||
LT.second.getVectorElementType() == MVT::bf16) &&
TLI->getOperationAction(ISDOpcode, LT.second) ==
TargetLoweringBase::LegalizeAction::Promote) {
MVT PromotedVT = TLI->getTypeToPromoteTo(ISDOpcode, LT.second);
Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Ty->getContext());
Type *LegalTy = EVT(LT.second).getTypeForEVT(Ty->getContext());
// Add cost of extending arguments
CastCost += LT.first * Args.size() *
getCastInstrCost(Instruction::FPExt, PromotedTy, LegalTy,
TTI::CastContextHint::None, CostKind);
// Add cost of truncating result
CastCost +=
LT.first * getCastInstrCost(Instruction::FPTrunc, LegalTy, PromotedTy,
TTI::CastContextHint::None, CostKind);
// Compute cost of op in promoted type
LT.second = PromotedVT;
}
auto getConstantMatCost =
[&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
if (OpInfo.isUniform() &&
canSplatOperand(Opcode, Operand, Intrinsic::not_intrinsic))
// Two sub-cases:
// * Has a 5 bit immediate operand which can be splatted.
// * Has a larger immediate which must be materialized in scalar register
// We return 0 for both as we currently ignore the cost of materializing
// scalar constants in GPRs.
return 0;
return getConstantPoolLoadCost(Ty, CostKind);
};
// Add the cost of materializing any constant vectors required.
InstructionCost ConstantMatCost = 0;
if (Op1Info.isConstant())
ConstantMatCost += getConstantMatCost(0, Op1Info);
if (Op2Info.isConstant())
ConstantMatCost += getConstantMatCost(1, Op2Info);
unsigned Op;
switch (ISDOpcode) {
case ISD::ADD:
case ISD::SUB:
Op = RISCV::VADD_VV;
break;
case ISD::SHL:
case ISD::SRL:
case ISD::SRA:
Op = RISCV::VSLL_VV;
break;
case ISD::AND:
case ISD::OR:
case ISD::XOR:
Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
break;
case ISD::MUL:
case ISD::MULHS:
case ISD::MULHU:
Op = RISCV::VMUL_VV;
break;
case ISD::SDIV:
case ISD::UDIV:
Op = RISCV::VDIV_VV;
break;
case ISD::SREM:
case ISD::UREM:
Op = RISCV::VREM_VV;
break;
case ISD::FADD:
case ISD::FSUB:
Op = RISCV::VFADD_VV;
break;
case ISD::FMUL:
Op = RISCV::VFMUL_VV;
break;
case ISD::FDIV:
Op = RISCV::VFDIV_VV;
break;
case ISD::FNEG:
Op = RISCV::VFSGNJN_VV;
break;
default:
// Assuming all other instructions have the same cost until a need arises to
// differentiate them.
return CastCost + ConstantMatCost +
BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
Args, CtxI);
}
InstructionCost InstrCost = getRISCVInstructionCost(Op, LT.second, CostKind);
// We use BasicTTIImpl to calculate scalar costs, which assumes floating point
// ops are twice as expensive as integer ops. Do the same for vectors so
// scalar floating point ops aren't cheaper than their vector equivalents.
if (Ty->isFPOrFPVectorTy())
InstrCost *= 2;
return CastCost + ConstantMatCost + LT.first * InstrCost;
}
// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
InstructionCost RISCVTTIImpl::getPointersChainCost(
ArrayRef<const Value *> Ptrs, const Value *Base,
const TTI::PointersChainInfo &Info, Type *AccessTy,
const TTI::TargetCostKind CostKind) const {
InstructionCost Cost = TTI::TCC_Free;
// In the basic model we take into account GEP instructions only
// (although here can come alloca instruction, a value, constants and/or
// constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
// pointer). Typically, if Base is a not a GEP-instruction and all the
// pointers are relative to the same base address, all the rest are
// either GEP instructions, PHIs, bitcasts or constants. When we have same
// base, we just calculate cost of each non-Base GEP as an ADD operation if
// any their index is a non-const.
// If no known dependencies between the pointers cost is calculated as a sum
// of costs of GEP instructions.
for (auto [I, V] : enumerate(Ptrs)) {
const auto *GEP = dyn_cast<GetElementPtrInst>(V);
if (!GEP)
continue;
if (Info.isSameBase() && V != Base) {
if (GEP->hasAllConstantIndices())
continue;
// If the chain is unit-stride and BaseReg + stride*i is a legal
// addressing mode, then presume the base GEP is sitting around in a
// register somewhere and check if we can fold the offset relative to
// it.
unsigned Stride = DL.getTypeStoreSize(AccessTy);
if (Info.isUnitStride() &&
isLegalAddressingMode(AccessTy,
/* BaseGV */ nullptr,
/* BaseOffset */ Stride * I,
/* HasBaseReg */ true,
/* Scale */ 0,
GEP->getType()->getPointerAddressSpace()))
continue;
Cost += getArithmeticInstrCost(Instruction::Add, GEP->getType(), CostKind,
{TTI::OK_AnyValue, TTI::OP_None},
{TTI::OK_AnyValue, TTI::OP_None}, {});
} else {
SmallVector<const Value *> Indices(GEP->indices());
Cost += getGEPCost(GEP->getSourceElementType(), GEP->getPointerOperand(),
Indices, CostKind, AccessTy);
}
}
return Cost;
}
void RISCVTTIImpl::getUnrollingPreferences(
Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
OptimizationRemarkEmitter *ORE) const {
// TODO: More tuning on benchmarks and metrics with changes as needed
// would apply to all settings below to enable performance.
if (ST->enableDefaultUnroll())
return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
// Enable Upper bound unrolling universally, not dependent upon the conditions
// below.
UP.UpperBound = true;
// Disable loop unrolling for Oz and Os.
UP.OptSizeThreshold = 0;
UP.PartialOptSizeThreshold = 0;
if (L->getHeader()->getParent()->hasOptSize())
return;
SmallVector<BasicBlock *, 4> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
LLVM_DEBUG(dbgs() << "Loop has:\n"
<< "Blocks: " << L->getNumBlocks() << "\n"
<< "Exit blocks: " << ExitingBlocks.size() << "\n");
// Only allow another exit other than the latch. This acts as an early exit
// as it mirrors the profitability calculation of the runtime unroller.
if (ExitingBlocks.size() > 2)
return;
// Limit the CFG of the loop body for targets with a branch predictor.
// Allowing 4 blocks permits if-then-else diamonds in the body.
if (L->getNumBlocks() > 4)
return;
// Scan the loop: don't unroll loops with calls as this could prevent
// inlining. Don't unroll auto-vectorized loops either, though do allow
// unrolling of the scalar remainder.
bool IsVectorized = getBooleanLoopAttribute(L, "llvm.loop.isvectorized");
InstructionCost Cost = 0;
for (auto *BB : L->getBlocks()) {
for (auto &I : *BB) {
// Both auto-vectorized loops and the scalar remainder have the
// isvectorized attribute, so differentiate between them by the presence
// of vector instructions.
if (IsVectorized && (I.getType()->isVectorTy() ||
llvm::any_of(I.operand_values(), [](Value *V) {
return V->getType()->isVectorTy();
})))
return;
if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
const Function *F = cast<CallBase>(I).getCalledFunction();
if (!F || isLoweredToCall(F))
return;
}
SmallVector<const Value *> Operands(I.operand_values());
Cost += getInstructionCost(&I, Operands,
TargetTransformInfo::TCK_SizeAndLatency);
}
}
LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
UP.Partial = true;
UP.Runtime = true;
UP.UnrollRemainder = true;
UP.UnrollAndJam = true;
// Force unrolling small loops can be very useful because of the branch
// taken cost of the backedge.
if (Cost < 12)
UP.Force = true;
}
void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
TTI::PeelingPreferences &PP) const {
BaseT::getPeelingPreferences(L, SE, PP);
}
bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
MemIntrinsicInfo &Info) const {
const DataLayout &DL = getDataLayout();
Intrinsic::ID IID = Inst->getIntrinsicID();
LLVMContext &C = Inst->getContext();
bool HasMask = false;
auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
bool IsWrite) -> int64_t {
if (auto *TarExtTy =
dyn_cast<TargetExtType>(II->getArgOperand(0)->getType()))
return TarExtTy->getIntParameter(0);
return 1;
};
switch (IID) {
case Intrinsic::riscv_vle_mask:
case Intrinsic::riscv_vse_mask:
case Intrinsic::riscv_vlseg2_mask:
case Intrinsic::riscv_vlseg3_mask:
case Intrinsic::riscv_vlseg4_mask:
case Intrinsic::riscv_vlseg5_mask:
case Intrinsic::riscv_vlseg6_mask:
case Intrinsic::riscv_vlseg7_mask:
case Intrinsic::riscv_vlseg8_mask:
case Intrinsic::riscv_vsseg2_mask:
case Intrinsic::riscv_vsseg3_mask:
case Intrinsic::riscv_vsseg4_mask:
case Intrinsic::riscv_vsseg5_mask:
case Intrinsic::riscv_vsseg6_mask:
case Intrinsic::riscv_vsseg7_mask:
case Intrinsic::riscv_vsseg8_mask:
HasMask = true;
[[fallthrough]];
case Intrinsic::riscv_vle:
case Intrinsic::riscv_vse:
case Intrinsic::riscv_vlseg2:
case Intrinsic::riscv_vlseg3:
case Intrinsic::riscv_vlseg4:
case Intrinsic::riscv_vlseg5:
case Intrinsic::riscv_vlseg6:
case Intrinsic::riscv_vlseg7:
case Intrinsic::riscv_vlseg8:
case Intrinsic::riscv_vsseg2:
case Intrinsic::riscv_vsseg3:
case Intrinsic::riscv_vsseg4:
case Intrinsic::riscv_vsseg5:
case Intrinsic::riscv_vsseg6:
case Intrinsic::riscv_vsseg7:
case Intrinsic::riscv_vsseg8: {
// Intrinsic interface:
// riscv_vle(merge, ptr, vl)
// riscv_vle_mask(merge, ptr, mask, vl, policy)
// riscv_vse(val, ptr, vl)
// riscv_vse_mask(val, ptr, mask, vl, policy)
// riscv_vlseg#(merge, ptr, vl, sew)
// riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
// riscv_vsseg#(val, ptr, vl, sew)
// riscv_vsseg#_mask(val, ptr, mask, vl, sew)
bool IsWrite = Inst->getType()->isVoidTy();
Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
// The results of segment loads are TargetExtType.
if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {
unsigned SEW =
1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))
->getZExtValue();
Ty = TarExtTy->getTypeParameter(0U);
Ty = ScalableVectorType::get(
IntegerType::get(C, SEW),
cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);
}
const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
unsigned VLIndex = RVVIInfo->VLOperand;
unsigned PtrOperandNo = VLIndex - 1 - HasMask;
MaybeAlign Alignment =
Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
Value *Mask = ConstantInt::getTrue(MaskType);
if (HasMask)
Mask = Inst->getArgOperand(VLIndex - 1);
Value *EVL = Inst->getArgOperand(VLIndex);
unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
// RVV uses contiguous elements as a segment.
if (SegNum > 1) {
unsigned ElemSize = Ty->getScalarSizeInBits();
auto *SegTy = IntegerType::get(C, ElemSize * SegNum);
Ty = VectorType::get(SegTy, cast<VectorType>(Ty));
}
Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,
Alignment, Mask, EVL);
return true;
}
case Intrinsic::riscv_vlse_mask:
case Intrinsic::riscv_vsse_mask:
case Intrinsic::riscv_vlsseg2_mask:
case Intrinsic::riscv_vlsseg3_mask:
case Intrinsic::riscv_vlsseg4_mask:
case Intrinsic::riscv_vlsseg5_mask:
case Intrinsic::riscv_vlsseg6_mask:
case Intrinsic::riscv_vlsseg7_mask:
case Intrinsic::riscv_vlsseg8_mask:
case Intrinsic::riscv_vssseg2_mask:
case Intrinsic::riscv_vssseg3_mask:
case Intrinsic::riscv_vssseg4_mask:
case Intrinsic::riscv_vssseg5_mask:
case Intrinsic::riscv_vssseg6_mask:
case Intrinsic::riscv_vssseg7_mask:
case Intrinsic::riscv_vssseg8_mask:
HasMask = true;
[[fallthrough]];
case Intrinsic::riscv_vlse:
case Intrinsic::riscv_vsse:
case Intrinsic::riscv_vlsseg2:
case Intrinsic::riscv_vlsseg3:
case Intrinsic::riscv_vlsseg4:
case Intrinsic::riscv_vlsseg5:
case Intrinsic::riscv_vlsseg6:
case Intrinsic::riscv_vlsseg7:
case Intrinsic::riscv_vlsseg8:
case Intrinsic::riscv_vssseg2:
case Intrinsic::riscv_vssseg3:
case Intrinsic::riscv_vssseg4:
case Intrinsic::riscv_vssseg5:
case Intrinsic::riscv_vssseg6:
case Intrinsic::riscv_vssseg7:
case Intrinsic::riscv_vssseg8: {
// Intrinsic interface:
// riscv_vlse(merge, ptr, stride, vl)
// riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
// riscv_vsse(val, ptr, stride, vl)
// riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
// riscv_vlsseg#(merge, ptr, offset, vl, sew)
// riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
// riscv_vssseg#(val, ptr, offset, vl, sew)
// riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
bool IsWrite = Inst->getType()->isVoidTy();
Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
// The results of segment loads are TargetExtType.
if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {
unsigned SEW =
1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))
->getZExtValue();
Ty = TarExtTy->getTypeParameter(0U);
Ty = ScalableVectorType::get(
IntegerType::get(C, SEW),
cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);
}
const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
unsigned VLIndex = RVVIInfo->VLOperand;
unsigned PtrOperandNo = VLIndex - 2 - HasMask;
MaybeAlign Alignment =
Inst->getArgOperand(PtrOperandNo)->getPointerAlignment(DL);
Value *Stride = Inst->getArgOperand(PtrOperandNo + 1);
// Use the pointer alignment as the element alignment if the stride is a
// multiple of the pointer alignment. Otherwise, the element alignment
// should be the greatest common divisor of pointer alignment and stride.
// For simplicity, just consider unalignment for elements.
unsigned PointerAlign = Alignment.valueOrOne().value();
if (!isa<ConstantInt>(Stride) ||
cast<ConstantInt>(Stride)->getZExtValue() % PointerAlign != 0)
Alignment = Align(1);
Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
Value *Mask = ConstantInt::getTrue(MaskType);
if (HasMask)
Mask = Inst->getArgOperand(VLIndex - 1);
Value *EVL = Inst->getArgOperand(VLIndex);
unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
// RVV uses contiguous elements as a segment.
if (SegNum > 1) {
unsigned ElemSize = Ty->getScalarSizeInBits();
auto *SegTy = IntegerType::get(C, ElemSize * SegNum);
Ty = VectorType::get(SegTy, cast<VectorType>(Ty));
}
Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,
Alignment, Mask, EVL, Stride);
return true;
}
case Intrinsic::riscv_vloxei_mask:
case Intrinsic::riscv_vluxei_mask:
case Intrinsic::riscv_vsoxei_mask:
case Intrinsic::riscv_vsuxei_mask:
case Intrinsic::riscv_vloxseg2_mask:
case Intrinsic::riscv_vloxseg3_mask:
case Intrinsic::riscv_vloxseg4_mask:
case Intrinsic::riscv_vloxseg5_mask:
case Intrinsic::riscv_vloxseg6_mask:
case Intrinsic::riscv_vloxseg7_mask:
case Intrinsic::riscv_vloxseg8_mask:
case Intrinsic::riscv_vluxseg2_mask:
case Intrinsic::riscv_vluxseg3_mask:
case Intrinsic::riscv_vluxseg4_mask:
case Intrinsic::riscv_vluxseg5_mask:
case Intrinsic::riscv_vluxseg6_mask:
case Intrinsic::riscv_vluxseg7_mask:
case Intrinsic::riscv_vluxseg8_mask:
case Intrinsic::riscv_vsoxseg2_mask:
case Intrinsic::riscv_vsoxseg3_mask:
case Intrinsic::riscv_vsoxseg4_mask:
case Intrinsic::riscv_vsoxseg5_mask:
case Intrinsic::riscv_vsoxseg6_mask:
case Intrinsic::riscv_vsoxseg7_mask:
case Intrinsic::riscv_vsoxseg8_mask:
case Intrinsic::riscv_vsuxseg2_mask:
case Intrinsic::riscv_vsuxseg3_mask:
case Intrinsic::riscv_vsuxseg4_mask:
case Intrinsic::riscv_vsuxseg5_mask:
case Intrinsic::riscv_vsuxseg6_mask:
case Intrinsic::riscv_vsuxseg7_mask:
case Intrinsic::riscv_vsuxseg8_mask:
HasMask = true;
[[fallthrough]];
case Intrinsic::riscv_vloxei:
case Intrinsic::riscv_vluxei:
case Intrinsic::riscv_vsoxei:
case Intrinsic::riscv_vsuxei:
case Intrinsic::riscv_vloxseg2:
case Intrinsic::riscv_vloxseg3:
case Intrinsic::riscv_vloxseg4:
case Intrinsic::riscv_vloxseg5:
case Intrinsic::riscv_vloxseg6:
case Intrinsic::riscv_vloxseg7:
case Intrinsic::riscv_vloxseg8:
case Intrinsic::riscv_vluxseg2:
case Intrinsic::riscv_vluxseg3:
case Intrinsic::riscv_vluxseg4:
case Intrinsic::riscv_vluxseg5:
case Intrinsic::riscv_vluxseg6:
case Intrinsic::riscv_vluxseg7:
case Intrinsic::riscv_vluxseg8:
case Intrinsic::riscv_vsoxseg2:
case Intrinsic::riscv_vsoxseg3:
case Intrinsic::riscv_vsoxseg4:
case Intrinsic::riscv_vsoxseg5:
case Intrinsic::riscv_vsoxseg6:
case Intrinsic::riscv_vsoxseg7:
case Intrinsic::riscv_vsoxseg8:
case Intrinsic::riscv_vsuxseg2:
case Intrinsic::riscv_vsuxseg3:
case Intrinsic::riscv_vsuxseg4:
case Intrinsic::riscv_vsuxseg5:
case Intrinsic::riscv_vsuxseg6:
case Intrinsic::riscv_vsuxseg7:
case Intrinsic::riscv_vsuxseg8: {
// Intrinsic interface (only listed ordered version):
// riscv_vloxei(merge, ptr, index, vl)
// riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
// riscv_vsoxei(val, ptr, index, vl)
// riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
// riscv_vloxseg#(merge, ptr, index, vl, sew)
// riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
// riscv_vsoxseg#(val, ptr, index, vl, sew)
// riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
bool IsWrite = Inst->getType()->isVoidTy();
Type *Ty = IsWrite ? Inst->getArgOperand(0)->getType() : Inst->getType();
// The results of segment loads are TargetExtType.
if (auto *TarExtTy = dyn_cast<TargetExtType>(Ty)) {
unsigned SEW =
1 << cast<ConstantInt>(Inst->getArgOperand(Inst->arg_size() - 1))
->getZExtValue();
Ty = TarExtTy->getTypeParameter(0U);
Ty = ScalableVectorType::get(
IntegerType::get(C, SEW),
cast<ScalableVectorType>(Ty)->getMinNumElements() * 8 / SEW);
}
const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IID);
unsigned VLIndex = RVVIInfo->VLOperand;
unsigned PtrOperandNo = VLIndex - 2 - HasMask;
Value *Mask;
if (HasMask) {
Mask = Inst->getArgOperand(VLIndex - 1);
} else {
// Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
// and casting that to scalar i64 triggers a vector/scalar mismatch
// assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
// via extractelement instead.
Type *MaskType = Ty->getWithNewType(Type::getInt1Ty(C));
Mask = ConstantInt::getTrue(MaskType);
}
Value *EVL = Inst->getArgOperand(VLIndex);
unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
// RVV uses contiguous elements as a segment.
if (SegNum > 1) {
unsigned ElemSize = Ty->getScalarSizeInBits();
auto *SegTy = IntegerType::get(C, ElemSize * SegNum);
Ty = VectorType::get(SegTy, cast<VectorType>(Ty));
}
Value *OffsetOp = Inst->getArgOperand(PtrOperandNo + 1);
Info.InterestingOperands.emplace_back(Inst, PtrOperandNo, IsWrite, Ty,
Align(1), Mask, EVL,
/* Stride */ nullptr, OffsetOp);
return true;
}
}
return false;
}
unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
if (Ty->isVectorTy()) {
// f16 with only zvfhmin and bf16 will be promoted to f32
Type *EltTy = cast<VectorType>(Ty)->getElementType();
if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
EltTy->isBFloatTy())
Ty = VectorType::get(Type::getFloatTy(Ty->getContext()),
cast<VectorType>(Ty));
TypeSize Size = DL.getTypeSizeInBits(Ty);
if (Size.isScalable() && ST->hasVInstructions())
return divideCeil(Size.getKnownMinValue(), RISCV::RVVBitsPerBlock);
if (ST->useRVVForFixedLengthVectors())
return divideCeil(Size, ST->getRealMinVLen());
}
return BaseT::getRegUsageForType(Ty);
}
unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
if (std::optional<unsigned> MaxVF = ST->getCLOpts().v_slp_max_vf)
return *MaxVF;
// Return how many elements can fit in getRegisterBitwidth. This is the
// same routine as used in LoopVectorizer. We should probably be
// accounting for whether we actually have instructions with the right
// lane type, but we don't have enough information to do that without
// some additional plumbing which hasn't been justified yet.
TypeSize RegWidth =
getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector);
// If no vector registers, or absurd element widths, disable
// vectorization by returning 1.
return std::max<unsigned>(1U, RegWidth.getFixedValue() / ElemWidth);
}
unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
return ST->getCLOpts().v_min_trip_count;
}
bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
return ST->enableUnalignedVectorMem();
}
TTI::AddressingModeKind
RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
ScalarEvolution *SE) const {
if (ST->hasVendorXCVmem() && !ST->is64Bit())
return TTI::AMK_PostIndexed;
return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
}
bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
const TargetTransformInfo::LSRCost &C2) const {
// RISC-V specific here are "instruction number 1st priority".
// If we need to emit adds inside the loop to add up base registers, then
// we need at least one extra temporary register.
unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
return std::tie(C1.Insns, C1NumRegs, C1.AddRecCost,
C1.NumIVMuls, C1.NumBaseAdds,
C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
std::tie(C2.Insns, C2NumRegs, C2.AddRecCost,
C2.NumIVMuls, C2.NumBaseAdds,
C2.ScaleCost, C2.ImmCost, C2.SetupCost);
}
bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
Align Alignment) const {
auto *VTy = dyn_cast<VectorType>(DataTy);
if (!VTy)
return false;
if (!isLegalMaskedLoadStore(DataTy, Alignment))
return false;
// FIXME: If it is an i8 vector and the element count exceeds 256, we should
// scalarize these types with LMUL >= maximum fixed-length LMUL.
if (VTy->getElementType()->isIntegerTy(8)) {
uint64_t MaxEltCount = VTy->getElementCount().getKnownMinValue();
if (VTy->isScalableTy())
MaxEltCount *= ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
// We can't yet split any widened indices type.
if (MaxEltCount > 256)
return getTypeLegalizationCost(
VTy->getWithNewType(Type::getInt16Ty(VTy->getContext())))
.first == 1;
}
return true;
}
bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
Align Alignment) const {
return isLegalMaskedLoadStore(DataTy, Alignment);
}
bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
ElementCount NumElements) const {
// Optimized zero-stride loads can be treated as broadcasts.
if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
return false;
return TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, ElementTy));
}
/// See if \p I should be considered for address type promotion. We check if \p
/// I is a sext with right type and used in memory accesses. If it used in a
/// "complex" getelementptr, we allow it to be promoted without finding other
/// sext instructions that sign extended the same initial value. A getelementptr
/// is considered as "complex" if it has more than 2 operands.
bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
bool Considerable = false;
AllowPromotionWithoutCommonHeader = false;
if (!isa<SExtInst>(&I))
return false;
Type *ConsideredSExtType =
Type::getInt64Ty(I.getParent()->getParent()->getContext());
if (I.getType() != ConsideredSExtType)
return false;
// See if the sext is the one with the right type and used in at least one
// GetElementPtrInst.
for (const User *U : I.users()) {
if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(U)) {
Considerable = true;
// A getelementptr is considered as "complex" if it has more than 2
// operands. We will promote a SExt used in such complex GEP as we
// expect some computation to be merged if they are done on 64 bits.
if (GEPInst->getNumOperands() > 2) {
AllowPromotionWithoutCommonHeader = true;
break;
}
}
}
return Considerable;
}
bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand,
Intrinsic::ID IntrinsicID) const {
switch (Opcode) {
case Instruction::Add:
case Instruction::Sub:
case Instruction::Mul:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
case Instruction::FAdd:
case Instruction::FSub:
case Instruction::FMul:
case Instruction::FDiv:
case Instruction::ICmp:
case Instruction::FCmp:
return true;
case Instruction::Shl:
case Instruction::LShr:
case Instruction::AShr:
case Instruction::UDiv:
case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem:
case Instruction::Select:
return Operand == 1;
case Instruction::Call:
switch (IntrinsicID) {
case Intrinsic::fma:
case Intrinsic::fmuladd:
return Operand == 0 || Operand == 1;
case Intrinsic::vp_udiv:
case Intrinsic::vp_sdiv:
case Intrinsic::vp_urem:
case Intrinsic::vp_srem:
case Intrinsic::ssub_sat:
case Intrinsic::usub_sat:
return Operand == 1;
// These intrinsics are commutative.
case Intrinsic::smin:
case Intrinsic::umin:
case Intrinsic::smax:
case Intrinsic::umax:
case Intrinsic::sadd_sat:
case Intrinsic::uadd_sat:
return Operand == 0 || Operand == 1;
default:
return false;
}
default:
return false;
}
}
bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
return false;
auto *II = dyn_cast<IntrinsicInst>(I);
Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
if (II)
IntrinsicID = II->getIntrinsicID();
return canSplatOperand(I->getOpcode(), Operand, IntrinsicID);
}
TargetTransformInfo::VectorInstrContext RISCVTTIImpl::getBuildVectorContextHint(
ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
function_ref<bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
GatherUseOps) const {
if (Scalars.empty() || !ST->hasVInstructions() || !ST->sinkSplatOperands() ||
!ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))
return VectorInstrContext::None;
const auto *SplatIt = find_if_not(Scalars, IsaPred<UndefValue>);
if (SplatIt == Scalars.end() || (*SplatIt)->getType()->isIntegerTy(1) ||
isa<VectorType>((*SplatIt)->getType()) ||
isa<ExtractElementInst>(*SplatIt))
return VectorInstrContext::None;
SmallVector<TargetTransformInfo::BuildVectorUseOp, 4> UserOps;
if (!GatherUseOps(UserOps) || UserOps.empty())
return VectorInstrContext::None;
if (all_of(UserOps,
[this](const TargetTransformInfo::BuildVectorUseOp &UserOp) {
return canSplatOperand(UserOp.Opcode, UserOp.OperandIndex,
UserOp.IntrinsicID);
}))
return VectorInstrContext::SplatOpFolded;
return VectorInstrContext::None;
}
/// Check if sinking \p I's operands to I's basic block is profitable, because
/// the operands can be folded into a target instruction, e.g.
/// splats of scalars can fold into vector instructions.
bool RISCVTTIImpl::isProfitableToSinkOperands(
Instruction *I, SmallVectorImpl<Use *> &Ops) const {
using namespace llvm::PatternMatch;
if (I->isBitwiseLogicOp()) {
if (!I->getType()->isVectorTy()) {
if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
for (auto &Op : I->operands()) {
// (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
if (match(Op.get(), m_Not(m_Value()))) {
Ops.push_back(&Op);
return true;
}
}
}
} else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
for (auto &Op : I->operands()) {
// (and X, (not Y)) -> (vandn.vv X, Y)
if (match(Op.get(), m_Not(m_Value()))) {
Ops.push_back(&Op);
return true;
}
// (and X, (splat (not Y))) -> (vandn.vx X, Y)
if (match(Op.get(), m_Shuffle(m_InsertElt(m_Value(), m_Not(m_Value()),
m_ZeroInt()),
m_Value(), m_ZeroMask()))) {
Use &InsertElt = cast<Instruction>(Op)->getOperandUse(0);
Use &Not = cast<Instruction>(InsertElt)->getOperandUse(1);
Ops.push_back(&Not);
Ops.push_back(&InsertElt);
Ops.push_back(&Op);
return true;
}
}
}
}
if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
return false;
// Don't sink splat operands if the target prefers it. Some targets requires
// S2V transfer buffers and we can run out of them copying the same value
// repeatedly.
// FIXME: It could still be worth doing if it would improve vector register
// pressure and prevent a vector spill.
if (!ST->sinkSplatOperands())
return false;
for (auto OpIdx : enumerate(I->operands())) {
if (!canSplatOperand(I, OpIdx.index()))
continue;
Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
// Make sure we are not already sinking this operand
if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
continue;
// We are looking for a splat that can be sunk.
if (!match(Op, m_Shuffle(m_InsertElt(m_Value(), m_Value(), m_ZeroInt()),
m_Value(), m_ZeroMask())))
continue;
// Don't sink i1 splats.
if (cast<VectorType>(Op->getType())->getElementType()->isIntegerTy(1))
continue;
// All uses of the shuffle should be sunk to avoid duplicating it across gpr
// and vector registers
for (Use &U : Op->uses()) {
Instruction *Insn = cast<Instruction>(U.getUser());
if (!canSplatOperand(Insn, U.getOperandNo()))
return false;
}
// Sink any fpexts since they might be used in a widening fp pattern.
Use *InsertEltUse = &Op->getOperandUse(0);
auto *InsertElt = cast<InsertElementInst>(InsertEltUse);
if (isa<FPExtInst>(InsertElt->getOperand(1)))
Ops.push_back(&InsertElt->getOperandUse(1));
Ops.push_back(InsertEltUse);
Ops.push_back(&OpIdx.value());
}
return true;
}
RISCVTTIImpl::TTI::MemCmpExpansionOptions
RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
TTI::MemCmpExpansionOptions Options;
if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
return Options;
Options.AllowOverlappingLoads = true;
Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
if (ST->is64Bit()) {
Options.LoadSizes = {8, 4, 2, 1};
Options.AllowedTailExpansions = {3, 5, 6};
} else {
Options.LoadSizes = {4, 2, 1};
Options.AllowedTailExpansions = {3};
}
if (IsZeroCmp && ST->hasVInstructions()) {
unsigned VLenB = ST->getRealMinVLen() / 8;
// The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
// `VLenB * MaxLMUL` so that it fits in a single register group.
unsigned MinSize = ST->getXLen() / 8 + 1;
unsigned MaxSize = VLenB * 8;
for (unsigned Size = MinSize; Size <= MaxSize; Size++)
Options.LoadSizes.insert(Options.LoadSizes.begin(), Size);
}
return Options;
}
bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
const Instruction *I) const {
if (ST->getCLOpts().or_like_select) {
// For the binary operators (e.g. or) we need to be more careful than
// selects, here we only transform them if they are already at a natural
// break point in the code - the end of a block with an unconditional
// terminator.
if (I->getOpcode() == Instruction::Or &&
isa<UncondBrInst>(I->getNextNode()))
return true;
if (I->getOpcode() == Instruction::Add ||
I->getOpcode() == Instruction::Sub)
return true;
}
return BaseT::shouldTreatInstructionLikeSelect(I);
}
bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
const Function *Caller, const Attribute &Attr) const {
// "interrupt" controls the prolog/epilog of interrupt handlers (and includes
// restrictions on their signatures). We can outline from the bodies of these
// handlers, but when we do we need to make sure we don't mark the outlined
// function as an interrupt handler too.
if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
return false;
return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
}
std::optional<Instruction *>
RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
// Attach a range return attribute describing the result of vsetvli/vsetvlimax
// so generic value analyses can reason about it. The verifier guarantees an
// XLen result and constant VSEW/VLMUL encoding a valid vtype, so no defensive
// validation is needed here.
if (is_contained({Intrinsic::riscv_vsetvli, Intrinsic::riscv_vsetvlimax},
II.getIntrinsicID())) {
// These intrinsics require the V extension; without it the VLEN queries
// below would assert. Such IR would fail isel anyway, so just bail out.
if (!ST->hasVInstructions())
return {};
bool HasAVL = II.getIntrinsicID() == Intrinsic::riscv_vsetvli;
unsigned Offset = HasAVL ? 1 : 0;
unsigned BitWidth = II.getType()->getIntegerBitWidth();
ConstantRange VLenRange(APInt(BitWidth, ST->getRealMinVLen()),
APInt(BitWidth, ST->getRealMaxVLen()) + 1);
uint64_t VSEW = cast<ConstantInt>(II.getArgOperand(Offset))->getZExtValue();
auto VLMUL = static_cast<RISCVVType::VLMUL>(
cast<ConstantInt>(II.getArgOperand(Offset + 1))->getZExtValue());
unsigned SEW = RISCVVType::decodeVSEW(VSEW);
unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMUL);
// VLMAX = VLEN / (SEW / LMUL), clamped to >= 1 for any usable vtype.
ConstantRange VLMAXRange =
VLenRange.udiv(ConstantRange(APInt(BitWidth, Ratio)))
.umax(ConstantRange(APInt(BitWidth, 1)));
// vsetvlimax returns exactly VLMAX; vsetvli returns vl with
// 0 <= vl <= min(AVL, VLMAX). vl == AVL only when AVL <= the smallest
// possible VLMAX; otherwise vl can shrink below VLMAX (to 0 at runtime), so
// only the VLMAX upper bound is sound.
ConstantRange VLRange = VLMAXRange;
if (HasAVL) {
// vl ≤ VLMAX
VLRange =
ConstantRange::makeAllowedICmpRegion(CmpInst::ICMP_ULE, VLMAXRange);
Value *AVL = II.getArgOperand(0);
ConstantRange AVLRange = computeConstantRangeIncludingKnownBits(
AVL, /*ForSigned=*/false,
IC.getSimplifyQuery().getWithInstruction(&II));
// vl = AVL if AVL ≤ VLMAX
if (AVLRange.icmp(CmpInst::ICMP_ULE, VLMAXRange))
return IC.replaceInstUsesWith(II, AVL);
// vl ≤ AVL
VLRange = VLRange.umin(AVLRange.getUnsignedMax());
// vl > 0 if AVL > 0
if (AVLRange.icmp(CmpInst::ICMP_UGT, APInt::getZero(BitWidth)))
VLRange = VLRange.umax(APInt(BitWidth, 1));
// vl = VLMAX if AVL ≥ (2 * VLMAX)
ConstantRange TwoVLMAX = VLMAXRange.multiply(APInt(BitWidth, 2));
if (AVLRange.icmp(CmpInst::ICMP_UGE, TwoVLMAX))
VLRange = VLRange.intersectWith(VLMAXRange);
// ceil(AVL / 2) ≤ vl ≤ VLMAX if AVL < (2 * VLMAX)
if (AVLRange.icmp(CmpInst::ICMP_ULT, TwoVLMAX))
VLRange = VLRange.umax(APIntOps::RoundingUDiv(AVLRange.getUnsignedMin(),
APInt(BitWidth, 2),
APInt::Rounding::UP));
}
ConstantRange OldRange =
II.getRange().value_or(ConstantRange::getFull(BitWidth));
ConstantRange NewRange = VLRange.intersectWith(OldRange);
if (NewRange != OldRange) {
II.addRangeRetAttr(NewRange);
return &II;
}
return {};
}
// If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
// the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
// creating redundant masks.
const DataLayout &DL = IC.getDataLayout();
if (II.user_empty())
return {};
auto *TargetVecTy = dyn_cast<ScalableVectorType>(II.user_back()->getType());
if (!TargetVecTy)
return {};
const APInt *Scalar;
uint64_t VL;
if (!match(&II, m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
m_Poison(), m_APInt(Scalar), m_ConstantInt(VL))) ||
!all_of(II.users(), [TargetVecTy](User *U) {
return U->getType() == TargetVecTy && match(U, m_BitCast(m_Value()));
}))
return {};
auto *SourceVecTy = cast<ScalableVectorType>(II.getType());
unsigned TargetEltBW = DL.getTypeSizeInBits(TargetVecTy->getElementType());
unsigned SourceEltBW = DL.getTypeSizeInBits(SourceVecTy->getElementType());
if (TargetEltBW % SourceEltBW)
return {};
unsigned TargetScale = TargetEltBW / SourceEltBW;
if (VL % TargetScale || TargetScale == 1)
return {};
Type *VLTy = II.getOperand(2)->getType();
ElementCount SourceEC = SourceVecTy->getElementCount();
unsigned NewEltBW = SourceEltBW * TargetScale;
if (!SourceEC.isKnownMultipleOf(TargetScale) ||
!DL.fitsInLegalInteger(NewEltBW))
return {};
auto *NewEltTy = IntegerType::get(II.getContext(), NewEltBW);
if (!TLI->isLegalElementTypeForRVV(TLI->getValueType(DL, NewEltTy)))
return {};
ElementCount NewEC = SourceEC.divideCoefficientBy(TargetScale);
Type *RetTy = VectorType::get(NewEltTy, NewEC);
assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
"Lossless bitcast between types expected");
APInt NewScalar = APInt::getSplat(NewEltBW, *Scalar);
return IC.replaceInstUsesWith(
II,
IC.Builder.CreateBitCast(
IC.Builder.CreateIntrinsic(
RetTy, Intrinsic::riscv_vmv_v_x,
{PoisonValue::get(RetTy), ConstantInt::get(NewEltTy, NewScalar),
ConstantInt::get(VLTy, VL / TargetScale)}),
SourceVecTy));
}