| //===-- Target.cpp ----------------------------------------------*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "../Target.h" |
| #include "../ParallelSnippetGenerator.h" |
| #include "../SerialSnippetGenerator.h" |
| #include "../SnippetGenerator.h" |
| |
| #include "MCTargetDesc/RISCVBaseInfo.h" |
| #include "MCTargetDesc/RISCVMCTargetDesc.h" |
| #include "MCTargetDesc/RISCVMatInt.h" |
| #include "RISCV.h" |
| #include "RISCVExegesisPasses.h" |
| #include "RISCVInstrInfo.h" |
| #include "RISCVRegisterInfo.h" |
| #include "llvm/Support/Regex.h" |
| #include "llvm/Support/raw_ostream.h" |
| |
| // include computeAvailableFeatures and computeRequiredFeatures. |
| #define GET_AVAILABLE_OPCODE_CHECKER |
| #include "RISCVGenInstrInfo.inc" |
| |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| |
| #include <vector> |
| |
| namespace llvm { |
| namespace exegesis { |
| |
| static cl::opt<bool> |
| OnlyUsesVLMAXForVL("riscv-vlmax-for-vl", |
| cl::desc("Only enumerate VLMAX for VL operand"), |
| cl::init(false), cl::Hidden); |
| |
| static cl::opt<bool> |
| EnumerateRoundingModes("riscv-enumerate-rounding-modes", |
| cl::desc("Enumerate different FRM and VXRM"), |
| cl::init(true), cl::Hidden); |
| |
| static cl::opt<std::string> |
| FilterConfig("riscv-filter-config", |
| cl::desc("Show only the configs matching this regex"), |
| cl::init(""), cl::Hidden); |
| |
| #include "RISCVGenExegesis.inc" |
| |
| namespace { |
| |
| template <class BaseT> class RISCVSnippetGenerator : public BaseT { |
| static void printRoundingMode(raw_ostream &OS, unsigned Val, bool UsesVXRM) { |
| if (UsesVXRM) { |
| assert(RISCVVXRndMode::isValidRoundingMode(Val)); |
| OS << RISCVVXRndMode::roundingModeToString( |
| static_cast<RISCVVXRndMode::RoundingMode>(Val)); |
| } else { |
| assert(RISCVFPRndMode::isValidRoundingMode(Val)); |
| OS << RISCVFPRndMode::roundingModeToString( |
| static_cast<RISCVFPRndMode::RoundingMode>(Val)); |
| } |
| } |
| |
| static constexpr unsigned MinSEW = 8; |
| // ELEN is basically SEW_max. |
| unsigned ELEN = 64; |
| |
| // We can't know the real min/max VLEN w/o a Function, so we're |
| // using the VLen from Zvl. |
| unsigned ZvlVLen = 32; |
| |
| /// Mask for registers that are NOT standalone registers like X0 and V0 |
| BitVector AggregateRegisters; |
| |
| // Returns true when opcode is available in any of the FBs. |
| static bool |
| isOpcodeAvailableIn(unsigned Opcode, |
| ArrayRef<RISCV_MC::SubtargetFeatureBits> FBs) { |
| FeatureBitset RequiredFeatures = RISCV_MC::computeRequiredFeatures(Opcode); |
| for (uint8_t FB : FBs) { |
| if (RequiredFeatures[FB]) |
| return true; |
| } |
| return false; |
| } |
| |
| static bool isRVVFloatingPointOp(unsigned Opcode) { |
| return isOpcodeAvailableIn(Opcode, |
| {RISCV_MC::Feature_HasVInstructionsAnyFBit}); |
| } |
| |
| // Get the element group width of each vector cryptor extension. |
| static unsigned getZvkEGWSize(unsigned Opcode, unsigned SEW) { |
| using namespace RISCV_MC; |
| if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvkgBit, |
| Feature_HasStdExtZvknedBit, |
| Feature_HasStdExtZvksedBit})) |
| return 128U; |
| if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvkshBit})) |
| return 256U; |
| if (isOpcodeAvailableIn(Opcode, {Feature_HasStdExtZvknhaOrZvknhbBit})) |
| // In Zvknh[ab], when SEW=64 is used (i.e. Zvknhb), EGW is 256. |
| // Otherwise it's 128. |
| return SEW == 64 ? 256U : 128U; |
| |
| llvm_unreachable("Unsupported opcode"); |
| } |
| |
| // A handy utility to multiply or divide an integer by LMUL. |
| template <typename T> static T multiplyLMul(T Val, RISCVVType::VLMUL VLMul) { |
| auto [LMul, IsFractional] = RISCVVType::decodeVLMUL(VLMul); |
| return IsFractional ? Val / LMul : Val * LMul; |
| } |
| |
| /// Return the denominator of the fractional (i.e. the `x` in .vfx suffix) or |
| /// nullopt if BaseOpcode is not a vector sext/zext. |
| static std::optional<unsigned> isRVVSignZeroExtend(unsigned BaseOpcode) { |
| switch (BaseOpcode) { |
| case RISCV::VSEXT_VF2: |
| case RISCV::VZEXT_VF2: |
| return 2; |
| case RISCV::VSEXT_VF4: |
| case RISCV::VZEXT_VF4: |
| return 4; |
| case RISCV::VSEXT_VF8: |
| case RISCV::VZEXT_VF8: |
| return 8; |
| default: |
| return std::nullopt; |
| } |
| } |
| |
| void annotateWithVType(const CodeTemplate &CT, const Instruction &Instr, |
| unsigned BaseOpcode, |
| const BitVector &ForbiddenRegisters, |
| std::vector<CodeTemplate> &Result) const; |
| |
| public: |
| RISCVSnippetGenerator(const LLVMState &State, |
| const SnippetGenerator::Options &Opts) |
| : BaseT(State, Opts), |
| AggregateRegisters(State.getRegInfo().getNumRegs(), /*initVal=*/true) { |
| // Initialize standalone registers mask. |
| const MCRegisterInfo &RegInfo = State.getRegInfo(); |
| const unsigned StandaloneRegClasses[] = { |
| RISCV::GPRRegClassID, RISCV::FPR16RegClassID, RISCV::VRRegClassID}; |
| |
| for (unsigned RegClassID : StandaloneRegClasses) |
| for (unsigned Reg : RegInfo.getRegClass(RegClassID)) |
| AggregateRegisters.reset(Reg); |
| |
| // Initialize ELEN and VLEN. |
| // FIXME: We could have obtained these two constants from RISCVSubtarget |
| // but in order to get that from TargetMachine, we need a Function. |
| const MCSubtargetInfo &STI = State.getSubtargetInfo(); |
| ELEN = STI.hasFeature(RISCV::FeatureStdExtZve64x) ? 64 : 32; |
| |
| const unsigned ZvlFeatures[] = { |
| RISCV::FeatureStdExtZvl32b, RISCV::FeatureStdExtZvl64b, |
| RISCV::FeatureStdExtZvl128b, RISCV::FeatureStdExtZvl256b, |
| RISCV::FeatureStdExtZvl512b, RISCV::FeatureStdExtZvl1024b, |
| RISCV::FeatureStdExtZvl2048b, RISCV::FeatureStdExtZvl4096b, |
| RISCV::FeatureStdExtZvl8192b, RISCV::FeatureStdExtZvl16384b, |
| RISCV::FeatureStdExtZvl32768b, RISCV::FeatureStdExtZvl65536b}; |
| for (auto [Idx, Feature] : enumerate(ZvlFeatures)) { |
| if (STI.hasFeature(Feature)) |
| ZvlVLen = std::max(ZvlVLen, 1u << (Idx + 5)); |
| } |
| } |
| |
| Expected<std::vector<CodeTemplate>> |
| generateCodeTemplates(InstructionTemplate Variant, |
| const BitVector &ForbiddenRegisters) const override; |
| }; |
| |
| static bool isMaskedSibling(unsigned MaskedOp, unsigned UnmaskedOp) { |
| const auto *RVVMasked = RISCV::getMaskedPseudoInfo(MaskedOp); |
| return RVVMasked && RVVMasked->UnmaskedPseudo == UnmaskedOp; |
| } |
| |
| // There are primarily two kinds of opcodes that are not eligible |
| // in a serial snippet: |
| // (1) Has a use operand that can not overlap with the def operand |
| // (i.e. early clobber). |
| // (2) The register file of the only use operand is different from |
| // that of the def operand. For instance, use operand is vector and |
| // the result is a scalar. |
| static bool isIneligibleOfSerialSnippets(unsigned BaseOpcode, |
| const Instruction &I) { |
| if (llvm::any_of(I.Operands, |
| [](const Operand &Op) { return Op.isEarlyClobber(); })) |
| return true; |
| |
| switch (BaseOpcode) { |
| case RISCV::VCOMPRESS_VM: |
| case RISCV::VCPOP_M: |
| case RISCV::VCPOP_V: |
| // The permutation instructions listed below cannot have destination |
| // overlapping with the source. |
| case RISCV::VRGATHEREI16_VV: |
| case RISCV::VRGATHER_VI: |
| case RISCV::VRGATHER_VV: |
| case RISCV::VRGATHER_VX: |
| case RISCV::VSLIDE1UP_VX: |
| case RISCV::VSLIDEUP_VI: |
| case RISCV::VSLIDEUP_VX: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| static bool isZvfhminZvfbfminOpcodes(unsigned BaseOpcode) { |
| switch (BaseOpcode) { |
| case RISCV::VFNCVT_F_F_W: |
| case RISCV::VFWCVT_F_F_V: |
| case RISCV::VFNCVTBF16_F_F_W: |
| case RISCV::VFWCVTBF16_F_F_V: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| static bool isVectorReduction(unsigned BaseOpcode) { |
| switch (BaseOpcode) { |
| case RISCV::VREDAND_VS: |
| case RISCV::VREDMAXU_VS: |
| case RISCV::VREDMAX_VS: |
| case RISCV::VREDMINU_VS: |
| case RISCV::VREDMIN_VS: |
| case RISCV::VREDOR_VS: |
| case RISCV::VREDSUM_VS: |
| case RISCV::VREDXOR_VS: |
| case RISCV::VWREDSUMU_VS: |
| case RISCV::VWREDSUM_VS: |
| case RISCV::VFREDMAX_VS: |
| case RISCV::VFREDMIN_VS: |
| case RISCV::VFREDOSUM_VS: |
| case RISCV::VFREDUSUM_VS: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| template <class BaseT> |
| void RISCVSnippetGenerator<BaseT>::annotateWithVType( |
| const CodeTemplate &OrigCT, const Instruction &Instr, unsigned BaseOpcode, |
| const BitVector &ForbiddenRegisters, |
| std::vector<CodeTemplate> &Result) const { |
| const MCSubtargetInfo &STI = SnippetGenerator::State.getSubtargetInfo(); |
| unsigned VPseudoOpcode = Instr.getOpcode(); |
| |
| bool IsSerial = std::is_same_v<BaseT, SerialSnippetGenerator>; |
| |
| const MCInstrDesc &MIDesc = Instr.Description; |
| const uint64_t TSFlags = MIDesc.TSFlags; |
| |
| RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags); |
| |
| const size_t StartingResultSize = Result.size(); |
| |
| SmallPtrSet<const Operand *, 4> VTypeOperands; |
| std::optional<AliasingConfigurations> SelfAliasing; |
| // Exegesis see instructions with tied operands being inherently serial. |
| // But for RVV instructions, those tied operands are passthru rather |
| // than real read operands. So we manually put dependency between |
| // destination (i.e. def) and any of the non-tied/SEW/policy/AVL/RM |
| // operands. |
| auto assignSerialRVVOperands = [&, this](InstructionTemplate &IT) { |
| // Initialize SelfAliasing on first use. |
| if (!SelfAliasing.has_value()) { |
| BitVector ExcludeRegs = ForbiddenRegisters; |
| ExcludeRegs |= AggregateRegisters; |
| SelfAliasing = AliasingConfigurations(Instr, Instr, ExcludeRegs); |
| bool EmptyUses = false; |
| for (auto &ARO : SelfAliasing->Configurations) { |
| auto &Uses = ARO.Uses; |
| for (auto ROA = Uses.begin(); ROA != Uses.end();) { |
| const Operand *Op = ROA->Op; |
| // Exclude tied operand(s). |
| if (Op->isTied()) { |
| ROA = Uses.erase(ROA); |
| continue; |
| } |
| |
| // Special handling for reduction operations: for a given reduction |
| // `vredop vd, vs2, vs1`, we don't want vd to be aliased with vs1 |
| // since we're only reading `vs1[0]` and many implementations |
| // optimize for this case (e.g. chaining). Instead, we're forcing |
| // it to create alias between vd and vs2. |
| if (isVectorReduction(BaseOpcode) && |
| // vs1's operand index is always 3. |
| Op->getIndex() == 3) { |
| ROA = Uses.erase(ROA); |
| continue; |
| } |
| |
| // Exclude any special operands like SEW and VL -- we've already |
| // assigned values to them. |
| if (VTypeOperands.count(Op)) { |
| ROA = Uses.erase(ROA); |
| continue; |
| } |
| ++ROA; |
| } |
| |
| // If any of the use operand candidate lists is empty, there is |
| // no point to assign self aliasing registers. |
| if (Uses.empty()) { |
| EmptyUses = true; |
| break; |
| } |
| } |
| if (EmptyUses) |
| SelfAliasing->Configurations.clear(); |
| } |
| |
| // This is a self aliasing instruction so defs and uses are from the same |
| // instance, hence twice IT in the following call. |
| if (!SelfAliasing->empty() && !SelfAliasing->hasImplicitAliasing()) |
| setRandomAliasing(*SelfAliasing, IT, IT); |
| }; |
| |
| // We are going to create a CodeTemplate (configuration) for each supported |
| // SEW, policy, and VL. |
| // FIXME: Account for EEW and EMUL. |
| SmallVector<std::optional<unsigned>, 4> Log2SEWs; |
| SmallVector<std::optional<unsigned>, 4> Policies; |
| SmallVector<std::optional<int>, 3> AVLs; |
| SmallVector<std::optional<unsigned>, 8> RoundingModes; |
| |
| bool HasSEWOp = RISCVII::hasSEWOp(TSFlags); |
| bool HasPolicyOp = RISCVII::hasVecPolicyOp(TSFlags); |
| bool HasVLOp = RISCVII::hasVLOp(TSFlags); |
| bool HasRMOp = RISCVII::hasRoundModeOp(TSFlags); |
| bool UsesVXRM = RISCVII::usesVXRM(TSFlags); |
| |
| if (HasSEWOp) { |
| const Operand &SEWOp = Instr.Operands[RISCVII::getSEWOpNum(MIDesc)]; |
| VTypeOperands.insert(&SEWOp); |
| |
| if (SEWOp.Info->OperandType == RISCVOp::OPERAND_SEW_MASK) { |
| // If it's a mask-producing instruction, the SEW operand is always zero. |
| Log2SEWs.push_back(0); |
| } else { |
| SmallVector<unsigned, 4> SEWCandidates; |
| |
| // (RVV spec 3.4.2) For fractional LMUL, the supported SEW are between |
| // [SEW_min, LMUL * ELEN]. |
| unsigned SEWUpperBound = |
| VLMul >= RISCVVType::LMUL_F8 ? multiplyLMul(ELEN, VLMul) : ELEN; |
| for (unsigned SEW = MinSEW; SEW <= SEWUpperBound; SEW <<= 1) { |
| SEWCandidates.push_back(SEW); |
| |
| // Some scheduling classes already integrate SEW; only put |
| // their corresponding SEW values at the SEW operands. |
| // NOTE: It is imperative to put this condition in the front, otherwise |
| // it is tricky and difficult to know if there is an integrated |
| // SEW after other rules are applied to filter the candidates. |
| const auto *RVVBase = |
| RISCVVInversePseudosTable::getBaseInfo(BaseOpcode, VLMul, SEW); |
| if (RVVBase && (RVVBase->Pseudo == VPseudoOpcode || |
| isMaskedSibling(VPseudoOpcode, RVVBase->Pseudo) || |
| isMaskedSibling(RVVBase->Pseudo, VPseudoOpcode))) { |
| // There is an integrated SEW, remove all but the SEW pushed last. |
| SEWCandidates.erase(SEWCandidates.begin(), SEWCandidates.end() - 1); |
| break; |
| } |
| } |
| |
| // Filter out some candidates. |
| for (auto SEW = SEWCandidates.begin(); SEW != SEWCandidates.end();) { |
| // For floating point operations, only select SEW of the supported FLEN. |
| if (isRVVFloatingPointOp(VPseudoOpcode)) { |
| bool Supported = false; |
| Supported |= isZvfhminZvfbfminOpcodes(BaseOpcode) && *SEW == 16; |
| Supported |= STI.hasFeature(RISCV::FeatureStdExtZvfh) && *SEW == 16; |
| Supported |= STI.hasFeature(RISCV::FeatureStdExtF) && *SEW == 32; |
| Supported |= STI.hasFeature(RISCV::FeatureStdExtD) && *SEW == 64; |
| if (!Supported) { |
| SEW = SEWCandidates.erase(SEW); |
| continue; |
| } |
| } |
| |
| // The EEW for source operand in VSEXT and VZEXT is a fraction |
| // of the SEW, hence only SEWs that will lead to valid EEW are allowed. |
| if (auto Frac = isRVVSignZeroExtend(BaseOpcode)) |
| if (*SEW / *Frac < MinSEW) { |
| SEW = SEWCandidates.erase(SEW); |
| continue; |
| } |
| |
| // Most vector crypto 1.0 instructions only work on SEW=32. |
| using namespace RISCV_MC; |
| if (isOpcodeAvailableIn(BaseOpcode, {Feature_HasStdExtZvkgBit, |
| Feature_HasStdExtZvknedBit, |
| Feature_HasStdExtZvknhaOrZvknhbBit, |
| Feature_HasStdExtZvksedBit, |
| Feature_HasStdExtZvkshBit})) { |
| if (*SEW != 32) |
| // Zvknhb supports SEW=64 as well. |
| if (*SEW != 64 || !STI.hasFeature(RISCV::FeatureStdExtZvknhb) || |
| !isOpcodeAvailableIn(BaseOpcode, |
| {Feature_HasStdExtZvknhaOrZvknhbBit})) { |
| SEW = SEWCandidates.erase(SEW); |
| continue; |
| } |
| |
| // We're also enforcing the requirement of `LMUL * VLEN >= EGW` here, |
| // because some of the extensions have SEW-dependant EGW. |
| unsigned EGW = getZvkEGWSize(BaseOpcode, *SEW); |
| if (multiplyLMul(ZvlVLen, VLMul) < EGW) { |
| SEW = SEWCandidates.erase(SEW); |
| continue; |
| } |
| } |
| |
| ++SEW; |
| } |
| |
| // We're not going to produce any result with zero SEW candidate. |
| if (SEWCandidates.empty()) |
| return; |
| |
| for (unsigned SEW : SEWCandidates) |
| Log2SEWs.push_back(Log2_32(SEW)); |
| } |
| } else { |
| Log2SEWs.push_back(std::nullopt); |
| } |
| |
| if (HasPolicyOp) { |
| VTypeOperands.insert(&Instr.Operands[RISCVII::getVecPolicyOpNum(MIDesc)]); |
| |
| Policies = {0, RISCVVType::TAIL_AGNOSTIC, RISCVVType::MASK_AGNOSTIC, |
| (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)}; |
| } else { |
| Policies.push_back(std::nullopt); |
| } |
| |
| if (HasVLOp) { |
| VTypeOperands.insert(&Instr.Operands[RISCVII::getVLOpNum(MIDesc)]); |
| |
| if (OnlyUsesVLMAXForVL) |
| AVLs.push_back(-1); |
| else |
| AVLs = {// 5-bit immediate value |
| 1, |
| // VLMAX |
| -1, |
| // Non-X0 register |
| 0}; |
| } else { |
| AVLs.push_back(std::nullopt); |
| } |
| |
| if (HasRMOp) { |
| VTypeOperands.insert(&Instr.Operands[RISCVII::getVLOpNum(MIDesc) - 1]); |
| |
| if (UsesVXRM) { |
| // Use RNU as the default VXRM. |
| RoundingModes = {RISCVVXRndMode::RNU}; |
| if (EnumerateRoundingModes) |
| RoundingModes.append( |
| {RISCVVXRndMode::RNE, RISCVVXRndMode::RDN, RISCVVXRndMode::ROD}); |
| } else { |
| if (EnumerateRoundingModes) |
| RoundingModes = {RISCVFPRndMode::RNE, RISCVFPRndMode::RTZ, |
| RISCVFPRndMode::RDN, RISCVFPRndMode::RUP, |
| RISCVFPRndMode::RMM}; |
| else |
| // If we're not enumerating FRM, use DYN to instruct |
| // RISCVInsertReadWriteCSRPass to insert nothing. |
| RoundingModes = {RISCVFPRndMode::DYN}; |
| } |
| } else { |
| RoundingModes = {std::nullopt}; |
| } |
| |
| std::set<std::tuple<std::optional<unsigned>, std::optional<int>, |
| std::optional<unsigned>, std::optional<unsigned>>> |
| Combinations; |
| for (auto AVL : AVLs) { |
| for (auto Log2SEW : Log2SEWs) |
| for (auto Policy : Policies) { |
| for (auto RM : RoundingModes) |
| Combinations.insert(std::make_tuple(RM, AVL, Log2SEW, Policy)); |
| } |
| } |
| |
| std::string ConfigStr; |
| SmallVector<std::pair<const Operand *, MCOperand>, 4> ValueAssignments; |
| for (const auto &[RM, AVL, Log2SEW, Policy] : Combinations) { |
| InstructionTemplate IT(&Instr); |
| |
| ListSeparator LS; |
| ConfigStr = "vtype = {"; |
| raw_string_ostream SS(ConfigStr); |
| |
| ValueAssignments.clear(); |
| |
| if (RM) { |
| const Operand &Op = Instr.Operands[RISCVII::getVLOpNum(MIDesc) - 1]; |
| ValueAssignments.push_back({&Op, MCOperand::createImm(*RM)}); |
| printRoundingMode(SS << LS << (UsesVXRM ? "VXRM" : "FRM") << ": ", *RM, |
| UsesVXRM); |
| } |
| |
| if (AVL) { |
| MCOperand OpVal; |
| if (*AVL < 0) { |
| // VLMAX |
| OpVal = MCOperand::createImm(-1); |
| SS << LS << "AVL: VLMAX"; |
| } else if (*AVL == 0) { |
| // A register holding AVL. |
| // TODO: Generate a random register. |
| OpVal = MCOperand::createReg(RISCV::X5); |
| OpVal.print(SS << LS << "AVL: "); |
| } else { |
| // A 5-bit immediate. |
| // The actual value assignment is deferred to |
| // RISCVExegesisTarget::randomizeTargetMCOperand. |
| SS << LS << "AVL: simm5"; |
| } |
| if (OpVal.isValid()) { |
| const Operand &Op = Instr.Operands[RISCVII::getVLOpNum(MIDesc)]; |
| ValueAssignments.push_back({&Op, OpVal}); |
| } |
| } |
| |
| if (Log2SEW) { |
| const Operand &Op = Instr.Operands[RISCVII::getSEWOpNum(MIDesc)]; |
| ValueAssignments.push_back({&Op, MCOperand::createImm(*Log2SEW)}); |
| SS << LS << "SEW: e" << (*Log2SEW ? 1 << *Log2SEW : 8); |
| } |
| |
| if (Policy) { |
| const Operand &Op = Instr.Operands[RISCVII::getVecPolicyOpNum(MIDesc)]; |
| ValueAssignments.push_back({&Op, MCOperand::createImm(*Policy)}); |
| SS << LS |
| << "Policy: " << (*Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") |
| << "/" << (*Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu"); |
| } |
| |
| SS << "}"; |
| |
| // Filter out some configurations, if needed. |
| if (!FilterConfig.empty()) { |
| if (!Regex(FilterConfig).match(ConfigStr)) |
| continue; |
| } |
| |
| CodeTemplate CT = OrigCT.clone(); |
| CT.Config = std::move(ConfigStr); |
| for (InstructionTemplate &IT : CT.Instructions) { |
| if (IsSerial) { |
| // Reset this template's value assignments and do it |
| // ourselves. |
| IT = InstructionTemplate(&Instr); |
| assignSerialRVVOperands(IT); |
| } |
| |
| for (const auto &[Op, OpVal] : ValueAssignments) |
| IT.getValueFor(*Op) = OpVal; |
| } |
| Result.push_back(std::move(CT)); |
| if (Result.size() - StartingResultSize >= |
| SnippetGenerator::Opts.MaxConfigsPerOpcode) |
| return; |
| } |
| } |
| |
| template <class BaseT> |
| Expected<std::vector<CodeTemplate>> |
| RISCVSnippetGenerator<BaseT>::generateCodeTemplates( |
| InstructionTemplate Variant, const BitVector &ForbiddenRegisters) const { |
| const Instruction &Instr = Variant.getInstr(); |
| |
| bool IsSerial = std::is_same_v<BaseT, SerialSnippetGenerator>; |
| |
| unsigned BaseOpcode = RISCV::getRVVMCOpcode(Instr.getOpcode()); |
| |
| // Bail out ineligible opcodes before generating base code templates since |
| // the latter is quite expensive. |
| if (IsSerial && BaseOpcode && isIneligibleOfSerialSnippets(BaseOpcode, Instr)) |
| return std::vector<CodeTemplate>{}; |
| |
| auto BaseCodeTemplates = |
| BaseT::generateCodeTemplates(Variant, ForbiddenRegisters); |
| if (!BaseCodeTemplates) |
| return BaseCodeTemplates.takeError(); |
| |
| if (!BaseOpcode) |
| return BaseCodeTemplates; |
| |
| // Specialize for RVV pseudo. |
| std::vector<CodeTemplate> ExpandedTemplates; |
| for (const auto &BaseCT : *BaseCodeTemplates) |
| annotateWithVType(BaseCT, Instr, BaseOpcode, ForbiddenRegisters, |
| ExpandedTemplates); |
| |
| return ExpandedTemplates; |
| } |
| |
| // Stores constant value to a general-purpose (integer) register. |
| static std::vector<MCInst> loadIntReg(const MCSubtargetInfo &STI, |
| MCRegister Reg, const APInt &Value) { |
| SmallVector<MCInst, 8> MCInstSeq; |
| MCRegister DestReg = Reg; |
| |
| RISCVMatInt::generateMCInstSeq(Value.getSExtValue(), STI, DestReg, MCInstSeq); |
| |
| std::vector<MCInst> MatIntInstrs(MCInstSeq.begin(), MCInstSeq.end()); |
| return MatIntInstrs; |
| } |
| |
| const MCPhysReg ScratchIntReg = RISCV::X30; // t5 |
| |
| // Stores constant bits to a floating-point register. |
| static std::vector<MCInst> loadFPRegBits(const MCSubtargetInfo &STI, |
| MCRegister Reg, const APInt &Bits, |
| unsigned FmvOpcode) { |
| std::vector<MCInst> Instrs = loadIntReg(STI, ScratchIntReg, Bits); |
| Instrs.push_back(MCInstBuilder(FmvOpcode).addReg(Reg).addReg(ScratchIntReg)); |
| return Instrs; |
| } |
| |
| // main idea is: |
| // we support APInt only if (represented as double) it has zero fractional |
| // part: 1.0, 2.0, 3.0, etc... then we can do the trick: write int to tmp reg t5 |
| // and then do FCVT this is only reliable thing in 32-bit mode, otherwise we |
| // need to use __floatsidf |
| static std::vector<MCInst> loadFP64RegBits32(const MCSubtargetInfo &STI, |
| MCRegister Reg, |
| const APInt &Bits) { |
| double D = Bits.bitsToDouble(); |
| double IPart; |
| double FPart = std::modf(D, &IPart); |
| |
| if (std::abs(FPart) > std::numeric_limits<double>::epsilon()) { |
| errs() << "loadFP64RegBits32 is not implemented for doubles like " << D |
| << ", please remove fractional part\n"; |
| return {}; |
| } |
| |
| std::vector<MCInst> Instrs = loadIntReg(STI, ScratchIntReg, Bits); |
| Instrs.push_back(MCInstBuilder(RISCV::FCVT_D_W) |
| .addReg(Reg) |
| .addReg(ScratchIntReg) |
| .addImm(RISCVFPRndMode::RNE)); |
| return Instrs; |
| } |
| |
| class ExegesisRISCVTarget : public ExegesisTarget { |
| // NOTE: Alternatively, we can use BitVector here, but the number of RVV MC |
| // opcodes is just a small portion of the entire opcode space, so I thought it |
| // would be a waste of space to use BitVector. |
| mutable SmallSet<unsigned, 16> RVVMCOpcodesWithPseudos; |
| |
| public: |
| ExegesisRISCVTarget(); |
| |
| bool matchesArch(Triple::ArchType Arch) const override; |
| |
| std::vector<MCInst> setRegTo(const MCSubtargetInfo &STI, MCRegister Reg, |
| const APInt &Value) const override; |
| |
| const char *getIgnoredOpcodeReasonOrNull(const LLVMState &State, |
| unsigned Opcode) const override { |
| // We don't want to support RVV instructions that depend on VTYPE, because |
| // those instructions by themselves don't carry any additional information |
| // for us to setup the proper VTYPE environment via VSETVL instructions. |
| // FIXME: Ideally, we should use RISCVVInversePseudosTable, but it requires |
| // LMUL and SEW and I don't think enumerating those combinations is any |
| // better than the ugly trick here that memorizes the corresponding MC |
| // opcodes of the RVV pseudo we have processed previously. This works most |
| // of the time because RVV pseudo opcodes are placed before any other RVV |
| // opcodes. Of course this doesn't work if we're asked to benchmark only a |
| // certain subset of opcodes. |
| if (RVVMCOpcodesWithPseudos.count(Opcode)) |
| return "The MC opcode of RVV instructions are ignored"; |
| |
| // We want to support all RVV pseudos. |
| if (unsigned MCOpcode = RISCV::getRVVMCOpcode(Opcode)) { |
| RVVMCOpcodesWithPseudos.insert(MCOpcode); |
| return nullptr; |
| } |
| |
| return ExegesisTarget::getIgnoredOpcodeReasonOrNull(State, Opcode); |
| } |
| |
| MCRegister getDefaultLoopCounterRegister(const Triple &) const override; |
| |
| void decrementLoopCounterAndJump(MachineBasicBlock &MBB, |
| MachineBasicBlock &TargetMBB, |
| const MCInstrInfo &MII, |
| MCRegister LoopRegister) const override; |
| |
| MCRegister getScratchMemoryRegister(const Triple &TT) const override; |
| |
| void fillMemoryOperands(InstructionTemplate &IT, MCRegister Reg, |
| unsigned Offset) const override; |
| |
| ArrayRef<MCPhysReg> getUnavailableRegisters() const override; |
| |
| bool allowAsBackToBack(const Instruction &Instr) const override { |
| return !Instr.Description.isPseudo(); |
| } |
| |
| Error randomizeTargetMCOperand(const Instruction &Instr, const Variable &Var, |
| MCOperand &AssignedValue, |
| const BitVector &ForbiddenRegs) const override; |
| |
| std::unique_ptr<SnippetGenerator> createSerialSnippetGenerator( |
| const LLVMState &State, |
| const SnippetGenerator::Options &Opts) const override { |
| return std::make_unique<RISCVSnippetGenerator<SerialSnippetGenerator>>( |
| State, Opts); |
| } |
| |
| std::unique_ptr<SnippetGenerator> createParallelSnippetGenerator( |
| const LLVMState &State, |
| const SnippetGenerator::Options &Opts) const override { |
| return std::make_unique<RISCVSnippetGenerator<ParallelSnippetGenerator>>( |
| State, Opts); |
| } |
| |
| std::vector<InstructionTemplate> |
| generateInstructionVariants(const Instruction &Instr, |
| unsigned MaxConfigsPerOpcode) const override; |
| |
| void addTargetSpecificPasses(PassManagerBase &PM) const override { |
| // Turn AVL operand of physical registers into virtual registers. |
| PM.add(exegesis::createRISCVPreprocessingPass()); |
| PM.add(createRISCVInsertVSETVLIPass()); |
| // Setting up the correct FRM. |
| PM.add(createRISCVInsertReadWriteCSRPass()); |
| PM.add(createRISCVInsertWriteVXRMPass()); |
| // This will assign physical register to the result of VSETVLI instructions |
| // that produce VLMAX. |
| PM.add(exegesis::createRISCVPostprocessingPass()); |
| // PseudoRET will be expanded by RISCVAsmPrinter; we have to expand |
| // PseudoMovImm with RISCVPostRAExpandPseudoPass though. |
| PM.add(createRISCVPostRAExpandPseudoPass()); |
| } |
| }; |
| |
| ExegesisRISCVTarget::ExegesisRISCVTarget() |
| : ExegesisTarget(RISCVCpuPfmCounters, RISCV_MC::isOpcodeAvailable) {} |
| |
| bool ExegesisRISCVTarget::matchesArch(Triple::ArchType Arch) const { |
| return Arch == Triple::riscv32 || Arch == Triple::riscv64; |
| } |
| |
| std::vector<MCInst> ExegesisRISCVTarget::setRegTo(const MCSubtargetInfo &STI, |
| MCRegister Reg, |
| const APInt &Value) const { |
| if (RISCV::GPRRegClass.contains(Reg)) |
| return loadIntReg(STI, Reg, Value); |
| if (RISCV::FPR16RegClass.contains(Reg)) |
| return loadFPRegBits(STI, Reg, Value, RISCV::FMV_H_X); |
| if (RISCV::FPR32RegClass.contains(Reg)) |
| return loadFPRegBits(STI, Reg, Value, RISCV::FMV_W_X); |
| if (RISCV::FPR64RegClass.contains(Reg)) { |
| if (STI.hasFeature(RISCV::Feature64Bit)) |
| return loadFPRegBits(STI, Reg, Value, RISCV::FMV_D_X); |
| return loadFP64RegBits32(STI, Reg, Value); |
| } |
| // TODO: Emit proper code to initialize other kinds of registers. |
| return {}; |
| } |
| |
| const MCPhysReg DefaultLoopCounterReg = RISCV::X31; // t6 |
| const MCPhysReg ScratchMemoryReg = RISCV::X10; // a0 |
| |
| MCRegister |
| ExegesisRISCVTarget::getDefaultLoopCounterRegister(const Triple &) const { |
| return DefaultLoopCounterReg; |
| } |
| |
| void ExegesisRISCVTarget::decrementLoopCounterAndJump( |
| MachineBasicBlock &MBB, MachineBasicBlock &TargetMBB, |
| const MCInstrInfo &MII, MCRegister LoopRegister) const { |
| BuildMI(&MBB, DebugLoc(), MII.get(RISCV::ADDI)) |
| .addDef(LoopRegister) |
| .addUse(LoopRegister) |
| .addImm(-1); |
| BuildMI(&MBB, DebugLoc(), MII.get(RISCV::BNE)) |
| .addUse(LoopRegister) |
| .addUse(RISCV::X0) |
| .addMBB(&TargetMBB); |
| } |
| |
| MCRegister |
| ExegesisRISCVTarget::getScratchMemoryRegister(const Triple &TT) const { |
| return ScratchMemoryReg; // a0 |
| } |
| |
| void ExegesisRISCVTarget::fillMemoryOperands(InstructionTemplate &IT, |
| MCRegister Reg, |
| unsigned Offset) const { |
| // TODO: for now we ignore Offset because have no way |
| // to detect it in instruction. |
| auto &I = IT.getInstr(); |
| |
| auto MemOpIt = |
| find_if(I.Operands, [](const Operand &Op) { return Op.isMemory(); }); |
| assert(MemOpIt != I.Operands.end() && |
| "Instruction must have memory operands"); |
| |
| const Operand &MemOp = *MemOpIt; |
| |
| assert(MemOp.isReg() && "Memory operand expected to be register"); |
| |
| IT.getValueFor(MemOp) = MCOperand::createReg(Reg); |
| } |
| |
| const MCPhysReg UnavailableRegisters[4] = {RISCV::X0, DefaultLoopCounterReg, |
| ScratchIntReg, ScratchMemoryReg}; |
| |
| ArrayRef<MCPhysReg> ExegesisRISCVTarget::getUnavailableRegisters() const { |
| return UnavailableRegisters; |
| } |
| |
| Error ExegesisRISCVTarget::randomizeTargetMCOperand( |
| const Instruction &Instr, const Variable &Var, MCOperand &AssignedValue, |
| const BitVector &ForbiddenRegs) const { |
| uint8_t OperandType = |
| Instr.getPrimaryOperand(Var).getExplicitOperandInfo().OperandType; |
| |
| switch (OperandType) { |
| case RISCVOp::OPERAND_FRMARG: |
| AssignedValue = MCOperand::createImm(RISCVFPRndMode::DYN); |
| break; |
| case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO: |
| AssignedValue = MCOperand::createImm(0b1 << 4); |
| break; |
| case RISCVOp::OPERAND_SIMM6_NONZERO: |
| case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO: |
| AssignedValue = MCOperand::createImm(1); |
| break; |
| case RISCVOp::OPERAND_SIMM5: |
| // 5-bit signed immediate value. |
| AssignedValue = MCOperand::createImm(randomIndex(31) - 16); |
| break; |
| case RISCVOp::OPERAND_AVL: |
| case RISCVOp::OPERAND_UIMM5: |
| // 5-bit unsigned immediate value. |
| AssignedValue = MCOperand::createImm(randomIndex(31)); |
| break; |
| default: |
| if (OperandType >= RISCVOp::OPERAND_FIRST_RISCV_IMM && |
| OperandType <= RISCVOp::OPERAND_LAST_RISCV_IMM) |
| AssignedValue = MCOperand::createImm(0); |
| } |
| return Error::success(); |
| } |
| |
| std::vector<InstructionTemplate> |
| ExegesisRISCVTarget::generateInstructionVariants( |
| const Instruction &Instr, unsigned int MaxConfigsPerOpcode) const { |
| InstructionTemplate IT{&Instr}; |
| for (const Operand &Op : Instr.Operands) |
| if (Op.isMemory()) { |
| IT.getValueFor(Op) = MCOperand::createReg(ScratchMemoryReg); |
| } |
| return {IT}; |
| } |
| |
| } // anonymous namespace |
| |
| static ExegesisTarget *getTheRISCVExegesisTarget() { |
| static ExegesisRISCVTarget Target; |
| return &Target; |
| } |
| |
| void InitializeRISCVExegesisTarget() { |
| ExegesisTarget::registerTarget(getTheRISCVExegesisTarget()); |
| } |
| |
| } // namespace exegesis |
| } // namespace llvm |
