| //===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file provides a simple and efficient mechanism for performing general | |
| // tree-based pattern matches on the VPlan values and recipes, based on | |
| // LLVM's IR pattern matchers. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H | |
| #define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H | |
| #include "VPlan.h" | |
| namespace llvm::VPlanPatternMatch { | |
| template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) { | |
| return P.match(V); | |
| } | |
| /// A match functor that can be used as a UnaryPredicate in functional | |
| /// algorithms like all_of. | |
| template <typename Val, typename Pattern> auto match_fn(const Pattern &P) { | |
| return bind_back<match<Val, Pattern>>(P); | |
| } | |
| template <typename Pattern> bool match(VPUser *U, const Pattern &P) { | |
| auto *R = dyn_cast<VPRecipeBase>(U); | |
| return R && match(R, P); | |
| } | |
| /// Match functor for VPUser. | |
| template <typename Pattern> auto match_fn(const Pattern &P) { | |
| return bind_back<match<Pattern>>(P); | |
| } | |
| template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) { | |
| return P.match(static_cast<const VPRecipeBase *>(R)); | |
| } | |
| template <typename... Classes> struct class_match { | |
| template <typename ITy> bool match(ITy *V) const { | |
| return isa<Classes...>(V); | |
| } | |
| }; | |
| /// Match an arbitrary VPValue and ignore it. | |
| inline class_match<VPValue> m_VPValue() { return class_match<VPValue>(); } | |
| template <typename Class> struct bind_ty { | |
| Class *&VR; | |
| bind_ty(Class *&V) : VR(V) {} | |
| template <typename ITy> bool match(ITy *V) const { | |
| if (auto *CV = dyn_cast<Class>(V)) { | |
| VR = CV; | |
| return true; | |
| } | |
| return false; | |
| } | |
| }; | |
| /// Match a specified VPValue. | |
| struct specificval_ty { | |
| const VPValue *Val; | |
| specificval_ty(const VPValue *V) : Val(V) {} | |
| bool match(VPValue *VPV) const { return VPV == Val; } | |
| }; | |
| inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; } | |
| /// Stores a reference to the VPValue *, not the VPValue * itself, | |
| /// thus can be used in commutative matchers. | |
| struct deferredval_ty { | |
| VPValue *const &Val; | |
| deferredval_ty(VPValue *const &V) : Val(V) {} | |
| bool match(VPValue *const V) const { return V == Val; } | |
| }; | |
| /// Like m_Specific(), but works if the specific value to match is determined | |
| /// as part of the same match() expression. For example: | |
| /// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will | |
| /// bind X before the pattern match starts. | |
| /// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against | |
| /// whichever value m_VPValue(X) populated. | |
| inline deferredval_ty m_Deferred(VPValue *const &V) { return V; } | |
| /// Match an integer constant or vector of constants if Pred::isValue returns | |
| /// true for the APInt. \p BitWidth optionally specifies the bitwidth the | |
| /// matched constant must have. If it is 0, the matched constant can have any | |
| /// bitwidth. | |
| template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty { | |
| Pred P; | |
| int_pred_ty(Pred P) : P(std::move(P)) {} | |
| int_pred_ty() : P() {} | |
| bool match(VPValue *VPV) const { | |
| auto *VPI = dyn_cast<VPInstruction>(VPV); | |
| if (VPI && VPI->getOpcode() == VPInstruction::Broadcast) | |
| VPV = VPI->getOperand(0); | |
| auto *CI = dyn_cast<VPConstantInt>(VPV); | |
| if (!CI) | |
| return false; | |
| if (BitWidth != 0 && CI->getBitWidth() != BitWidth) | |
| return false; | |
| return P.isValue(CI->getAPInt()); | |
| } | |
| }; | |
| /// Match a specified integer value or vector of all elements of that | |
| /// value. \p BitWidth optionally specifies the bitwidth the matched constant | |
| /// must have. If it is 0, the matched constant can have any bitwidth. | |
| struct is_specific_int { | |
| APInt Val; | |
| is_specific_int(APInt Val) : Val(std::move(Val)) {} | |
| bool isValue(const APInt &C) const { return APInt::isSameValue(Val, C); } | |
| }; | |
| template <unsigned Bitwidth = 0> | |
| using specific_intval = int_pred_ty<is_specific_int, Bitwidth>; | |
| inline specific_intval<0> m_SpecificInt(uint64_t V) { | |
| return specific_intval<0>(is_specific_int(APInt(64, V))); | |
| } | |
| inline specific_intval<1> m_False() { | |
| return specific_intval<1>(is_specific_int(APInt(64, 0))); | |
| } | |
| inline specific_intval<1> m_True() { | |
| return specific_intval<1>(is_specific_int(APInt(64, 1))); | |
| } | |
| struct is_all_ones { | |
| bool isValue(const APInt &C) const { return C.isAllOnes(); } | |
| }; | |
| /// Match an integer or vector with all bits set. | |
| /// For vectors, this includes constants with undefined elements. | |
| inline int_pred_ty<is_all_ones> m_AllOnes() { | |
| return int_pred_ty<is_all_ones>(); | |
| } | |
| struct is_zero_int { | |
| bool isValue(const APInt &C) const { return C.isZero(); } | |
| }; | |
| struct is_one { | |
| bool isValue(const APInt &C) const { return C.isOne(); } | |
| }; | |
| /// Match an integer 0 or a vector with all elements equal to 0. | |
| /// For vectors, this includes constants with undefined elements. | |
| inline int_pred_ty<is_zero_int> m_ZeroInt() { | |
| return int_pred_ty<is_zero_int>(); | |
| } | |
| /// Match an integer 1 or a vector with all elements equal to 1. | |
| /// For vectors, this includes constants with undefined elements. | |
| inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); } | |
| struct bind_apint { | |
| const APInt *&Res; | |
| bind_apint(const APInt *&Res) : Res(Res) {} | |
| bool match(VPValue *VPV) const { | |
| auto *CI = dyn_cast<VPConstantInt>(VPV); | |
| if (!CI) | |
| return false; | |
| Res = &CI->getAPInt(); | |
| return true; | |
| } | |
| }; | |
| inline bind_apint m_APInt(const APInt *&C) { return C; } | |
| struct bind_const_int { | |
| uint64_t &Res; | |
| bind_const_int(uint64_t &Res) : Res(Res) {} | |
| bool match(VPValue *VPV) const { | |
| const APInt *APConst; | |
| if (!bind_apint(APConst).match(VPV)) | |
| return false; | |
| if (auto C = APConst->tryZExtValue()) { | |
| Res = *C; | |
| return true; | |
| } | |
| return false; | |
| } | |
| }; | |
| struct match_poison { | |
| bool match(VPValue *V) const { | |
| return isa<VPIRValue>(V) && | |
| isa<PoisonValue>(cast<VPIRValue>(V)->getValue()); | |
| } | |
| }; | |
| /// Match a VPIRValue that's poison. | |
| inline match_poison m_Poison() { return match_poison(); } | |
| /// Match a plain integer constant no wider than 64-bits, capturing it if we | |
| /// match. | |
| inline bind_const_int m_ConstantInt(uint64_t &C) { return C; } | |
| /// Matching combinators | |
| template <typename LTy, typename RTy> struct match_combine_or { | |
| LTy L; | |
| RTy R; | |
| match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | |
| template <typename ITy> bool match(ITy *V) const { | |
| return L.match(V) || R.match(V); | |
| } | |
| }; | |
| template <typename LTy, typename RTy> struct match_combine_and { | |
| LTy L; | |
| RTy R; | |
| match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {} | |
| template <typename ITy> bool match(ITy *V) const { | |
| return L.match(V) && R.match(V); | |
| } | |
| }; | |
| /// Combine two pattern matchers matching L || R | |
| template <typename LTy, typename RTy> | |
| inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) { | |
| return match_combine_or<LTy, RTy>(L, R); | |
| } | |
| /// Combine two pattern matchers matching L && R | |
| template <typename LTy, typename RTy> | |
| inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) { | |
| return match_combine_and<LTy, RTy>(L, R); | |
| } | |
| /// Match a VPValue, capturing it if we match. | |
| inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; } | |
| /// Match a VPIRValue. | |
| inline bind_ty<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; } | |
| /// Match a VPInstruction, capturing if we match. | |
| inline bind_ty<VPInstruction> m_VPInstruction(VPInstruction *&V) { return V; } | |
| template <typename Ops_t, unsigned Opcode, bool Commutative, | |
| typename... RecipeTys> | |
| struct Recipe_match { | |
| Ops_t Ops; | |
| template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) { | |
| static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) && | |
| "number of operands in constructor doesn't match Ops_t"); | |
| static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) && | |
| "only binary ops can be commutative"); | |
| } | |
| bool match(const VPValue *V) const { | |
| auto *DefR = V->getDefiningRecipe(); | |
| return DefR && match(DefR); | |
| } | |
| bool match(const VPSingleDefRecipe *R) const { | |
| return match(static_cast<const VPRecipeBase *>(R)); | |
| } | |
| bool match(const VPRecipeBase *R) const { | |
| if (std::tuple_size_v<Ops_t> == 0) { | |
| auto *VPI = dyn_cast<VPInstruction>(R); | |
| return VPI && VPI->getOpcode() == Opcode; | |
| } | |
| if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...)) | |
| return false; | |
| if (R->getNumOperands() < std::tuple_size<Ops_t>::value) { | |
| [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(R); | |
| assert(((isa<VPInstruction>(R) && | |
| cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) || | |
| (RepR && std::tuple_size_v<Ops_t> == | |
| RepR->getNumOperands() - RepR->isPredicated())) && | |
| "non-variadic recipe with matched opcode does not have the " | |
| "expected number of operands"); | |
| return false; | |
| } | |
| // If the recipe has more operands than expected, we only support matching | |
| // masked VPInstructions where the number of operands of the matcher is the | |
| // same as the number of operands excluding mask. | |
| if (R->getNumOperands() > std::tuple_size<Ops_t>::value) { | |
| auto *VPI = dyn_cast<VPInstruction>(R); | |
| if (!VPI || !VPI->isMasked() || | |
| VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value) | |
| return false; | |
| } | |
| auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>(); | |
| if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) { | |
| return Op.match(R->getOperand(Idx)); | |
| })) | |
| return true; | |
| return Commutative && | |
| all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) { | |
| return Op.match(R->getOperand(R->getNumOperands() - Idx - 1)); | |
| }); | |
| } | |
| private: | |
| template <typename RecipeTy> | |
| static bool matchRecipeAndOpcode(const VPRecipeBase *R) { | |
| auto *DefR = dyn_cast<RecipeTy>(R); | |
| // Check for recipes that do not have opcodes. | |
| if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> || | |
| std::is_same_v<RecipeTy, VPCanonicalIVPHIRecipe> || | |
| std::is_same_v<RecipeTy, VPDerivedIVRecipe> || | |
| std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>) | |
| return DefR; | |
| else | |
| return DefR && DefR->getOpcode() == Opcode; | |
| } | |
| /// Helper to check if predicate \p P holds on all tuple elements in Ops using | |
| /// the provided index sequence. | |
| template <typename Fn, std::size_t... Is> | |
| bool all_of_tuple_elements(std::index_sequence<Is...>, Fn P) const { | |
| return (P(std::get<Is>(Ops), Is) && ...); | |
| } | |
| }; | |
| template <unsigned Opcode, typename... OpTys> | |
| using AllRecipe_match = | |
| Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false, | |
| VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe, | |
| VPInstruction>; | |
| template <unsigned Opcode, typename... OpTys> | |
| using AllRecipe_commutative_match = | |
| Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true, | |
| VPWidenRecipe, VPReplicateRecipe, VPInstruction>; | |
| template <unsigned Opcode, typename... OpTys> | |
| using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode, | |
| /*Commutative*/ false, VPInstruction>; | |
| template <unsigned Opcode, typename... OpTys> | |
| inline VPInstruction_match<Opcode, OpTys...> | |
| m_VPInstruction(const OpTys &...Ops) { | |
| return VPInstruction_match<Opcode, OpTys...>(Ops...); | |
| } | |
| /// BuildVector is matches only its opcode, w/o matching its operands as the | |
| /// number of operands is not fixed. | |
| inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() { | |
| return m_VPInstruction<VPInstruction::BuildVector>(); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<Instruction::Freeze, Op0_t> | |
| m_Freeze(const Op0_t &Op0) { | |
| return m_VPInstruction<Instruction::Freeze>(Op0); | |
| } | |
| inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() { | |
| return m_VPInstruction<VPInstruction::BranchOnCond>(); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t> | |
| m_BranchOnCond(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::BranchOnCond>(Op0); | |
| } | |
| inline VPInstruction_match<VPInstruction::BranchOnTwoConds> | |
| m_BranchOnTwoConds() { | |
| return m_VPInstruction<VPInstruction::BranchOnTwoConds>(); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t> | |
| m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::Broadcast, Op0_t> | |
| m_Broadcast(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::Broadcast>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t> | |
| m_EVL(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t> | |
| m_ExtractLastLane(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t> | |
| m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t> | |
| m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t> | |
| m_ExtractLastPart(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match< | |
| VPInstruction::ExtractLastLane, | |
| VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>> | |
| m_ExtractLastLaneOfLastPart(const Op0_t &Op0) { | |
| return m_ExtractLastLane(m_ExtractLastPart(Op0)); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t> | |
| m_ExtractPenultimateElement(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0); | |
| } | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t> | |
| m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { | |
| return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2); | |
| } | |
| inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() { | |
| return m_VPInstruction<VPInstruction::BranchOnCount>(); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t> | |
| m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1); | |
| } | |
| inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() { | |
| return m_VPInstruction<VPInstruction::AnyOf>(); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::AnyOf, Op0_t> | |
| m_AnyOf(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::AnyOf>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t> | |
| m_FirstActiveLane(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t> | |
| m_LastActiveLane(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::LastActiveLane>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t> | |
| m_ComputeReductionResult(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0); | |
| } | |
| /// Match FindIV result pattern: | |
| /// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel), | |
| /// ComputeReductionResult(ReducedIV), Start. | |
| template <typename Op0_t, typename Op1_t> | |
| inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) { | |
| return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE, | |
| m_ComputeReductionResult(ReducedIV), | |
| m_VPValue()), | |
| m_ComputeReductionResult(ReducedIV), Start)); | |
| } | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| inline VPInstruction_match<VPInstruction::ComputeAnyOfResult, Op0_t, Op1_t, | |
| Op2_t> | |
| m_ComputeAnyOfResult(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { | |
| return m_VPInstruction<VPInstruction::ComputeAnyOfResult>(Op0, Op1, Op2); | |
| } | |
| template <typename Op0_t> | |
| inline VPInstruction_match<VPInstruction::Reverse, Op0_t> | |
| m_Reverse(const Op0_t &Op0) { | |
| return m_VPInstruction<VPInstruction::Reverse>(Op0); | |
| } | |
| inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() { | |
| return m_VPInstruction<VPInstruction::StepVector>(); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t, Op1_t> | |
| m_ExitingIVValue(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0, Op1); | |
| } | |
| template <unsigned Opcode, typename Op0_t> | |
| inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) { | |
| return AllRecipe_match<Opcode, Op0_t>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) { | |
| return m_Unary<Instruction::Trunc, Op0_t>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t> | |
| m_TruncOrSelf(const Op0_t &Op0) { | |
| return m_CombineOr(m_Trunc(Op0), Op0); | |
| } | |
| template <typename Op0_t> | |
| inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) { | |
| return m_Unary<Instruction::ZExt, Op0_t>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) { | |
| return m_Unary<Instruction::SExt, Op0_t>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) { | |
| return m_Unary<Instruction::FPExt, Op0_t>(Op0); | |
| } | |
| template <typename Op0_t> | |
| inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, | |
| AllRecipe_match<Instruction::SExt, Op0_t>> | |
| m_ZExtOrSExt(const Op0_t &Op0) { | |
| return m_CombineOr(m_ZExt(Op0), m_SExt(Op0)); | |
| } | |
| template <typename Op0_t> | |
| inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t> | |
| m_ZExtOrSelf(const Op0_t &Op0) { | |
| return m_CombineOr(m_ZExt(Op0), Op0); | |
| } | |
| template <unsigned Opcode, typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <unsigned Opcode, typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t> | |
| m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) { | |
| return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t> | |
| m_c_Add(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t> | |
| m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t> | |
| m_FMul(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t> | |
| m_FAdd(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t> | |
| m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t> | |
| m_UDiv(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| /// Match a binary AND operation. | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t> | |
| m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| /// Match a binary OR operation. Note that while conceptually the operands can | |
| /// be matched commutatively, \p Commutative defaults to false in line with the | |
| /// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative | |
| /// version of the matcher. | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t> | |
| m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t> | |
| m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| /// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison | |
| /// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or | |
| /// both. | |
| template <typename Op0_t, typename Op1_t, unsigned... Opcodes> | |
| struct Cmp_match { | |
| static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) && | |
| "Expected one or two opcodes"); | |
| static_assert( | |
| ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) && | |
| "Expected a compare instruction opcode"); | |
| CmpPredicate *Predicate = nullptr; | |
| Op0_t Op0; | |
| Op1_t Op1; | |
| Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) | |
| : Predicate(&Pred), Op0(Op0), Op1(Op1) {} | |
| Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {} | |
| bool match(const VPValue *V) const { | |
| auto *DefR = V->getDefiningRecipe(); | |
| return DefR && match(DefR); | |
| } | |
| bool match(const VPRecipeBase *V) const { | |
| if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) { | |
| if (Predicate) | |
| *Predicate = cast<VPRecipeWithIRFlags>(V)->getPredicate(); | |
| return true; | |
| } | |
| return false; | |
| } | |
| }; | |
| /// SpecificCmp_match is a variant of Cmp_match that matches the comparison | |
| /// predicate, instead of binding it. | |
| template <typename Op0_t, typename Op1_t, unsigned... Opcodes> | |
| struct SpecificCmp_match { | |
| const CmpPredicate Predicate; | |
| Op0_t Op0; | |
| Op1_t Op1; | |
| SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS) | |
| : Predicate(Pred), Op0(LHS), Op1(RHS) {} | |
| bool match(const VPValue *V) const { | |
| auto *DefR = V->getDefiningRecipe(); | |
| return DefR && match(DefR); | |
| } | |
| bool match(const VPRecipeBase *V) const { | |
| CmpPredicate CurrentPred; | |
| return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1) | |
| .match(V) && | |
| CmpPredicate::getMatching(CurrentPred, Predicate); | |
| } | |
| }; | |
| template <typename Op0_t, typename Op1_t> | |
| inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> | |
| m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) { | |
| return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp> | |
| m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) { | |
| return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0, | |
| Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp> | |
| m_Cmp(const Op0_t &Op0, const Op1_t &Op1) { | |
| return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0, | |
| Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp> | |
| m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) { | |
| return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>( | |
| Pred, Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp> | |
| m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) { | |
| return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>( | |
| MatchPred, Op0, Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| using GEPLikeRecipe_match = match_combine_or< | |
| Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr, | |
| /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>, | |
| match_combine_or< | |
| VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>, | |
| VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>>>; | |
| template <typename Op0_t, typename Op1_t> | |
| inline GEPLikeRecipe_match<Op0_t, Op1_t> m_GetElementPtr(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return m_CombineOr( | |
| Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr, | |
| /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>( | |
| Op0, Op1), | |
| m_CombineOr( | |
| VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1), | |
| VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, | |
| Op1))); | |
| } | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t> | |
| m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { | |
| return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>( | |
| {Op0, Op1, Op2}); | |
| } | |
| template <typename Op0_t> | |
| inline match_combine_or<VPInstruction_match<VPInstruction::Not, Op0_t>, | |
| AllRecipe_commutative_match< | |
| Instruction::Xor, int_pred_ty<is_all_ones>, Op0_t>> | |
| m_Not(const Op0_t &Op0) { | |
| return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0), | |
| m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0)); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline match_combine_or< | |
| VPInstruction_match<VPInstruction::LogicalAnd, Op0_t, Op1_t>, | |
| AllRecipe_match<Instruction::Select, Op0_t, Op1_t, specific_intval<1>>> | |
| m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_CombineOr( | |
| m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1), | |
| m_Select(Op0, Op1, m_False())); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| inline AllRecipe_match<Instruction::Select, Op0_t, specific_intval<1>, Op1_t> | |
| m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) { | |
| return m_Select(Op0, m_True(), Op1); | |
| } | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| using VPScalarIVSteps_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, | |
| false, VPScalarIVStepsRecipe>; | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t> | |
| m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { | |
| return VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2}); | |
| } | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| using VPDerivedIV_match = | |
| Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false, VPDerivedIVRecipe>; | |
| template <typename Op0_t, typename Op1_t, typename Op2_t> | |
| inline VPDerivedIV_match<Op0_t, Op1_t, Op2_t> | |
| m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) { | |
| return VPDerivedIV_match<Op0_t, Op1_t, Op2_t>({Op0, Op1, Op2}); | |
| } | |
| template <typename Addr_t, typename Mask_t> struct Load_match { | |
| Addr_t Addr; | |
| Mask_t Mask; | |
| Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {} | |
| template <typename OpTy> bool match(const OpTy *V) const { | |
| auto *Load = dyn_cast<VPWidenLoadRecipe>(V); | |
| if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() || | |
| !Mask.match(Load->getMask())) | |
| return false; | |
| return true; | |
| } | |
| }; | |
| /// Match a (possibly reversed) masked load. | |
| template <typename Addr_t, typename Mask_t> | |
| inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr, | |
| const Mask_t &Mask) { | |
| return Load_match<Addr_t, Mask_t>(Addr, Mask); | |
| } | |
| template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match { | |
| Addr_t Addr; | |
| Val_t Val; | |
| Mask_t Mask; | |
| Store_match(Addr_t Addr, Val_t Val, Mask_t Mask) | |
| : Addr(Addr), Val(Val), Mask(Mask) {} | |
| template <typename OpTy> bool match(const OpTy *V) const { | |
| auto *Store = dyn_cast<VPWidenStoreRecipe>(V); | |
| if (!Store || !Addr.match(Store->getAddr()) || | |
| !Val.match(Store->getStoredValue()) || !Store->isMasked() || | |
| !Mask.match(Store->getMask())) | |
| return false; | |
| return true; | |
| } | |
| }; | |
| /// Match a (possibly reversed) masked store. | |
| template <typename Addr_t, typename Val_t, typename Mask_t> | |
| inline Store_match<Addr_t, Val_t, Mask_t> | |
| m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) { | |
| return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask); | |
| } | |
| template <typename Op0_t, typename Op1_t> | |
| using VectorEndPointerRecipe_match = | |
| Recipe_match<std::tuple<Op0_t, Op1_t>, 0, | |
| /*Commutative*/ false, VPVectorEndPointerRecipe>; | |
| template <typename Op0_t, typename Op1_t> | |
| VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0, | |
| const Op1_t &Op1) { | |
| return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1); | |
| } | |
| /// Match a call argument at a given argument index. | |
| template <typename Opnd_t> struct Argument_match { | |
| /// Call argument index to match. | |
| unsigned OpI; | |
| Opnd_t Val; | |
| Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {} | |
| template <typename OpTy> bool match(OpTy *V) const { | |
| if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V)) | |
| return Val.match(R->getOperand(OpI)); | |
| if (const auto *R = dyn_cast<VPWidenCallRecipe>(V)) | |
| return Val.match(R->getOperand(OpI)); | |
| if (const auto *R = dyn_cast<VPReplicateRecipe>(V)) | |
| if (R->getOpcode() == Instruction::Call) | |
| return Val.match(R->getOperand(OpI)); | |
| if (const auto *R = dyn_cast<VPInstruction>(V)) | |
| if (R->getOpcode() == Instruction::Call) | |
| return Val.match(R->getOperand(OpI)); | |
| return false; | |
| } | |
| }; | |
| /// Match a call argument. | |
| template <unsigned OpI, typename Opnd_t> | |
| inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) { | |
| return Argument_match<Opnd_t>(OpI, Op); | |
| } | |
| /// Intrinsic matchers. | |
| struct IntrinsicID_match { | |
| unsigned ID; | |
| IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {} | |
| template <typename OpTy> bool match(OpTy *V) const { | |
| if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V)) | |
| return R->getVectorIntrinsicID() == ID; | |
| if (const auto *R = dyn_cast<VPWidenCallRecipe>(V)) | |
| return R->getCalledScalarFunction()->getIntrinsicID() == ID; | |
| auto MatchCalleeIntrinsic = [&](VPValue *CalleeOp) { | |
| if (!isa<VPIRValue>(CalleeOp)) | |
| return false; | |
| auto *F = cast<Function>(CalleeOp->getLiveInIRValue()); | |
| return F->getIntrinsicID() == ID; | |
| }; | |
| if (const auto *R = dyn_cast<VPReplicateRecipe>(V)) | |
| if (R->getOpcode() == Instruction::Call) { | |
| // The mask is always the last operand if predicated. | |
| return MatchCalleeIntrinsic( | |
| R->getOperand(R->getNumOperands() - 1 - R->isPredicated())); | |
| } | |
| if (const auto *R = dyn_cast<VPInstruction>(V)) | |
| if (R->getOpcode() == Instruction::Call) | |
| return MatchCalleeIntrinsic(R->getOperand(R->getNumOperands() - 1)); | |
| return false; | |
| } | |
| }; | |
| /// Intrinsic matches are combinations of ID matchers, and argument | |
| /// matchers. Higher arity matcher are defined recursively in terms of and-ing | |
| /// them with lower arity matchers. Here's some convenient typedefs for up to | |
| /// several arguments, and more can be added as needed | |
| template <typename T0 = void, typename T1 = void, typename T2 = void, | |
| typename T3 = void> | |
| struct m_Intrinsic_Ty; | |
| template <typename T0> struct m_Intrinsic_Ty<T0> { | |
| using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>; | |
| }; | |
| template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> { | |
| using Ty = | |
| match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>; | |
| }; | |
| template <typename T0, typename T1, typename T2> | |
| struct m_Intrinsic_Ty<T0, T1, T2> { | |
| using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty, | |
| Argument_match<T2>>; | |
| }; | |
| template <typename T0, typename T1, typename T2, typename T3> | |
| struct m_Intrinsic_Ty { | |
| using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty, | |
| Argument_match<T3>>; | |
| }; | |
| /// Match intrinsic calls like this: | |
| /// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...) | |
| template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() { | |
| return IntrinsicID_match(IntrID); | |
| } | |
| /// Match intrinsic calls with a runtime intrinsic ID. | |
| inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) { | |
| return IntrinsicID_match(IntrID); | |
| } | |
| template <Intrinsic::ID IntrID, typename T0> | |
| inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) { | |
| return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0)); | |
| } | |
| template <Intrinsic::ID IntrID, typename T0, typename T1> | |
| inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0, | |
| const T1 &Op1) { | |
| return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1)); | |
| } | |
| template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2> | |
| inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty | |
| m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) { | |
| return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2)); | |
| } | |
| template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2, | |
| typename T3> | |
| inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty | |
| m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) { | |
| return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3)); | |
| } | |
| inline auto m_LiveIn() { return class_match<VPIRValue, VPSymbolicValue>(); } | |
| /// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe) | |
| /// and bind the source element type and operands. | |
| struct GetElementPtr_match { | |
| Type *&SourceElementType; | |
| ArrayRef<VPValue *> &Operands; | |
| GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands) | |
| : SourceElementType(SourceElementType), Operands(Operands) {} | |
| template <typename ITy> bool match(ITy *V) const { | |
| return matchRecipeAndBind<VPWidenGEPRecipe>(V) || | |
| matchRecipeAndBind<VPInstruction>(V) || | |
| matchRecipeAndBind<VPReplicateRecipe>(V); | |
| } | |
| private: | |
| template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const { | |
| auto *DefR = dyn_cast<RecipeTy>(V); | |
| if (!DefR) | |
| return false; | |
| if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) { | |
| SourceElementType = DefR->getSourceElementType(); | |
| } else if (DefR->getOpcode() == Instruction::GetElementPtr) { | |
| SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr()) | |
| ->getSourceElementType(); | |
| } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) { | |
| if (DefR->getOpcode() == VPInstruction::PtrAdd) { | |
| // PtrAdd is a byte-offset GEP with i8 element type. | |
| LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext(); | |
| SourceElementType = Type::getInt8Ty(Ctx); | |
| } else { | |
| return false; | |
| } | |
| } else { | |
| return false; | |
| } | |
| Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end()); | |
| return true; | |
| } | |
| }; | |
| /// Match a GEP recipe with any number of operands and bind source element type | |
| /// and operands. | |
| inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType, | |
| ArrayRef<VPValue *> &Operands) { | |
| return GetElementPtr_match(SourceElementType, Operands); | |
| } | |
| template <typename SubPattern_t> struct OneUse_match { | |
| SubPattern_t SubPattern; | |
| OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {} | |
| template <typename OpTy> bool match(OpTy *V) { | |
| return V->hasOneUse() && SubPattern.match(V); | |
| } | |
| }; | |
| template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) { | |
| return SubPattern; | |
| } | |
| inline bind_ty<VPReductionPHIRecipe> m_ReductionPhi(VPReductionPHIRecipe *&V) { | |
| return V; | |
| } | |
| } // namespace llvm::VPlanPatternMatch | |
| #endif |